bes  Updated for version 3.20.10
NgapContainer.cc
1 // NgapContainer.cc
2 
3 // -*- mode: c++; c-basic-offset:4 -*-
4 
5 // This file is part of ngap_module, A C++ module that can be loaded in to
6 // the OPeNDAP Back-End Server (BES) and is able to handle remote requests.
7 
8 // Copyright (c) 2020 OPeNDAP, Inc.
9 // Author: Nathan Potter <ndp@opendap.org>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 // Authors:
27 // ndp Nathan Potter <ndp@opendap.org>
28 
29 #include "config.h"
30 
31 #include <cstdio>
32 #include <map>
33 #include <sstream>
34 #include <string>
35 #include <fstream>
36 #include <streambuf>
37 #include <time.h>
38 
39 #include "BESStopWatch.h"
40 #include "BESLog.h"
41 #include "BESSyntaxUserError.h"
42 #include "BESNotFoundError.h"
43 #include "BESInternalError.h"
44 #include "BESDebug.h"
45 #include "BESUtil.h"
46 #include "TheBESKeys.h"
47 #include "AllowedHosts.h"
48 #include "BESContextManager.h"
49 #include "CurlUtils.h"
50 #include "HttpUtils.h"
51 #include "RemoteResource.h"
52 #include "url_impl.h"
53 
54 #include "NgapContainer.h"
55 #include "NgapApi.h"
56 #include "NgapNames.h"
57 
58 #define prolog std::string("NgapContainer::").append(__func__).append("() - ")
59 
60 using namespace std;
61 using namespace bes;
62 
63 namespace ngap {
64 
75 NgapContainer::NgapContainer(const string &sym_name,
76  const string &real_name,
77  const string &type) :
78  BESContainer(sym_name, real_name, type),
79  d_dmrpp_rresource(0) {
80  initialize();
81 }
82 
83 
84 void NgapContainer::initialize()
85 {
86  BESDEBUG(MODULE, prolog << "BEGIN (obj_addr: "<< (void *) this << ")" << endl);
87  BESDEBUG(MODULE, prolog << "sym_name: "<< get_symbolic_name() << endl);
88  BESDEBUG(MODULE, prolog << "real_name: "<< get_real_name() << endl);
89  BESDEBUG(MODULE, prolog << "type: "<< get_container_type() << endl);
90 
91  bool found;
92 
93  NgapApi ngap_api;
94  if (get_container_type().empty())
95  set_container_type("ngap");
96 
97  string uid = BESContextManager::TheManager()->get_context(EDL_UID_KEY, found);
98  BESDEBUG(MODULE, prolog << "EDL_UID_KEY(" << EDL_UID_KEY << "): " << uid << endl);
99 
100  string data_access_url = ngap_api.convert_ngap_resty_path_to_data_access_url(get_real_name(), uid);
101 
102  set_real_name(data_access_url);
103 // Because we know the name is really a URL, then we know the "relative_name" is meaningless
104 // So we set it to be the same as "name"
105  set_relative_name(data_access_url);
106  BESDEBUG(MODULE, prolog << "END (obj_addr: "<< (void *) this << ")" << endl);
107 
108 }
109 
113 NgapContainer::NgapContainer(const NgapContainer &copy_from) :
114  BESContainer(copy_from),
115  d_dmrpp_rresource(copy_from.d_dmrpp_rresource) {
116  BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << " Copying from: " << (void *) &copy_from << endl);
117  // we can not make a copy of this container once the request has
118  // been made
119  if (d_dmrpp_rresource) {
120  string err = (string) "The Container has already been accessed, "
121  + "can not create a copy of this container.";
122  throw BESInternalError(err, __FILE__, __LINE__);
123  }
124  BESDEBUG(MODULE, prolog << "object address: "<< (void *) this << endl);
125 }
126 
127 void NgapContainer::_duplicate(NgapContainer &copy_to) {
128  if (copy_to.d_dmrpp_rresource) {
129  string err = (string) "The Container has already been accessed, "
130  + "can not duplicate this resource.";
131  throw BESInternalError(err, __FILE__, __LINE__);
132  }
133  BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << " Copying to: " << (void *) &copy_to << endl);
134  copy_to.d_dmrpp_rresource = d_dmrpp_rresource;
135  BESContainer::_duplicate(copy_to);
136 }
137 
138 BESContainer *
140  NgapContainer *container = new NgapContainer;
141  _duplicate(*container);
142  BESDEBUG(MODULE, prolog << "object address: "<< (void *) this << " to: " << (void *)container << endl);
143  return container;
144 }
145 
146 NgapContainer::~NgapContainer() {
147  BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << endl);
148  if (d_dmrpp_rresource) {
149  release();
150  }
151  BESDEBUG(MODULE, prolog << "END object address: "<< (void *) this << endl);
152 }
153 
154 
155 
162  BESDEBUG(MODULE, prolog << "BEGIN (obj_addr: "<< (void *) this << ")" << endl);
163 
164  // Since this the ngap we know that the real_name is a URL.
165  string data_access_url_str = get_real_name();
166 
167  // And we know that the dmr++ file should "right next to it" (side-car)
168  string dmrpp_url_str = data_access_url_str + ".dmrpp";
169 
170  // And if there's a missing data file (side-car) it should be "right there" too.
171  string missing_data_url_str = data_access_url_str + ".missing";
172 
173  BESDEBUG(MODULE, prolog << " data_access_url: " << data_access_url_str << endl);
174  BESDEBUG(MODULE, prolog << " dmrpp_url: " << dmrpp_url_str << endl);
175  BESDEBUG(MODULE, prolog << "missing_data_url: " << missing_data_url_str << endl);
176 
177  // TODO 10/8/21 Is this a syntax error? Should the \" be after 'true'. jhrg
178  string trusted_url_hack="\" dmrpp:trust=\"true";
179  string data_access_url_with_trusted_attr_str = data_access_url_str + trusted_url_hack;
180  string dmrpp_url_with_trusted_attr_str = dmrpp_url_str + trusted_url_hack;
181  string missing_data_url_with_trusted_attr_str = missing_data_url_str + trusted_url_hack;
182 
183  BESDEBUG(MODULE, prolog << " data_access_url_with_trusted_attr_str: " << data_access_url_with_trusted_attr_str << endl);
184  BESDEBUG(MODULE, prolog << " dmrpp_url_with_trusted_attr_str: " << dmrpp_url_with_trusted_attr_str << endl);
185  BESDEBUG(MODULE, prolog << "missing_data_url_with_trusted_attr_str: " << missing_data_url_with_trusted_attr_str << endl);
186 
187  string type = get_container_type();
188  if (type == "ngap")
189  type = "";
190 
191  if (!d_dmrpp_rresource) {
192  BESDEBUG(MODULE, prolog << "Building new RemoteResource (dmr++)." << endl);
193  map<string,string> content_filters;
194  if (inject_data_url()) {
195  content_filters.insert(pair<string,string>(DATA_ACCESS_URL_KEY,data_access_url_with_trusted_attr_str));
196  content_filters.insert(pair<string,string>(MISSING_DATA_ACCESS_URL_KEY,missing_data_url_with_trusted_attr_str));
197  }
198  shared_ptr<http::url> dmrpp_url(new http::url(dmrpp_url_str, true));
199  {
200  d_dmrpp_rresource = new http::RemoteResource(dmrpp_url);
201  BESStopWatch besTimer;
202  if (BESISDEBUG(MODULE) || BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()){
203  besTimer.start("DMR++ retrieval: "+ dmrpp_url->str());
204  }
205  d_dmrpp_rresource->retrieveResource(content_filters);
206  }
207  BESDEBUG(MODULE, prolog << "Retrieved remote resource: " << dmrpp_url->str() << endl);
208  }
209 
210  // TODO This file should be read locked before leaving this method.
211  // 10/8/21 I think the RemoteResource should do that. jhrg
212  string cachedResource = d_dmrpp_rresource->getCacheFileName();
213  BESDEBUG(MODULE, prolog << "Using local cache file: " << cachedResource << endl);
214 
215  type = d_dmrpp_rresource->getType();
216  set_container_type(type);
217  BESDEBUG(MODULE, prolog << "Type: " << type << endl);
218  BESDEBUG(MODULE, prolog << "Done retrieving: " << dmrpp_url_str << " returning cached file " << cachedResource << endl);
219  BESDEBUG(MODULE, prolog << "END (obj_addr: "<< (void *) this << ")" << endl);
220 
221  return cachedResource; // this should return the dmr++ file name from the NgapCache
222 }
223 
224 
232  // TODO The cache file (that will be) read locked in the access() method must be unlocked here.
233  // If we make that part of the RemoteResource dtor, the unlock will happen here. jhrg
234  if (d_dmrpp_rresource) {
235  BESDEBUG(MODULE, prolog << "Releasing RemoteResource" << endl);
236  delete d_dmrpp_rresource;
237  d_dmrpp_rresource = 0;
238  }
239 
240  BESDEBUG(MODULE, prolog << "Done releasing Ngap response" << endl);
241  return true;
242 }
243 
251 void NgapContainer::dump(ostream &strm) const {
252  strm << BESIndent::LMarg << "NgapContainer::dump - (" << (void *) this
253  << ")" << endl;
254  BESIndent::Indent();
255  BESContainer::dump(strm);
256  if (d_dmrpp_rresource) {
257  strm << BESIndent::LMarg << "RemoteResource.getCacheFileName(): " << d_dmrpp_rresource->getCacheFileName()
258  << endl;
259  strm << BESIndent::LMarg << "response headers: ";
260  vector<string> *hdrs = d_dmrpp_rresource->getResponseHeaders();
261  if (hdrs) {
262  strm << endl;
263  BESIndent::Indent();
264  vector<string>::const_iterator i = hdrs->begin();
265  vector<string>::const_iterator e = hdrs->end();
266  for (; i != e; i++) {
267  string hdr_line = (*i);
268  strm << BESIndent::LMarg << hdr_line << endl;
269  }
270  BESIndent::UnIndent();
271  } else {
272  strm << "none" << endl;
273  }
274  } else {
275  strm << BESIndent::LMarg << "response not yet obtained" << endl;
276  }
277  BESIndent::UnIndent();
278 }
279 
280 bool NgapContainer::inject_data_url(){
281  bool result = false;
282  bool found;
283  string key_value;
284  TheBESKeys::TheKeys()->get_value(NGAP_INJECT_DATA_URL_KEY, key_value, found);
285  if (found && key_value == "true") {
286  result = true;
287  }
288  BESDEBUG(MODULE, prolog << "NGAP_INJECT_DATA_URL_KEY(" << NGAP_INJECT_DATA_URL_KEY << "): " << result << endl);
289  return result;
290 }
291 }
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
void set_container_type(const std::string &type)
set the type of data that this container represents, such as cedar or netcdf.
Definition: BESContainer.h:161
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual void dump(std::ostream &strm) const
dumps information about this object
Definition: BESContainer.cc:73
void set_real_name(const std::string &real_name)
set the real name for this container, such as a file name if reading a data file.
Definition: BESContainer.h:146
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
void set_relative_name(const std::string &relative)
Set the relative name of the object in this container.
Definition: BESContainer.h:152
void _duplicate(BESContainer &copy_to)
duplicate this instance into the passed container
Definition: BESContainer.cc:54
std::string get_real_name() const
retrieve the real name for this container, such as a file name.
Definition: BESContainer.h:180
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
exception thrown if internal error encountered
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
std::string getCacheFileName()
std::vector< std::string > * getResponseHeaders()
std::string getType()
std::string convert_ngap_resty_path_to_data_access_url(const std::string &restified_path, const std::string &uid="")
Converts an NGAP restified granule path into a CMR metadata query for the granule.
Definition: NgapApi.cc:423
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual std::string access()
access the remote target response by making the remote request
virtual BESContainer * ptr_duplicate()
pure abstract method to duplicate this instances of BESContainer
virtual bool release()
release the resources