bes  Updated for version 3.19.1
BESDapResponseBuilder.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 #include <signal.h>
28 #include <unistd.h>
29 #include <sys/stat.h>
30 
31 #ifdef HAVE_UUID_UUID_H
32 #include <uuid/uuid.h> // used to build CID header value for data ddx
33 #elif defined(HAVE_UUID_H)
34 #include <uuid.h>
35 #else
36 #error "Could not find UUID library header"
37 #endif
38 
39 
40 #ifndef WIN32
41 #include <sys/wait.h>
42 #else
43 #include <io.h>
44 #include <fcntl.h>
45 #include <process.h>
46 #endif
47 
48 #include <iostream>
49 #include <string>
50 #include <sstream>
51 #include <fstream>
52 
53 #include <cstring>
54 #include <ctime>
55 
56 //#define DODS_DEBUG
57 #define CLEAR_LOCAL_DATA
58 #undef USE_LOCAL_TIMEOUT_SCHEME
59 
60 #include <DAS.h>
61 #include <DDS.h>
62 #include <Structure.h>
63 #include <ConstraintEvaluator.h>
64 #include <DDXParserSAX2.h>
65 #include <Ancillary.h>
66 #include <XDRStreamMarshaller.h>
67 #include <XDRFileUnMarshaller.h>
68 
69 #include <DMR.h>
70 #include <D4Group.h>
71 #include <XMLWriter.h>
72 #include <D4AsyncUtil.h>
73 #include <D4StreamMarshaller.h>
74 #include <chunked_ostream.h>
75 #include <chunked_istream.h>
76 #include <D4ConstraintEvaluator.h>
77 #include <D4FunctionEvaluator.h>
78 #include <D4BaseTypeFactory.h>
79 
80 #include <ServerFunctionsList.h>
81 
82 #include <mime_util.h> // for last_modified_time() and rfc_822_date()
83 #include <escaping.h>
84 #include <util.h>
85 #if USE_LOCAL_TIMEOUT_SCHEME
86 #ifndef WIN32
87 #include <SignalHandler.h>
88 #include <EventHandler.h>
89 #include <AlarmHandler.h>
90 #endif
91 #endif
92 
93 #include "TheBESKeys.h"
94 #include "BESDapResponseBuilder.h"
95 #include "BESContextManager.h"
96 #include "BESDapFunctionResponseCache.h"
97 #include "BESStoredDapResultCache.h"
98 
99 #include "BESResponseObject.h"
100 #include "BESDDSResponse.h"
101 #include "BESDataDDSResponse.h"
102 #include "BESDataHandlerInterface.h"
103 #include "BESInternalFatalError.h"
104 #include "BESDataNames.h"
105 
106 #include "BESUtil.h"
107 #include "BESDebug.h"
108 #include "BESStopWatch.h"
109 #include "DapFunctionUtils.h"
110 
111 using namespace std;
112 using namespace libdap;
113 
114 const string CRLF = "\r\n"; // Change here, expr-test.cc
115 const string BES_KEY_TIMEOUT_CANCEL = "BES.CancelTimeoutOnSend";
116 
122 {
123  bool found = false;
124  string cancel_timeout_on_send = "";
125  TheBESKeys::TheKeys()->get_value(BES_KEY_TIMEOUT_CANCEL, cancel_timeout_on_send, found);
126  if (found && !cancel_timeout_on_send.empty()) {
127  // The default value is false.
128  downcase(cancel_timeout_on_send);
129  if (cancel_timeout_on_send == "yes" || cancel_timeout_on_send == "true")
130  d_cancel_timeout_on_send = true;
131  }
132 }
133 
134 BESDapResponseBuilder::~BESDapResponseBuilder()
135 {
136 #if USE_LOCAL_TIMEOUT_SCHEME
137  // If an alarm was registered, delete it. The register code in SignalHandler
138  // always deletes the old alarm handler object, so only the one returned by
139  // remove_handler needs to be deleted at this point.
140  delete dynamic_cast<AlarmHandler*>(SignalHandler::instance()->remove_handler(SIGALRM));
141 #endif
142 }
143 
151 {
152  return d_dap2ce;
153 }
154 
166 {
167  d_dap2ce = www2id(_ce, "%", "%20");
168 }
169 
174 {
175  return d_dap4ce;
176 }
177 
189 {
190  d_dap4ce = www2id(_ce, "%", "%20");
191 }
192 
197 {
198  return d_dap4function;
199 }
200 
213 {
214  d_dap4function = www2id(_func, "%", "%20");
215 }
216 
217 std::string BESDapResponseBuilder::get_store_result() const
218 {
219  return d_store_result;
220 }
221 
222 void BESDapResponseBuilder::set_store_result(std::string _sr)
223 {
224  d_store_result = _sr;
225  BESDEBUG("dap", "BESDapResponseBuilder::set_store_result() - store_result: " << _sr << endl);
226 }
227 
228 std::string BESDapResponseBuilder::get_async_accepted() const
229 {
230  return d_async_accepted;
231 }
232 
233 void BESDapResponseBuilder::set_async_accepted(std::string _aa)
234 {
235  d_async_accepted = _aa;
236  BESDEBUG("dap", "BESDapResponseBuilder::set_async_accepted() - async_accepted: " << _aa << endl);
237 }
238 
248 {
249  return d_dataset;
250 }
251 
263 {
264  d_dataset = www2id(ds, "%", "%20");
265 }
266 
273 {
274  d_timeout = t;
275 }
276 
279 {
280  return d_timeout;
281 }
282 
289 void
291 {
292 #if USE_LOCAL_TIMEOUT_SCHEME
293 #ifndef WIN32
294  alarm(d_timeout);
295 #endif
296 #endif
297 }
298 
304 void
306 {
307 #if USE_LOCAL_TIMEOUT_SCHEME
308 #ifndef WIN32
309  alarm(0);
310 #endif
311 #endif
312 }
313 
329 {
330  if (d_cancel_timeout_on_send)
331  alarm(0);
332 }
333 
343 {
344 #if USE_LOCAL_TIMEOUT_SCHEME
345 #ifndef WIN32
346  SignalHandler *sh = SignalHandler::instance();
347  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
348  delete old_eh;
349 #endif
350 #endif
351 }
352 
353 
360 {
361 #if USE_LOCAL_TIMEOUT_SCHEME
362 #ifndef WIN32
363  if (d_timeout > 0) {
364  SignalHandler *sh = SignalHandler::instance();
365  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
366  delete old_eh;
367  alarm(d_timeout);
368  }
369 #endif
370 #endif
371 }
372 
387 static string::size_type find_closing_paren(const string &ce, string::size_type pos)
388 {
389  // Iterate over the string finding all ( or ) characters until the matching ) is found.
390  // For each ( found, increment count. When a ) is found and count is zero, it is the
391  // matching closing paren, otherwise, decrement count and keep looking.
392  int count = 1;
393  do {
394  pos = ce.find_first_of("()", pos + 1);
395  if (pos == string::npos)
396  throw Error(malformed_expr, "Expected to find a matching closing parenthesis in " + ce);
397 
398  if (ce[pos] == '(')
399  ++count;
400  else
401  --count; // must be ')'
402 
403  } while (count > 0);
404 
405  return pos;
406 }
407 
414 void BESDapResponseBuilder::split_ce(ConstraintEvaluator &eval, const string &expr)
415 {
416  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - source expression: " << expr << endl);
417 
418  string ce;
419  if (!expr.empty())
420  ce = expr;
421  else
422  ce = d_dap2ce;
423 
424  string btp_function_ce = "";
425  string::size_type pos = 0;
426 
427  // This hack assumes that the functions are listed first. Look for the first
428  // open paren and the last closing paren to accommodate nested function calls
429  string::size_type first_paren = ce.find("(", pos);
430  string::size_type closing_paren = string::npos;
431  if (first_paren != string::npos) closing_paren = find_closing_paren(ce, first_paren); //ce.find(")", pos);
432 
433  while (first_paren != string::npos && closing_paren != string::npos) {
434  // Maybe a BTP function; get the name of the potential function
435  string name = ce.substr(pos, first_paren - pos);
436 
437  // is this a BTP function
438  btp_func f;
439  if (eval.find_function(name, &f)) {
440  // Found a BTP function
441  if (!btp_function_ce.empty()) btp_function_ce += ",";
442  btp_function_ce += ce.substr(pos, closing_paren + 1 - pos);
443  ce.erase(pos, closing_paren + 1 - pos);
444  if (ce[pos] == ',') ce.erase(pos, 1);
445  }
446  else {
447  pos = closing_paren + 1;
448  // exception?
449  if (pos < ce.length() && ce.at(pos) == ',') ++pos;
450  }
451 
452  first_paren = ce.find("(", pos);
453  closing_paren = ce.find(")", pos);
454  }
455 
456  d_dap2ce = ce;
457  d_btp_func_ce = btp_function_ce;
458 
459  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - Modified constraint: " << d_dap2ce << endl);
460  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - BTP Function part: " << btp_function_ce << endl);
461  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - END" << endl);
462 }
463 
478 void BESDapResponseBuilder::send_das(ostream &out, DAS &das, bool with_mime_headers) const
479 {
480  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
481 
482  das.print(out);
483 
484  out << flush;
485 }
486 
504 void BESDapResponseBuilder::send_das(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
505  bool with_mime_headers)
506 {
507 #if USE_LOCAL_TIMEOUT_SCHEME
508  // Set up the alarm.
509  establish_timeout(out);
510  dds.set_timeout(d_timeout);
511 #endif
512  if (!constrained) {
513  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
514 
515  conditional_timeout_cancel();
516 
517  (*dds)->print_das(out);
518  out << flush;
519 
520  return;
521  }
522 
523  split_ce(eval);
524 
525  // If there are functions, parse them and eval.
526  // Use that DDS and parse the non-function ce
527  // Serialize using the second ce and the second dds
528  if (!d_btp_func_ce.empty()) {
529  ConstraintEvaluator func_eval;
530  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
531 
532  DDS *fdds = 0; // nulll_ptr
533  if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
534  fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
535  }
536  else {
537  func_eval.parse_constraint(get_btp_func_ce(), **dds);
538  fdds = func_eval.eval_function_clauses(**dds);
539  }
540 
541  delete *dds; *dds = 0;
542  *dds = fdds;
543 
544  if (with_mime_headers)
545  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
546 
547  conditional_timeout_cancel();
548 
549  (*dds)->print_das(out);
550  }
551  else {
552  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
553 
554  if (with_mime_headers)
555  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
556 
557  conditional_timeout_cancel();
558 
559  (*dds)->print_das(out);
560  }
561 
562  out << flush;
563 }
564 
565 
584 void BESDapResponseBuilder::send_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
585  bool with_mime_headers)
586 {
587  if (!constrained) {
588  if (with_mime_headers)
589  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
590 
591  conditional_timeout_cancel();
592 
593  (*dds)->print(out);
594  out << flush;
595  return;
596  }
597 
598 #if USE_LOCAL_TIMEOUT_SCHEME
599  // Set up the alarm.
600  establish_timeout(out);
601  dds.set_timeout(d_timeout);
602 #endif
603 
604  // Split constraint into two halves
605  split_ce(eval);
606 
607  // If there are functions, parse them and eval.
608  // Use that DDS and parse the non-function ce
609  // Serialize using the second ce and the second dds
610  if (!d_btp_func_ce.empty()) {
611  ConstraintEvaluator func_eval;
612 
613  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
614 
615  DDS *fdds = 0; // nulll_ptr
616  if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
617  fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
618  }
619  else {
620  func_eval.parse_constraint(get_btp_func_ce(), **dds);
621  fdds = func_eval.eval_function_clauses(**dds);
622  }
623 
624  delete *dds; *dds = 0;
625  *dds = fdds;
626 
627  // Server functions might mark variables to use their read()
628  // methods. Clear that so the CE in d_dap2ce will control what is
629  // sent. If that is empty (there was only a function call) all
630  // of the variables in the intermediate DDS (i.e., the function
631  // result) will be sent.
632  (*dds)->mark_all(false);
633 
634  // This next step utilizes a well known static method (so really it's a function;),
635  // promote_function_output_structures() to look for
636  // one or more top level Structures whose name indicates (by way of ending with
637  // "_uwrap") that their contents should be promoted (aka moved) to the top level.
638  // This is in support of a hack around the current API where server side functions
639  // may only return a single DAP object and not a collection of objects. The name suffix
640  // "_unwrap" is used as a signal from the function to the the various response
641  // builders and transmitters that the representation needs to be altered before
642  // transmission, and that in fact is what happens in our friend
643  // promote_function_output_structures()
644  promote_function_output_structures(*dds);
645 
646  eval.parse_constraint(d_dap2ce, **dds);
647 
648  if (with_mime_headers)
649  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
650 
651 
652  conditional_timeout_cancel();
653 
654  (*dds)->print_constrained(out);
655  }
656  else {
657  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
658 
659  if (with_mime_headers)
660  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset),(*dds)->get_dap_version());
661 
662  conditional_timeout_cancel();
663 
664  (*dds)->print_constrained(out);
665  }
666 
667  out << flush;
668 }
669 
670 #ifdef DAP2_STORED_RESULTS
671 
685 bool BESDapResponseBuilder::store_dap2_result(ostream &out, DDS &dds, ConstraintEvaluator &eval)
686 {
687  if (get_store_result().empty()) return false;
688 
689  string serviceUrl = get_store_result();
690 
691  XMLWriter xmlWrtr;
692  D4AsyncUtil d4au;
693 
694  // FIXME Keys should be read in initialize(). Also, I think the D4AsyncUtil should
695  // be removed from libdap - it is much more about how the BES processes these kinds
696  // of operations. Change this when working on the response caching for ODSIP. But...
697  // do we really need to put the style sheet in the bes.conf file? Should it be baked
698  // into the code (because we don't want people to change it)?
699  bool found;
700  string *stylesheet_ref = 0, ss_ref_value;
701  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
702  if (found && ss_ref_value.length() > 0) {
703  stylesheet_ref = &ss_ref_value;
704  }
705 
707  if (resultCache == NULL) {
708 
714  string msg = "The Stored Result request cannot be serviced. ";
715  msg += "Unable to acquire StoredResultCache instance. ";
716  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
717 
718  BESDEBUG("dap", "[WARNING] " << msg << endl);
719 
720  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
721  out << xmlWrtr.get_doc();
722  out << flush;
723 
724  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - Sent AsyncRequestRejected" << endl);
725  }
726  else if (get_async_accepted().length() != 0) {
727 
731  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - serviceUrl="<< serviceUrl << endl);
732 
734  string storedResultId = "";
735  storedResultId = resultCache->store_dap2_result(dds, get_ce(), this, &eval);
736 
737  BESDEBUG("dap",
738  "BESDapResponseBuilder::store_dap2_result() - storedResultId='"<< storedResultId << "'" << endl);
739 
740  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
741  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - targetURL='"<< targetURL << "'" << endl);
742 
743  XMLWriter xmlWrtr;
744  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
745  out << xmlWrtr.get_doc();
746  out << flush;
747 
748  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - sent DAP4 AsyncAccepted response" << endl);
749  }
750  else {
755  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
756  out << xmlWrtr.get_doc();
757  out << flush;
758 
759  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - sent DAP4 AsyncRequired response" << endl);
760  }
761 
762  return true;
763 
764 }
765 #endif
766 
770 void BESDapResponseBuilder::serialize_dap2_data_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool ce_eval)
771 {
772  BESStopWatch sw;
773  if (BESISDEBUG(TIMING_LOG)) sw.start("BESDapResponseBuilder::serialize_dap2_data_dds", "");
774 
775  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap2_data_dds() - BEGIN" << endl);
776 
777  (*dds)->print_constrained(out);
778  out << "Data:\n";
779  out << flush;
780 
781  XDRStreamMarshaller m(out);
782 
783  // This only has an effect when the timeout in BESInterface::execute_request()
784  // is set. Otherwise it does nothing.
785  conditional_timeout_cancel();
786 
787  // Send all variables in the current projection (send_p())
788  for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
789  if ((*i)->send_p()) {
790  (*i)->serialize(eval, **dds, m, ce_eval);
791 #ifdef CLEAR_LOCAL_DATA
792  (*i)->clear_local_data();
793 #endif
794  }
795  }
796 
797  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap2_data_dds() - END" << endl);
798 }
799 
800 #ifdef DAP2_STORED_RESULTS
801 
809 void BESDapResponseBuilder::serialize_dap2_data_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval,
810  const string &boundary, const string &start, bool ce_eval)
811 {
812  BESDEBUG("dap", __PRETTY_FUNCTION__ << " BEGIN" << endl);
813 
814  // Write the MPM headers for the DDX (text/xml) part of the response
815  libdap::set_mime_ddx_boundary(out, boundary, start, dods_ddx, x_plain);
816 
817  // Make cid
818  uuid_t uu;
819  uuid_generate(uu);
820  char uuid[37];
821  uuid_unparse(uu, &uuid[0]);
822  char domain[256];
823  if (getdomainname(domain, 255) != 0 || strlen(domain) == 0) strncpy(domain, "opendap.org", 255);
824 
825  string cid = string(&uuid[0]) + "@" + string(&domain[0]);
826 
827  // Send constrained DDX with a data blob reference.
828  // Note: CID passed but ignored jhrg 10/20/15
829  (*dds)->print_xml_writer(out, true, cid);
830 
831  // write the data part mime headers here
832  set_mime_data_boundary(out, boundary, cid, dods_data_ddx /* old value dap4_data*/, x_plain);
833 
834  XDRStreamMarshaller m(out);
835 
836  conditional_timeout_cancel();
837 
838 
839  // Send all variables in the current projection (send_p()).
840  for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
841  if ((*i)->send_p()) {
842  (*i)->serialize(eval, **dds, m, ce_eval);
843 #ifdef CLEAR_LOCAL_DATA
844  (*i)->clear_local_data();
845 #endif
846  }
847  }
848 
849  BESDEBUG("dap", __PRETTY_FUNCTION__ << " END" << endl);
850 }
851 #endif
852 
870 {
871 #if USE_LOCAL_TIMEOUT_SCHEME
872  alarm(0);
873 #endif
874 }
875 
889 libdap::DDS *
891 {
892  BESDEBUG("dap", "BESDapResponseBuilder::process_dap2_dds() - BEGIN"<< endl);
893 
894  dhi.first_container();
895 
896  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(obj);
897  if (!bdds) throw BESInternalFatalError("Expected a BESDDSResponse instance", __FILE__, __LINE__);
898 
899  DDS *dds = bdds->get_dds();
900 
901  set_dataset_name(dds->filename());
902  set_ce(dhi.data[POST_CONSTRAINT]);
903  set_async_accepted(dhi.data[ASYNC]);
904  set_store_result(dhi.data[STORE_RESULT]);
905 
906  ConstraintEvaluator &eval = bdds->get_ce();
907 
908  // Split constraint into two halves
909  split_ce(eval);
910 
911  // If there are functions, parse them and eval.
912  // Use that DDS and parse the non-function ce
913  // Serialize using the second ce and the second dds
914  if (!d_btp_func_ce.empty()) {
915  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
916 
917  ConstraintEvaluator func_eval;
918  DDS *fdds = 0; // nulll_ptr
919  if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
920  fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
921  }
922  else {
923  func_eval.parse_constraint(get_btp_func_ce(), *dds);
924  fdds = func_eval.eval_function_clauses(*dds);
925  }
926 
927  delete dds; // Delete so that we can ...
928  bdds->set_dds(fdds); // Transfer management responsibility
929  dds = fdds;
930 
931  dds->mark_all(false);
932 
933  promote_function_output_structures(dds);
934  }
935 
936  eval.parse_constraint(d_dap2ce, *dds); // Throws Error if the ce doesn't parse.
937 
938  return dds;
939 }
940 
958 libdap::DDS *
960 {
961  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap2_data() - BEGIN"<< endl);
962 
963  dhi.first_container();
964 
965  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(obj);
966  if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
967 
968  DDS *dds = bdds->get_dds();
969 
970  set_dataset_name(dds->filename());
971  set_ce(dhi.data[POST_CONSTRAINT]);
972  set_async_accepted(dhi.data[ASYNC]);
973  set_store_result(dhi.data[STORE_RESULT]);
974 
975  ConstraintEvaluator &eval = bdds->get_ce();
976 
977  // Split constraint into two halves; stores the function and non-function parts in this instance.
978  split_ce(eval);
979 
980  // If there are functions, parse them and eval.
981  // Use that DDS and parse the non-function ce
982  // Serialize using the second ce and the second dds
983  if (!get_btp_func_ce().empty()) {
984  BESDEBUG("dap",
985  "BESDapResponseBuilder::intern_dap2_data() - Found function(s) in CE: " << get_btp_func_ce() << endl);
986 
987  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
988 
989  ConstraintEvaluator func_eval;
990  DDS *fdds = 0; // nulll_ptr
991  if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
992  fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
993  }
994  else {
995  func_eval.parse_constraint(get_btp_func_ce(), *dds);
996  fdds = func_eval.eval_function_clauses(*dds);
997  }
998 
999  delete dds; // Delete so that we can ...
1000  bdds->set_dds(fdds); // Transfer management responsibility
1001  dds = fdds;
1002 
1003  // Server functions might mark (i.e. setting send_p) so variables will use their read()
1004  // methods. Clear that so the CE in d_dap2ce will control what is
1005  // sent. If that is empty (there was only a function call) all
1006  // of the variables in the intermediate DDS (i.e., the function
1007  // result) will be sent.
1008  dds->mark_all(false);
1009 
1010  // Look for one or more top level Structures whose name indicates (by way of ending with
1011  // "_uwrap") that their contents should be moved to the top level.
1012  //
1013  // This is in support of a hack around the current API where server side functions
1014  // may only return a single DAP object and not a collection of objects. The name suffix
1015  // "_unwrap" is used as a signal from the function to the the various response
1016  // builders and transmitters that the representation needs to be altered before
1017  // transmission, and that in fact is what happens in our friend
1018  // promote_function_output_structures()
1019  promote_function_output_structures(dds);
1020  }
1021 
1022  // evaluate the rest of the CE - the part that follows the function calls.
1023  eval.parse_constraint(get_ce(), *dds);
1024 
1025  dds->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1026 
1027  if (dds->get_response_limit() != 0 && dds->get_request_size(true) > dds->get_response_limit()) {
1028  string msg = "The Request for " + long_to_string(dds->get_request_size(true) / 1024)
1029  + "KB is too large; requests for this user are limited to "
1030  + long_to_string(dds->get_response_limit() / 1024) + "KB.";
1031  throw Error(msg);
1032  }
1033 
1034  // Iterate through the variables in the DataDDS and read
1035  // in the data if the variable has the send flag set.
1036  for (DDS::Vars_iter i = dds->var_begin(), e = dds->var_end(); i != e; ++i) {
1037  if ((*i)->send_p()) {
1038  (*i)->intern_data(eval, *dds);
1039  }
1040  }
1041 
1042  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap2_data() - END"<< endl);
1043 
1044  return dds;
1045 }
1046 
1047 
1060 void BESDapResponseBuilder::send_dap2_data(ostream &data_stream, DDS **dds, ConstraintEvaluator &eval,
1061  bool with_mime_headers)
1062 {
1063  BESDEBUG("dap", "BESDapResponseBuilder::send_dap2_data() - BEGIN"<< endl);
1064 
1065 #if USE_LOCAL_TIMEOUT_SCHEME
1066  // Set up the alarm.
1067  establish_timeout(data_stream);
1068  dds.set_timeout(d_timeout);
1069 #endif
1070 
1071  // Split constraint into two halves
1072  split_ce(eval);
1073 
1074  // If there are functions, parse them and eval.
1075  // Use that DDS and parse the non-function ce
1076  // Serialize using the second ce and the second dds
1077  if (!get_btp_func_ce().empty()) {
1078  BESDEBUG("dap",
1079  "BESDapResponseBuilder::send_dap2_data() - Found function(s) in CE: " << get_btp_func_ce() << endl);
1080 
1081  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1082 
1083  ConstraintEvaluator func_eval;
1084  DDS *fdds = 0; // nulll_ptr
1085  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1086  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1087  }
1088  else {
1089  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1090  fdds = func_eval.eval_function_clauses(**dds);
1091  }
1092 
1093  delete *dds; *dds = 0;
1094  *dds = fdds;
1095 
1096  (*dds)->mark_all(false);
1097 
1098  promote_function_output_structures(*dds);
1099 
1100  // evaluate the rest of the CE - the part that follows the function calls.
1101  eval.parse_constraint(get_ce(), **dds);
1102 
1103  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1104 
1105  if ((*dds)->get_response_limit() != 0 && (*dds)->get_request_size(true) > (*dds)->get_response_limit()) {
1106  string msg = "The Request for " + long_to_string((*dds)->get_request_size(true) / 1024)
1107  + "KB is too large; requests for this user are limited to "
1108  + long_to_string((*dds)->get_response_limit() / 1024) + "KB.";
1109  throw Error(msg);
1110  }
1111 
1112  if (with_mime_headers)
1113  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1114 
1115 #if STORE_DAP2_RESULT_FEATURE
1116  // This means: if we are not supposed to store the result, then serialize it.
1117  if (!store_dap2_result(data_stream, **dds, eval)) {
1118  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1119  }
1120 #else
1121  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1122 #endif
1123 
1124  }
1125  else {
1126  BESDEBUG("dap", "BESDapResponseBuilder::send_dap2_data() - Simple constraint" << endl);
1127 
1128  eval.parse_constraint(get_ce(), **dds); // Throws Error if the ce doesn't parse.
1129 
1130  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1131 
1132  if ((*dds)->get_response_limit() != 0 && (*dds)->get_request_size(true) > (*dds)->get_response_limit()) {
1133  string msg = "The Request for " + long_to_string((*dds)->get_request_size(true) / 1024)
1134  + "KB is too large; requests for this user are limited to "
1135  + long_to_string((*dds)->get_response_limit() / 1024) + "KB.";
1136  throw Error(msg);
1137  }
1138 
1139  if (with_mime_headers)
1140  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1141 
1142 #if STORE_DAP2_RESULT_FEATURE
1143  // This means: if we are not supposed to store the result, then serialize it.
1144  if (!store_dap2_result(data_stream, **dds, eval)) {
1145  serialize_dap2_data_dds(data_stream, dds, eval);
1146  }
1147 #else
1148  serialize_dap2_data_dds(data_stream, dds, eval);
1149 #endif
1150  }
1151 
1152  data_stream << flush;
1153 
1154  BESDEBUG("dap", "BESDapResponseBuilder::send_dap2_data() - END"<< endl);
1155 
1156 }
1157 
1158 
1172 void BESDapResponseBuilder::send_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool with_mime_headers)
1173 {
1174  if (d_dap2ce.empty()) {
1175  if (with_mime_headers)
1176  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1177 
1178  (*dds)->print_xml_writer(out, false /*constrained */, "");
1179  //dds.print(out);
1180  out << flush;
1181  return;
1182  }
1183 
1184 #if USE_LOCAL_TIMEOUT_SCHEME
1185  // Set up the alarm.
1186  establish_timeout(out);
1187  dds.set_timeout(d_timeout);
1188 #endif
1189 
1190  // Split constraint into two halves
1191  split_ce(eval);
1192 
1193  // If there are functions, parse them and eval.
1194  // Use that DDS and parse the non-function ce
1195  // Serialize using the second ce and the second dds
1196  if (!d_btp_func_ce.empty()) {
1197  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1198 
1199  ConstraintEvaluator func_eval;
1200  DDS *fdds = 0; // nulll_ptr
1201  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1202  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1203  }
1204  else {
1205  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1206  fdds = func_eval.eval_function_clauses(**dds);
1207  }
1208 
1209  delete *dds; *dds = 0;
1210  *dds = fdds;
1211 
1212  (*dds)->mark_all(false);
1213 
1214  promote_function_output_structures(*dds);
1215 
1216  eval.parse_constraint(d_dap2ce, **dds);
1217 
1218  if (with_mime_headers)
1219  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1220 
1221  conditional_timeout_cancel();
1222 
1223  (*dds)->print_xml_writer(out, true, "");
1224  }
1225  else {
1226  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
1227 
1228  if (with_mime_headers)
1229  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1230 
1231  conditional_timeout_cancel();
1232 
1233 
1234  // dds.print_constrained(out);
1235  (*dds)->print_xml_writer(out, true, "");
1236  }
1237 
1238  out << flush;
1239 }
1240 
1241 void BESDapResponseBuilder::send_dmr(ostream &out, DMR &dmr, bool with_mime_headers)
1242 {
1243  // If the CE is not empty, parse it. The projections, etc., are set as a side effect.
1244  // If the parser returns false, the expression did not parse. The parser may also
1245  // throw Error
1246  if (!d_dap4ce.empty()) {
1247 
1248  BESDEBUG("dap", "BESDapResponseBuilder::send_dmr() - Parsing DAP4 constraint: '"<< d_dap4ce << "'"<< endl);
1249 
1250  D4ConstraintEvaluator parser(&dmr);
1251  bool parse_ok = parser.parse(d_dap4ce);
1252  if (!parse_ok) throw Error(malformed_expr, "Constraint Expression (" + d_dap4ce + ") failed to parse.");
1253  }
1254  // with an empty CE, send everything. Even though print_dap4() and serialize()
1255  // don't need this, other code may depend on send_p being set. This may change
1256  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1257  else {
1258  dmr.root()->set_send_p(true);
1259  }
1260 
1261  if (with_mime_headers) set_mime_text(out, dap4_dmr, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1262 
1263  conditional_timeout_cancel();
1264 
1265 
1266  XMLWriter xml;
1267  dmr.print_dap4(xml, /*constrained &&*/!d_dap4ce.empty() /* true == constrained */);
1268  out << xml.get_doc() << flush;
1269 }
1270 
1271 void BESDapResponseBuilder::send_dap4_data_using_ce(ostream &out, DMR &dmr, bool with_mime_headers)
1272 {
1273  if (!d_dap4ce.empty()) {
1274  D4ConstraintEvaluator parser(&dmr);
1275  bool parse_ok = parser.parse(d_dap4ce);
1276  if (!parse_ok) throw Error(malformed_expr, "Constraint Expression (" + d_dap4ce + ") failed to parse.");
1277  }
1278  // with an empty CE, send everything. Even though print_dap4() and serialize()
1279  // don't need this, other code may depend on send_p being set. This may change
1280  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1281  else {
1282  dmr.root()->set_send_p(true);
1283  }
1284 
1285  if (dmr.response_limit() != 0 && dmr.request_size(true) > dmr.response_limit()) {
1286  string msg = "The Request for " + long_to_string(dmr.request_size(true) / 1024)
1287  + "MB is too large; requests for this user are limited to " + long_to_string(dmr.response_limit() / 1024)
1288  + "MB.";
1289  throw Error(msg);
1290  }
1291 
1292  if (!store_dap4_result(out, dmr)) {
1293  serialize_dap4_data(out, dmr, with_mime_headers);
1294  }
1295 }
1296 
1297 void BESDapResponseBuilder::send_dap4_data(ostream &out, DMR &dmr, bool with_mime_headers)
1298 {
1299  // If a function was passed in with this request, evaluate it and use that DMR
1300  // for the remainder of this request.
1301  // TODO Add caching for these function invocations
1302  if (!d_dap4function.empty()) {
1303  D4BaseTypeFactory d4_factory;
1304  DMR function_result(&d4_factory, "function_results");
1305 
1306  // Function modules load their functions onto this list. The list is
1307  // part of libdap, not the BES.
1308  if (!ServerFunctionsList::TheList())
1309  throw Error(
1310  "The function expression could not be evaluated because there are no server functions defined on this server");
1311 
1312  D4FunctionEvaluator parser(&dmr, ServerFunctionsList::TheList());
1313  bool parse_ok = parser.parse(d_dap4function);
1314  if (!parse_ok) throw Error("Function Expression (" + d_dap4function + ") failed to parse.");
1315 
1316  parser.eval(&function_result);
1317 
1318  // Now use the results of running the functions for the remainder of the
1319  // send_data operation.
1320  send_dap4_data_using_ce(out, function_result, with_mime_headers);
1321  }
1322  else {
1323  send_dap4_data_using_ce(out, dmr, with_mime_headers);
1324  }
1325 }
1326 
1330 void BESDapResponseBuilder::serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers)
1331 {
1332  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap4_data() - BEGIN" << endl);
1333 
1334  if (with_mime_headers) set_mime_binary(out, dap4_data, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1335 
1336  // Write the DMR
1337  XMLWriter xml;
1338  dmr.print_dap4(xml, !d_dap4ce.empty());
1339 
1340  // now make the chunked output stream; set the size to be at least chunk_size
1341  // but make sure that the whole of the xml plus the CRLF can fit in the first
1342  // chunk. (+2 for the CRLF bytes).
1343  chunked_ostream cos(out, max((unsigned int) CHUNK_SIZE, xml.get_doc_size() + 2));
1344 
1345  conditional_timeout_cancel();
1346 
1347  // using flush means that the DMR and CRLF are in the first chunk.
1348  cos << xml.get_doc() << CRLF << flush;
1349 
1350  // Write the data, chunked with checksums
1351  D4StreamMarshaller m(cos);
1352  dmr.root()->serialize(m, dmr, !d_dap4ce.empty());
1353 #ifdef CLEAR_LOCAL_DATA
1354  dmr.root()->clear_local_data();
1355 #endif
1356  cos << flush;
1357 
1358  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap4_data() - END" << endl);
1359 }
1360 
1375 bool BESDapResponseBuilder::store_dap4_result(ostream &out, libdap::DMR &dmr)
1376 {
1377  if (get_store_result().length() != 0) {
1378  string serviceUrl = get_store_result();
1379 
1380  D4AsyncUtil d4au;
1381  XMLWriter xmlWrtr;
1382 
1383  // FIXME See above comment for store dap2 result
1384  bool found;
1385  string *stylesheet_ref = 0, ss_ref_value;
1386  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
1387  if (found && ss_ref_value.length() > 0) {
1388  stylesheet_ref = &ss_ref_value;
1389  }
1390 
1392  if (resultCache == NULL) {
1393 
1399  string msg = "The Stored Result request cannot be serviced. ";
1400  msg += "Unable to acquire StoredResultCache instance. ";
1401  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
1402 
1403  BESDEBUG("dap", "[WARNING] " << msg << endl);
1404  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
1405  out << xmlWrtr.get_doc();
1406  out << flush;
1407  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - Sent AsyncRequestRejected" << endl);
1408 
1409  return true;
1410  }
1411 
1412  if (get_async_accepted().length() != 0) {
1413 
1417  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - serviceUrl="<< serviceUrl << endl);
1418 
1419  string storedResultId = "";
1420  storedResultId = resultCache->store_dap4_result(dmr, get_ce(), this);
1421 
1422  BESDEBUG("dap",
1423  "BESDapResponseBuilder::store_dap4_result() - storedResultId='"<< storedResultId << "'" << endl);
1424 
1425  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
1426  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - targetURL='"<< targetURL << "'" << endl);
1427 
1428  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
1429  out << xmlWrtr.get_doc();
1430  out << flush;
1431  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - sent AsyncAccepted" << endl);
1432 
1433  }
1434  else {
1439  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
1440  out << xmlWrtr.get_doc();
1441  out << flush;
1442  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - sent AsyncAccepted" << endl);
1443  }
1444 
1445  return true;
1446  }
1447 
1448  return false;
1449 }
void set_dds(libdap::DDS *ddsIn)
virtual libdap::DDS * intern_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
exception thrown if an internal error is found and is fatal to the BES
Holds a DDS object within the BES.
void set_dds(libdap::DDS *ddsIn)
libdap::DDS * get_dds()
virtual void send_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool constrained=false, bool with_mime_headers=true)
Transmit a DDS.
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:422
virtual bool start(string name)
Definition: BESStopWatch.cc:57
virtual void serialize_dap2_data_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool ce_eval=true)
virtual std::string get_dataset_name() const
Get the dataset name.
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:757
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:62
virtual void remove_timeout() const
Transmit data.
static BESStoredDapResultCache * get_instance()
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
libdap::ConstraintEvaluator & get_ce()
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual void establish_timeout(std::ostream &stream) const
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_timeout(int timeout=0)
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual void set_dap4function(std::string _func)
Structure storing information used by the BES to handle the request.
map< string, string > data
the map of string data that will be required for the current request.
virtual std::string get_ce() const
Get the constraint expression.
virtual void send_dap2_data(std::ostream &data_stream, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool with_mime_headers=true)
virtual std::string get_dap4function() const
Get the DAP4 server side function expression.
void first_container()
set the container pointer to the first container in the containers list
virtual void set_ce(std::string _ce)
Cache the results from server functions.
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void send_ddx(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool with_mime_headers=true)
virtual bool store_dap4_result(ostream &out, libdap::DMR &dmr)
virtual std::string get_dap4ce() const
Get the DAP4 constraint expression.
virtual void set_dap4ce(std::string _ce)
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)
virtual libdap::DDS * process_dap2_dds(BESResponseObject *obj, BESDataHandlerInterface &dhi)
Process a DDS (i.e., apply a constraint) for a non-DAP transmitter.