bes  Updated for version 3.20.10
util_ff.cc
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of ff_handler a FreeForm API handler for the OPeNDAP
5 // DAP2 data server.
6 
7 // Copyright (c) 2005 OPeNDAP, Inc.
8 // Author: James Gallagher <jgallagher@opendap.org>
9 //
10 // This is free software; you can redistribute it and/or modify it under the
11 // terms of the GNU Lesser General Public License as published by the Free
12 // Software Foundation; either version 2.1 of the License, or (at your
13 // option) any later version.
14 //
15 // This software is distributed in the hope that it will be useful, but
16 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18 // License for more details.
19 //
20 // You should have received a copy of the GNU Lesser General Public
21 // License along with this library; if not, write to the Free Software
22 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 //
24 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25 
26 // (c) COPYRIGHT URI/MIT 1997-99
27 // Please read the full copyright statement in the file COPYRIGHT.
28 //
29 // Authors: reza (Reza Nekovei)
30 
31 // Utility functions for the FreeForm data server.
32 //
33 // jhrg 3/29/96
34 
35 #include "config_ff.h"
36 
37 static char rcsid[] not_used =
38  { "$Id$" };
39 
40 #ifndef WIN32
41 #include <unistd.h> // for access
42 #else
43 #define F_OK 0
44 #endif
45 
46 #include <iostream>
47 #include <sstream>
48 #include <fstream>
49 #include <string>
50 #include <vector>
51 #include <cstdlib>
52 
53 #include <BESRegex.h>
54 #include <BESDebug.h>
55 
56 #include <libdap/BaseType.h>
57 #include <libdap/Byte.h>
58 #include <libdap/Int16.h>
59 #include <libdap/Int32.h>
60 #include <libdap/UInt16.h>
61 #include <libdap/UInt32.h>
62 #include <libdap/Float32.h>
63 #include <libdap/Float64.h>
64 #include <libdap/InternalErr.h>
65 #include <libdap/dods-limits.h>
66 #include <libdap/util.h>
67 #include <libdap/debug.h>
68 
69 #include "BESRegex.h"
70 #include "BESDebug.h"
71 
72 #include "FFRequestHandler.h"
73 #include "util_ff.h"
74 
75 using namespace std;
76 
90 static string &remove_paths(string &src)
91 {
92  size_t p1 = src.find_first_of('/');
93  if (p1 == string::npos)
94  return src;
95  size_t p2 = src.find_last_of('/');
96  // The string has one '/', not a big deal
97  if (p2 == p1)
98  return src;
99 
100  src.erase(p1, p2-p1+1);
101  return src;
102 }
103 
104 // These two functions are defined in FFND/error.c. They were originally
105 // static functions. I used them to read error strings since FreeForm
106 // did not have a good way to get the error text. jhrg 9/11/12
107 extern "C" FF_ERROR_PTR pull_error(void);
108 extern "C" BOOLEAN is_a_warning(FF_ERROR_PTR error);
109 
123 static string freeform_error_message()
124 {
125  FF_ERROR_PTR error = pull_error();
126  if (!error)
127  throw BESInternalError("Called the FreeForm error message code, but there was no error.", __FILE__, __LINE__);
128 
129  ostringstream oss;
130  do {
131  if (is_a_warning(error))
132  oss << "Warning: ";
133  else
134  oss << "Error: ";
135 
136  // if either of these contain a pathname, remove it
137  string problem = error->problem;
138  string message = error->message;
139  oss << remove_paths(problem) << ": " << remove_paths(message) << endl;
140 
141  ff_destroy_error (error);
142  error = pull_error();
143  } while (error);
144 
145  return oss.str();
146 }
147 
160 long read_ff(const char *dataset, const char *if_file, const char *o_format, char *o_buffer, unsigned long bsize)
161 {
162  FF_BUFSIZE_PTR newform_log = NULL;
163  FF_STD_ARGS_PTR std_args = NULL;
164 
165  try {
166  std_args = ff_create_std_args();
167  if (!std_args)
168  throw BESInternalError("FreeForm could not allocate a 'stdargs' object.", __FILE__, __LINE__);
169 
170  // set the std_arg structure values - cast away const for dataset, if_file,
171  // and o_format.
172  std_args->error_prompt = FALSE;
173  std_args->user.is_stdin_redirected = 0;
174  std_args->input_file = (char*) (dataset);
175  std_args->input_format_file = (char*) (if_file);
176  std_args->output_file = NULL;
177  std_args->output_format_buffer = (char*) (o_format);
178  std_args->log_file = (char *) "/dev/null";
179 #if 0
180  // This just doesn't seem to work within the BES framework. jhrg 9/11/12
181  std_args->log_file = (char *)"/tmp/ffdods.log";
182 #endif
183 
184  // memory freed automatically on exit
185  vector<char> l_bufsz(sizeof(FF_BUFSIZE));
186  //bufsz = (FF_BUFSIZE *)&l_bufsz[0];
187  FF_BUFSIZE_PTR bufsz = reinterpret_cast<FF_BUFSIZE_PTR>(&l_bufsz[0]);
188 
189  bufsz->usage = 1;
190  bufsz->buffer = o_buffer;
191  bufsz->total_bytes = (FF_BSS_t) bsize;
192  bufsz->bytes_used = (FF_BSS_t) 0;
193 
194  std_args->output_bufsize = bufsz;
195 
196  newform_log = ff_create_bufsize(SCRATCH_QUANTA);
197  if (!newform_log)
198  throw BESInternalError("FreeForm could not allocate a 'newform_log' object.", __FILE__, __LINE__);
199 
200  // passing 0 for the FILE* param is a wash since a non-null
201  // value for newform_log selects that as the 'logging' sink.
202  // jhrg 9/11/12
203  int status = newform(std_args, newform_log, 0 /*stderr*/);
204 
205  BESDEBUG("ff", "FreeForm: newform returns " << status << endl);
206 
207  if (err_count()) {
208  string message = freeform_error_message();
209  BESDEBUG("ff", "FreeForm: error message " << message << endl);
210  throw BESError(message, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
211  }
212 
213  ff_destroy_bufsize(newform_log);
214  ff_destroy_std_args(std_args);
215 
216  return bufsz->bytes_used;
217  }
218  catch (...) {
219  if (newform_log)
220  ff_destroy_bufsize(newform_log);
221  if (std_args)
222  ff_destroy_std_args(std_args);
223 
224  throw;
225  }
226 
227  return 0;
228 }
229 
236 void free_ff_char_vector(char **v, int len)
237 {
238  for (int i = 0; i < len; ++i)
239  if (v && v[i])
240  free (v[i]);
241  if (v && len > 0)
242  free (v);
243 }
244 
245 // Given the name of a DODS data type, return the name of the corresponding
246 // FreeForm data type.
247 //
248 // Returns: a const char * if the DODS type maps into a FreeForm type,
249 // otherwise NULL.
250 
251 const string ff_types(Type dods_type)
252 {
253  switch (dods_type) {
254  case dods_byte_c:
255  return "uint8";
256  case dods_int16_c:
257  return "int16";
258  case dods_uint16_c:
259  return "uint16";
260  case dods_int32_c:
261  return "int32";
262  case dods_uint32_c:
263  return "uint32";
264  case dods_float32_c:
265  return "float32";
266  case dods_float64_c:
267  return "float64";
268  case dods_str_c:
269  return "text";
270  case dods_url_c:
271  return "text";
272  default:
273  throw Error("ff_types: DODS type " + D2type_name(dods_type) + " does not map to a FreeForm type.");
274  }
275 }
276 
277 // Given the name of a DODS data type, return the precision of the
278 // corresponding FreeForm data type.
279 //
280 // Returns: a positive integer if the DODS type maps into a FreeForm type,
281 // otherwise -1.
282 
283 int ff_prec(Type dods_type)
284 {
285  switch (dods_type) {
286  case dods_byte_c:
287  case dods_int16_c:
288  case dods_uint16_c:
289  case dods_int32_c:
290  case dods_uint32_c:
291  return 0;
292  case dods_float32_c:
293  return DODS_FLT_DIG;
294  case dods_float64_c:
295  return DODS_DBL_DIG;
296  case dods_str_c:
297  case dods_url_c:
298  return 0;
299  default:
300  throw Error("ff_prec: DODS type " + D2type_name(dods_type) + " does not map to a FreeForm type.");
301  }
302 }
303 
309 const string
310 make_output_format(const string & name, Type type, const int width)
311 {
312  ostringstream str;
313 
314  str << "binary_output_data \"DODS binary output data\"" << endl;
315  str << name << " 1 " << width << " " << ff_types(type)
316  << " " << ff_prec(type) << endl;
317 
318  return str.str();
319 }
320 
321 // format for multi-dimension array
322 const string
323 makeND_output_format(const string & name, Type type, const int width,
324  int ndim, const long *start, const long *edge, const
325  long *stride, string * dname)
326 {
327  ostringstream str;
328  str << "binary_output_data \"DODS binary output data\"" << endl;
329  str << name << " 1 " << width << " ARRAY";
330 
331  for (int i = 0; i < ndim; i++)
332  str << "[" << "\"" << dname[i] << "\" " << start[i] + 1 << " to "
333  << start[i] + (edge[i] - 1) * stride[i] +
334  1 << " by " << stride[i] << " ]";
335 
336  str << " of " << ff_types(type) << " " << ff_prec(type) << endl;
337 
338  DBG(cerr << "ND output format: " << str.str() << endl);
339 
340  return str.str();
341 }
342 
348 const string & format_delimiter(const string & new_delimiter)
349 {
350  static string delimiter = ".";
351 
352  if (new_delimiter != "")
353  delimiter = new_delimiter;
354 
355  return delimiter;
356 }
357 
364 const string & format_extension(const string & new_extension)
365 {
366  static string extension = ".fmt";
367 
368  if (new_extension != "")
369  extension = new_extension;
370 
371  return extension;
372 }
373 
376 static bool
377 cmp_array_conduit(FF_ARRAY_CONDUIT_PTR src_conduit,
378  FF_ARRAY_CONDUIT_PTR trg_conduit)
379 {
380  if (src_conduit->input && trg_conduit->input)
381  return (bool) ff_format_comp(src_conduit->input->fd->format,
382  trg_conduit->input->fd->format);
383  else if (src_conduit->output && trg_conduit->output)
384  return (bool) ff_format_comp(src_conduit->output->fd->format,
385  trg_conduit->output->fd->format);
386  else
387  return false;
388 }
389 
390 static int merge_redundant_conduits(FF_ARRAY_CONDUIT_LIST conduit_list)
391 {
392  int error = 0;
393  error = list_replace_items((pgenobj_cmp_t) cmp_array_conduit,
394  conduit_list);
395  return (error);
396 }
397 
406 int SetDodsDB(FF_STD_ARGS_PTR std_args, DATA_BIN_HANDLE dbin_h, char *Msgt)
407 {
408  FORMAT_DATA_LIST format_data_list = NULL;
409  int error = 0;
410 
411  assert(dbin_h);
412 
413  if (!dbin_h) {
414  snprintf(Msgt, Msgt_size, "Error: NULL DATA_BIN_HANDLE in %s", ROUTINE_NAME);
415  return (ERR_API);
416  }
417 
418  if (!*dbin_h) {
419  *dbin_h = db_make(std_args->input_file);
420 
421  if (!*dbin_h) {
422  snprintf(Msgt, Msgt_size, "Error in Standard Data Bin");
423  return (ERR_MEM_LACK);
424  }
425  }
426 
427  /* Now set the formats and the auxiliary files */
428 
429  if (db_set(*dbin_h, DBSET_READ_EQV, std_args->input_file)) {
430  snprintf(Msgt, Msgt_size, "Error making name table for %s",
431  std_args->input_file);
432  return (DBSET_READ_EQV);
433  }
434 
435  if (db_set(*dbin_h,
436  DBSET_INPUT_FORMATS,
437  std_args->input_file,
438  std_args->output_file,
439  std_args->input_format_file,
440  std_args->input_format_buffer,
441  std_args->input_format_title, &format_data_list)) {
442  if (format_data_list)
443  dll_free_holdings(format_data_list);
444 
445  snprintf(Msgt, Msgt_size, "Error setting an input format for %s",
446  std_args->input_file);
447  return (DBSET_INPUT_FORMATS);
448  }
449 
450  error =
451  db_set(*dbin_h, DBSET_CREATE_CONDUITS, std_args, format_data_list);
452  dll_free_holdings(format_data_list);
453  if (error) {
454  snprintf(Msgt, Msgt_size, "Error creating array information for %s",
455  std_args->input_file);
456  return (DBSET_CREATE_CONDUITS);
457  }
458 
459  if (db_set(*dbin_h, DBSET_HEADER_FILE_NAMES, FFF_INPUT,
460  std_args->input_file)) {
461  snprintf(Msgt, Msgt_size, "Error determining input header file names for %s",
462  std_args->input_file);
463  return (DBSET_HEADER_FILE_NAMES);
464  }
465 
466  if (db_set(*dbin_h, DBSET_HEADERS)) {
467  snprintf(Msgt, Msgt_size, "getting header file for %s", std_args->input_file);
468  return (DBSET_HEADERS);
469  }
470 
471 
472  if (db_set(*dbin_h, DBSET_INIT_CONDUITS, FFF_DATA,
473  std_args->records_to_read)) {
474  snprintf(Msgt, Msgt_size, "Error creating array information for %s",
475  std_args->input_file);
476  return (DBSET_INIT_CONDUITS);
477  }
478 
479  error = merge_redundant_conduits((*dbin_h)->array_conduit_list);
480  if (error)
481  snprintf(Msgt, Msgt_size, "Error merging redundent conduits");
482 
483  return (error);
484 }
485 
491 long Records(const string &filename)
492 {
493  int error = 0;
494  DATA_BIN_PTR dbin = NULL;
495  FF_STD_ARGS_PTR SetUps = NULL;
496  PROCESS_INFO_LIST pinfo_list = NULL;
497  PROCESS_INFO_PTR pinfo = NULL;
498  static char Msgt[255];
499 
500  SetUps = ff_create_std_args();
501  if (!SetUps) {
502  return -1;
503  }
504 
506  SetUps->user.is_stdin_redirected = 0;
507  SetUps->input_file = const_cast<char*>(filename.c_str());
508 
509  SetUps->output_file = NULL;
510 
511  error = SetDodsDB(SetUps, &dbin, Msgt);
512  if (error && error < ERR_WARNING_ONLY) {
513  ff_destroy_std_args(SetUps);
514  db_destroy(dbin);
515  return -1;
516  }
517 
518  ff_destroy_std_args(SetUps);
519 
520  error = db_ask(dbin, DBASK_PROCESS_INFO, FFF_INPUT | FFF_DATA, &pinfo_list);
521  if (error)
522  return (-1);
523 
524  pinfo_list = dll_first(pinfo_list);
525 
526  pinfo = ((PROCESS_INFO_PTR) (pinfo_list)->data.u.pi);
527 
528  long num_records = PINFO_SUPER_ARRAY_ELS(pinfo);
529 
530  ff_destroy_process_info_list(pinfo_list);
531  db_destroy(dbin);
532 
533  return num_records;
534 }
535 
536 
537 bool file_exist(const char *filename)
538 {
539  return access(filename, F_OK) == 0;
540 }
541 
554 const string
555 find_ancillary_rss_formats(const string & dataset, const string & /* delimiter */,
556  const string & /* extension */)
557 {
558  string FormatFile;
559  string FormatPath = FFRequestHandler::get_RSS_format_files();
560  string BaseName;
561  string FileName;
562 
563  // Separate the filename from the pathname, for both plain files
564  // and cached decompressed files (ones with '#' in their names).
565  size_t delim = dataset.rfind("#");
566  if (delim != string::npos)
567  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
568  else {
569  delim = dataset.rfind("/");
570  if (delim != string::npos)
571  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
572  else
573  FileName = dataset;
574  }
575 
576  // The file/dataset name has to have an underscore...
577  delim = FileName.find("_");
578  if ( delim != string::npos ) {
579  BaseName = FileName.substr(0,delim+1);
580  }
581  else {
582  throw Error("Could not find input format for: " + dataset);
583  }
584 
585  // Now determine if this is files holds averaged or daily data.
586  string DatePart = FileName.substr(delim+1, FileName.length()-delim+1);
587 
588  if (FormatPath[FormatPath.length()-1] != '/')
589  FormatPath.append("/");
590 
591  if ( (DatePart.find("_") != string::npos) || (DatePart.length() < 10) )
592  FormatFile = FormatPath + BaseName + "averaged.fmt";
593  else
594  FormatFile = FormatPath + BaseName + "daily.fmt";
595 
596  return string(FormatFile);
597 }
598 
611 const string
612 find_ancillary_rss_das(const string & dataset, const string & /* delimiter */,
613  const string & /* extension */)
614 {
615  string FormatFile;
616  string FormatPath = FFRequestHandler::get_RSS_format_files();
617  string BaseName;
618  string FileName;
619 
620  size_t delim = dataset.rfind("#");
621  if (delim != string::npos)
622  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
623  else {
624  delim = dataset.rfind("/");
625  if (delim != string::npos)
626  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
627  else
628  FileName = dataset;
629  }
630 
631  delim = FileName.find("_");
632  if ( delim != string::npos ) {
633  BaseName = FileName.substr(0,delim+1);
634  }
635  else {
636  string msg = "Could not find input format for: ";
637  msg += dataset;
638  throw InternalErr(msg);
639  }
640 
641  string DatePart = FileName.substr(delim+1, FileName.length()-delim+1);
642 
643  if (FormatPath[FormatPath.length()-1] != '/')
644  FormatPath.append("/");
645 
646  if ( (DatePart.find("_") != string::npos) || (DatePart.length() < 10) )
647  FormatFile = FormatPath + BaseName + "averaged.das";
648  else
649  FormatFile = FormatPath + BaseName + "daily.das";
650 
651  return string(FormatFile);
652 }
653 
654 // These functions are used by the Date/Time Factory classes but they might
655 // be generally useful in writing server-side functions. 1/21/2002 jhrg
656 
657 bool is_integer_type(BaseType * btp)
658 {
659  switch (btp->type()) {
660  case dods_null_c:
661  return false;
662 
663  case dods_byte_c:
664  case dods_int16_c:
665  case dods_uint16_c:
666  case dods_int32_c:
667  case dods_uint32_c:
668  return true;
669 
670  case dods_float32_c:
671  case dods_float64_c:
672  case dods_str_c:
673  case dods_url_c:
674  case dods_array_c:
675  case dods_structure_c:
676  case dods_sequence_c:
677  case dods_grid_c:
678  default:
679  return false;
680  }
681 }
682 
683 bool is_float_type(BaseType * btp)
684 {
685  switch (btp->type()) {
686  case dods_null_c:
687  case dods_byte_c:
688  case dods_int16_c:
689  case dods_uint16_c:
690  case dods_int32_c:
691  case dods_uint32_c:
692  return false;
693 
694  case dods_float32_c:
695  case dods_float64_c:
696  return true;
697 
698  case dods_str_c:
699  case dods_url_c:
700  case dods_array_c:
701  case dods_structure_c:
702  case dods_sequence_c:
703  case dods_grid_c:
704  default:
705  return false;
706  }
707 }
708 
713 dods_uint32 get_integer_value(BaseType * var) throw(InternalErr)
714 {
715  if (!var)
716  return 0;
717 
718  switch (var->type()) {
719  case dods_byte_c:
720  return static_cast<Byte*>(var)->value();
721 
722  case dods_int16_c:
723  return static_cast<Int16*>(var)->value();
724 
725  case dods_int32_c:
726  return static_cast<Int32*>(var)->value();
727 
728  case dods_uint16_c:
729  return static_cast<UInt16*>(var)->value();
730 
731  case dods_uint32_c:
732  return static_cast<UInt32*>(var)->value();
733 
734  default:
735  throw InternalErr(__FILE__, __LINE__,
736  "Tried to get an integer value for a non-integer datatype!");
737  }
738 }
739 
740 dods_float64 get_float_value(BaseType * var) throw(InternalErr)
741 {
742  if (!var)
743  return 0.0;
744 
745  switch (var->type()) {
746  case dods_int16_c:
747  case dods_uint16_c:
748  case dods_int32_c:
749  case dods_uint32_c:
750  return get_integer_value(var);
751 
752  case dods_float32_c:
753  return static_cast<Float32*>(var)->value();
754 
755  case dods_float64_c:
756  return static_cast<Float64*>(var)->value();
757 
758  default:
759  throw InternalErr(__FILE__, __LINE__,
760  "Tried to get an float value for a non-numeric datatype!");
761  }
762 }
763 
764 string get_Regex_format_file(const string & filename)
765 {
766  string::size_type found = filename.find_last_of("/\\");
767  string base_name = filename.substr(found+1);
768  string retVal = "";
769  std::map<string,string> mapFF = FFRequestHandler::get_fmt_regex_map();
770  for (auto rgx = mapFF.begin(); rgx != mapFF.end(); ++ rgx) {
771  BESDEBUG("ff", "get_Regex_format_file() - filename: '" << filename << "'" <<
772  " regex: '" << (*rgx).first << "'" <<
773  " format: '" << (*rgx).second << "'" << endl);
774  BESRegex regex(((*rgx).first).c_str());
775  if ( (unsigned long) regex.match(base_name.c_str(), base_name.length()) == base_name.length() ){
776  retVal = string((*rgx).second);
777  break;
778  }
779  }
780  BESDEBUG("ff", "get_Regex_format_file() - returning format filename: '"<< retVal << "'" << endl);
781  return retVal;
782 }
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
exception thrown if internal error encountered
Regular expression matching.
Definition: BESRegex.h:53
Type
Type of JSON value.
Definition: rapidjson.h:664