39 #include <sys/types.h>
44 #include <libdap/DMR.h>
45 #include <libdap/D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
50 #include <libdap/mime_util.h>
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <libdap/Ancillary.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
71 #include <BESStopWatch.h>
73 #include "config_hdf5.h"
75 #define HDF5_NAME "h5"
81 #define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
84 void get_attr_contents(AttrTable* temp_table);
88 void write_das_to_file(DAS*das_ptr,FILE* das_file);
91 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
94 void write_container_name_to_file(
const string&,FILE* das_file);
97 void write_das_attr_info(AttrTable* dtp,
const string&,
const string&,FILE * das_file);
100 char* copy_str(
char *temp_ptr,
const string & str);
104 char* obtain_str(
char*temp_ptr,
string & str);
107 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at);
113 static unsigned int get_uint_key(
const string &key,
unsigned int def_val);
114 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val);
117 static float get_float_key(
const string &key,
float def_val);
120 static string get_beskeys(
const string&);
123 bool obtain_beskeys_info(
const string &,
bool &);
126 extern void read_cfdas(DAS &das,
const string & filename,hid_t fileid);
127 extern void read_cfdds(DDS &dds,
const string & filename,hid_t fileid);
128 extern void read_cfdmr(DMR *dmr,
const string & filename,hid_t fileid);
133 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
134 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
135 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
136 float HDF5RequestHandler::_cache_purge_level = 0.2;
141 ObjMemCache *HDF5RequestHandler::datadds_cache = 0;
144 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
145 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
151 bool HDF5RequestHandler::_usecf =
true;
152 bool HDF5RequestHandler::_pass_fileid =
false;
153 bool HDF5RequestHandler::_disable_structmeta =
true;
154 bool HDF5RequestHandler::_disable_ecsmeta =
false;
155 bool HDF5RequestHandler::_keep_var_leading_underscore =
false;
156 bool HDF5RequestHandler::_check_name_clashing =
false;
157 bool HDF5RequestHandler::_add_path_attrs =
true;
158 bool HDF5RequestHandler::_drop_long_string =
true;
159 bool HDF5RequestHandler::_fillvalue_check =
true;
160 bool HDF5RequestHandler::_check_ignore_obj =
false;
161 bool HDF5RequestHandler::_flatten_coor_attr =
true;
162 bool HDF5RequestHandler::_default_handle_dimension =
true;
163 bool HDF5RequestHandler::_eos5_rm_convention_attr_path =
true;
164 bool HDF5RequestHandler::_dmr_long_int =
true;
165 bool HDF5RequestHandler::_no_zero_size_fullnameattr =
false;
166 bool HDF5RequestHandler::_enable_coord_attr_add_path =
true;
168 bool HDF5RequestHandler::_usecfdmr =
true;
170 bool HDF5RequestHandler::_common_cache_dirs =
false;
172 bool HDF5RequestHandler::_use_disk_cache =
false;
173 bool HDF5RequestHandler::_use_disk_dds_cache =
false;
174 string HDF5RequestHandler::_disk_cache_dir =
"";
175 string HDF5RequestHandler::_disk_cachefile_prefix =
"";
176 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
179 bool HDF5RequestHandler::_disk_cache_comp_data =
false;
180 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =
false;
181 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
182 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
184 bool HDF5RequestHandler::_use_disk_meta_cache =
false;
185 string HDF5RequestHandler::_disk_meta_cache_path =
"";
187 bool HDF5RequestHandler::_use_latlon_disk_cache =
false;
188 long HDF5RequestHandler::_latlon_disk_cache_size =0;
189 string HDF5RequestHandler::_latlon_disk_cache_dir =
"";
190 string HDF5RequestHandler::_latlon_disk_cachefile_prefix=
"";
192 DMR* HDF5RequestHandler::dmr_int64 = 0;
199 string HDF5RequestHandler::_stp_east_filename;
200 string HDF5RequestHandler::_stp_north_filename;
201 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
202 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
203 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
209 HDF5RequestHandler::HDF5RequestHandler(
const string & name)
213 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
215 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
216 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
217 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
218 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
219 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
221 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
222 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
224 #if !(DYNAMIC_CONFIG_ENABLED)
228 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
231 HDF5RequestHandler::~HDF5RequestHandler()
236 delete datadds_cache;
238 delete lrdata_mem_cache;
239 delete srdata_mem_cache;
246 void HDF5RequestHandler::load_config()
248 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
250 if (BESISDEBUG(HDF5_NAME)){
251 sw.
start(prolog,
"ClockTheBESKeys");
255 HDF5RequestHandler::_mdcache_entries = get_uint_key(
"H5.MetaDataMemCacheEntries", 0);
256 HDF5RequestHandler::_lrdcache_entries = get_uint_key(
"H5.LargeDataMemCacheEntries", 0);
257 HDF5RequestHandler::_srdcache_entries = get_uint_key(
"H5.SmallDataMemCacheEntries", 0);
258 HDF5RequestHandler::_cache_purge_level = get_float_key(
"H5.CachePurgeLevel", 0.2);
260 if (get_mdcache_entries()) {
261 das_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
262 dds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
263 datadds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
264 dmr_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
276 bool has_key =
false;
277 bool key_value = obtain_beskeys_info(
"H5.EnableCF",has_key);
280 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCF: " << (_usecf?
"true":
"false") << endl);
284 key_value = obtain_beskeys_info(
"H5.DefaultHandleDimension",has_key);
286 _default_handle_dimension = key_value;
287 BESDEBUG(HDF5_NAME, prolog <<
"H5.DefaultHandleDimension: " << (_default_handle_dimension?
"true":
"false") << endl);
291 key_value = obtain_beskeys_info(
"H5.EnablePassFileID",has_key);
293 _pass_fileid = key_value;
294 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnablePassFileID: " << (_pass_fileid?
"true":
"false") << endl);
297 key_value = obtain_beskeys_info(
"H5.DisableStructMetaAttr",has_key);
299 _disable_structmeta = key_value;
300 BESDEBUG(HDF5_NAME, prolog <<
"H5.DisableStructMetaAttr: " << (_disable_structmeta?
"true":
"false") << endl);
303 key_value = obtain_beskeys_info(
"H5.DisableECSMetaAttr",has_key);
305 _disable_ecsmeta = key_value;
306 BESDEBUG(HDF5_NAME, prolog <<
"H5.DisableECSMetaAttr: " << (_disable_ecsmeta?
"true":
"false") << endl);
309 key_value = obtain_beskeys_info(
"H5.KeepVarLeadingUnderscore",has_key);
311 _keep_var_leading_underscore = key_value;
312 BESDEBUG(HDF5_NAME, prolog <<
"H5.KeepVarLeadingUnderscore: " << (_keep_var_leading_underscore?
"true":
"false") << endl);
315 key_value = obtain_beskeys_info(
"H5.EnableCheckNameClashing",has_key);
317 _check_name_clashing = key_value;
318 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCheckNameClashing: " << (_check_name_clashing?
"true":
"false") << endl);
321 key_value = obtain_beskeys_info(
"H5.EnableAddPathAttrs",has_key);
323 _add_path_attrs = key_value;
324 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableAddPathAttrs: " << (_add_path_attrs?
"true":
"false") << endl);
327 key_value = obtain_beskeys_info(
"H5.EnableDropLongString",has_key);
329 _drop_long_string = key_value;
330 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDropLongString: " << (_drop_long_string?
"true":
"false") << endl);
333 key_value = obtain_beskeys_info(
"H5.EnableFillValueCheck",has_key);
335 _fillvalue_check = key_value;
336 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableFillValueCheck: " << (_fillvalue_check?
"true":
"false") << endl);
340 key_value = obtain_beskeys_info(
"H5.CheckIgnoreObj",has_key);
342 _check_ignore_obj = key_value;
343 BESDEBUG(HDF5_NAME, prolog <<
"H5.CheckIgnoreObj: " << (_check_ignore_obj?
"true":
"false") << endl);
346 key_value = obtain_beskeys_info(
"H5.ForceFlattenNDCoorAttr",has_key);
348 _flatten_coor_attr = key_value;
349 BESDEBUG(HDF5_NAME, prolog <<
"H5.ForceFlattenNDCoorAttr: " << (_flatten_coor_attr?
"true":
"false") << endl);
353 key_value = obtain_beskeys_info(
"H5.RmConventionAttrPath",has_key);
355 _eos5_rm_convention_attr_path = key_value;
356 BESDEBUG(HDF5_NAME, prolog <<
"H5.RmConventionAttrPath: " << (_eos5_rm_convention_attr_path?
"true":
"false") << endl);
359 key_value = obtain_beskeys_info(
"H5.EnableDMR64bitInt",has_key);
361 _dmr_long_int = key_value;
362 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDMR64bitInt: " << (_dmr_long_int?
"true":
"false") << endl);
365 key_value = obtain_beskeys_info(
"H5.NoZeroSizeFullnameAttr",has_key);
367 _no_zero_size_fullnameattr = key_value;
368 BESDEBUG(HDF5_NAME, prolog <<
"H5.NoZeroSizeFullnameAttr: " << (_no_zero_size_fullnameattr?
"true":
"false") << endl);
371 key_value = obtain_beskeys_info(
"H5.EnableCoorattrAddPath",has_key);
373 _enable_coord_attr_add_path = key_value;
374 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCoorattrAddPath: " << (_enable_coord_attr_add_path?
"true":
"false") << endl);
377 key_value = obtain_beskeys_info(
"H5.EnableCFDMR",has_key);
379 _usecfdmr = key_value;
380 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCFDMR: " << (_usecfdmr?
"true":
"false") << endl);
383 key_value = obtain_beskeys_info(
"H5.EnableDiskDataCache",has_key);
385 _use_disk_cache = key_value;
386 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDiskDataCache: " << (_use_disk_cache?
"true":
"false") << endl);
387 _disk_cache_dir = get_beskeys(
"H5.DiskCacheDataPath");
388 _disk_cachefile_prefix = get_beskeys(
"H5.DiskCacheFilePrefix");
389 _disk_cache_size = get_ulong_key(
"H5.DiskCacheSize",0);
392 key_value = obtain_beskeys_info(
"H5.DiskCacheComp",has_key);
394 _disk_cache_comp_data = key_value;
395 BESDEBUG(HDF5_NAME, prolog <<
"H5.DiskCacheComp: " << (_disk_cache_comp_data?
"true":
"false") << endl);
398 key_value = obtain_beskeys_info(
"H5.DiskCacheFloatOnlyComp",has_key);
400 _disk_cache_float_only_comp_data = key_value;
401 BESDEBUG(HDF5_NAME, prolog <<
"H5.DiskCacheFloatOnlyComp: " << (_disk_cache_float_only_comp_data?
"true":
"false") << endl);
402 _disk_cache_comp_threshold = get_float_key(
"H5.DiskCacheCompThreshold",1.0);
403 _disk_cache_var_size = 1024*get_uint_key(
"H5.DiskCacheCompVarSize",0);
406 key_value = obtain_beskeys_info(
"H5.EnableDiskMetaDataCache",has_key);
408 _use_disk_meta_cache = key_value;
409 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDiskMetaDataCache: " << (_use_disk_meta_cache?
"true":
"false") << endl);
412 key_value = obtain_beskeys_info(
"H5.EnableDiskDDSCache",has_key);
414 _use_disk_dds_cache = key_value;
415 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableDiskDDSCache: " << (_use_disk_dds_cache?
"true":
"false") << endl);
416 _disk_meta_cache_path = get_beskeys(
"H5.DiskMetaDataCachePath");
419 key_value = obtain_beskeys_info(
"H5.EnableEOSGeoCacheFile",has_key);
421 _use_latlon_disk_cache = key_value;
422 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableEOSGeoCacheFile: " << (_use_latlon_disk_cache?
"true":
"false") << endl);
423 _latlon_disk_cache_size = get_uint_key(
"H5.Cache.latlon.size",0);
424 _latlon_disk_cache_dir = get_beskeys(
"H5.Cache.latlon.path");
425 _latlon_disk_cachefile_prefix= get_beskeys(
"H5.Cache.latlon.prefix");
429 key_value = obtain_beskeys_info(
"H5.DefaultHandleDimension",has_key);
431 _default_handle_dimension = key_value;
432 BESDEBUG(HDF5_NAME, prolog <<
"H5.DefaultHandleDimension: " << (_default_handle_dimension?
"true":
"false") << endl);
436 if(get_lrdcache_entries()) {
437 lrdata_mem_cache =
new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
438 bool has_LFMC_config =
false;
439 key_value = obtain_beskeys_info(
"H5.LargeDataMemCacheConfig",has_key);
441 has_LFMC_config = key_value;
442 BESDEBUG(HDF5_NAME, prolog <<
"H5.LargeDataMemCacheConfig: " << (has_LFMC_config?
"true":
"false") << endl);
443 if(
true == has_LFMC_config) {
444 _common_cache_dirs =obtain_lrd_common_cache_dirs();
446 if(
false == _common_cache_dirs)
447 cerr<<
"No specific cache info"<<endl;
451 if(get_srdcache_entries()) {
453 BESDEBUG(HDF5_NAME, prolog <<
"Generate memory cache for smaller coordinate variables" << endl);
454 srdata_mem_cache =
new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
458 if(_disk_cache_comp_data ==
true && _use_disk_cache ==
true) {
459 if(_disk_cache_comp_threshold < 1.0) {
461 ss<< _disk_cache_comp_threshold;
462 string _comp_threshold_str(ss.str());
463 string invalid_comp_threshold =
"The Compression Threshold is the total size of the variable array";
464 invalid_comp_threshold+=
" divided by the storage size of compressed array. It should always be >1";
465 invalid_comp_threshold+=
" The current threhold set at h5.conf is ";
466 invalid_comp_threshold+=_comp_threshold_str;
467 invalid_comp_threshold+=
" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
471 _stp_east_filename = get_beskeys(
"H5.STPEastFileName");
472 _stp_north_filename = get_beskeys(
"H5.STPNorthFileName");
474 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
481 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
482 #if DYNAMIC_CONFIG_ENABLED
487 hid_t cf_fileid = -1;
502 DAS *das = bdas->get_das();
505 DAS *cached_das_ptr = 0;
506 bool use_das_cache =
false;
508 cached_das_ptr =
static_cast<DAS*
>(das_cache->
get(filename));
510 use_das_cache =
true;
512 if (
true == use_das_cache) {
515 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
516 *das = *cached_das_ptr;
520 bool das_from_dc =
false;
521 string das_cache_fname;
524 if(_use_disk_meta_cache ==
true) {
526 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
527 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
529 if(access(das_cache_fname.c_str(),F_OK) !=-1)
535 if(
true == das_from_dc) {
536 read_das_from_disk_cache(das_cache_fname,das);
541 BESDEBUG(HDF5_NAME, prolog <<
"HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
542 das_cache->
add(
new DAS(*das), filename);
547 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
548 if (
true == _usecf) {
550 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
552 string invalid_file_msg=
"Could not open this HDF5 file ";
553 invalid_file_msg +=filename;
554 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
555 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
556 invalid_file_msg +=
" distributor.";
560 if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
561 HDF5RequestHandler::set_dmr_64bit_int(NULL);
562 read_cfdas( *das,filename,cf_fileid);
568 string invalid_file_msg=
"Could not open this HDF5 file ";
569 invalid_file_msg +=filename;
570 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
571 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
572 invalid_file_msg +=
" distributor.";
582 Ancillary::read_ancillary_das( *das, filename ) ;
586 AttrTable* top_table = das->get_top_level_attributes();
587 get_attr_contents(top_table);
590 AttrTable::Attr_iter start_aiter=das->var_begin();
591 AttrTable::Attr_iter it = start_aiter;
592 AttrTable::Attr_iter end_aiter = das->var_end();
593 while(it != end_aiter) {
594 AttrTable* temp_table = das->get_table(it);
596 cerr<<
"var_begin"<<endl;
597 temp_table->print(cerr);
605 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
606 das_cache->
add(
new DAS(*das), filename);
611 if(das_cache_fname!=
"") {
612 BESDEBUG(HDF5_NAME, prolog <<
"HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
613 write_das_to_disk_cache(das_cache_fname,das);
623 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
626 catch(InternalErr & e) {
630 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
637 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
644 string s =
"unknown exception caught building HDF5 DAS";
648 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
655 void HDF5RequestHandler::get_dds_with_attributes(
BESDDSResponse*bdds,
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename,
const string &dds_cache_fname,
const string &das_cache_fname,
bool dds_from_dc,
bool das_from_dc,
bool build_data)
658 if(
true == build_data)
659 dds = data_bdds->get_dds();
664 hid_t cf_fileid = -1;
669 DDS* cached_dds_ptr = 0;
670 bool use_dds_cache =
false;
672 cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename));
674 use_dds_cache =
true;
675 if (
true == use_dds_cache) {
678 BESDEBUG(HDF5_NAME, prolog <<
"DDS Metadata Cached hit for : " << filename << endl);
679 *dds = *cached_dds_ptr;
681 else if (
true ==dds_from_dc) {
682 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
685 BESDEBUG(HDF5_NAME, prolog <<
"Build DDS from the HDF5 file. " << filename << endl);
686 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
687 dds->filename(filename);
692 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
694 string invalid_file_msg=
"Could not open this HDF5 file ";
695 invalid_file_msg +=filename;
696 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
697 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
698 invalid_file_msg +=
" distributor.";
703 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
704 HDF5RequestHandler::set_dmr_64bit_int(NULL);
705 read_cfdds(*dds,filename,cf_fileid);
711 string invalid_file_msg=
"Could not open this HDF5 file ";
712 invalid_file_msg +=filename;
713 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
714 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
715 invalid_file_msg +=
" distributor.";
719 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
723 if (!dds->check_semantics()) {
725 throw InternalErr(__FILE__, __LINE__,
726 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
729 Ancillary::read_ancillary_dds( *dds, filename ) ;
732 if(dds_cache_fname!=
"" && dds_from_dc ==
false)
733 write_dds_to_disk_cache(dds_cache_fname,dds);
742 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
748 BESDEBUG(HDF5_NAME, prolog <<
"DDS added to the cache for : " << filename << endl);
749 dds_cache->
add(
new DDS(*dds), filename);
760 catch(InternalErr & e) {
768 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
778 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
788 string s =
"unknown exception caught building HDF5 DDS";
794 void HDF5RequestHandler::get_dds_without_attributes_datadds(
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename)
796 DDS *dds = data_bdds->get_dds();
800 hid_t cf_fileid = -1;
805 DDS* cached_dds_ptr = 0;
806 bool use_datadds_cache =
false;
808 cached_dds_ptr =
static_cast<DDS*
>(datadds_cache->
get(filename));
810 use_datadds_cache =
true;
811 if (
true == use_datadds_cache) {
814 BESDEBUG(HDF5_NAME, prolog <<
"DataDDS Metadata Cached hit for : " << filename << endl);
815 *dds = *cached_dds_ptr;
818 BESDEBUG(HDF5_NAME, prolog <<
"Build DDS from the HDF5 file. " << filename << endl);
819 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
820 dds->filename(filename);
825 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
827 string invalid_file_msg=
"Could not open this HDF5 file ";
828 invalid_file_msg +=filename;
829 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
830 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
831 invalid_file_msg +=
" distributor.";
836 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
837 HDF5RequestHandler::set_dmr_64bit_int(NULL);
838 read_cfdds(*dds,filename,cf_fileid);
844 string invalid_file_msg=
"Could not open this HDF5 file ";
845 invalid_file_msg +=filename;
846 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
847 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
848 invalid_file_msg +=
" distributor.";
852 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
856 if (!dds->check_semantics()) {
858 throw InternalErr(__FILE__, __LINE__,
859 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
862 Ancillary::read_ancillary_dds( *dds, filename ) ;
872 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
879 BESDEBUG(HDF5_NAME, prolog <<
"DataDDS added to the cache for : " << filename << endl);
880 datadds_cache->
add(
new DDS(*dds), filename);
889 BESDEBUG(HDF5_NAME, prolog <<
"Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
890 data_bdds->set_ia_flag(
false);
893 catch(InternalErr & e) {
901 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
911 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
921 string s =
"unknown exception caught building HDF5 DDS";
931 void HDF5RequestHandler::get_dds_with_attributes(
const string &filename,
const string &container_name, DDS*dds) {
935 hid_t cf_fileid = -1;
940 DDS* cached_dds_ptr = 0;
941 if (dds_cache && (cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename)))) {
944 BESDEBUG(HDF5_NAME, prolog <<
"DDS Cached hit for : " << filename << endl);
945 *dds = *cached_dds_ptr;
950 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
951 if (!container_name.empty())
952 dds->container_name(container_name);
953 dds->filename(filename);
959 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
960 string dds_filename =
"/tmp/"+base_filename+
"_dds";
961 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
962 cerr<<
"before parsing "<<endl;
964 DDS tdds(&tf,name_path(filename),
"3.2");
965 tdds.filename(filename);
967 tdds.parse(dds_file);
969 cache_dds =
new DDS(tdds);
975 cerr<<
"after parsing "<<endl;
984 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
986 string invalid_file_msg=
"Could not open this HDF5 file ";
987 invalid_file_msg +=filename;
988 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
989 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
990 invalid_file_msg +=
" distributor.";
994 read_cfdds(*dds,filename,cf_fileid);
997 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
998 string dds_filename =
"/tmp/"+base_filename+
"_dds";
999 FILE *dds_file = fopen(dds_filename.c_str(),
"w");
1000 dds->print(dds_file);
1008 string invalid_file_msg=
"Could not open this HDF5 file ";
1009 invalid_file_msg +=filename;
1010 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1011 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1012 invalid_file_msg +=
" distributor.";
1016 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
1022 if (!dds->check_semantics()) {
1024 throw InternalErr(__FILE__, __LINE__,
1025 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
1028 Ancillary::read_ancillary_dds( *dds, filename ) ;
1034 if (das_cache && (das =
static_cast<DAS*
>(das_cache->
get(filename)))) {
1035 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
1036 dds->transfer_attributes(das);
1043 if (!container_name.empty())
1044 das->container_name(container_name);
1046 if (
true == _usecf) {
1049 read_cfdas( *das,filename,cf_fileid);
1060 H5Fclose(cf_fileid);
1062 Ancillary::read_ancillary_das( *das, filename ) ;
1064 dds->transfer_attributes(das);
1070 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
1072 das_cache->
add(das, filename);
1081 BESDEBUG(HDF5_NAME, prolog <<
"DDS added to the cache for : " << filename << endl);
1082 dds_cache->
add(
new DDS(*dds), filename);
1090 catch(InternalErr & e) {
1093 H5Fclose(cf_fileid);
1098 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1099 __FILE__, __LINE__);
1104 H5Fclose(cf_fileid);
1108 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1109 __FILE__, __LINE__);
1114 H5Fclose(cf_fileid);
1118 string s =
"unknown exception caught building HDF5 DDS";
1128 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1129 #if DYNAMIC_CONFIG_ENABLED
1145 bool dds_from_dc =
false;
1146 bool das_from_dc =
false;
1147 bool build_data =
false;
1148 string dds_cache_fname;
1149 string das_cache_fname;
1151 if(_use_disk_meta_cache) {
1153 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1156 if(_use_disk_dds_cache) {
1157 dds_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_dds";
1158 if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1162 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1164 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1169 get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1173 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1174 string dds_filename =
"/tmp/"+base_filename+
"_dds";
1177 DDS tdds(&tf,name_path(filename),
"3.2");
1178 tdds.filename(filename);
1181 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
1182 tdds.parse(dds_file);
1184 DDS* cache_dds =
new DDS(tdds);
1196 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1199 catch(InternalErr & e) {
1201 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1202 __FILE__, __LINE__);
1206 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1207 __FILE__, __LINE__);
1211 string s =
"unknown exception caught building HDF5 DDS";
1215 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1221 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1222 #if DYNAMIC_CONFIG_ENABLED
1228 if(
true == _pass_fileid)
1229 return hdf5_build_data_with_IDs(dhi);
1243 bool dds_from_dc =
false;
1244 bool das_from_dc =
false;
1245 bool build_data =
true;
1246 string dds_cache_fname;
1247 string das_cache_fname;
1251 if(_use_disk_meta_cache ==
true) {
1253 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1254 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1256 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1262 get_dds_without_attributes_datadds(bdds,container_name,filename);
1269 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1272 catch(InternalErr & e) {
1274 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1275 __FILE__, __LINE__);
1279 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1280 __FILE__, __LINE__);
1284 string s =
"unknown exception caught building HDF5 DDS";
1288 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1295 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1296 #if DYNAMIC_CONFIG_ENABLED
1300 BESDEBUG(HDF5_NAME,prolog <<
"Building DataDDS by passing file IDs. "<<endl);
1301 hid_t cf_fileid = -1;
1305 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1306 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1308 string invalid_file_msg=
"Could not open this HDF5 file ";
1309 invalid_file_msg +=filename;
1310 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1311 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1312 invalid_file_msg +=
" distributor.";
1326 delete bdds->get_dds();
1330 hdds->setHDF5Dataset(cf_fileid);
1332 read_cfdds( *hdds,filename,cf_fileid);
1334 if (!hdds->check_semantics()) {
1336 throw InternalErr(__FILE__, __LINE__,
1337 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1340 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1342 DAS *das =
new DAS ;
1345 read_cfdas( *das,filename,cf_fileid);
1346 Ancillary::read_ancillary_das( *das, filename ) ;
1348 hdds->transfer_attributes(das);
1356 H5Fclose(cf_fileid);
1357 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1360 catch(InternalErr & e) {
1362 H5Fclose(cf_fileid);
1363 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1364 __FILE__, __LINE__);
1368 H5Fclose(cf_fileid);
1369 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1370 __FILE__, __LINE__);
1374 H5Fclose(cf_fileid);
1375 string s =
"unknown exception caught building HDF5 DataDDS";
1379 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1385 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1386 #if DYNAMIC_CONFIG_ENABLED
1397 DMR *dmr = bes_dmr_response.get_dmr();
1401 hid_t cf_fileid = -1;
1405 DMR* cached_dmr_ptr = 0;
1407 BESDEBUG(HDF5_NAME, prolog <<
"Checking DMR cache for : " << filename << endl);
1408 cached_dmr_ptr =
static_cast<DMR*
>(dmr_cache->
get(filename));
1411 if (cached_dmr_ptr) {
1413 BESDEBUG(HDF5_NAME, prolog <<
"DMR cache hit for : " << filename << endl);
1414 *dmr = *cached_dmr_ptr;
1419 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1420 D4BaseTypeFactory MyD4TypeFactory;
1421 dmr->set_factory(&MyD4TypeFactory);
1425 if(
true == _usecfdmr) {
1426 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1428 string invalid_file_msg=
"Could not open this HDF5 file ";
1429 invalid_file_msg +=filename;
1430 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1431 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1432 invalid_file_msg +=
" distributor.";
1435 read_cfdmr(dmr,filename,cf_fileid);
1436 H5Fclose(cf_fileid);
1439 dmr->set_factory(0);
1441 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1446 if(
true == _pass_fileid)
1447 return hdf5_build_dmr_with_IDs(dhi);
1449 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1451 string invalid_file_msg=
"Could not open this HDF5 file ";
1452 invalid_file_msg +=filename;
1453 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1454 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1455 invalid_file_msg +=
" distributor.";
1460 BaseTypeFactory factory;
1461 DDS dds(&factory, name_path(filename),
"3.2");
1462 dds.filename(filename);
1468 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1469 read_cfdds( dds,filename,cf_fileid);
1470 if (!dds.check_semantics()) {
1472 throw InternalErr(__FILE__, __LINE__,
1473 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1476 read_cfdas(das,filename,cf_fileid);
1477 Ancillary::read_ancillary_das( das, filename ) ;
1479 dds.transfer_attributes(&das);
1483 H5Fclose(cf_fileid);
1485 dmr->build_using_dds(dds);
1493 string invalid_file_msg=
"Could not open this HDF5 file ";
1494 invalid_file_msg +=filename;
1495 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1496 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1497 invalid_file_msg +=
" distributor.";
1501 bool use_dimscale =
false;
1502 if(
true == _default_handle_dimension)
1503 use_dimscale = check_dimscale(fileid);
1504 dmr->set_name(name_path(filename));
1505 dmr->set_filename(name_path(filename));
1510 D4Group* root_grp = dmr->root();
1511 BESDEBUG(
"h5",
"use_dimscale is "<< use_dimscale <<endl);
1519 vector<link_info_t> hdf5_hls;
1520 breadth_first(fileid, fileid,(
char*)
"/",root_grp,filename.c_str(),use_dimscale,hdf5_hls);
1522 BESDEBUG(
"h5",
"build_dmr - before obtain dimensions"<< endl);
1523 D4Dimensions *root_dims = root_grp->dims();
1524 for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1525 BESDEBUG(
"fonc",
"transform_dap4() - check dimensions"<< endl);
1526 BESDEBUG(
"fonc",
"transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1527 BESDEBUG(
"fonc",
"transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1528 BESDEBUG(
"fonc",
"transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1532 BESDEBUG(
"h5",
"build_dmr - after obtain dimensions"<< endl);
1536 if(
true == use_dimscale)
1538 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),
true);
1540 depth_first(fileid,(
char*)
"/",root_grp,filename.c_str());
1551 BESDEBUG(HDF5_NAME, prolog <<
"DMR added to the cache for : " << filename << endl);
1552 dmr_cache->
add(
new DMR(*dmr), filename);
1558 H5Fclose(cf_fileid);
1561 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1564 catch(InternalErr & e) {
1567 H5Fclose(cf_fileid);
1571 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1572 __FILE__, __LINE__);
1577 H5Fclose(cf_fileid);
1580 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1581 __FILE__, __LINE__);
1586 H5Fclose(cf_fileid);
1589 string s =
"unknown exception caught building HDF5 DMR";
1602 dmr->set_factory(0);
1604 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1611 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1612 #if DYNAMIC_CONFIG_ENABLED
1616 BESDEBUG(
"h5",
"Building DMR with passing file IDs. "<<endl);
1618 hid_t cf_fileid = -1;
1620 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1621 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1623 string invalid_file_msg=
"Could not open this HDF5 file ";
1624 invalid_file_msg +=filename;
1625 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1626 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1627 invalid_file_msg +=
" distributor.";
1631 BaseTypeFactory factory;
1632 DDS dds(&factory, name_path(filename),
"3.2");
1633 dds.filename(filename);
1641 read_cfdds( dds,filename,cf_fileid);
1643 if (!dds.check_semantics()) {
1645 throw InternalErr(__FILE__, __LINE__,
1646 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1649 Ancillary::read_ancillary_dds( dds, filename ) ;
1652 read_cfdas(das,filename,cf_fileid);
1654 Ancillary::read_ancillary_das( das, filename ) ;
1656 dds.transfer_attributes(&das);
1665 H5Fclose(cf_fileid);
1666 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1669 catch(InternalErr & e) {
1672 H5Fclose(cf_fileid);
1674 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1675 __FILE__, __LINE__);
1680 H5Fclose(cf_fileid);
1682 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1683 __FILE__, __LINE__);
1688 H5Fclose(cf_fileid);
1690 string s =
"unknown exception caught building HDF5 DataDDS";
1704 DMR *dmr = bes_dmr.get_dmr();
1705 D4BaseTypeFactory MyD4TypeFactory;
1706 dmr->set_factory(&MyD4TypeFactory);
1707 dmr->build_using_dds(dds);
1710 hdf5_dmr->setHDF5Dataset(cf_fileid);
1712 bes_dmr.set_dmr(hdf5_dmr);
1721 hdf5_dmr->set_factory(0);
1723 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1734 string add_info=
"Just for Test";
1736 map<string,string> attrs ;
1737 attrs[
"name"] = MODULE_NAME ;
1738 attrs[
"version"] = MODULE_VERSION ;
1739 list<string> services ;
1740 BESServiceRegistry::TheRegistry()->
services_handled( HDF5_NAME, services );
1741 if( services.size() > 0 )
1744 attrs[
"handles"] = handles ;
1746 info->begin_tag(
"module", &attrs ) ;
1747 info->end_tag(
"module" ) ;
1760 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1766 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1768 string lrd_config_fpath;
1769 string lrd_config_fname;
1772 lrd_config_fpath = get_beskeys(
"H5.DataCachePath");
1775 lrd_config_fname = get_beskeys(
"H5.LargeDataMemCacheFileName");
1778 if(lrd_config_fpath==
"" || lrd_config_fname==
"")
1785 string mcache_config_fname = lrd_config_fpath+
"/"+lrd_config_fname;
1789 ifstream mcache_config_file(mcache_config_fname.c_str());
1792 if(mcache_config_file.is_open()==
false){
1793 BESDEBUG(HDF5_NAME, prolog <<
"The large data memory cache configure file "<<mcache_config_fname );
1794 BESDEBUG(HDF5_NAME, prolog <<
" cannot be opened."<<endl);
1799 while(getline(mcache_config_file,temp_line)) {
1802 if(temp_line.size()>1 && temp_line.at(1)==
' ') {
1804 string subline = temp_line.substr(2);
1805 vector<string> temp_name_list;
1808 if(temp_line.at(0)==
'1') {
1809 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1811 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1814 else if(temp_line.at(0)==
'0'){
1815 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1817 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1820 else if(temp_line.at(0)==
'2') {
1826 for(
unsigned int i = 0; i<subline.size();i++){
1827 if(subline[i]==
'"') {
1828 dq_pos.push_back(i);
1830 else if(subline[i]==
'\'')
1831 sq_pos.push_back(i);
1833 if(dq_pos.size()==0 && sq_pos.size()==0)
1834 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1835 else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1836 unsigned int dq_index= 0;
1837 while(dq_index < dq_pos.size()){
1838 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1839 temp_name_list.push_back
1840 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1842 dq_index = dq_index + 2;
1845 else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1846 unsigned int sq_index= 0;
1847 while(sq_index < sq_pos.size()){
1848 if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1849 temp_name_list.push_back
1850 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1852 sq_index = sq_index+2;
1856 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1864 for(
int i =0; i<lrd_cache_dir_list.size();i++)
1865 cerr<<
"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1866 for(
int i =0; i<lrd_non_cache_dir_list.size();i++)
1867 cerr<<
"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1868 for(
int i =0; i<lrd_var_cache_file_list.size();i++)
1869 cerr<<
"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1873 mcache_config_file.close();
1874 if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1881 bool HDF5RequestHandler::read_das_from_disk_cache(
const string & cache_filename,DAS *das_ptr) {
1883 BESDEBUG(HDF5_NAME, prolog <<
"Coming to read_das_from_disk_cache() " << cache_filename << endl);
1884 bool ret_value =
true;
1885 FILE *md_file = NULL;
1886 md_file = fopen(cache_filename.c_str(),
"rb");
1888 if(NULL == md_file) {
1889 string bes_error =
"An error occurred trying to open a metadata cache file " + cache_filename;
1894 int fd_md = fileno(md_file);
1896 l_md = lock(F_RDLCK);
1899 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1902 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1909 if(stat(cache_filename.c_str(),&sb) != 0) {
1910 string bes_error =
"An error occurred trying to stat a metadata cache file size " + cache_filename;
1916 size_t bytes_expected_read=(size_t)sb.st_size;
1917 BESDEBUG(HDF5_NAME, prolog <<
"DAS Disk cache file size is " << bytes_expected_read << endl);
1920 buf.resize(bytes_expected_read);
1921 size_t bytes_to_read =fread((
void*)&buf[0],1,bytes_expected_read,md_file);
1922 if(bytes_to_read != bytes_expected_read)
1923 throw InternalErr(__FILE__,__LINE__,
"Fail to read the data from the das cache file.");
1925 char* temp_pointer =&buf[0];
1927 AttrTable*at = NULL;
1930 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1936 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1938 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1942 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a das cache file.");
1946 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1948 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1957 bool HDF5RequestHandler::write_dds_to_disk_cache(
const string& dds_cache_fname,DDS *dds_ptr) {
1959 BESDEBUG(HDF5_NAME, prolog <<
"Write DDS to disk cache " << dds_cache_fname << endl);
1960 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"w");
1962 if(NULL == dds_file) {
1963 string bes_error =
"An error occurred trying to open a metadata cache file " + dds_cache_fname;
1968 int fd_md = fileno(dds_file);
1970 l_md = lock(F_WRLCK);
1973 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1976 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1981 dds_ptr->print(dds_file);
1984 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1986 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1990 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
1993 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1995 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2005 bool HDF5RequestHandler::write_das_to_disk_cache(
const string & das_cache_fname, DAS *das_ptr) {
2007 BESDEBUG(HDF5_NAME, prolog <<
"Write DAS to disk cache " << das_cache_fname << endl);
2008 FILE *das_file = fopen(das_cache_fname.c_str(),
"wb");
2009 if(NULL == das_file) {
2010 string bes_error =
"An error occurred trying to open a metadata cache file " + das_cache_fname;
2014 int fd_md = fileno(das_file);
2016 l_md = lock(F_WRLCK);
2019 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
2022 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
2027 write_das_to_file(das_ptr,das_file);
2030 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2032 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2036 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
2039 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2041 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2053 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
2056 uint8_t category_flag = 2;
2057 AttrTable* top_table = das_ptr->get_top_level_attributes();
2058 write_das_table_to_file(top_table,das_file);
2061 fwrite((
const void*)&category_flag,1,1,das_file);
2067 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
2069 if(temp_table !=NULL) {
2072 uint8_t category_flag = 2;
2075 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2076 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2077 AttrTable::Attr_iter top_it = top_startit;
2078 while(top_it !=top_endit) {
2079 AttrType atype = temp_table->get_attr_type(top_it);
2080 if(atype == Attr_unknown)
2081 throw InternalErr(__FILE__,__LINE__,
"Unsupported DAS Attribute type");
2082 else if(atype!=Attr_container) {
2083 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr name is: "
2084 << temp_table->get_name(top_it) << endl);
2085 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr type is: "
2086 << temp_table->get_type(top_it) << endl);
2095 write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
2098 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr container name is: "
2099 << (*top_it)->name << endl);
2101 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2102 write_container_name_to_file(sub_table->get_name(),das_file);
2103 write_das_table_to_file(sub_table,das_file);
2106 fwrite((
const void*)&category_flag,1,1,das_file);
2116 void write_container_name_to_file(
const string& cont_name,FILE *das_file) {
2119 uint8_t category_flag = 1;
2121 size_t bytes_to_write = cont_name.size()+
sizeof(size_t)+1;
2122 buf.resize(bytes_to_write);
2123 char*temp_pointer =&buf[0];
2124 memcpy((
void*)temp_pointer,(
void*)&category_flag,1);
2126 temp_pointer=copy_str(temp_pointer,cont_name);
2128 size_t bytes_to_be_written = fwrite((
const void*)&buf[0],1,bytes_to_write,das_file);
2129 if(bytes_to_be_written != bytes_to_write)
2130 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS container name to a cache");
2136 void write_das_attr_info(AttrTable* dtp,
const string& attr_name,
const string & attr_type,FILE * das_file) {
2139 uint8_t category_flag = 0;
2141 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
2142 vector<string> attr_values;
2143 size_t total_attr_values_size = 0;
2144 for (
unsigned int i = 0; i <num_attr_elems;i++){
2145 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
2146 total_attr_values_size += attr_values[i].size();
2150 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2*
sizeof(size_t);
2156 bytes_to_write_attr +=
sizeof(
unsigned int) + num_attr_elems*
sizeof(
size_t)+total_attr_values_size;
2158 vector<char>attr_buf;
2159 attr_buf.resize(bytes_to_write_attr);
2160 char* temp_attrp =&attr_buf[0];
2163 memcpy((
void*)temp_attrp,(
void*)&category_flag,1);
2167 temp_attrp=copy_str(temp_attrp,attr_name);
2168 temp_attrp=copy_str(temp_attrp,attr_type);
2171 memcpy((
void*)temp_attrp,(
void*)&num_attr_elems,
sizeof(
unsigned int));
2172 temp_attrp+=
sizeof(
unsigned int);
2175 for (
unsigned int i = 0; i <num_attr_elems;i++)
2176 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2178 size_t bytes_to_be_written = fwrite((
const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
2179 if(bytes_to_be_written != bytes_to_write_attr)
2180 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS attribute to a cache");
2188 bool build_data,
const string & container_name,
const string & h5_fname,
2189 const string & dds_cache_fname,
const string &das_cache_fname, hid_t h5_fd,
2193 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2196 if(
true == build_data)
2197 dds = data_bdds->get_dds();
2203 DDS tdds(&tf,name_path(h5_fname),
"3.2");
2204 tdds.filename(h5_fname);
2206 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"r");
2207 tdds.parse(dds_file);
2208 DDS* cache_dds =
new DDS(tdds);
2210 cerr<<
"before dds "<<endl;
2212 cerr<<
"after dds "<<endl;
2213 cerr<<
"before tdds "<<endl;
2214 cache_dds->dump(cerr);
2215 cerr<<
"after tdds "<<endl;
2220 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2222 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2223 if(
true == build_data)
2224 data_bdds->
set_dds(cache_dds);
2231 BESDEBUG(HDF5_NAME, prolog <<
"For memory cache, DDS added to the cache for : " << h5_fname << endl);
2232 dds_cache->
add(
new DDS(*cache_dds), h5_fname);
2238 void HDF5RequestHandler::add_das_to_dds(DDS *dds,
const string &,
const string &filename,
2239 const string &das_cache_fname, hid_t h5_fd,
bool das_from_dc) {
2241 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
2245 bool use_das_cache =
false;
2247 das =
static_cast<DAS*
>(das_cache->
get(filename));
2249 use_das_cache =
true;
2251 if (
true == use_das_cache) {
2252 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
2253 dds->transfer_attributes(das);
2260 if (!container_name.empty())
2261 das->container_name(container_name);
2263 if(das_from_dc ==
true)
2264 read_das_from_disk_cache(das_cache_fname,das);
2267 bool h5_file_open =
true;
2269 h5_file_open =
false;
2270 if (
true == _usecf) {
2272 if(h5_file_open ==
false)
2273 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2275 read_cfdas( *das,filename,h5_fd);
2276 if(h5_file_open ==
false)
2280 if(h5_file_open ==
false)
2284 if(h5_file_open ==
false)
2288 Ancillary::read_ancillary_das( *das, filename ) ;
2290 if(das_cache_fname!=
"" && das_from_dc ==
false)
2291 write_das_to_disk_cache(das_cache_fname,das);
2294 dds->transfer_attributes(das);
2298 BESDEBUG(HDF5_NAME, prolog <<
"For memory cache, DAS added to the cache for : " << filename << endl);
2299 das_cache->
add(
new DAS(*das), filename);
2307 bool obtain_beskeys_info(
const string& key,
bool & has_key) {
2309 bool ret_value =
false;
2313 const string dosettrue =
"true";
2314 const string dosetyes =
"yes";
2316 ret_value = (dosettrue == doset || dosetyes == doset);
2322 bool is_beskeys_set_true(
const string& bes_value) {
2324 const string dosettrue =
"true";
2325 const string dosetyes =
"yes";
2327 return (dosettrue == doset || dosetyes == doset);
2330 bool check_and_set_beskeys(
const string key) {
2334 const string dosettrue =
"true";
2335 const string dosetyes =
"yes";
2338 if(
true == found ) {
2341 BESDEBUG(HDF5_NAME, prolog <<
"Key: " << key << (found?(
" was found. value: "+doset):
" was not found.") << endl);
2342 return found && (dosettrue == doset || dosetyes == doset);
2349 static unsigned int get_uint_key(
const string &key,
unsigned int def_val)
2355 if (
true == found) {
2357 return atoi(doset.c_str());
2364 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val)
2370 if (
true == found) {
2372 return atol(doset.c_str());
2378 static float get_float_key(
const string &key,
float def_val)
2384 if (
true == found) {
2385 return atof(doset.c_str());
2392 static string get_beskeys(
const string &key) {
2395 string ret_value =
"";
2403 char* copy_str(
char*temp_ptr,
const string & str) {
2405 size_t str_size=str.size();
2406 memcpy((
void*)temp_ptr,(
void*)&str_size,
sizeof(
size_t));
2407 temp_ptr+=
sizeof(size_t);
2408 vector<char>temp_vc2(str.begin(),str.end());
2409 memcpy((
void*)temp_ptr,(
void*)&temp_vc2[0],str.size());
2410 temp_ptr+=str.size();
2419 char* obtain_str(
char*temp_ptr,
string & str) {
2421 size_t oname_size = *((
size_t *)temp_ptr);
2422 temp_ptr = temp_ptr +
sizeof(size_t);
2424 for(
unsigned int i =0; i<oname_size; i++){
2425 oname.push_back(*temp_ptr);
2436 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at_par) {
2441 flag = *((uint8_t*)(temp_pointer));
2442 BESDEBUG(HDF5_NAME, prolog <<
"Build DAS from the disk cache file flag: "
2443 <<
" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2444 <<
" flag = 3; the initial value to get the attribute retrieval process started."
2445 <<
" The flag value is "
2446 << (
int)flag <<endl);
2450 string container_name;
2451 temp_pointer = obtain_str(temp_pointer,container_name);
2452 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, container name is " << container_name << endl);
2455 AttrTable*temp_at_par = at_par;
2457 at_par = das->add_table(container_name,
new AttrTable);
2459 at_par = at_par->append_container(container_name);
2461 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2463 at_par = temp_at_par;
2466 else if(flag == 0) {
2469 throw BESInternalError(
"The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2473 temp_pointer = obtain_str(temp_pointer,attr_name);
2474 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attr name is: " << attr_name << endl);
2478 temp_pointer = obtain_str(temp_pointer,attr_type);
2479 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attr type is: " << attr_type << endl);
2482 unsigned int num_values = *((
unsigned int*)(temp_pointer));
2483 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, number of attribute values is: " << num_values << endl);
2484 temp_pointer+=
sizeof(
unsigned int);
2486 vector <string> attr_values;
2488 for(
unsigned int i = 0; i<num_values; i++) {
2490 temp_pointer = obtain_str(temp_pointer,attr_value);
2491 attr_values.push_back(attr_value);
2492 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attribute value is: " << attr_value << endl);
2495 at_par->append_attr(attr_name,attr_type,&attr_values);
2499 return temp_pointer;
2504 void get_attr_contents(AttrTable*temp_table) {
2505 if(temp_table !=NULL) {
2506 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2507 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2508 AttrTable::Attr_iter top_it = top_startit;
2509 while(top_it !=top_endit) {
2510 AttrType atype = temp_table->get_attr_type(top_it);
2511 if(atype == Attr_unknown)
2512 cerr<<
"unsupported DAS attributes" <<endl;
2513 else if(atype!=Attr_container) {
2515 cerr<<
"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2516 cerr<<
"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2517 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2518 cerr<<
"Attribute values are "<<endl;
2519 for (
unsigned int i = 0; i <num_attrs;i++)
2520 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<
" ";
2524 cerr<<
"Coming to the attribute container. "<<endl;
2525 cerr<<
"container name is "<<(*top_it)->name <<endl;
2526 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2527 cerr<<
"container table name is "<<sub_table->get_name() <<endl;
2528 get_attr_contents(sub_table);
2544 DDS *dds = bdds->get_dds();
2548 bool das_from_mcache =
false;
2550 das =
static_cast<DAS*
>(das_cache->
get(filename));
2552 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
2553 dds->transfer_attributes(das);
2554 das_from_mcache =
true;
2558 if(
false == das_from_mcache) {
2562 if (!container_name.empty()) das->container_name(container_name);
2565 if (
true == _usecf) {
2567 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2569 read_cfdas( *das,filename,h5_fd);
2581 Ancillary::read_ancillary_das(*das, filename);
2583 dds->transfer_attributes(das);
2588 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
2589 das_cache->
add(das, filename);
2595 BESDEBUG(HDF5_NAME, prolog <<
"Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2596 bdds->set_ia_flag(
true);
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
Abstract exception class for the BES with basic string message.
virtual std::string get_message()
get the error message for this exception
informational response object
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
static std::string lowercase(const std::string &s)
static std::string implode(const std::list< std::string > &values, char delim)
An in-memory cache for DapObj (DAS, DDS, ...) objects.
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
void find_gloattr(hid_t file, DAS &das)
bool breadth_first(const hid_t file_id, hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
hid_t get_fileid(const char *filename)
void close_fileid(hid_t fid)
The main header of the HDF5 OPeNDAP handler.