32 #ifdef HAVE_TR1_FUNCTIONAL 33 #include <tr1/functional> 41 #include <DapXmlNamespaces.h> 42 #include <ConstraintEvaluator.h> 43 #include <DDXParserSAX2.h> 47 #include <D4EnumDefs.h> 48 #include <D4Dimensions.h> 51 #include <D4ParserSax2.h> 52 #include <XDRStreamMarshaller.h> 53 #include <XDRStreamUnMarshaller.h> 54 #include <chunked_istream.h> 55 #include <D4StreamUnMarshaller.h> 58 #include <mime_util.h> 61 #include "BESStoredDapResultCache.h" 62 #include "BESDapResponseBuilder.h" 63 #include "BESInternalError.h" 66 #include "TheBESKeys.h" 69 #ifdef HAVE_TR1_FUNCTIONAL 70 #define HASH_OBJ std::tr1::hash 72 #define HASH_OBJ std::hash 76 #define BES_DATA_ROOT "BES.Data.RootDirectory" 77 #define BES_CATALOG_ROOT "BES.Catalog.catalog.RootDirectory" 83 const string BESStoredDapResultCache::SUBDIR_KEY =
"DAP.StoredResultsCache.subdir";
84 const string BESStoredDapResultCache::PREFIX_KEY =
"DAP.StoredResultsCache.prefix";
85 const string BESStoredDapResultCache::SIZE_KEY =
"DAP.StoredResultsCache.size";
87 unsigned long BESStoredDapResultCache::getCacheSizeFromConfig(){
91 unsigned long size_in_megabytes = 0;
94 istringstream iss(size);
95 iss >> size_in_megabytes;
98 string msg =
"[ERROR] BESStoredDapResultCache::getCacheSize() - The BES Key " + SIZE_KEY +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
99 BESDEBUG(
"cache", msg << endl);
102 return size_in_megabytes;
105 string BESStoredDapResultCache::getSubDirFromConfig(){
111 string msg =
"[ERROR] BESStoredDapResultCache::getSubDirFromConfig() - The BES Key " + SUBDIR_KEY +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
112 BESDEBUG(
"cache", msg << endl);
116 while(*subdir.begin() ==
'/' && subdir.length()>0){
117 subdir = subdir.substr(1);
127 string BESStoredDapResultCache::getResultPrefixFromConfig(){
135 string msg =
"[ERROR] BESStoredDapResultCache::getResultPrefix() - The BES Key " + PREFIX_KEY +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
136 BESDEBUG(
"cache", msg << endl);
144 string BESStoredDapResultCache::getBesDataRootDirFromConfig(){
146 string cacheDir =
"";
151 string msg = ((string)
"[ERROR] BESStoredDapResultCache::getStoredResultsDir() - Neither the BES Key ") + BES_CATALOG_ROOT +
152 "or the BES key " + BES_DATA_ROOT +
" have been set! One MUST be set to utilize the Stored Result Caching system. ";
153 BESDEBUG(
"cache", msg << endl);
162 string BESStoredDapResultCache::assemblePath(
const string &firstPart,
const string &secondPart,
bool addLeadingSlash){
168 string firstPathFragment = firstPart;
169 string secondPathFragment = secondPart;
173 if(*firstPathFragment.begin() !=
'/')
174 firstPathFragment =
"/" + firstPathFragment;
178 while(*firstPathFragment.rbegin() ==
'/' && firstPathFragment.length()>0){
179 firstPathFragment = firstPathFragment.substr(0,firstPathFragment.length()-1);
184 if(*firstPathFragment.rbegin() !=
'/'){
185 firstPathFragment +=
"/";
190 while(*secondPathFragment.begin() ==
'/' && secondPathFragment.length()>0){
191 secondPathFragment = secondPathFragment.substr(1);
196 string newPath = firstPathFragment + secondPathFragment;
205 BESStoredDapResultCache::BESStoredDapResultCache(){
206 BESDEBUG(
"cache",
"BESStoredDapResultCache::BESStoredDapResultCache() - BEGIN" << endl);
208 d_storedResultsSubdir = getSubDirFromConfig();
209 d_dataRootDir = getBesDataRootDirFromConfig();
212 d_resultFilePrefix = getResultPrefixFromConfig();
213 d_maxCacheSize = getCacheSizeFromConfig();
215 BESDEBUG(
"cache",
"BESStoredDapResultCache() - Stored results cache configuration params: " << resultsDir <<
", " << d_resultFilePrefix <<
", " << d_maxCacheSize << endl);
217 initialize(resultsDir, d_resultFilePrefix, d_maxCacheSize);
219 BESDEBUG(
"cache",
"BESStoredDapResultCache::BESStoredDapResultCache() - END" << endl);
226 BESStoredDapResultCache::BESStoredDapResultCache(
const string &data_root_dir,
const string &stored_results_subdir,
const string &result_file_prefix,
unsigned long long max_cache_size){
228 d_storedResultsSubdir = stored_results_subdir;
229 d_dataRootDir = data_root_dir;
230 d_resultFilePrefix = result_file_prefix;
231 d_maxCacheSize = max_cache_size;
232 initialize(
BESUtil::assemblePath(d_dataRootDir,stored_results_subdir), d_resultFilePrefix, d_maxCacheSize);
239 if (d_instance == 0){
240 if(dir_exists(data_root_dir)){
242 d_instance =
new BESStoredDapResultCache(data_root_dir, stored_results_subdir, result_file_prefix, max_cache_size);
244 atexit(delete_instance);
248 BESDEBUG(
"cache",
"[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
261 if (d_instance == 0) {
265 atexit(delete_instance);
269 BESDEBUG(
"cache",
"[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
286 bool BESStoredDapResultCache::is_valid(
const string &cache_file_name,
const string &dataset)
291 off_t entry_size = 0;
292 time_t entry_time = 0;
294 if (stat(cache_file_name.c_str(), &buf) == 0) {
295 entry_size = buf.st_size;
296 entry_time = buf.st_mtime;
305 time_t dataset_time = entry_time;
306 if (stat(dataset.c_str(), &buf) == 0) {
307 dataset_time = buf.st_mtime;
315 if (dataset_time > entry_time)
332 bool BESStoredDapResultCache::read_dap2_data_from_cache(
const string &cache_file_name, DDS *fdds)
334 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap2_data_from_cache() - Opening cache file: " << cache_file_name << endl);
339 if (get_read_lock(cache_file_name, fd)) {
341 ifstream data(cache_file_name.c_str());
344 string mime = get_next_mime_header(data);
345 while (!mime.empty()) {
346 mime = get_next_mime_header(data);
350 DDXParser ddx_parser(fdds->get_factory());
353 string boundary = read_multipart_boundary(data);
354 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap2_data_from_cache() - MPM Boundary: " << boundary << endl);
356 read_multipart_headers(data,
"text/xml", dods_ddx);
358 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap2_data_from_cache() - Read the multipart haeaders" << endl);
364 ddx_parser.intern_stream(data, fdds, data_cid, boundary);
365 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap2_data_from_cache() - Dataset name: " << fdds->get_dataset_name() << endl);
368 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap2_data_from_cache() - DDX Parser Error: " << e.get_error_message() << endl);
373 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (before): " << data_cid << endl);
374 data_cid = cid_to_header_value(data_cid);
375 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (after): " << data_cid << endl);
379 read_multipart_headers(data,
"application/octet-stream", dods_data_ddx, data_cid);
384 XDRStreamUnMarshaller um(data);
385 for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
386 (*i)->deserialize(um, fdds);
390 unlock_and_close(cache_file_name );
394 BESDEBUG(
"cache",
"BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
400 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl );
403 unlock_and_close(cache_file_name );
419 bool BESStoredDapResultCache::read_dap4_data_from_cache(
const string &cache_file_name, libdap::DMR *dmr)
421 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - BEGIN" << endl);
427 if (get_read_lock(cache_file_name, fd)) {
428 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - Opening cache file: " << cache_file_name << endl);
429 fstream in(cache_file_name.c_str(), ios::in|ios::binary);
440 chunked_istream cis(in, CHUNK_SIZE);
447 int chunk_size = cis.read_next_chunk();
449 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - First chunk_size: " << chunk_size << endl);
452 if(chunk_size == EOF){
453 throw InternalErr(__FILE__, __LINE__,
"BESStoredDapResultCache::read_dap4_data_from_cache() - Failed to read first chunk from file. Chunk size = EOF (aka " + libdap::long_to_string(EOF) +
")");
457 char chunk[chunk_size];
458 cis.read(chunk, chunk_size);
459 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - Read first chunk." << endl);
464 parser.intern(chunk, chunk_size-2, dmr, debug);
465 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - Parsed first chunk." << endl);
467 D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
469 dmr->root()->deserialize(um, *dmr);
470 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - Deserialized data." << endl);
472 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - END" << endl);
475 unlock_and_close(cache_file_name );
481 BESDEBUG(
"cache",
"BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
488 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl );
491 unlock_and_close(cache_file_name );
503 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cached_dap2_data_ddx() - Reading cache for " << cache_file_name << endl);
505 DDS *fdds =
new DDS(factory);
508 if(read_dap2_data_from_cache(cache_file_name, fdds)){
511 fdds->filename(filename) ;
514 BESDEBUG(
"cache",
"DDS Filename: " << fdds->filename() << endl);
515 BESDEBUG(
"cache",
"DDS Dataset name: " << fdds->get_dataset_name() << endl);
517 fdds->set_factory( 0 ) ;
521 DDS::Vars_iter i = fdds->var_begin();
522 while(i != fdds->var_end()) {
523 (*i)->set_read_p(
true );
524 (*i++)->set_send_p(
true);
545 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cached_dap4_data() - Reading cache for " << cache_file_name << endl);
547 DMR *fdmr =
new DMR(factory);
549 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cached_dap4_data() - DMR Filename: " << fdmr->filename() << endl);
550 fdmr->set_filename(filename) ;
552 if(read_dap4_data_from_cache(cache_file_name, fdmr)){
553 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cached_dap4_data() - DMR Dataset name: " << fdmr->name() << endl);
555 fdmr->set_factory( 0 ) ;
559 fdmr->root()->set_send_p(
true);
560 fdmr->root()->set_read_p(
true);
574 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - BEGIN" << endl );
576 BaseTypeFactory factory;
580 string local_id = get_stored_result_local_id(dds.filename(), constraint, DAP_3_2);
581 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - local_id: "<< local_id << endl );
582 string cache_file_name = get_cache_file_name(local_id,
false);
583 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - cache_file_name: "<< cache_file_name << endl );
590 if (!is_valid(cache_file_name, dds.filename()))
591 purge_file(cache_file_name);
593 if (get_read_lock(cache_file_name, fd)) {
594 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
596 else if (create_and_lock(cache_file_name, fd)) {
599 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - cache_file_name " << cache_file_name <<
", constraint: " << constraint << endl);
603 eval->parse_constraint(constraint, *fdds);
605 if (eval->function_clauses()) {
606 DDS *temp_fdds = eval->eval_function_clauses(*fdds);
611 ofstream data_stream(cache_file_name.c_str());
613 throw InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to write cached response.");
615 string start=
"dataddx_cache_start", boundary=
"dataddx_cache_boundary";
619 ConstraintEvaluator eval;
623 fdds->set_dap_version(
"3.2");
630 set_mime_multipart(data_stream, boundary, start, dods_data_ddx, x_plain, last_modified_time(rb->
get_dataset_name()));
635 data_stream << CRLF <<
"--" << boundary <<
"--" << CRLF;
642 exclusive_to_shared_lock(fd);
647 unsigned long long size = update_cache_info(cache_file_name);
648 if (cache_too_big(size))
649 update_and_purge(cache_file_name);
657 else if (get_read_lock(cache_file_name, fd)) {
658 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
661 throw InternalErr(__FILE__, __LINE__,
"BESStoredDapResultCache::store_dap2_result() - Cache error during function invocation.");
664 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - unlocking and closing cache file "<< cache_file_name << endl );
665 unlock_and_close(cache_file_name);
668 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - caught exception, unlocking cache and re-throw." << endl );
674 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - END (local_id=`"<< local_id <<
"')" << endl );
686 BESStoredDapResultCache::get_stored_result_local_id(
const string &dataset,
const string &ce, libdap::DAPVersion version )
688 BESDEBUG(
"cache",
"get_stored_result_local_id() - BEGIN. dataset: " << dataset <<
", ce: " << ce << endl);
689 std::ostringstream ostr;
690 HASH_OBJ<std::string> str_hash;
691 string name = dataset +
"#" + ce;
692 ostr << str_hash(name);
693 string hashed_name = ostr.str();
694 BESDEBUG(
"cache",
"get_stored_result_local_id() - hashed_name: " << hashed_name << endl);
703 suffix =
".data_ddx";
710 throw BESInternalError(
"BESStoredDapResultCache::get_stored_result_local_id() - Unrecognized DAP version!!", __FILE__, __LINE__);
714 BESDEBUG(
"cache",
"get_stored_result_local_id() - Data file suffix: " << suffix << endl);
717 string local_id = d_resultFilePrefix + hashed_name + suffix;
718 BESDEBUG(
"cache",
"get_stored_result_local_id() - file: " << local_id << endl);
722 BESDEBUG(
"cache",
"get_stored_result_local_id() - END. local_id: " << local_id << endl);
742 string BESStoredDapResultCache::get_cache_file_name(
const string &local_id,
bool mangle)
744 if(local_id.empty()){
745 throw BESInternalError(
"BESStoredDapResultCache: The target cache file name must not be an empty string. Srsly.", __FILE__, __LINE__);
748 string cacheFile = assemblePath(d_dataRootDir,local_id);
750 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cache_file_name() - local_id: '" << local_id <<
"'" << endl);
751 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cache_file_name() - cacheDir: '" << cacheFile <<
"'" << endl);
754 BESDEBUG(
"cache",
"[WARNING] BESStoredDapResultCache::get_cache_file_name() - The parameter 'mangle' is ignored!" << endl);
767 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - BEGIN" << endl );
769 BaseTypeFactory factory;
773 string local_id = get_stored_result_local_id(dmr.filename(), constraint, DAP_4_0);
774 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - local_id: "<< local_id << endl );
775 string cache_file_name = get_cache_file_name(local_id,
false);
776 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - cache_file_name: "<< cache_file_name << endl );
783 if (!is_valid(cache_file_name, dmr.filename())){
784 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
785 purge_file(cache_file_name);
788 if (get_read_lock(cache_file_name, fd)) {
789 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
791 else if (create_and_lock(cache_file_name, fd)) {
794 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - cache_file_name: " << cache_file_name <<
", constraint: " << constraint << endl);
796 ofstream data_stream(cache_file_name.c_str());
798 throw InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to write cached response.");
809 exclusive_to_shared_lock(fd);
814 unsigned long long size = update_cache_info(cache_file_name);
815 if (cache_too_big(size))
816 update_and_purge(cache_file_name);
820 else if (get_read_lock(cache_file_name, fd)) {
821 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - Couldn't create and lock file, But I got a read lock. " 822 "Result may have been created by another process. " 823 "Not rewriting file: " << cache_file_name << endl);
826 throw InternalErr(__FILE__, __LINE__,
"BESStoredDapResultCache::store_dap4_result() - Cache error during function invocation.");
829 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - unlocking and closing cache file "<< cache_file_name << endl );
830 unlock_and_close(cache_file_name);
833 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - caught exception, unlocking cache and re-throw." << endl );
839 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - END (local_id=`"<< local_id <<
"')" << endl );
exception thrown if inernal error encountered
static string lowercase(const string &s)
virtual std::string get_message()
get the error message for this exception
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
libdap::DMR * get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory, const string &filename)
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
static BESStoredDapResultCache * get_instance()
libdap::DDS * get_cached_dap2_data_ddx(const std::string &cache_file_name, libdap::BaseTypeFactory *factory, const std::string &dataset)
static bool IsSet(const string &flagName)
see if the debug context flagName is set to true
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
virtual void serialize_dap2_data_ddx(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, const std::string &boundary, const std::string &start, bool ce_eval=true)
static BESKeys * TheKeys()
virtual std::string get_dataset_name() const
Get the dataset name.
virtual string store_dap2_result(libdap::DDS &dds, const std::string &constraint, BESDapResponseBuilder *rb, libdap::ConstraintEvaluator *eval)
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)