bes  Updated for version 3.20.10
h5get.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
36 
37 #include "h5get.h"
38 #include "HDF5Int32.h"
39 #include "HDF5UInt32.h"
40 #include "HDF5UInt16.h"
41 #include "HDF5Int16.h"
42 #include "HDF5Byte.h"
43 #include "HDF5Int8.h"
44 #include "HDF5Int64.h"
45 #include "HDF5UInt64.h"
46 #include "HDF5Array.h"
47 #include "HDF5Str.h"
48 #include "HDF5Float32.h"
49 #include "HDF5Float64.h"
50 #include "HDF5Url.h"
51 #include "HDF5Structure.h"
52 #include "HDF5RequestHandler.h"
53 
54 #include <BESDebug.h>
55 #include <math.h>
56 #include <sstream>
57 
58 using namespace libdap;
59 
60 
61 // H5Lvisit call back function. After finding all the hard links, return 1.
62 static int visit_link_cb(hid_t group_id, const char *name, const H5L_info_t *oinfo,
63  void *_op_data);
64 
65 // H5OVISIT call back function. When finding the dimension scale attributes, return 1.
66 static int
67 visit_obj_cb(hid_t o_id, const char *name, const H5O_info_t *oinfo,
68  void *_op_data);
69 
70 // H5Aiterate2 call back function, check if having the dimension scale attributes.
71 static herr_t attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata);
72 
73 
90 hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t * attr_inst_ptr,
91  bool *ignore_attr_ptr)
92 {
93 
94  hid_t attrid = -1;
95 
96  // Always assume that we don't ignore any attributes.
97  *ignore_attr_ptr = false;
98 
99  if ((attrid = H5Aopen_by_idx(dset, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC,(hsize_t)index, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
100  string msg = "unable to open attribute by index ";
101  throw InternalErr(__FILE__, __LINE__, msg);
102  }
103 
104  // Obtain the size of attribute name.
105  ssize_t name_size = H5Aget_name(attrid, 0, NULL);
106  if (name_size < 0) {
107  H5Aclose(attrid);
108  string msg = "unable to obtain the size of the hdf5 attribute name ";
109  throw InternalErr(__FILE__, __LINE__, msg);
110  }
111 
112  vector<char> attr_name;
113  attr_name.resize(name_size+1);
114  // Obtain the attribute name.
115  if ((H5Aget_name(attrid, name_size+1, &attr_name[0])) < 0) {
116  H5Aclose(attrid);
117  string msg = "unable to obtain the hdf5 attribute name ";
118  throw InternalErr(__FILE__, __LINE__, msg);
119  }
120 
121  // Obtain the type of the attribute.
122  hid_t ty_id = -1;
123  if ((ty_id = H5Aget_type(attrid)) < 0) {
124  string msg = "unable to obtain hdf5 datatype for the attribute ";
125  string attrnamestr(attr_name.begin(),attr_name.end());
126  msg += attrnamestr;
127  H5Aclose(attrid);
128  throw InternalErr(__FILE__, __LINE__, msg);
129  }
130 
131  H5T_class_t ty_class = H5Tget_class(ty_id);
132  if (ty_class < 0) {
133  string msg = "cannot get hdf5 attribute datatype class for the attribute ";
134  string attrnamestr(attr_name.begin(),attr_name.end());
135  msg += attrnamestr;
136  H5Aclose(attrid);
137  throw InternalErr(__FILE__, __LINE__, msg);
138  }
139 
140  // The following datatype will not be supported for mapping to DAS for both DAP2 and DAP4.
141  // Note:DAP4 explicitly states that the data should be defined atomic datatype(int, string).
142  // 1-D variable length of string can also be mapped to both DAS and DDS.
143  // The variable length string class is H5T_STRING rather than H5T_VLEN,
144  // So safe here.
145  // We also ignore the mapping of integer 64 bit for DAP2 since DAP2 doesn't
146  // support 64-bit integer. In theory, DAP2 doesn't support long double
147  // (128-bit or 92-bit floating point type), since this rarely happens
148  // in DAP application, we simply don't consider here.
149  // However, DAP4 supports 64-bit integer.
150  if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
151  || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM)
152  || (ty_class == H5T_REFERENCE) ||(ty_class == H5T_COMPOUND)
153  || (ty_class == H5T_VLEN) || (ty_class == H5T_ARRAY)){
154 
155  *ignore_attr_ptr = true;
156  H5Tclose(ty_id);
157  return attrid;
158  }
159 
160  // Ignore 64-bit integer for DAP2.
161  // The nested if is better to understand the code. Don't combine
162  if (false == is_dap4) {
163  if((ty_class == H5T_INTEGER) && (H5Tget_size(ty_id)== 8)) {//64-bit int
164  *ignore_attr_ptr = true;
165  H5Tclose(ty_id);
166  return attrid;
167  }
168  }
169 
170  // Here we ignore netCDF-4 specific attributes for DAP4 to make filenetCDF-4 work
171  if (true == is_dap4 && HDF5RequestHandler::get_default_handle_dimension() == true) {
172  // Remove the NULLTERM etc.
173  string attr_name_str(attr_name.begin(),attr_name.end()-1);
174  if(attr_name_str == "CLASS" || attr_name_str == "NAME" || attr_name_str == "_Netcdf4Dimid"
175  || attr_name_str == "_nc3_strict" || attr_name_str=="_NCProperties" || attr_name_str=="_Netcdf4Coordinates") {
176  *ignore_attr_ptr = true;
177  H5Tclose(ty_id);
178  return attrid;
179  }
180  }
181 
182  hid_t aspace_id = -1;
183  if ((aspace_id = H5Aget_space(attrid)) < 0) {
184  string msg = "cannot get hdf5 dataspace id for the attribute ";
185  string attrnamestr(attr_name.begin(),attr_name.end());
186  msg += attrnamestr;
187  H5Aclose(attrid);
188  throw InternalErr(__FILE__, __LINE__, msg);
189  }
190 
191  // It is better to use the dynamic allocation of the array.
192  // However, since the DODS_MAX_RANK is not big and it is also
193  // used in other location, we still keep the original code.
194  // KY 2011-11-16
195 
196  int ndims = H5Sget_simple_extent_ndims(aspace_id);
197  if (ndims < 0) {
198  string msg = "cannot get hdf5 dataspace number of dimension for attribute ";
199  string attrnamestr(attr_name.begin(),attr_name.end());
200  msg += attrnamestr;
201  H5Sclose(aspace_id);
202  H5Aclose(attrid);
203  throw InternalErr(__FILE__, __LINE__, msg);
204  }
205 
206  // Check if the dimension size exceeds the maximum number of dimension DAP supports
207  if (ndims > DODS_MAX_RANK) {
208  string msg = "number of dimensions exceeds allowed for attribute ";
209  string attrnamestr(attr_name.begin(),attr_name.end());
210  msg += attrnamestr;
211  H5Sclose(aspace_id);
212  H5Aclose(attrid);
213  throw InternalErr(__FILE__, __LINE__, msg);
214  }
215 
216  hsize_t size[DODS_MAX_RANK];
217  hsize_t maxsize[DODS_MAX_RANK];
218 
219  //The HDF5 attribute should not have unlimited dimension,
220  // maxsize is only a place holder.
221  if (H5Sget_simple_extent_dims(aspace_id, size, maxsize)<0){
222  string msg = "cannot obtain the dim. info for the attribute ";
223  string attrnamestr(attr_name.begin(),attr_name.end());
224  msg += attrnamestr;
225  H5Sclose(aspace_id);
226  H5Aclose(attrid);
227  throw InternalErr(__FILE__, __LINE__, msg);
228  }
229 
230  // Return ndims and size[ndims].
231  hsize_t nelmts = 1;
232  if (ndims) {
233  for (int j = 0; j < ndims; j++)
234  nelmts *= size[j];
235  }
236 
237  size_t ty_size = H5Tget_size(ty_id);
238  if (ty_size == 0) {
239  string msg = "cannot obtain the dtype size for the attribute ";
240  string attrnamestr(attr_name.begin(),attr_name.end());
241  msg += attrnamestr;
242  H5Sclose(aspace_id);
243  H5Aclose(attrid);
244  throw InternalErr(__FILE__, __LINE__, msg);
245  }
246 
247  size_t need = nelmts * H5Tget_size(ty_id);
248 
249  // We want to save memory type in the struct.
250  hid_t memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
251  if (memtype < 0){
252  string msg = "cannot obtain the memory dtype for the attribute ";
253  string attrnamestr(attr_name.begin(),attr_name.end());
254  msg += attrnamestr;
255  H5Sclose(aspace_id);
256  H5Aclose(attrid);
257  throw InternalErr(__FILE__, __LINE__, msg);
258  }
259 
260  // Save the information to the struct
261  (*attr_inst_ptr).type = memtype;
262  (*attr_inst_ptr).ndims = ndims;
263  (*attr_inst_ptr).nelmts = nelmts;
264  (*attr_inst_ptr).need = need;
265  strncpy((*attr_inst_ptr).name, &attr_name[0], name_size+1);
266 
267  for (int j = 0; j < ndims; j++) {
268  (*attr_inst_ptr).size[j] = size[j];
269  }
270 
271  if(H5Sclose(aspace_id)<0) {
272  H5Aclose(attrid);
273  throw InternalErr(__FILE__,__LINE__,"Cannot close HDF5 dataspace ");
274  }
275 
276  return attrid;
277 }
278 
291 string get_dap_type(hid_t type, bool is_dap4)
292 {
293  size_t size = 0;
294  H5T_sign_t sign;
295  BESDEBUG("h5", ">get_dap_type(): type=" << type << endl);
296  H5T_class_t class_t = H5Tget_class(type);
297  if (H5T_NO_CLASS == class_t)
298  throw InternalErr(__FILE__, __LINE__,
299  "The HDF5 datatype doesn't belong to any Class.");
300  switch (class_t) {
301 
302  case H5T_INTEGER:
303 
304  size = H5Tget_size(type);
305  if (size == 0){
306  throw InternalErr(__FILE__, __LINE__,
307  "size of datatype is invalid");
308  }
309 
310  sign = H5Tget_sign(type);
311  if (sign < 0){
312  throw InternalErr(__FILE__, __LINE__,
313  "sign of datatype is invalid");
314  }
315 
316  BESDEBUG("h5", "=get_dap_type(): H5T_INTEGER" <<
317  " sign = " << sign <<
318  " size = " << size <<
319  endl);
320  if (size == 1){
321  // DAP2 doesn't have signed 8-bit integer, so we need map it to INT16.
322  if(true == is_dap4) {
323  if (sign == H5T_SGN_NONE)
324  return BYTE;
325  else
326  return INT8;
327  }
328  else {
329  if (sign == H5T_SGN_NONE)
330  return BYTE;
331  else
332  return INT16;
333  }
334  }
335 
336  if (size == 2) {
337  if (sign == H5T_SGN_NONE)
338  return UINT16;
339  else
340  return INT16;
341  }
342 
343  if (size == 4) {
344  if (sign == H5T_SGN_NONE)
345  return UINT32;
346  else
347  return INT32;
348  }
349 
350  if(size == 8) {
351  // DAP4 supports 64-bit integer.
352  if (true == is_dap4) {
353  if (sign == H5T_SGN_NONE)
354  return UINT64;
355  else
356  return INT64;
357  }
358  else
359  return INT_ELSE;
360  }
361 
362  return INT_ELSE;
363 
364  case H5T_FLOAT:
365  size = H5Tget_size(type);
366  if (size == 0){
367  throw InternalErr(__FILE__, __LINE__,
368  "size of the datatype is invalid");
369  }
370 
371  BESDEBUG("h5", "=get_dap_type(): FLOAT size = " << size << endl);
372  if (size == 4)
373  return FLOAT32;
374  if (size == 8)
375  return FLOAT64;
376 
377  return FLOAT_ELSE;
378 
379  case H5T_STRING:
380  BESDEBUG("h5", "<get_dap_type(): H5T_STRING" << endl);
381  return STRING;
382 
383  case H5T_REFERENCE:
384  BESDEBUG("h5", "<get_dap_type(): H5T_REFERENCE" << endl);
385  return URL;
386  // Note: Currently DAP2 and DAP4 only support defined atomic types.
387  // So the H5T_COMPOUND and H5_ARRAY cases should never be reached for attribute handling.
388  // However, this function may be used for variable handling.
389  case H5T_COMPOUND:
390  BESDEBUG("h5", "<get_dap_type(): COMPOUND" << endl);
391  return COMPOUND;
392 
393  case H5T_ARRAY:
394  return ARRAY;
395 
396  default:
397  BESDEBUG("h5", "<get_dap_type(): Unmappable Type" << endl);
398  return "Unmappable Type";
399  }
400 }
401 
411 hid_t get_fileid(const char *filename)
412 {
413  hid_t fileid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);
414  if (fileid < 0){
415  string msg = "cannot open the HDF5 file ";
416  string filenamestr(filename);
417  msg += filenamestr;
418  throw InternalErr(__FILE__, __LINE__, msg);
419  }
420 
421  return fileid;
422 }
423 
433 void close_fileid(hid_t fid)
434 {
435  if (H5Fclose(fid) < 0)
436  throw Error(unknown_error,
437  string("Could not close the HDF5 file."));
438 
439 }
440 
451 
452 void get_dataset(hid_t pid, const string &dname, DS_t * dt_inst_ptr)
453 {
454  BESDEBUG("h5", ">get_dataset()" << endl);
455 
456  // Obtain the dataset ID
457  hid_t dset = -1;
458  if ((dset = H5Dopen(pid, dname.c_str(),H5P_DEFAULT)) < 0) {
459  string msg = "cannot open the HDF5 dataset ";
460  msg += dname;
461  throw InternalErr(__FILE__, __LINE__, msg);
462  }
463 
464  // Obtain the datatype ID
465  hid_t dtype = -1;
466  if ((dtype = H5Dget_type(dset)) < 0) {
467  H5Dclose(dset);
468  string msg = "cannot get the the datatype of HDF5 dataset ";
469  msg += dname;
470  throw InternalErr(__FILE__, __LINE__, msg);
471  }
472 
473  // Obtain the datatype class
474  H5T_class_t ty_class = H5Tget_class(dtype);
475  if (ty_class < 0) {
476  H5Tclose(dtype);
477  H5Dclose(dset);
478  string msg = "cannot get the datatype class of HDF5 dataset ";
479  msg += dname;
480  throw InternalErr(__FILE__, __LINE__, msg);
481  }
482 
483  // These datatype classes are unsupported. Note we do support
484  // variable length string and the variable length string class is
485  // H5T_STRING rather than H5T_VLEN.
486  if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
487  || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM) || (ty_class == H5T_VLEN)) {
488  string msg = "unexpected datatype of HDF5 dataset ";
489  msg += dname;
490  throw InternalErr(__FILE__, __LINE__, msg);
491  }
492 
493  hid_t dspace = -1;
494  if ((dspace = H5Dget_space(dset)) < 0) {
495  H5Tclose(dtype);
496  H5Dclose(dset);
497  string msg = "cannot get the the dataspace of HDF5 dataset ";
498  msg += dname;
499  throw InternalErr(__FILE__, __LINE__, msg);
500  }
501 
502  // It is better to use the dynamic allocation of the array.
503  // However, since the DODS_MAX_RANK is not big and it is also
504  // used in other location, we still keep the original code.
505  // KY 2011-11-17
506 
507  int ndims = H5Sget_simple_extent_ndims(dspace);
508  if (ndims < 0) {
509  H5Tclose(dtype);
510  H5Sclose(dspace);
511  H5Dclose(dset);
512  string msg = "cannot get hdf5 dataspace number of dimension for dataset ";
513  msg += dname;
514  throw InternalErr(__FILE__, __LINE__, msg);
515  }
516 
517  // Check if the dimension size exceeds the maximum number of dimension DAP supports
518  if (ndims > DODS_MAX_RANK) {
519  string msg = "number of dimensions exceeds allowed for dataset ";
520  msg += dname;
521  H5Tclose(dtype);
522  H5Sclose(dspace);
523  H5Dclose(dset);
524  throw InternalErr(__FILE__, __LINE__, msg);
525  }
526 
527  hsize_t size[DODS_MAX_RANK];
528  hsize_t maxsize[DODS_MAX_RANK];
529 
530  // Retrieve size. DAP4 doesn't have a convention to support multi-unlimited dimension yet.
531  if (H5Sget_simple_extent_dims(dspace, size, maxsize)<0){
532  string msg = "cannot obtain the dim. info for the dataset ";
533  msg += dname;
534  H5Tclose(dtype);
535  H5Sclose(dspace);
536  H5Dclose(dset);
537  throw InternalErr(__FILE__, __LINE__, msg);
538  }
539 
540  // return ndims and size[ndims].
541  hsize_t nelmts = 1;
542  if (ndims !=0) {
543  for (int j = 0; j < ndims; j++)
544  nelmts *= size[j];
545  }
546 
547  size_t dtype_size = H5Tget_size(dtype);
548  if (dtype_size == 0) {
549  string msg = "cannot obtain the data type size for the dataset ";
550  msg += dname;
551  H5Tclose(dtype);
552  H5Sclose(dspace);
553  H5Dclose(dset);
554  throw InternalErr(__FILE__, __LINE__, msg);
555  }
556 
557  size_t need = nelmts * dtype_size;
558 
559  hid_t memtype = H5Tget_native_type(dtype, H5T_DIR_ASCEND);
560  if (memtype < 0){
561  string msg = "cannot obtain the memory data type for the dataset ";
562  msg += dname;
563  H5Tclose(dtype);
564  H5Sclose(dspace);
565  H5Dclose(dset);
566  throw InternalErr(__FILE__, __LINE__, msg);
567  }
568 
569  (*dt_inst_ptr).type = memtype;
570  (*dt_inst_ptr).ndims = ndims;
571  (*dt_inst_ptr).nelmts = nelmts;
572  (*dt_inst_ptr).need = need;
573  strncpy((*dt_inst_ptr).name, dname.c_str(), dname.length());
574  (*dt_inst_ptr).name[dname.length()] = '\0';
575  for (int j = 0; j < ndims; j++)
576  (*dt_inst_ptr).size[j] = size[j];
577 
578  if(H5Tclose(dtype)<0) {
579  H5Sclose(dspace);
580  H5Dclose(dset);
581  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
582  }
583 
584  if(H5Sclose(dspace)<0) {
585  H5Dclose(dset);
586  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataspace.");
587  }
588 
589  if(H5Dclose(dset)<0) {
590  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataset.");
591  }
592 
593 }
594 
609 void get_dataset_dmr(const hid_t file_id, hid_t pid, const string &dname, DS_t * dt_inst_ptr,bool use_dimscale, bool &is_pure_dim, vector<link_info_t> &hdf5_hls)
610 {
611 
612  BESDEBUG("h5", ">get_dataset()" << endl);
613 
614  // Obtain the dataset ID
615  hid_t dset = -1;
616  if ((dset = H5Dopen(pid, dname.c_str(),H5P_DEFAULT)) < 0) {
617  string msg = "cannot open the HDF5 dataset ";
618  msg += dname;
619  throw InternalErr(__FILE__, __LINE__, msg);
620  }
621 
622  // Obtain the datatype ID
623  hid_t dtype = -1;
624  if ((dtype = H5Dget_type(dset)) < 0) {
625  H5Dclose(dset);
626  string msg = "cannot get the the datatype of HDF5 dataset ";
627  msg += dname;
628  throw InternalErr(__FILE__, __LINE__, msg);
629  }
630 
631  // Obtain the datatype class
632  H5T_class_t ty_class = H5Tget_class(dtype);
633  if (ty_class < 0) {
634  H5Tclose(dtype);
635  H5Dclose(dset);
636  string msg = "cannot get the datatype class of HDF5 dataset ";
637  msg += dname;
638  throw InternalErr(__FILE__, __LINE__, msg);
639  }
640 
641  // These datatype classes are unsupported. Note we do support
642  // variable length string and the variable length string class is
643  // H5T_STRING rather than H5T_VLEN.
644  if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
645  || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM) || (ty_class == H5T_VLEN)) {
646  string msg = "unexpected datatype of HDF5 dataset ";
647  msg += dname;
648  throw InternalErr(__FILE__, __LINE__, msg);
649  }
650 
651  hid_t dspace = -1;
652  if ((dspace = H5Dget_space(dset)) < 0) {
653  H5Tclose(dtype);
654  H5Dclose(dset);
655  string msg = "cannot get the the dataspace of HDF5 dataset ";
656  msg += dname;
657  throw InternalErr(__FILE__, __LINE__, msg);
658  }
659 
660  // It is better to use the dynamic allocation of the array.
661  // However, since the DODS_MAX_RANK is not big and it is also
662  // used in other location, we still keep the original code.
663  // KY 2011-11-17
664 
665  int ndims = H5Sget_simple_extent_ndims(dspace);
666  if (ndims < 0) {
667  H5Tclose(dtype);
668  H5Sclose(dspace);
669  H5Dclose(dset);
670  string msg = "cannot get hdf5 dataspace number of dimension for dataset ";
671  msg += dname;
672  throw InternalErr(__FILE__, __LINE__, msg);
673  }
674 
675  // Check if the dimension size exceeds the maximum number of dimension DAP supports
676  if (ndims > DODS_MAX_RANK) {
677  string msg = "number of dimensions exceeds allowed for dataset ";
678  msg += dname;
679  H5Tclose(dtype);
680  H5Sclose(dspace);
681  H5Dclose(dset);
682  throw InternalErr(__FILE__, __LINE__, msg);
683  }
684 
685  hsize_t size[DODS_MAX_RANK];
686  hsize_t maxsize[DODS_MAX_RANK];
687 
688  // Retrieve size. DAP4 doesn't have a convention to support multi-unlimited dimension yet.
689  if (H5Sget_simple_extent_dims(dspace, size, maxsize)<0){
690  string msg = "cannot obtain the dim. info for the dataset ";
691  msg += dname;
692  H5Tclose(dtype);
693  H5Sclose(dspace);
694  H5Dclose(dset);
695  throw InternalErr(__FILE__, __LINE__, msg);
696  }
697 
698  // return ndims and size[ndims].
699  hsize_t nelmts = 1;
700  if (ndims !=0) {
701  for (int j = 0; j < ndims; j++)
702  nelmts *= size[j];
703  }
704 
705  size_t dtype_size = H5Tget_size(dtype);
706  if (dtype_size == 0) {
707  string msg = "cannot obtain the data type size for the dataset ";
708  msg += dname;
709  H5Tclose(dtype);
710  H5Sclose(dspace);
711  H5Dclose(dset);
712  throw InternalErr(__FILE__, __LINE__, msg);
713  }
714 
715  size_t need = nelmts * dtype_size;
716 
717  hid_t memtype = H5Tget_native_type(dtype, H5T_DIR_ASCEND);
718  if (memtype < 0){
719  string msg = "cannot obtain the memory data type for the dataset ";
720  msg += dname;
721  H5Tclose(dtype);
722  H5Sclose(dspace);
723  H5Dclose(dset);
724  throw InternalErr(__FILE__, __LINE__, msg);
725  }
726 
727  (*dt_inst_ptr).type = memtype;
728  (*dt_inst_ptr).ndims = ndims;
729  (*dt_inst_ptr).nelmts = nelmts;
730  (*dt_inst_ptr).need = need;
731  strncpy((*dt_inst_ptr).name, dname.c_str(), dname.length());
732  (*dt_inst_ptr).name[dname.length()] = '\0';
733  for (int j = 0; j < ndims; j++)
734  (*dt_inst_ptr).size[j] = size[j];
735 
736  // For DAP4 when dimension scales are used.
737  if(true == use_dimscale) {
738  BESDEBUG("h5", "<h5get.cc: get_dataset() use dim scale is true." << endl);
739 
740  // Some HDF5 datasets are dimension scale datasets; some are not. We need to distinguish.
741  bool is_dimscale = false;
742 
743  // Dimension scales must be 1-D.
744  if(1 == ndims) {
745 
746  bool has_ds_attr = false;
747 
748  try{
749  has_ds_attr = has_dimscale_attr(dset);
750  }
751  catch(...) {
752  H5Tclose(dtype);
753  H5Sclose(dspace);
754  H5Dclose(dset);
755  throw InternalErr(__FILE__, __LINE__, "Fail to check dim. scale.");
756  }
757 
758  if(true == has_ds_attr) {
759 
760  //int count = 0;
761  //vector<bool>dim_attr_mark;
762  //dim_attr_mark.resize(4);
763  //bool dim_attr_mark[4];
764  int dim_attr_mark[3];
765  for(int i = 0;i<3;i++)
766  dim_attr_mark[i] = 0;
767 
768  // This will check if "NAME" and "REFERENCE_LIST" exists.
769  //herr_t ret = H5Aiterate2(dset, H5_INDEX_NAME, H5_ITER_INC, NULL, attr_info, &dim_attr_mark[0]);
770  herr_t ret = H5Aiterate2(dset, H5_INDEX_NAME, H5_ITER_INC, NULL, attr_info_dimscale, dim_attr_mark);
771  if(ret < 0) {
772  string msg = "cannot interate the attributes of the dataset ";
773  msg += dname;
774  H5Tclose(dtype);
775  H5Sclose(dspace);
776  H5Dclose(dset);
777  throw InternalErr(__FILE__, __LINE__, msg);
778  }
779 
780  for (int i = 0; i<3;i++)
781  BESDEBUG("h5","dim_attr_mark is "<<dim_attr_mark[i] <<endl);
782  // Find the dimension scale. DIM*SCALE is a must. Then NAME=VARIABLE or (REFERENCE_LIST and not PURE DIM)
783  // Here a little bias towards files created by the netCDF-4 APIs.
784  // If we don't have RERERENCE_LIST in a dataset that has CLASS=DIMENSION_SCALE attribute,
785  // we will ignore this orphage dimension sale since it is not associated with other datasets.
786  // However, it is an orphage dimension scale created by the netCDF-4 APIs, we think
787  // it must have a purpose to do this way by data creator. So keep this as a dimension scale.
788  //
789  if (((dim_attr_mark[0] && !dim_attr_mark[1]) || dim_attr_mark[2]))
790  is_dimscale =true;
791  else if(dim_attr_mark[1])
792  is_pure_dim = true;
793  }
794  }
795 
796  if(true == is_dimscale) {
797  BESDEBUG("h5", "<h5get.cc: dname is " << dname << endl);
798  BESDEBUG("h5", "<h5get.cc: get_dataset() this is dim scale." << endl);
799  BESDEBUG("h5", "<h5get.cc: dataset storage size is: " <<H5Dget_storage_size(dset)<< endl);
800  //if(H5Dget_storage_size(dset)!=0) {
801  // Save the dimension names.We Only need to provide the dimension name(not the full path).
802  // We still need the dimension name fullpath for distinguishing the different dimension that
803  // has the same dimension name but in the different path
804  // TODO; pure dimension doesn't work for all cases. See https://jira.hdfgroup.org/browse/HFVHANDLER-340
805  (*dt_inst_ptr).dimnames.push_back(dname.substr(dname.find_last_of("/")+1));
806  (*dt_inst_ptr).dimnames_path.push_back(dname);
807  //}
808  //else
809  //is_pure_dim = true;
810  is_pure_dim = false;
811  }
812 
813  else if(false == is_pure_dim) // Except pure dimension,we need to save all dimension names in this dimension.
814  obtain_dimnames(file_id,dset,ndims,dt_inst_ptr,hdf5_hls);
815  }
816 
817  if(H5Tclose(dtype)<0) {
818  H5Sclose(dspace);
819  H5Dclose(dset);
820  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
821  }
822 
823  if(H5Sclose(dspace)<0) {
824  H5Dclose(dset);
825  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataspace.");
826  }
827 
828  if(H5Dclose(dset)<0) {
829  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataset.");
830  }
831 
832 }
833 
842 bool check_h5str(hid_t h5type)
843 {
844  if (H5Tget_class(h5type) == H5T_STRING)
845  return true;
846  else
847  return false;
848 }
849 
850 
863 string print_attr(hid_t type, int loc, void *sm_buf) {
864  union {
865  unsigned char* ucp;
866  char *tcp;
867  short *tsp;
868  unsigned short *tusp;
869  int *tip;
870  unsigned int*tuip;
871  long *tlp;
872  unsigned long*tulp;
873  float *tfp;
874  double *tdp;
875  } gp;
876 
877  vector<char> rep;
878 
879  switch (H5Tget_class(type)) {
880 
881  case H5T_INTEGER: {
882 
883  size_t size = H5Tget_size(type);
884  if (size == 0){
885  throw InternalErr(__FILE__, __LINE__,
886  "size of datatype is invalid");
887  }
888 
889  H5T_sign_t sign = H5Tget_sign(type);
890  if (sign < 0){
891  throw InternalErr(__FILE__, __LINE__,
892  "sign of datatype is invalid");
893  }
894 
895  BESDEBUG("h5", "=get_dap_type(): H5T_INTEGER" <<
896  " sign = " << sign <<
897  " size = " << size <<
898  endl);
899 
900  // change void pointer into the corresponding integer datatype.
901  // 32 should be long enough to hold one integer and one
902  // floating point number.
903  rep.resize(32);
904 
905  if (size == 1){
906 
907  if(sign == H5T_SGN_NONE) {
908  gp.ucp = (unsigned char *) sm_buf;
909  unsigned char tuchar = *(gp.ucp + loc);
910  snprintf(&rep[0], 32, "%u", tuchar);
911  }
912 
913  else {
914  gp.tcp = (char *) sm_buf;
915  snprintf(&rep[0], 32, "%d", *(gp.tcp + loc));
916  }
917  }
918 
919  else if (size == 2) {
920 
921  if(sign == H5T_SGN_NONE) {
922  gp.tusp = (unsigned short *) sm_buf;
923  snprintf(&rep[0], 32, "%hu", *(gp.tusp + loc));
924 
925  }
926  else {
927  gp.tsp = (short *) sm_buf;
928  snprintf(&rep[0], 32, "%hd", *(gp.tsp + loc));
929 
930  }
931  }
932 
933  else if (size == 4) {
934 
935  if(sign == H5T_SGN_NONE) {
936  gp.tuip = (unsigned int *) sm_buf;
937  snprintf(&rep[0], 32, "%u", *(gp.tuip + loc));
938 
939  }
940  else {
941  gp.tip = (int *) sm_buf;
942  snprintf(&rep[0], 32, "%d", *(gp.tip + loc));
943  }
944  }
945  else if (size == 8) {
946 
947  if(sign == H5T_SGN_NONE) {
948  gp.tulp = (unsigned long *) sm_buf;
949  snprintf(&rep[0], 32, "%lu", *(gp.tulp + loc));
950  }
951  else {
952  gp.tlp = (long *) sm_buf;
953  snprintf(&rep[0], 32, "%ld", *(gp.tlp + loc));
954  }
955  }
956  else
957  throw InternalErr(__FILE__, __LINE__,"Unsupported integer type, check the size of datatype.");
958 
959  break;
960  }
961 
962  case H5T_FLOAT: {
963  rep.resize(32);
964  char gps[32];
965 
966  if (H5Tget_size(type) == 4) {
967 
968  float attr_val = *(float*)sm_buf;
969  bool is_a_fin = isfinite(attr_val);
970  // Represent the float number.
971  // Some space may be wasted. But it is okay.
972  gp.tfp = (float *) sm_buf;
973  int ll = snprintf(gps, 30, "%.10g", *(gp.tfp + loc));
974  //int ll = strlen(gps);
975 
976  // Add the dot to assure this is a floating number
977  if (!strchr(gps, '.') && !strchr(gps, 'e') && !strchr(gps,'E')
978  && (true == is_a_fin)){
979  gps[ll++] = '.';
980  }
981 
982  gps[ll] = '\0';
983  snprintf(&rep[0], 32, "%s", gps);
984  }
985  else if (H5Tget_size(type) == 8) {
986 
987  double attr_val = *(double*)sm_buf;
988  bool is_a_fin = isfinite(attr_val);
989  gp.tdp = (double *) sm_buf;
990  int ll = snprintf(gps, 30, "%.17g", *(gp.tdp + loc));
991 #if 0
992  //int ll = strlen(gps);
993 #endif
994  if (!strchr(gps, '.') && !strchr(gps, 'e')&& !strchr(gps,'E')
995  && (true == is_a_fin)) {
996  gps[ll++] = '.';
997  }
998  gps[ll] = '\0';
999  snprintf(&rep[0], 32, "%s", gps);
1000  }
1001  else if (H5Tget_size(type) == 0){
1002  throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1003  }
1004  break;
1005  }
1006 
1007  case H5T_STRING: {
1008  int str_size = H5Tget_size(type);
1009  if(H5Tis_variable_str(type)>0) {
1010  throw InternalErr(__FILE__, __LINE__,
1011  "print_attr function doesn't handle variable length string, variable length string should be handled separately.");
1012  }
1013  if (str_size == 0){
1014  throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1015  }
1016  BESDEBUG("h5", "=print_attr(): H5T_STRING sm_buf=" << (char *) sm_buf
1017  << " size=" << str_size << endl);
1018  char *buf = NULL;
1019  // This try/catch block is here to protect the allocation of buf.
1020  try {
1021  buf = new char[str_size + 1];
1022  strncpy(buf, (char *) sm_buf, str_size);
1023  buf[str_size] = '\0';
1024  // Not necessarily allocate 3 more bytes.
1025  rep.resize(str_size+3);
1026  snprintf(&rep[0], str_size + 3, "%s", buf);
1027  rep[str_size + 2] = '\0';
1028  delete[] buf; buf = 0;
1029  }
1030  catch (...) {
1031  if( buf ) delete[] buf;
1032  throw;
1033  }
1034  break;
1035  }
1036 
1037  default:
1038  break;
1039  } // switch(H5Tget_class(type))
1040 
1041  string rep_str(rep.begin(),rep.end());
1042  return rep_str;
1043 }
1044 
1045 D4AttributeType daptype_strrep_to_dap4_attrtype(std::string s){
1046 
1047  if (s == "Byte")
1048  return attr_byte_c;
1049  else if (s == "Int8")
1050  return attr_int8_c;
1051  else if (s == "UInt8") // This may never be used.
1052  return attr_uint8_c;
1053  else if (s == "Int16")
1054  return attr_int16_c;
1055  else if (s == "UInt16")
1056  return attr_uint16_c;
1057  else if (s == "Int32")
1058  return attr_int32_c;
1059  else if (s == "UInt32")
1060  return attr_uint32_c;
1061  else if (s == "Int64")
1062  return attr_int64_c;
1063  else if (s == "UInt64")
1064  return attr_uint64_c;
1065  else if (s == "Float32")
1066  return attr_float32_c;
1067  else if (s == "Float64")
1068  return attr_float64_c;
1069  else if (s == "String")
1070  return attr_str_c;
1071  else if (s == "Url")
1072  return attr_url_c;
1073  else
1074  return attr_null_c;
1075 
1076 
1077 }
1078 
1079 
1093 //static BaseType *Get_bt(const string &vname,
1094 BaseType *Get_bt(const string &vname,
1095  const string &vpath,
1096  const string &dataset,
1097  hid_t datatype, bool is_dap4)
1098 {
1099  BaseType *btp = NULL;
1100 
1101  try {
1102 
1103  BESDEBUG("h5", ">Get_bt varname=" << vname << " datatype=" << datatype
1104  << endl);
1105 
1106  size_t size = 0;
1107  H5T_sign_t sign = H5T_SGN_ERROR;
1108  switch (H5Tget_class(datatype)) {
1109 
1110  case H5T_INTEGER:
1111  {
1112  size = H5Tget_size(datatype);
1113  sign = H5Tget_sign(datatype);
1114  BESDEBUG("h5", "=Get_bt() H5T_INTEGER size = " << size << " sign = "
1115  << sign << endl);
1116 
1117  if (sign == H5T_SGN_ERROR) {
1118  throw InternalErr(__FILE__, __LINE__, "cannot retrieve the sign type of the integer");
1119  }
1120  if (size == 0) {
1121  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1122  }
1123  else if (size == 1) { // Either signed char or unsigned char
1124  // DAP2 doesn't support signed char, it maps to DAP int16.
1125  if (sign == H5T_SGN_2) {
1126  if (false == is_dap4) // signed char to DAP2 int16
1127  btp = new HDF5Int16(vname, vpath, dataset);
1128  else
1129  btp = new HDF5Int8(vname,vpath,dataset);
1130  }
1131  else
1132  btp = new HDF5Byte(vname, vpath,dataset);
1133  }
1134  else if (size == 2) {
1135  if (sign == H5T_SGN_2)
1136  btp = new HDF5Int16(vname, vpath,dataset);
1137  else
1138  btp = new HDF5UInt16(vname,vpath, dataset);
1139  }
1140  else if (size == 4) {
1141  if (sign == H5T_SGN_2){
1142  btp = new HDF5Int32(vname, vpath,dataset);
1143  }
1144  else
1145  btp = new HDF5UInt32(vname,vpath, dataset);
1146  }
1147  else if (size == 8) {
1148  if(true == is_dap4) {
1149  if(sign == H5T_SGN_2)
1150  btp = new HDF5Int64(vname,vpath, dataset);
1151  else
1152  btp = new HDF5UInt64(vname,vpath, dataset);
1153  }
1154  else {
1155  throw
1156  InternalErr(__FILE__, __LINE__,
1157  string("Unsupported HDF5 64-bit Integer type:")
1158  + vname);
1159  }
1160  }
1161  }
1162  break;
1163 
1164  case H5T_FLOAT:
1165  {
1166  size = H5Tget_size(datatype);
1167  BESDEBUG("h5", "=Get_bt() H5T_FLOAT size = " << size << endl);
1168 
1169  if (size == 0) {
1170  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1171  }
1172  else if (size == 4) {
1173  btp = new HDF5Float32(vname,vpath, dataset);
1174  }
1175  else if (size == 8) {
1176  btp = new HDF5Float64(vname,vpath, dataset);
1177  }
1178  }
1179  break;
1180 
1181  case H5T_STRING:
1182  btp = new HDF5Str(vname, vpath,dataset);
1183  break;
1184 
1185  // The array datatype is rarely,rarely used. So this
1186  // part of code is not reviewed.
1187 #if 0
1188  case H5T_ARRAY: {
1189  BaseType *ar_bt = 0;
1190  try {
1191  BESDEBUG("h5",
1192  "=Get_bt() H5T_ARRAY datatype = " << datatype
1193  << endl);
1194 
1195  // Get the base datatype of the array
1196  hid_t dtype_base = H5Tget_super(datatype);
1197  ar_bt = Get_bt(vname, dataset, dtype_base);
1198  btp = new HDF5Array(vname, dataset, ar_bt);
1199  delete ar_bt; ar_bt = 0;
1200 
1201  // Set the size of the array.
1202  int ndim = H5Tget_array_ndims(datatype);
1203  size = H5Tget_size(datatype);
1204  int nelement = 1;
1205 
1206  if (dtype_base < 0) {
1207  throw InternalErr(__FILE__, __LINE__, "cannot return the base datatype");
1208  }
1209  if (ndim < 0) {
1210  throw InternalErr(__FILE__, __LINE__, "cannot return the rank of the array datatype");
1211  }
1212  if (size == 0) {
1213  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1214  }
1215  BESDEBUG(cerr
1216  << "=Get_bt()" << " Dim = " << ndim
1217  << " Size = " << size
1218  << endl);
1219 
1220  hsize_t size2[DODS_MAX_RANK];
1221  if(H5Tget_array_dims(datatype, size2) < 0){
1222  throw
1223  InternalErr(__FILE__, __LINE__,
1224  string("Could not get array dims for: ")
1225  + vname);
1226  }
1227 
1228 
1229  HDF5Array &h5_ar = static_cast < HDF5Array & >(*btp);
1230  for (int dim_index = 0; dim_index < ndim; dim_index++) {
1231  h5_ar.append_dim(size2[dim_index]);
1232  BESDEBUG("h5", "=Get_bt() " << size2[dim_index] << endl);
1233  nelement = nelement * size2[dim_index];
1234  }
1235 
1236  h5_ar.set_did(dt_inst.dset);
1237  // Assign the array datatype id.
1238  h5_ar.set_tid(datatype);
1239  h5_ar.set_memneed(size);
1240  h5_ar.set_numdim(ndim);
1241  h5_ar.set_numelm(nelement);
1242  h5_ar.set_length(nelement);
1243  h5_ar.d_type = H5Tget_class(dtype_base);
1244  if (h5_ar.d_type == H5T_NO_CLASS){
1245  throw InternalErr(__FILE__, __LINE__, "cannot return the datatype class identifier");
1246  }
1247  }
1248  catch (...) {
1249  if( ar_bt ) delete ar_bt;
1250  if( btp ) delete btp;
1251  throw;
1252  }
1253  break;
1254  }
1255 #endif
1256 
1257  // Reference map to DAP URL, check the technical note.
1258  case H5T_REFERENCE:
1259  btp = new HDF5Url(vname, vpath,dataset);
1260  break;
1261 
1262  default:
1263  throw InternalErr(__FILE__, __LINE__,
1264  string("Unsupported HDF5 type: ") + vname);
1265  }
1266  }
1267  catch (...) {
1268  if( btp ) delete btp;
1269  throw;
1270  }
1271 
1272  if (!btp)
1273  throw InternalErr(__FILE__, __LINE__,
1274  string("Could not make a DAP variable for: ")
1275  + vname);
1276 
1277  BESDEBUG("h5", "<Get_bt()" << endl);
1278  return btp;
1279 }
1280 
1281 
1298 //static Structure *Get_structure(const string &varname,
1299 Structure *Get_structure(const string &varname,const string &vpath,
1300  const string &dataset,
1301  hid_t datatype,bool is_dap4)
1302 {
1303  HDF5Structure *structure_ptr = NULL;
1304  char* memb_name = NULL;
1305  hid_t memb_type = -1;
1306 
1307  BESDEBUG("h5", ">Get_structure()" << datatype << endl);
1308 
1309  if (H5Tget_class(datatype) != H5T_COMPOUND)
1310  throw InternalErr(__FILE__, __LINE__,
1311  string("Compound-to-structure mapping error for ")
1312  + varname);
1313 
1314  try {
1315  structure_ptr = new HDF5Structure(varname, vpath, dataset);
1316 
1317  // Retrieve member types
1318  int nmembs = H5Tget_nmembers(datatype);
1319  BESDEBUG("h5", "=Get_structure() has " << nmembs << endl);
1320  if (nmembs < 0){
1321  throw InternalErr(__FILE__, __LINE__, "cannot retrieve the number of elements");
1322  }
1323  for (int i = 0; i < nmembs; i++) {
1324  memb_name = H5Tget_member_name(datatype, i);
1325  H5T_class_t memb_cls = H5Tget_member_class(datatype, i);
1326  memb_type = H5Tget_member_type(datatype, i);
1327  if (memb_name == NULL){
1328  throw InternalErr(__FILE__, __LINE__, "cannot retrieve the name of the member");
1329  }
1330  if ((memb_cls < 0) || (memb_type < 0)) {
1331  throw InternalErr(__FILE__, __LINE__,
1332  string("Type mapping error for ")
1333  + string(memb_name) );
1334  }
1335 
1336  if (memb_cls == H5T_COMPOUND) {
1337  Structure *s = Get_structure(memb_name, memb_name, dataset, memb_type,is_dap4);
1338  structure_ptr->add_var(s);
1339  delete s; s = 0;
1340  }
1341  else if(memb_cls == H5T_ARRAY) {
1342 
1343  BaseType *ar_bt = 0;
1344  BaseType *btp = 0;
1345  Structure *s = 0;
1346  hid_t dtype_base = 0;
1347 
1348  try {
1349 
1350  // Get the base memb_type of the array
1351  dtype_base = H5Tget_super(memb_type);
1352 
1353  // Set the size of the array.
1354  int ndim = H5Tget_array_ndims(memb_type);
1355  size_t size = H5Tget_size(memb_type);
1356  int nelement = 1;
1357 
1358  if (dtype_base < 0) {
1359  throw InternalErr(__FILE__, __LINE__, "cannot return the base memb_type");
1360  }
1361  if (ndim < 0) {
1362  throw InternalErr(__FILE__, __LINE__, "cannot return the rank of the array memb_type");
1363  }
1364  if (size == 0) {
1365  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the memb_type");
1366  }
1367 
1368  hsize_t size2[DODS_MAX_RANK];
1369  if(H5Tget_array_dims(memb_type, size2) < 0){
1370  throw
1371  InternalErr(__FILE__, __LINE__,
1372  string("Could not get array dims for: ")
1373  + string(memb_name));
1374  }
1375 
1376  H5T_class_t array_memb_cls = H5Tget_class(dtype_base);
1377  if(array_memb_cls == H5T_NO_CLASS) {
1378  throw InternalErr(__FILE__, __LINE__,
1379  string("cannot get the correct class for compound type member")
1380  + string(memb_name));
1381  }
1382  if(H5T_COMPOUND == array_memb_cls) {
1383 
1384  s = Get_structure(memb_name, memb_name,dataset, dtype_base,is_dap4);
1385  HDF5Array *h5_ar = new HDF5Array(memb_name, dataset, s);
1386 
1387  for (int dim_index = 0; dim_index < ndim; dim_index++) {
1388  h5_ar->append_dim(size2[dim_index]);
1389  nelement = nelement * size2[dim_index];
1390  }
1391 
1392  h5_ar->set_memneed(size);
1393  h5_ar->set_numdim(ndim);
1394  h5_ar->set_numelm(nelement);
1395  h5_ar->set_length(nelement);
1396 
1397  structure_ptr->add_var(h5_ar);
1398  delete h5_ar;
1399 
1400  }
1401  else if (H5T_INTEGER == array_memb_cls || H5T_FLOAT == array_memb_cls || H5T_STRING == array_memb_cls) {
1402  ar_bt = Get_bt(memb_name, memb_name,dataset, dtype_base,is_dap4);
1403  HDF5Array *h5_ar = new HDF5Array(memb_name,dataset,ar_bt);
1404 
1405  for (int dim_index = 0; dim_index < ndim; dim_index++) {
1406  h5_ar->append_dim(size2[dim_index]);
1407  nelement = nelement * size2[dim_index];
1408  }
1409 
1410  h5_ar->set_memneed(size);
1411  h5_ar->set_numdim(ndim);
1412  h5_ar->set_numelm(nelement);
1413  h5_ar->set_length(nelement);
1414 
1415  structure_ptr->add_var(h5_ar);
1416  delete h5_ar;
1417  }
1418  if( ar_bt ) delete ar_bt;
1419  if( btp ) delete btp;
1420  if(s) delete s;
1421  H5Tclose(dtype_base);
1422 
1423  }
1424  catch (...) {
1425  if( ar_bt ) delete ar_bt;
1426  if( btp ) delete btp;
1427  if(s) delete s;
1428  H5Tclose(dtype_base);
1429  throw;
1430  }
1431 
1432  }
1433  else if (memb_cls == H5T_INTEGER || memb_cls == H5T_FLOAT || memb_cls == H5T_STRING) {
1434  BaseType *bt = Get_bt(memb_name, memb_name,dataset, memb_type,is_dap4);
1435  structure_ptr->add_var(bt);
1436  delete bt; bt = 0;
1437  }
1438  else {
1439  free(memb_name);
1440  memb_name = NULL;
1441  throw InternalErr(__FILE__, __LINE__, "unsupported field datatype inside a compound datatype");
1442  }
1443  // Caller needs to free the memory allocated by the library for memb_name.
1444  if(memb_name != NULL)
1445  free(memb_name);
1446  }
1447  }
1448  catch (...) {
1449  if( structure_ptr )
1450  delete structure_ptr;
1451  if(memb_name!= NULL)
1452  free(memb_name);
1453  if(memb_type != -1)
1454  H5Tclose(memb_type);
1455  throw;
1456  }
1457 
1458  BESDEBUG("h5", "<Get_structure()" << endl);
1459 
1460  return structure_ptr;
1461 }
1462 
1479 
1480 // Function to use H5OVISIT to check if dimension scale attributes exist.
1481 bool check_dimscale(hid_t fileid) {
1482 
1483  bool ret_value = false;
1484  herr_t ret_o= H5OVISIT(fileid, H5_INDEX_NAME, H5_ITER_INC, visit_obj_cb, NULL);
1485  if(ret_o < 0)
1486  throw InternalErr(__FILE__, __LINE__, "H5OVISIT fails");
1487  else
1488  ret_value =(ret_o >0)?true:false;
1489 
1490  return ret_value;
1491 }
1492 
1493 static int
1494 visit_obj_cb(hid_t group_id, const char *name, const H5O_info_t *oinfo,
1495  void *_op_data)
1496 {
1497 
1498  int ret_value = 0;
1499  if(oinfo->type == H5O_TYPE_DATASET) {
1500 
1501  hid_t dataset = -1;
1502  dataset = H5Dopen2(group_id,name,H5P_DEFAULT);
1503  if(dataset <0)
1504  throw InternalErr(__FILE__, __LINE__, "H5Dopen2 fails in the H5OVISIT call back function.");
1505 
1506  hid_t dspace = -1;
1507  dspace = H5Dget_space(dataset);
1508  if(dspace <0) {
1509  H5Dclose(dataset);
1510  throw InternalErr(__FILE__, __LINE__, "H5Dget_space fails in the H5OVISIT call back function.");
1511  }
1512 
1513  // We only support netCDF-4 like dimension scales, that is the dimension scale dataset is 1-D dimension.
1514  if(H5Sget_simple_extent_ndims(dspace) == 1) {
1515  try {
1516  if(true == has_dimscale_attr(dataset))
1517  ret_value = 1;
1518  }
1519  catch(...) {
1520  H5Sclose(dspace);
1521  H5Dclose(dataset);
1522  throw;
1523  }
1524 
1525 #if 0
1526  //vector<bool>dim_attr_mark;
1527  //dim_attr_mark.resize(4);
1528  //bool dim_attr_mark[4];
1529  int dim_attr_mark[4];
1530  for(int i =0;i<4;i++)
1531  dim_attr_mark[i] = 0;
1532  //int count = 0;
1533  // Check if having "class = DIMENSION_SCALE" and REFERENCE_LIST attributes.
1534  //herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, NULL, attr_info, &count);
1535  //herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, NULL, attr_info, &dim_attr_mark[0]);
1536  herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, NULL, attr_info, dim_attr_mark);
1537  if(ret < 0) {
1538  H5Sclose(dspace);
1539  H5Dclose(dataset);
1540  throw InternalErr(__FILE__, __LINE__, "H5Aiterate2 fails in the H5OVISIT call back function.");
1541  }
1542 
1543  BESDEBUG("h5", "<dset name is " << name <<endl);
1544  //BESDEBUG("h5", "<count is " << count <<endl);
1545  // Find it.
1546  if (dim_attr_mark[0] && dim_attr_mark[1]){
1547  //if (2==count || 3==count) {
1548  if(dspace != -1)
1549  H5Sclose(dspace);
1550  if(dataset != -1)
1551  H5Dclose(dataset);
1552  return 1;
1553  //}
1554  }
1555 #endif
1556  }
1557  if(dspace != -1)
1558  H5Sclose(dspace);
1559  if(dataset != -1)
1560  H5Dclose(dataset);
1561  }
1562  return ret_value;
1563 }
1564 
1565 bool has_dimscale_attr(hid_t dataset) {
1566 
1567  bool ret_value = false;
1568  string dimscale_attr_name="CLASS";
1569  string dimscale_attr_value="DIMENSION_SCALE";
1570  htri_t dimscale_attr_exist = H5Aexists_by_name(dataset,".",dimscale_attr_name.c_str(),H5P_DEFAULT);
1571  if(dimscale_attr_exist <0)
1572  throw InternalErr(__FILE__, __LINE__, "H5Aexists_by_name fails when checking the CLASS attribute.");
1573  else if(dimscale_attr_exist > 0) {//Attribute CLASS exists
1574 
1575  hid_t attr_id = -1;
1576  hid_t atype_id = -1;
1577 
1578  // Open the attribute
1579  attr_id = H5Aopen(dataset,dimscale_attr_name.c_str(), H5P_DEFAULT);
1580  if(attr_id < 0)
1581  throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1582 
1583  // Get attribute datatype and dataspace
1584  atype_id = H5Aget_type(attr_id);
1585  if(atype_id < 0) {
1586  H5Aclose(attr_id);
1587  throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1588  }
1589 
1590  try {
1591  // Check if finding the attribute CLASS and the value is "DIMENSION_SCALE".
1592  if (H5T_STRING == H5Tget_class(atype_id))
1593  ret_value = check_str_attr_value(attr_id,atype_id,dimscale_attr_value,false);
1594  }
1595  catch(...) {
1596  if(atype_id != -1)
1597  H5Tclose(atype_id);
1598  if(attr_id != -1)
1599  H5Aclose(attr_id);
1600  throw;
1601  }
1602  // Close IDs.
1603  if(atype_id != -1)
1604  H5Tclose(atype_id);
1605  if(attr_id != -1)
1606  H5Aclose(attr_id);
1607  }
1608  return ret_value;
1609 
1610 }
1611 #if 0
1624 
1625 static herr_t
1626 attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata)
1627 {
1628  int *countp = (int*)opdata;
1629 
1630  hid_t attr_id = -1;
1631  hid_t atype_id = -1;
1632 
1633  // Open the attribute
1634  attr_id = H5Aopen(loc_id, name, H5P_DEFAULT);
1635  if(attr_id < 0)
1636  throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1637 
1638  // Get attribute datatype and dataspace
1639  atype_id = H5Aget_type(attr_id);
1640  if(atype_id < 0) {
1641  H5Aclose(attr_id);
1642  throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1643  }
1644 
1645  try {
1646 
1647 #if 0
1648  // If finding the "REFERENCE_LIST", increases the countp.
1649  if ((H5T_COMPOUND == H5Tget_class(atype_id)) && (strcmp(name,"REFERENCE_LIST")==0)) {
1650  //(*countp)++;
1651  *dimattr_p = 1;
1652  //*dimattr_p = true;
1653  }
1654 #endif
1655  // Check if finding the attribute CLASS and the value is "DIMENSION_SCALE".
1656  if (H5T_STRING == H5Tget_class(atype_id)) {
1657  if (strcmp(name,"CLASS") == 0) {
1658  string dim_scale_mark = "DIMENSION_SCALE";
1659  bool is_dim_scale = check_str_attr_value(attr_id,atype_id,dim_scale_mark,false);
1660  if(true == is_dim_scale)
1661  (*countp)++;
1662  }
1663  }
1664  }
1665  catch(...) {
1666  if(atype_id != -1)
1667  H5Tclose(atype_id);
1668  if(attr_id != -1)
1669  H5Aclose(attr_id);
1670  throw;
1671  }
1672 
1673  // Close IDs.
1674  if(atype_id != -1)
1675  H5Tclose(atype_id);
1676  if(attr_id != -1)
1677  H5Aclose(attr_id);
1678 
1679  return 0;
1680 }
1681 
1682 #endif
1695 
1696 static herr_t
1697 attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata)
1698 {
1699  //bool *countp = (bool*)opdata;
1700  //bool *dimattr_p = (bool*)opdata;
1701  int *dimattr_p = (int*)opdata;
1702 
1703 #if 0
1704  bool has_reference_list = false;
1705  bool has_dimscale = false;
1706  bool has_name_as_var = false;
1707  bool has_name_as_nc4_purdim = false;
1708 #endif
1709 
1710 
1711  hid_t attr_id = -1;
1712  hid_t atype_id = -1;
1713 
1714  // Open the attribute
1715  attr_id = H5Aopen(loc_id, name, H5P_DEFAULT);
1716  if(attr_id < 0)
1717  throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1718 
1719  // Get attribute datatype and dataspace
1720  atype_id = H5Aget_type(attr_id);
1721  if(atype_id < 0) {
1722  H5Aclose(attr_id);
1723  throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1724  }
1725 
1726  try {
1727 
1728 //#if 0
1729  // If finding the "REFERENCE_LIST", increases the countp.
1730  if ((H5T_COMPOUND == H5Tget_class(atype_id)) && (strcmp(name,"REFERENCE_LIST")==0)) {
1731  //(*countp)++;
1732  *dimattr_p = 1;
1733  //*dimattr_p = true;
1734  }
1735 //#endif
1736  // Check if finding the CLASS attribute.
1737  if (H5T_STRING == H5Tget_class(atype_id)) {
1738  if (strcmp(name,"NAME") == 0) {
1739 
1740  string pure_dimname_mark = "This is a netCDF dimension but not a netCDF variable";
1741  bool is_pure_dim = check_str_attr_value(attr_id,atype_id,pure_dimname_mark,true);
1742 
1743  BESDEBUG("h5","pure dimension name yes" << is_pure_dim <<endl);
1744  if(true == is_pure_dim)
1745  *(dimattr_p+1) =1;
1746  //*(dimattr_p+2) =true;
1747  else {
1748  // netCDF save the variable name in the "NAME" attribute.
1749  // We need to retrieve the variable name first.
1750  ssize_t objnamelen = -1;
1751  if ((objnamelen= H5Iget_name(loc_id,NULL,0))<=0) {
1752  string msg = "Cannot obtain the variable name length." ;
1753  throw InternalErr(__FILE__,__LINE__,msg);
1754  }
1755  vector<char> objname;
1756  objname.resize(objnamelen+1);
1757  if ((objnamelen= H5Iget_name(loc_id,&objname[0],objnamelen+1))<=0) {
1758  string msg = "Cannot obtain the variable name." ;
1759  throw InternalErr(__FILE__,__LINE__,msg);
1760  }
1761 
1762  string objname_str = string(objname.begin(),objname.end());
1763 
1764  // Must trim the string delimter.
1765  objname_str = objname_str.substr(0,objnamelen);
1766  // Remove the path
1767  string normal_dimname_mark = objname_str.substr(objname_str.find_last_of("/")+1);
1768  bool is_normal_dim = check_str_attr_value(attr_id,atype_id,normal_dimname_mark,false);
1769  if(true == is_normal_dim)
1770  *(dimattr_p+2) = 1;
1771  //*(dimattr_p+3) = true;
1772  }
1773  }
1774  }
1775 
1776 #if 0
1777  H5T_str_t str_pad = H5Tget_strpad(atype_id);
1778 
1779  hid_t aspace_id = -1;
1780  aspace_id = H5Aget_space(attr_id);
1781  if(aspace_id < 0)
1782  throw InternalErr(__FILE__, __LINE__, "H5Aget_space fails in the attr_info call back function.");
1783 
1784  // CLASS is a variable-length string
1785  int ndims = H5Sget_simple_extent_ndims(aspace_id);
1786  hsize_t nelmts = 1;
1787 
1788  // if it is a scalar attribute, just define number of elements to be 1.
1789  if (ndims != 0) {
1790 
1791  vector<hsize_t> asize;
1792  vector<hsize_t> maxsize;
1793  asize.resize(ndims);
1794  maxsize.resize(ndims);
1795 
1796  // DAP applications don't care about the unlimited dimensions
1797  // since the applications only care about retrieving the data.
1798  // So we don't check the maxsize to see if it is the unlimited dimension
1799  // attribute.
1800  if (H5Sget_simple_extent_dims(aspace_id, &asize[0], &maxsize[0])<0) {
1801  H5Sclose(aspace_id);
1802  throw InternalErr(__FILE__, __LINE__, "Cannot obtain the dim. info in the H5Aiterate2 call back function.");
1803  }
1804 
1805  // Return ndims and size[ndims].
1806  for (int j = 0; j < ndims; j++)
1807  nelmts *= asize[j];
1808  } // if(ndims != 0)
1809 
1810  size_t ty_size = H5Tget_size(atype_id);
1811  if (0 == ty_size) {
1812  H5Sclose(aspace_id);
1813  throw InternalErr(__FILE__, __LINE__, "Cannot obtain the type size in the H5Aiterate2 call back function.");
1814  }
1815 
1816  size_t total_bytes = nelmts * ty_size;
1817  string total_vstring ="";
1818  if(H5Tis_variable_str(atype_id) > 0) {
1819 
1820  // Variable length string attribute values only store pointers of the actual string value.
1821  vector<char> temp_buf;
1822  temp_buf.resize(total_bytes);
1823 
1824  if (H5Aread(attr_id, atype_id, &temp_buf[0]) < 0){
1825  H5Sclose(aspace_id);
1826  throw InternalErr(__FILE__,__LINE__,"Cannot read the attribute in the H5Aiterate2 call back function");
1827  }
1828 
1829  char *temp_bp = NULL;
1830  temp_bp = &temp_buf[0];
1831  char* onestring = NULL;
1832 
1833  for (unsigned int temp_i = 0; temp_i <nelmts; temp_i++) {
1834 
1835  // This line will assure that we get the real variable length string value.
1836  onestring =*(char **)temp_bp;
1837 
1838  if(onestring!= NULL)
1839  total_vstring +=string(onestring);
1840 
1841  // going to the next value.
1842  temp_bp +=ty_size;
1843  }
1844 
1845  if ((&temp_buf[0]) != NULL) {
1846  // Reclaim any VL memory if necessary.
1847  if (H5Dvlen_reclaim(atype_id,aspace_id,H5P_DEFAULT,&temp_buf[0]) < 0) {
1848  H5Sclose(aspace_id);
1849  throw InternalErr(__FILE__,__LINE__,"Cannot reclaim VL memory in the H5Aiterate2 call back function.");
1850  }
1851  }
1852 
1853  }
1854  else {// Fixed-size string, need to retrieve the string value.
1855 
1856  // string attribute values
1857  vector<char> temp_buf;
1858  temp_buf.resize(total_bytes);
1859  if (H5Aread(attr_id, atype_id, &temp_buf[0]) < 0){
1860  H5Sclose(aspace_id);
1861  throw InternalErr(__FILE__,__LINE__,"Cannot read the attribute in the H5Aiterate2 call back function");
1862  }
1863  string temp_buf_string(temp_buf.begin(),temp_buf.end());
1864  total_vstring = temp_buf_string.substr(0,total_bytes);
1865 
1866  // Note: we need to remove the string pad or term to find DIMENSION_SCALE.
1867  if(str_pad != H5T_STR_ERROR)
1868  total_vstring = total_vstring.substr(0,total_vstring.size()-1);
1869  }
1870 
1871  // Close attribute data space ID.
1872  if(aspace_id != -1)
1873  H5Sclose(aspace_id);
1874  if(total_vstring == "DIMENSION_SCALE"){
1875  (*countp)++;
1876  }
1877 #endif
1878 
1879  }
1880  catch(...) {
1881  if(atype_id != -1)
1882  H5Tclose(atype_id);
1883  if(attr_id != -1)
1884  H5Aclose(attr_id);
1885  throw;
1886  }
1887 
1888  // Close IDs.
1889  if(atype_id != -1)
1890  H5Tclose(atype_id);
1891  if(attr_id != -1)
1892  H5Aclose(attr_id);
1893 
1894  return 0;
1895 }
1896 
1897 
1905 
1907 void obtain_dimnames(const hid_t file_id,hid_t dset,int ndims, DS_t *dt_inst_ptr,vector<link_info_t> & hdf5_hls) {
1908 
1909  htri_t has_dimension_list = -1;
1910 
1911  string dimlist_name = "DIMENSION_LIST";
1912  has_dimension_list = H5Aexists(dset,dimlist_name.c_str());
1913 
1914  if(has_dimension_list > 0 && ndims > 0) {
1915 
1916  hobj_ref_t rbuf;
1917  vector<hvl_t> vlbuf;
1918  vlbuf.resize(ndims);
1919 
1920  hid_t attr_id = -1;
1921  hid_t atype_id = -1;
1922  hid_t amemtype_id = -1;
1923  hid_t aspace_id = -1;
1924  hid_t ref_dset = -1;
1925 
1926  try {
1927  attr_id = H5Aopen(dset,dimlist_name.c_str(),H5P_DEFAULT);
1928  if (attr_id <0 ) {
1929  string msg = "Cannot open the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1930  throw InternalErr(__FILE__, __LINE__, msg);
1931  }
1932 
1933  atype_id = H5Aget_type(attr_id);
1934  if (atype_id <0) {
1935  string msg = "Cannot get the datatype of the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1936  throw InternalErr(__FILE__, __LINE__, msg);
1937  }
1938 
1939  amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
1940  if (amemtype_id < 0) {
1941  string msg = "Cannot get the memory datatype of the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1942  throw InternalErr(__FILE__, __LINE__, msg);
1943 
1944  }
1945 
1946  if (H5Aread(attr_id,amemtype_id,&vlbuf[0]) <0) {
1947  string msg = "Cannot obtain the referenced object for the variable " + string(dt_inst_ptr->name);
1948  throw InternalErr(__FILE__, __LINE__, msg);
1949  }
1950 
1951  vector<char> objname;
1952 
1953  // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
1954  for (int i = 0; i < ndims; i++) {
1955 
1956  if(vlbuf[i].p == NULL) {
1957  stringstream sindex ;
1958  sindex <<i;
1959  string msg = "For variable " + string(dt_inst_ptr->name) + "; ";
1960  msg = msg + "the dimension of which the index is "+ sindex.str() + " doesn't exist. ";
1961  throw InternalErr(__FILE__, __LINE__, msg);
1962  }
1963 
1964  rbuf =((hobj_ref_t*)vlbuf[i].p)[0];
1965 
1966  // Note: TODO: H5Rget_name may be used to replace H5RDEREFERENCE and H5Iget_name in the future. KY 2016-06-28
1967  if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0) {
1968  string msg = "Cannot dereference from the DIMENSION_LIST attribute for the variable " + string(dt_inst_ptr->name);
1969  throw InternalErr(__FILE__, __LINE__, msg);
1970  }
1971 
1972  ssize_t objnamelen = -1;
1973  if ((objnamelen= H5Iget_name(ref_dset,NULL,0))<=0) {
1974  string msg = "Cannot obtain the dimension name length for the variable " + string(dt_inst_ptr->name);
1975  throw InternalErr(__FILE__,__LINE__,msg);
1976  }
1977 
1978  objname.resize(objnamelen+1);
1979  if ((objnamelen= H5Iget_name(ref_dset,&objname[0],objnamelen+1))<=0) {
1980  H5Dclose(ref_dset);
1981  string msg = "Cannot obtain the dimension name for the variable " + string(dt_inst_ptr->name);
1982  throw InternalErr(__FILE__,__LINE__,msg);
1983  }
1984 
1985  string objname_str = string(objname.begin(),objname.end());
1986 
1987  // Must trim the string delimter.
1988  string trim_objname = objname_str.substr(0,objnamelen);
1989 
1990  // We need to check if there are hardlinks for this variable.
1991  // If yes, we need to find the hardlink that has the shortest path and at the ancestor group
1992  // of all links.
1993  H5O_info_t obj_info;
1994  if(H5OGET_INFO(ref_dset,&obj_info)<0) {
1995  H5Dclose(ref_dset);
1996  string msg = "Cannot obtain the object info for the dimension variable " + objname_str;
1997  throw InternalErr(__FILE__,__LINE__,msg);
1998  }
1999 
2000  // This dimension indeed has hard links.
2001  if(obj_info.rc > 1) {
2002 
2003  // 1. Search the hdf5_hls to see if the address is inside
2004  // if yes,
2005  // obtain the hard link which has the shortest path, use this as the dimension name.
2006  // else
2007  // search all the hardlinks with the callback.
2008  // obtain the shortest path, add this to hdf5_hls.
2009 
2010  bool link_find = false;
2011 
2012  // If finding the object in the hdf5_hls, obtain the hardlink and make it the dimension name(trim_objname).
2013  for (int i = 0; i <hdf5_hls.size();i++) {
2014 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2015  int token_cmp = -1;
2016  if(H5Otoken_cmp(ref_dset,&(obj_info.token),&(hdf5_hls[i].link_addr),&token_cmp) <0)
2017  throw InternalErr(__FILE__,__LINE__,"H5Otoken_cmp failed");
2018  if(!token_cmp) {
2019 #else
2020  if(obj_info.addr == hdf5_hls[i].link_addr) {
2021 #endif
2022  trim_objname = '/'+hdf5_hls[i].slink_path;
2023  link_find = true;
2024  break;
2025  }
2026  }
2027 
2028  // The hard link is not in the hdf5_hls, need to iterate all objects and find those hardlinks.
2029  if(link_find == false) {
2030 
2031 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2032  typedef struct {
2033  unsigned link_unvisited;
2034  H5O_token_t link_addr;
2035  vector<string> hl_names;
2036  } t_link_info_t;
2037 #else
2038  typedef struct {
2039  unsigned link_unvisited;
2040  haddr_t link_addr;
2041  vector<string> hl_names;
2042  } t_link_info_t;
2043 #endif
2044 
2045  t_link_info_t t_li_info;
2046  t_li_info.link_unvisited = obj_info.rc;
2047 
2048 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2049  memcpy(&t_li_info.link_addr,&obj_info.token,sizeof(H5O_token_t));
2050 #else
2051  t_li_info.link_addr = obj_info.addr;
2052 #endif
2053 
2054  if(H5Lvisit(file_id, H5_INDEX_NAME, H5_ITER_NATIVE, visit_link_cb, (void*)&t_li_info) < 0) {
2055  H5Dclose(ref_dset);
2056  string err_msg;
2057  err_msg = "Find all hardlinks: H5Lvisit failed to iterate all the objects";
2058  throw InternalErr(__FILE__,__LINE__,err_msg);
2059  }
2060 #if 0
2061 for(int i = 0; i<t_li_info.hl_names.size();i++)
2062  cerr<<"hl name is "<<t_li_info.hl_names[i] <<endl;
2063 #endif
2064 
2065  string shortest_hl = obtain_shortest_ancestor_path(t_li_info.hl_names);
2066 //cerr<<"shortest_hl is "<<shortest_hl <<endl;
2067  if(shortest_hl =="") {
2068  H5Dclose(ref_dset);
2069  string err_msg;
2070  err_msg = "The shortest hardlink is not located under an ancestor group of all links.";
2071  err_msg +="This is not supported by netCDF4 data model and the current Hyrax DAP4 implementation.";
2072  throw InternalErr(__FILE__,__LINE__,err_msg);
2073  }
2074 
2075  // Save this link that holds the shortest path for future use.
2076  link_info_t new_hdf5_hl;
2077 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2078  memcpy(&new_hdf5_hl.link_addr,&obj_info.token,sizeof(H5O_token_t));
2079 #else
2080  new_hdf5_hl.link_addr = obj_info.addr;
2081 #endif
2082  new_hdf5_hl.slink_path = shortest_hl;
2083  hdf5_hls.push_back(new_hdf5_hl);
2084  trim_objname = '/'+shortest_hl;
2085 
2086  }
2087  }
2088  // Need to save the dimension names without the path
2089 
2090  dt_inst_ptr->dimnames.push_back(trim_objname.substr(trim_objname.find_last_of("/")+1));
2091  dt_inst_ptr->dimnames_path.push_back(trim_objname);
2092 
2093  if(H5Dclose(ref_dset)<0) {
2094  throw InternalErr(__FILE__,__LINE__,"Cannot close the HDF5 dataset in the function obtain_dimnames().");
2095  }
2096  objname.clear();
2097  }// for (vector<Dimension *>::iterator ird is var->dims.begin()
2098  if(vlbuf.empty()== false) {
2099 
2100  if ((aspace_id = H5Aget_space(attr_id)) < 0) {
2101  string msg = "Cannot close the HDF5 attribute space successfully for <DIMENSION_LIST> of the variable "+string(dt_inst_ptr->name);
2102  throw InternalErr(__FILE__,__LINE__,msg);
2103  }
2104 
2105  if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)&vlbuf[0])<0) {
2106  throw InternalErr(__FILE__,__LINE__,"Cannot reclaim the variable length memory in the function obtain_dimnames()");
2107  }
2108 
2109  H5Sclose(aspace_id);
2110 
2111  }
2112 
2113  H5Tclose(atype_id);
2114  H5Tclose(amemtype_id);
2115  H5Aclose(attr_id);
2116 
2117  }
2118 
2119  catch(...) {
2120 
2121  if(atype_id != -1)
2122  H5Tclose(atype_id);
2123 
2124  if(amemtype_id != -1)
2125  H5Tclose(amemtype_id);
2126 
2127  if(aspace_id != -1)
2128  H5Sclose(aspace_id);
2129 
2130  if(attr_id != -1)
2131  H5Aclose(attr_id);
2132 
2133  throw;
2134  }
2135 
2136  }
2137  return ;
2138 }
2139 
2140 void write_vlen_str_attrs(hid_t attr_id,hid_t ty_id, DSattr_t * attr_inst_ptr,D4Attribute *d4_attr, AttrTable* d2_attr,bool is_dap4){
2141 
2142  BESDEBUG("h5","attribute name " << attr_inst_ptr->name <<endl);
2143  BESDEBUG("h5","attribute size " <<attr_inst_ptr->need <<endl);
2144  BESDEBUG("h5","attribute type size " <<(int)(H5Tget_size(ty_id))<<endl);
2145 
2146  hid_t temp_space_id = H5Aget_space(attr_id);
2147  BESDEBUG("h5","attribute calculated size "<<(int)(H5Tget_size(ty_id)) *(int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
2148  if(temp_space_id <0) {
2149  H5Tclose(ty_id);
2150  H5Aclose(attr_id);
2151  if(d4_attr)
2152  delete d4_attr;
2153  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
2154  }
2155 
2156  vector<char> temp_buf;
2157  // Variable length string attribute values only store pointers of the actual string value.
2158  temp_buf.resize((size_t)attr_inst_ptr->need);
2159 
2160  if (H5Aread(attr_id, ty_id, &temp_buf[0]) < 0) {
2161  H5Tclose(ty_id);
2162  H5Aclose(attr_id);
2163  H5Sclose(temp_space_id);
2164  if(d4_attr)
2165  delete d4_attr;
2166  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
2167  }
2168 
2169  char *temp_bp;
2170  temp_bp = &temp_buf[0];
2171  char* onestring;
2172  for (unsigned int temp_i = 0; temp_i <attr_inst_ptr->nelmts; temp_i++) {
2173 
2174  // This line will assure that we get the real variable length string value.
2175  onestring =*(char **)temp_bp;
2176 
2177  // Change the C-style string to C++ STD string just for easy appending the attributes in DAP.
2178  if (onestring !=NULL) {
2179  string tempstring(onestring);
2180  if(true == is_dap4)
2181  d4_attr->add_value(tempstring);
2182  else
2183  d2_attr->append_attr(attr_inst_ptr->name,"String",tempstring);
2184  }
2185 
2186  temp_bp +=H5Tget_size(ty_id);
2187  }
2188  if (temp_buf.empty() != true) {
2189 
2190  // Reclaim any VL memory if necessary.
2191  herr_t ret_vlen_claim;
2192  ret_vlen_claim = H5Dvlen_reclaim(ty_id,temp_space_id,H5P_DEFAULT,&temp_buf[0]);
2193  if(ret_vlen_claim < 0){
2194  H5Tclose(ty_id);
2195  H5Aclose(attr_id);
2196  H5Sclose(temp_space_id);
2197  if(d4_attr)
2198  delete d4_attr;
2199  throw InternalErr(__FILE__, __LINE__, "Cannot reclaim the memory buffer of the HDF5 variable length string.");
2200  }
2201 
2202  temp_buf.clear();
2203  }
2204  H5Sclose(temp_space_id);
2205 }
2206 
2207 bool check_str_attr_value(hid_t attr_id,hid_t atype_id,const string & value_to_compare,bool check_substr) {
2208 
2209  bool ret_value = false;
2210 
2211  H5T_str_t str_pad = H5Tget_strpad(atype_id);
2212  if(str_pad == H5T_STR_ERROR)
2213  throw InternalErr(__FILE__, __LINE__, "Fail to obtain string pad.");
2214 
2215  hid_t aspace_id = -1;
2216  aspace_id = H5Aget_space(attr_id);
2217  if(aspace_id < 0)
2218  throw InternalErr(__FILE__, __LINE__, "Fail to obtain attribute space.");
2219 
2220  int ndims = H5Sget_simple_extent_ndims(aspace_id);
2221  if(ndims < 0) {
2222  H5Sclose(aspace_id);
2223  throw InternalErr(__FILE__, __LINE__, "Fail to obtain number of dimensions.");
2224  }
2225 
2226  hsize_t nelmts = 1;
2227 
2228  // if it is a scalar attribute, just define number of elements to be 1.
2229  if (ndims != 0) {
2230 
2231  vector<hsize_t> asize;
2232  asize.resize(ndims);
2233  if (H5Sget_simple_extent_dims(aspace_id, &asize[0], NULL)<0) {
2234  H5Sclose(aspace_id);
2235  throw InternalErr(__FILE__, __LINE__, "Fail to obtain the dimension info.");
2236  }
2237 
2238  // Return ndims and size[ndims].
2239  for (int j = 0; j < ndims; j++)
2240  nelmts *= asize[j];
2241  } // if(ndims != 0)
2242 
2243  size_t ty_size = H5Tget_size(atype_id);
2244  if (0 == ty_size) {
2245  H5Sclose(aspace_id);
2246  throw InternalErr(__FILE__, __LINE__, "Fail to obtain the type size.");
2247  }
2248 
2249  size_t total_bytes = nelmts * ty_size;
2250  string total_vstring ="";
2251  if(H5Tis_variable_str(atype_id) > 0) {
2252 
2253  // Variable length string attribute values only store pointers of the actual string value.
2254  vector<char> temp_buf;
2255  temp_buf.resize(total_bytes);
2256 
2257  if (H5Aread(attr_id, atype_id, &temp_buf[0]) < 0){
2258  H5Sclose(aspace_id);
2259  throw InternalErr(__FILE__,__LINE__,"Fail to read the attribute.");
2260  }
2261 
2262  char *temp_bp = NULL;
2263  temp_bp = &temp_buf[0];
2264  char* onestring = NULL;
2265 
2266  for (unsigned int temp_i = 0; temp_i <nelmts; temp_i++) {
2267 
2268  // This line will assure that we get the real variable length string value.
2269  onestring =*(char **)temp_bp;
2270 
2271  if(onestring!= NULL)
2272  total_vstring +=string(onestring);
2273 
2274  // going to the next value.
2275  temp_bp +=ty_size;
2276  }
2277 
2278  if ((&temp_buf[0]) != NULL) {
2279  // Reclaim any VL memory if necessary.
2280  if (H5Dvlen_reclaim(atype_id,aspace_id,H5P_DEFAULT,&temp_buf[0]) < 0) {
2281  H5Sclose(aspace_id);
2282  throw InternalErr(__FILE__,__LINE__,"Fail to reclaim VL memory.");
2283  }
2284  }
2285 
2286  }
2287  else {// Fixed-size string, need to retrieve the string value.
2288 
2289  // string attribute values
2290  vector<char> temp_buf;
2291  temp_buf.resize(total_bytes);
2292  if (H5Aread(attr_id, atype_id, &temp_buf[0]) < 0){
2293  H5Sclose(aspace_id);
2294  throw InternalErr(__FILE__,__LINE__,"Fail to read the attribute.");
2295  }
2296  string temp_buf_string(temp_buf.begin(),temp_buf.end());
2297  total_vstring = temp_buf_string.substr(0,total_bytes);
2298 
2299  // Note: we need to remove the string pad or term to find DIMENSION_SCALE.
2300  if(str_pad != H5T_STR_ERROR)
2301  total_vstring = total_vstring.substr(0,total_vstring.size()-1);
2302  }
2303 
2304  // Close attribute data space ID.
2305  if(aspace_id != -1)
2306  H5Sclose(aspace_id);
2307 
2308  if(false == check_substr) {
2309  if(total_vstring == value_to_compare)
2310  ret_value = true;
2311  }
2312  else {
2313  if(total_vstring.size()>=value_to_compare.size()) {
2314  if( 0 == total_vstring.compare(0,value_to_compare.size(),value_to_compare))
2315  ret_value = true;
2316  }
2317  }
2318  return ret_value;
2319 }
2320 
2321 // Call back function used by H5Lvisit that iterates all HDF5 objects.
2322 static int
2323 visit_link_cb(hid_t group_id, const char *name, const H5L_info_t *linfo,
2324  void *_op_data)
2325 {
2326 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2327  typedef struct {
2328  unsigned link_unvisited;
2329  H5O_token_t link_addr;
2330  vector<string> hl_names;
2331  } t_link_info_t;
2332 #else
2333  typedef struct {
2334  unsigned link_unvisited;
2335  haddr_t link_addr;
2336  vector<string> hl_names;
2337  } t_link_info_t;
2338 #endif
2339 
2340  t_link_info_t *op_data = (t_link_info_t *)_op_data;
2341  int ret = 0;
2342 
2343  // We only need the hard link info.
2344  if(linfo->type == H5L_TYPE_HARD) {
2345 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2346  int token_cmp = -1;
2347  if(H5Otoken_cmp(group_id,&(op_data->link_addr),&(linfo->u.token),&token_cmp) <0)
2348  throw InternalErr(__FILE__,__LINE__,"H5Otoken_cmp failed");
2349  if(!token_cmp) {
2350 #else
2351  if(op_data->link_addr == linfo->u.address) {
2352 #endif
2353  op_data->link_unvisited = op_data->link_unvisited -1;
2354  string tmp_str(name,name+strlen(name));
2355  op_data->hl_names.push_back(tmp_str);
2356  // Once visiting all hard links, stop.
2357  if(op_data->link_unvisited == 0)
2358  ret = 1;
2359  }
2360 
2361  }
2362  return ret;
2363 
2364 }
2365 
2366 // Obtain the shortest path of all hard links of the object.
2367 std::string obtain_shortest_ancestor_path(const std::vector<std::string> & hls) {
2368 
2369  vector<string> hls_path;
2370  char slash = '/';
2371  bool hl_under_root = false;
2372  string ret_str ="";
2373  unsigned i = 0;
2374 
2375  for (i= 0; i<hls.size(); i++) {
2376 
2377  size_t path_pos = hls[i].find_last_of(slash);
2378 
2379  // The hard link may be under the root group,
2380  // This is the shortest path, we will return this path.
2381  if(path_pos == std::string::npos) {
2382  //Found
2383  hl_under_root = true;
2384  break;
2385  }
2386  else {
2387  string tmp_str = hls[i].substr(0,path_pos+1);
2388  hls_path.push_back(tmp_str);
2389  }
2390  }
2391 
2392  if(hl_under_root)
2393  ret_str = hls[i];
2394 
2395  else {
2396  // We just need to find the minimum size.
2397  unsigned short_path_index = 0;
2398  unsigned min_path_size = hls_path[0].size();
2399 
2400  // Find the shortest path index
2401  for(unsigned j = 1; j <hls_path.size();j++) {
2402  if(min_path_size>hls_path[j].size()) {
2403  min_path_size = hls_path[j].size();
2404  short_path_index = j;
2405  }
2406  }
2407  string tmp_sp = hls_path[short_path_index];
2408  ret_str = hls[short_path_index];
2409 
2410  //check if all hardlinks have a common ancestor link
2411  // If not, set the return value be the empty string.
2412  for(unsigned j = 0; j <hls_path.size();j++) {
2413  if(hls_path[j].find(tmp_sp)!=0) {
2414  ret_str ="";
2415  break;
2416  }
2417  }
2418  }
2419  return ret_str;
2420 
2421 
2422 }
A class for handling all types of array in HDF5 for the default option.
This class provides a way to map HDF5 byte to DAP Byte for the default option.
A class for mapping HDF5 32-bit float to DAP for the default option.
A class for mapping HDF5 64-bit float to DAP for the default option.
A class for HDF5 signed 16 bit integer type.
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
This class provides a way to map HDF5 Int64 to DAP Int64 for the default option.
This class provides a way to map HDF5 int8 to DAP Int8 for the default option.
include the entry functions to execute the handlers
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
This class provides a way to map HDF5 uint64 to DAP UInt64 for the default option.
This class generates DAP URL type for the default option.
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1737
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1741
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1733
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:863
bool check_h5str(hid_t h5type)
Definition: h5get.cc:842
hid_t get_fileid(const char *filename)
Definition: h5get.cc:411
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:291
void close_fileid(hid_t fid)
Definition: h5get.cc:433
void get_dataset(hid_t pid, const string &dname, DS_t *dt_inst_ptr)
Definition: h5get.cc:452
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:90
const int DODS_MAX_RANK
Maximum number of dimensions in an array(default option only).
Definition: hdf5_handler.h:63
A structure for DDS generation.
Definition: hdf5_handler.h:71
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:73
A structure for DAS generation.
Definition: hdf5_handler.h:94
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:96
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:104
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:106