bes  Updated for version 3.20.10
h5dmr.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2007-2015 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
36 // the correct DAP4 DMR layout(group's variables first and then the group).
39 
40 #include <sstream>
41 #include "config_hdf5.h"
42 
43 #include <libdap/InternalErr.h>
44 #include <BESDebug.h>
45 
46 #include <libdap/mime_util.h>
47 
48 #include "hdf5_handler.h"
49 #include "HDF5Int32.h"
50 #include "HDF5UInt32.h"
51 #include "HDF5UInt16.h"
52 #include "HDF5Int16.h"
53 #include "HDF5Byte.h"
54 #include "HDF5Array.h"
55 #include "HDF5Str.h"
56 #include "HDF5Float32.h"
57 #include "HDF5Float64.h"
58 #include "HDF5Url.h"
59 #include "HDF5Structure.h"
60 
61 // The HDF5CFUtil.h includes the utility function obtain_string_after_lastslash.
62 #include "HDF5CFUtil.h"
63 #include "h5dmr.h"
64 
65 using namespace std;
66 using namespace libdap;
69 
70 
72 static DS_t dt_inst;
73 
75 void map_h5_attrs_to_dap4(hid_t oid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag);
76 
77 #if 0
83 // \param par_grp DAP4 parent group
93 
94 //bool depth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname)
95 bool depth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname)
96 {
97  BESDEBUG("h5",
98  ">depth_first() for dmr "
99  << " pid: " << pid
100  << " gname: " << gname
101  << " fname: " << fname
102  << endl);
103 
105  int slinkindex = 0;
106 
107  H5G_info_t g_info;
108  hsize_t nelems = 0;
109 
111  if(H5Gget_info(pid,&g_info) <0) {
112  string msg =
113  "h5_dmr handler: counting hdf5 group elements error for ";
114  msg += gname;
115  throw InternalErr(__FILE__, __LINE__, msg);
116  }
117 
118  nelems = g_info.nlinks;
119 
120  ssize_t oname_size = 0;
121 
122  // Iterate through the file to see the members of the group from the root.
123  for (hsize_t i = 0; i < nelems; i++) {
124 
125  vector <char>oname;
126 
127  // Query the length of object name.
128  oname_size =
129  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
130  (size_t)DODS_NAMELEN, H5P_DEFAULT);
131  if (oname_size <= 0) {
132  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
133  msg += gname;
134  throw InternalErr(__FILE__, __LINE__, msg);
135  }
136 
137  // Obtain the name of the object
138  oname.resize((size_t) oname_size + 1);
139 
140  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
141  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
142  string msg =
143  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
144  msg += gname;
145  throw InternalErr(__FILE__, __LINE__, msg);
146  }
147 
148  // Check if it is the hard link or the soft link
149  H5L_info_t linfo;
150  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
151  string msg = "hdf5 link name error from: ";
152  msg += gname;
153  throw InternalErr(__FILE__, __LINE__, msg);
154  }
155 
156  // Information of soft links are stored as attributes
157  if(linfo.type == H5L_TYPE_SOFT) {
158  slinkindex++;
159  size_t val_size = linfo.u.val_size;
160  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
161  //get_softlink(par_grp,pid,gname,&oname[0],slinkindex,val_size);
162  continue;
163  }
164 
165  // Ignore external links
166  if(linfo.type == H5L_TYPE_EXTERNAL)
167  continue;
168 
169  // Obtain the object type, such as group or dataset.
170  H5O_info_t oinfo;
171 
172  if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
173  i, &oinfo, H5P_DEFAULT)<0) {
174  string msg = "h5_dmr handler: Error obtaining the info for the object";
175  msg += string(oname.begin(),oname.end());
176  throw InternalErr(__FILE__, __LINE__, msg);
177  }
178 
179  H5O_type_t obj_type = oinfo.type;
180 
181  switch (obj_type) {
182 
183  case H5O_TYPE_GROUP:
184  {
185 
186  // Obtain the full path name
187  string full_path_name =
188  string(gname) + string(oname.begin(),oname.end()-1) + "/";
189 
190  BESDEBUG("h5", "=depth_first dmr ():H5G_GROUP " << full_path_name
191  << endl);
192 
193  vector <char>t_fpn;
194  t_fpn.resize(full_path_name.length()+1);
195  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
196  t_fpn[full_path_name.length()] = '\0';
197 
198  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
199  if (cgroup < 0){
200  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
201  }
202 
203  string grp_name = string(oname.begin(),oname.end()-1);
204 
205  // Check the hard link loop and break the loop if it exists.
206  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
207  if (oid == "") {
208  try {
209  D4Group* tem_d4_cgroup = new D4Group(grp_name);
210  // Map the HDF5 cgroup attributes to DAP4 group attributes.
211  // Note the last flag of map_h5_attrs_to_dap4 must be 0 for the group attribute mapping.
212  map_h5_attrs_to_dap4(cgroup,tem_d4_cgroup,NULL,NULL,0);
213 
214  // Add this new DAP4 group
215  par_grp->add_group_nocopy(tem_d4_cgroup);
216 
217  // Continue searching the objects under this group
218  //depth_first(cgroup, &t_fpn[0], dmr, tem_d4_cgroup,fname);
219  depth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname);
220  }
221  catch(...) {
222  H5Gclose(cgroup);
223  throw;
224  }
225  }
226  else {
227  // This group has been visited.
228  // Add the attribute table with the attribute name as HDF5_HARDLINK.
229  // The attribute value is the name of the group when it is first visited.
230  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
231 
232  // Note attr_str_c is the DAP4 attribute string datatype
233  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
234 
235  d4_hlinfo->add_value(obj_paths.get_name(oid));
236  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
237  par_grp->add_group_nocopy(tem_d4_cgroup);
238 
239  }
240 
241  if (H5Gclose(cgroup) < 0){
242  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
243  }
244  break;
245  }
246 
247  case H5O_TYPE_DATASET:
248  {
249 
250  // Obtain the absolute path of the HDF5 dataset
251  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
252 
253  // TOOOODOOOO
254  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
255  // All the metadata information in the handler is stored in dt_inst.
256  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
257  // Note: depth_first is for building DMR of an HDF5 file that doesn't use dim. scale.
258  // so passing the last parameter as false.
259  get_dataset(pid, full_path_name, &dt_inst,false);
260 
261  // Here we open the HDF5 dataset again to use the dataset id for dataset attributes.
262  // This is not necessary for DAP2 since DAS and DDS are separated.
263  hid_t dset_id = -1;
264  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
265  string msg = "cannot open the HDF5 dataset ";
266  msg += full_path_name;
267  throw InternalErr(__FILE__, __LINE__, msg);
268  }
269 
270  try {
271  read_objects(par_grp, full_path_name, fname,dset_id);
272  }
273  catch(...) {
274  H5Dclose(dset_id);
275  throw;
276  }
277  if(H5Dclose(dset_id)<0) {
278  string msg = "cannot close the HDF5 dataset ";
279  msg += full_path_name;
280  throw InternalErr(__FILE__, __LINE__, msg);
281  }
282  }
283  break;
284 
285  case H5O_TYPE_NAMED_DATATYPE:
286  // ignore the named datatype
287  break;
288  default:
289  break;
290  }// switch(obj_type)
291  } // for i is 0 ... nelems
292 
293  BESDEBUG("h5", "<depth_first() for dmr" << endl);
294  return true;
295 }
296 #endif
303 // \param par_grp DAP4 parent group
316 
317 
318 // The reason to use breadth_first is that the DMR representation needs to show the dimension names and the variables under the group first and then the group names.
319 // So we use this search. In the future, we may just use the breadth_first search for all cases.??
320 //bool breadth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname,bool use_dimscale)
321 bool breadth_first(const hid_t file_id, hid_t pid, char *gname, D4Group* par_grp, const char *fname,bool use_dimscale,vector<link_info_t> & hdf5_hls )
322 {
323  BESDEBUG("h5",
324  ">breadth_first() for dmr "
325  << " pid: " << pid
326  << " gname: " << gname
327  << " fname: " << fname
328  << endl);
329 
331  int slinkindex = 0;
332 
333  // Obtain the number of objects in this group
334  H5G_info_t g_info;
335  hsize_t nelems = 0;
336  if(H5Gget_info(pid,&g_info) <0) {
337  string msg =
338  "h5_dmr handler: counting hdf5 group elements error for ";
339  msg += gname;
340  throw InternalErr(__FILE__, __LINE__, msg);
341  }
342 
343  nelems = g_info.nlinks;
344 
345  ssize_t oname_size;
346 
347  // First iterate through the HDF5 datasets under the group.
348  for (hsize_t i = 0; i < nelems; i++) {
349 
350  vector <char>oname;
351 
352  // Query the length of object name.
353  oname_size =
354  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
355  (size_t)DODS_NAMELEN, H5P_DEFAULT);
356  if (oname_size <= 0) {
357  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
358  msg += gname;
359  throw InternalErr(__FILE__, __LINE__, msg);
360  }
361 
362  // Obtain the name of the object
363  oname.resize((size_t) oname_size + 1);
364 
365  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
366  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
367  string msg =
368  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
369  msg += gname;
370  throw InternalErr(__FILE__, __LINE__, msg);
371  }
372 
373  // Check if it is the hard link or the soft link
374  H5L_info_t linfo;
375  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
376  string msg = "hdf5 link name error from: ";
377  msg += gname;
378  throw InternalErr(__FILE__, __LINE__, msg);
379  }
380 
381  // Information of soft links are stored as attributes
382  if(linfo.type == H5L_TYPE_SOFT) {
383  slinkindex++;
384 
385  // Size of a soft link value
386  size_t val_size = linfo.u.val_size;
387  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
388  continue;
389  }
390 
391  // Ignore external links
392  if(linfo.type == H5L_TYPE_EXTERNAL)
393  continue;
394 
395  // Obtain the object type, such as group or dataset.
396  H5O_info_t oinfo;
397 
398  if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
399  i, &oinfo, H5P_DEFAULT)<0) {
400  string msg = "h5_dmr handler: Error obtaining the info for the object";
401  msg += string(oname.begin(),oname.end());
402  throw InternalErr(__FILE__, __LINE__, msg);
403  }
404 
405  H5O_type_t obj_type = oinfo.type;
406 
407  if(H5O_TYPE_DATASET == obj_type) {
408 
409  // Obtain the absolute path of the HDF5 dataset
410  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
411 
412  // TOOOODOOOO
413  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
414  // All the metadata information in the handler is stored in dt_inst.
415  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
416  // Dimension scale is handled in this routine. So need to keep it. KY 2020-06-10
417  bool is_pure_dim = false;
418  get_dataset_dmr(file_id, pid, full_path_name, &dt_inst,use_dimscale,is_pure_dim,hdf5_hls);
419 
420  if(false == is_pure_dim) {
421  hid_t dset_id = -1;
422  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
423  string msg = "cannot open the HDF5 dataset ";
424  msg += full_path_name;
425  throw InternalErr(__FILE__, __LINE__, msg);
426  }
427 
428  try {
429  read_objects(par_grp, full_path_name, fname,dset_id);
430  }
431  catch(...) {
432  H5Dclose(dset_id);
433  throw;
434  }
435  if(H5Dclose(dset_id)<0) {
436  string msg = "cannot close the HDF5 dataset ";
437  msg += full_path_name;
438  throw InternalErr(__FILE__, __LINE__, msg);
439  }
440  }
441  else {
442  //Need to add this pure dimension to the corresponding DAP4 group
443  D4Dimensions *d4_dims = par_grp->dims();
444  string d4dim_name = string(oname.begin(),oname.end()-1);
445  D4Dimension *d4_dim = d4_dims->find_dim(d4dim_name);
446  if(d4_dim == NULL) {
447  d4_dim = new D4Dimension(d4dim_name,dt_inst.nelmts);
448  d4_dims->add_dim_nocopy(d4_dim);
449  }
450  BESDEBUG("h5", "<h5dmr.cc: pure dimension: dataset name." << d4dim_name << endl);
451  if(H5Tclose(dt_inst.type)<0) {
452  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
453  }
454  }
455 
456  }
457  }
458 
459  // The attributes of this group. Doing this order to follow ncdump's way (variable,attribute then groups)
460  map_h5_attrs_to_dap4(pid,par_grp,NULL,NULL,0);
461 
462  // Then HDF5 child groups
463  for (hsize_t i = 0; i < nelems; i++) {
464 
465  vector <char>oname;
466 
467  // Query the length of object name.
468  oname_size =
469  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
470  (size_t)DODS_NAMELEN, H5P_DEFAULT);
471  if (oname_size <= 0) {
472  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
473  msg += gname;
474  throw InternalErr(__FILE__, __LINE__, msg);
475  }
476 
477  // Obtain the name of the object
478  oname.resize((size_t) oname_size + 1);
479 
480  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
481  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
482  string msg =
483  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
484  msg += gname;
485  throw InternalErr(__FILE__, __LINE__, msg);
486  }
487 
488  // Check if it is the hard link or the soft link
489  H5L_info_t linfo;
490  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
491  string msg = "hdf5 link name error from: ";
492  msg += gname;
493  throw InternalErr(__FILE__, __LINE__, msg);
494  }
495 
496  // Information of soft links are handled already, the softlinks need to be ignored, otherwise
497  // the group it links will be mapped again in the block of if obj_type is H5O_TYPE_GROUP
498  if(linfo.type == H5L_TYPE_SOFT) {
499  continue;
500  }
501 
502  // Ignore external links
503  if(linfo.type == H5L_TYPE_EXTERNAL)
504  continue;
505 
506  // Obtain the object type, such as group or dataset.
507  H5O_info_t oinfo;
508 
509  if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
510  i, &oinfo, H5P_DEFAULT)<0) {
511  string msg = "h5_dmr handler: Error obtaining the info for the object in the breadth_first.";
512  throw InternalErr(__FILE__, __LINE__, msg);
513  }
514 
515  H5O_type_t obj_type = oinfo.type;
516 
517 
518  if(obj_type == H5O_TYPE_GROUP) {
519 
520  // Obtain the full path name
521  string full_path_name =
522  string(gname) + string(oname.begin(),oname.end()-1) + "/";
523 
524  BESDEBUG("h5", "=breadth_first dmr ():H5G_GROUP " << full_path_name
525  << endl);
526 
527  vector <char>t_fpn;
528  t_fpn.resize(full_path_name.length()+1);
529  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
530  t_fpn[full_path_name.length()] = '\0';
531 
532  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
533  if (cgroup < 0){
534  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
535  }
536 
537  string grp_name = string(oname.begin(),oname.end()-1);
538 
539  // Check the hard link loop and break the loop if it exists.
540  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
541  if (oid == "") {
542  try {
543  D4Group* tem_d4_cgroup = new D4Group(grp_name);
544 
545  // Add this new DAP4 group
546  par_grp->add_group_nocopy(tem_d4_cgroup);
547 
548  // Continue searching the objects under this group
549  breadth_first(file_id,cgroup, &t_fpn[0], tem_d4_cgroup,fname,use_dimscale,hdf5_hls);
550  }
551  catch(...) {
552  H5Gclose(cgroup);
553  throw;
554  }
555  }
556  else {
557  // This group has been visited.
558  // Add the attribute table with the attribute name as HDF5_HARDLINK.
559  // The attribute value is the name of the group when it is first visited.
560  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
561 
562  // Note attr_str_c is the DAP4 attribute string datatype
563  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
564 
565  d4_hlinfo->add_value(obj_paths.get_name(oid));
566  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
567  par_grp->add_group_nocopy(tem_d4_cgroup);
568  }
569 
570  if (H5Gclose(cgroup) < 0){
571  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
572  }
573  }// end if
574  } // for i is 0 ... nelems
575 
576  BESDEBUG("h5", "<breadth_first() " << endl);
577  return true;
578 }
579 
595 //
596 void
597 read_objects( D4Group * d4_grp, const string &varname, const string &filename, const hid_t dset_id)
598 {
599 
600  switch (H5Tget_class(dt_inst.type)) {
601 
602  // HDF5 compound maps to DAP structure.
603  case H5T_COMPOUND:
604  read_objects_structure(d4_grp, varname, filename,dset_id);
605  break;
606 
607  case H5T_ARRAY:
608  H5Tclose(dt_inst.type);
609  throw InternalErr(__FILE__, __LINE__, "Currently don't support accessing data of Array datatype when array datatype is not inside the compound.");
610 
611  default:
612  read_objects_base_type(d4_grp,varname, filename,dset_id);
613  break;
614  }
615  // We must close the datatype obtained in the get_dataset routine since this is the end of reading DDS.
616  if(H5Tclose(dt_inst.type)<0) {
617  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
618  }
619 }
620 
635 //
636 
637 //void
638 //read_objects_base_type(DMR & dmr, D4Group * d4_grp,const string & varname,
639 void
640 read_objects_base_type(D4Group * d4_grp,const string & varname,
641  const string & filename,hid_t dset_id)
642 {
643 
644  // Obtain the relative path of the variable name under the leaf group
645  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
646 
647  // Get a base type. It should be an HDF5 atomic datatype
648  // datatype.
649  BaseType *bt = Get_bt(newvarname, varname,filename, dt_inst.type,true);
650  if (!bt) {
651  throw
652  InternalErr(__FILE__, __LINE__,
653  "Unable to convert hdf5 datatype to dods basetype");
654  }
655 
656  // First deal with scalar data.
657  if (dt_inst.ndims == 0) {
658  // transform the DAP2 to DAP4 for this DAP base type and add it to d4_grp
659  bt->transform_to_dap4(d4_grp,d4_grp);
660  // Get it back - this may return null because the underlying type
661  // may have no DAP2 manifestation.
662  BaseType* new_var = d4_grp->var(bt->name());
663  if(new_var){
664  // Map the HDF5 dataset attributes to DAP4
665  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
666  // If this variable is a hardlink, stores the HARDLINK info. as an attribute.
667  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
668  }
669  delete bt;
670  bt = 0;
671  }
672  else {
673  // Next, deal with Array data. This 'else clause' runs to
674  // the end of the method.
675  HDF5Array *ar = new HDF5Array(newvarname, filename, bt);
676  delete bt; bt = 0;
677 
678  // set number of elements and variable name values.
679  // This essentially stores in the struct.
680  ar->set_memneed(dt_inst.need);
681  ar->set_numdim(dt_inst.ndims);
682  ar->set_numelm((int) (dt_inst.nelmts));
683  ar->set_varpath(varname);
684 
685 
686  // If we have dimension names(dimension scale is used.),we will see if we can add the names.
687  int dimnames_size = 0;
688  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
689  {
690  delete ar;
691  throw
692  InternalErr(__FILE__, __LINE__,
693  "number of dimensions: overflow");
694  }
695  dimnames_size = (int)(dt_inst.dimnames.size());
696 //cerr<<"dimnames_size is "<<dimnames_size <<endl;
697 //cerr<<"ndims is "<<dt_inst.ndims <<endl;
698 
699  if(dimnames_size ==dt_inst.ndims) {
700 
701  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
702  if(dt_inst.dimnames[dim_index] !="")
703  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
704  else
705  ar->append_dim(dt_inst.size[dim_index]);
706  // D4dimension has to have a name. If no name, no D4dimension(from comments libdap4: Array.cc)
707  }
708  dt_inst.dimnames.clear();
709  }
710  else {
711  // For DAP4, no need to add dimension if no dimension name
712  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
713  ar->append_dim(dt_inst.size[dim_index]);
714  }
715 
716  // We need to transform dimension info. to DAP4 group
717  BaseType* new_var = NULL;
718  try {
719  new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
720  }
721  catch(...) {
722  delete ar;
723  throw;
724  }
725 
726  // clear DAP4 dimnames_path vector
727  dt_inst.dimnames_path.clear();
728 
729  // Map HDF5 dataset attributes to DAP4
730  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
731 
732  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
733  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
734 #if 0
735  // Test the attribute
736  D4Attribute *test_attr = new D4Attribute("DAP4_test",attr_str_c);
737  test_attr->add_value("test_grp_attr");
738  new_var->attributes()->add_attribute_nocopy(test_attr);
739 #endif
740  // Add this var to DAP4 group.
741  d4_grp->add_var_nocopy(new_var);
742  delete ar; ar = 0;
743  }
744  BESDEBUG("h5", "<read_objects_base_type(dmr)" << endl);
745 
746 }
747 
761 void
762 read_objects_structure(D4Group *d4_grp, const string & varname,
763  const string & filename,hid_t dset_id)
764 {
765  // Obtain the relative path of the variable name under the leaf group
766  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
767 
768  // Map HDF5 compound datatype to Structure
769  Structure *structure = Get_structure(newvarname, varname,filename, dt_inst.type,true);
770 
771  try {
772  BESDEBUG("h5", "=read_objects_structure(): Dimension is "
773  << dt_inst.ndims << endl);
774 
775  if (dt_inst.ndims != 0) { // Array of Structure
776  BESDEBUG("h5", "=read_objects_structure(): array of size " <<
777  dt_inst.nelmts << endl);
778  BESDEBUG("h5", "=read_objects_structure(): memory needed = " <<
779  dt_inst.need << endl);
780 
781  // Create the Array of structure.
782  HDF5Array *ar = new HDF5Array(newvarname, filename, structure);
783  delete structure; structure = 0;
784 
785 
786  // These parameters are used in the data read function.
787  ar->set_memneed(dt_inst.need);
788  ar->set_numdim(dt_inst.ndims);
789  ar->set_numelm((int) (dt_inst.nelmts));
790  ar->set_length((int) (dt_inst.nelmts));
791  ar->set_varpath(varname);
792 
793  // If having dimension names, add the dimension names to DAP.
794  int dimnames_size = 0;
795  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
796  {
797  delete ar;
798  throw
799  InternalErr(__FILE__, __LINE__,
800  "number of dimensions: overflow");
801  }
802  dimnames_size = (int)(dt_inst.dimnames.size());
803 
804 
805  if(dimnames_size ==dt_inst.ndims) {
806  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
807  if(dt_inst.dimnames[dim_index] !="")
808  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
809  else
810  ar->append_dim(dt_inst.size[dim_index]);
811  }
812  dt_inst.dimnames.clear();
813  }
814  else {
815  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
816  ar->append_dim(dt_inst.size[dim_index]);
817 
818  }
819 
820  // We need to transform dimension info. to DAP4 group
821  BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
822  dt_inst.dimnames_path.clear();
823 
824  // Map HDF5 dataset attributes to DAP4
825  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
826 
827  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
828  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
829 
830  // Add this var to DAP4 group
831  if(new_var)
832  d4_grp->add_var_nocopy(new_var);
833  delete ar; ar = 0;
834  }// end if
835  else {// A scalar structure
836 
837  structure->set_is_dap4(true);
838  map_h5_attrs_to_dap4(dset_id,NULL,NULL,structure,2);
839  map_h5_dset_hardlink_to_d4(dset_id,varname,NULL,structure,2);
840  if(structure)
841  d4_grp->add_var_nocopy(structure);
842  }
843  } // try Structure
844  catch (...) {
845  delete structure;
846  throw;
847  }
848 }
849 
850 
864 //
865 
866 void map_h5_attrs_to_dap4(hid_t h5_objid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag) {
867 
868  // Get the object info
869  H5O_info_t obj_info;
870  if (H5OGET_INFO(h5_objid, &obj_info) <0) {
871  string msg = "Fail to obtain the HDF5 object info. .";
872  throw InternalErr(__FILE__, __LINE__, msg);
873  }
874 
875  // Obtain the number of attributes
876  int num_attr = obj_info.num_attrs;
877  if (num_attr < 0 ) {
878  string msg = "Fail to get the number of attributes for the HDF5 object. ";
879  throw InternalErr(__FILE__, __LINE__,msg);
880  }
881 
882  string print_rep;
883  vector<char>temp_buf;
884 
885  bool ignore_attr = false;
886  hid_t attr_id = -1;
887  for (int j = 0; j < num_attr; j++) {
888 
889  // Obtain attribute information.
890  DSattr_t attr_inst;
891 
892  // Ignore the attributes of which the HDF5 datatype
893  // cannot be mapped to DAP4. The ignored attribute datatypes can be found
894  // at function get_attr_info in h5get.cc.
895  attr_id = get_attr_info(h5_objid, j, true,&attr_inst, &ignore_attr);
896  if (true == ignore_attr) {
897  H5Aclose(attr_id);
898  continue;
899  }
900 
901  // Get the corresponding DAP data type of the HDF5 datatype.
902  // The following line doesn't work in HDF5 1.10.
903 #if 0
904  //hid_t ty_id = attr_inst.type;
905 #endif
906  hid_t ty_id = H5Aget_type(attr_id);
907  if(ty_id <0) {
908  H5Aclose(attr_id);
909  throw InternalErr(__FILE__, __LINE__, "Cannot retrieve HDF5 attribute datatype successfully.");
910  }
911 
912  string dap_type = get_dap_type(ty_id,true);
913 
914  // Need to have DAP4 representation of the attribute type
915  D4AttributeType dap4_attr_type = daptype_strrep_to_dap4_attrtype(dap_type);
916 
917  // We encounter an unsupported DAP4 attribute type.
918  if(attr_null_c == dap4_attr_type) {
919  H5Tclose(ty_id);
920  H5Aclose(attr_id);
921  throw InternalErr(__FILE__, __LINE__, "unsupported DAP4 attribute type");
922  }
923 
924  string attr_name = attr_inst.name;
925  BESDEBUG("h5", "arttr_name= " << attr_name << endl);
926 
927  // Create the DAP4 attribute mapped from HDF5
928  D4Attribute *d4_attr = new D4Attribute(attr_name,dap4_attr_type);
929 
930  // We have to handle variable length string differently.
931  if (H5Tis_variable_str(ty_id)) {
932  write_vlen_str_attrs(attr_id,ty_id,&attr_inst,d4_attr,NULL,true);
933  }// if (H5Tis_variable_str(ty_id)
934  else {
935 
936  vector<char> value;
937  value.resize(attr_inst.need + sizeof(char));
938  //value.resize(attr_inst.need);
939  BESDEBUG("h5", "arttr_inst.need=" << attr_inst.need << endl);
940 
941  // Need to obtain the memtype since we still find BE data.
942  hid_t memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
943  // Read HDF5 attribute data.
944  if (H5Aread(attr_id, memtype, (void *) (&value[0])) < 0) {
945  delete d4_attr;
946  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
947  }
948  H5Aclose(memtype);
949 
950  // For scalar data, just read data once.
951  if (attr_inst.ndims == 0) {
952  for (int loc = 0; loc < (int) attr_inst.nelmts; loc++) {
953  print_rep = print_attr(ty_id, loc, &value[0]);
954  if (print_rep.c_str() != NULL) {
955  d4_attr->add_value(print_rep);
956  }
957  }
958 
959  }
960  else {// The number of dimensions is > 0
961 
962  // Get the attribute datatype size
963  int elesize = (int) H5Tget_size(ty_id);
964  if (elesize == 0) {
965  H5Tclose(ty_id);
966  H5Aclose(attr_id);
967  delete d4_attr;
968  throw InternalErr(__FILE__, __LINE__, "unable to get attibute size");
969  }
970 
971  // Due to the implementation of print_attr, the attribute value will be
972  // written one by one.
973  char *tempvalue = &value[0];
974 
975  // Write this value. the "loc" can always be set to 0 since
976  // tempvalue will be moved to the next value.
977  for( hsize_t temp_index = 0; temp_index < attr_inst.nelmts; temp_index ++) {
978  //print_rep = print_attr(ty_id, 0, (void*)&value[0]);
979  print_rep = print_attr(ty_id, 0, tempvalue);
980  if (print_rep.c_str() != NULL) {
981 
982  BESDEBUG("h5", "print_rep= " << print_rep << endl);
983 
984  d4_attr->add_value(print_rep);
985  tempvalue = tempvalue + elesize;
986  BESDEBUG("h5",
987  "tempvalue= " << tempvalue
988  << "elesize=" << elesize
989  << endl);
990 
991  }
992  else {
993  H5Tclose(ty_id);
994  H5Aclose(attr_id);
995  delete d4_attr;
996  throw InternalErr(__FILE__, __LINE__, "unable to convert attibute value to DAP");
997  }
998  }//for(hsize_t temp_index=0; .....
999  } // if attr_inst.ndims != 0
1000  }
1001  if(H5Tclose(ty_id) < 0) {
1002  H5Aclose(attr_id);
1003  delete d4_attr;
1004  throw InternalErr(__FILE__, __LINE__, "unable to close HDF5 type id");
1005  }
1006  if (H5Aclose(attr_id) < 0) {
1007  delete d4_attr;
1008  throw InternalErr(__FILE__, __LINE__, "unable to close attibute id");
1009  }
1010 
1011  if(0 == flag) // D4group
1012  d4g->attributes()->add_attribute_nocopy(d4_attr);
1013  else if (1 == flag) // HDF5 dataset with atomic datatypes
1014  d4b->attributes()->add_attribute_nocopy(d4_attr);
1015  else if ( 2 == flag) // HDF5 dataset with compound datatype
1016  d4s->attributes()->add_attribute_nocopy(d4_attr);
1017  else {
1018  stringstream sflag;
1019  sflag << flag;
1020  string msg ="The add_dap4_attr flag has to be either 0,1 or 2.";
1021  msg+="The current flag is "+sflag.str();
1022  delete d4_attr;
1023  throw InternalErr(__FILE__, __LINE__, msg);
1024  }
1025  } // for (int j = 0; j < num_attr; j++)
1026 
1027  return;
1028 }
1029 
1043 
1044 
1045 void map_h5_dset_hardlink_to_d4(hid_t h5_dsetid,const string & full_path, BaseType* d4b,Structure * d4s,int flag) {
1046 
1047  // Obtain the unique object number info. If no hardlinks, empty string will return.
1048  string oid = get_hardlink_dmr(h5_dsetid, full_path);
1049 
1050  // Find that this is a hardlink,add the hardlink info to a DAP4 attribute.
1051  if(false == oid.empty()) {
1052 
1053  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
1054  d4_hlinfo->add_value(obj_paths.get_name(oid));
1055 
1056  if (1 == flag)
1057  d4b->attributes()->add_attribute_nocopy(d4_hlinfo);
1058  else if ( 2 == flag)
1059  d4s->attributes()->add_attribute_nocopy(d4_hlinfo);
1060  else
1061  delete d4_hlinfo;
1062  }
1063 
1064 }
1065 
1078 void get_softlink(D4Group* par_grp, hid_t h5obj_id, const string & oname, int index, size_t val_size)
1079 {
1080  BESDEBUG("h5", "dap4 >get_softlink():" << oname << endl);
1081 
1082  ostringstream oss;
1083  oss << string("HDF5_SOFTLINK");
1084  oss << "_";
1085  oss << index;
1086  string temp_varname = oss.str();
1087 
1088 
1089  BESDEBUG("h5", "dap4->get_softlink():" << temp_varname << endl);
1090  D4Attribute *d4_slinfo = new D4Attribute;
1091  d4_slinfo->set_name(temp_varname);
1092 
1093  // Make the type as a container
1094  d4_slinfo->set_type(attr_container_c);
1095 
1096  string softlink_name = "linkname";
1097 
1098  D4Attribute *softlink_src = new D4Attribute(softlink_name,attr_str_c);
1099  softlink_src->add_value(oname);
1100 
1101  d4_slinfo->attributes()->add_attribute_nocopy(softlink_src);
1102  string softlink_value_name ="LINKTARGET";
1103 
1104  // Get the link target information. We always return the link value in a string format.
1105  D4Attribute *softlink_tgt = 0;
1106 
1107  try {
1108  vector<char> buf;
1109  buf.resize(val_size + 1);
1110 
1111  // get link target name
1112  if (H5Lget_val(h5obj_id, oname.c_str(), (void*) &buf[0], val_size + 1, H5P_DEFAULT) < 0) {
1113  throw InternalErr(__FILE__, __LINE__, "unable to get link value");
1114  }
1115  softlink_tgt = new D4Attribute(softlink_value_name, attr_str_c);
1116  string link_target_name = string(buf.begin(), buf.end());
1117  softlink_tgt->add_value(link_target_name);
1118 
1119  d4_slinfo->attributes()->add_attribute_nocopy(softlink_tgt);
1120  }
1121  catch (...) {
1122  delete softlink_tgt;
1123  throw;
1124  }
1125 
1126  par_grp->attributes()->add_attribute_nocopy(d4_slinfo);
1127 }
1128 
1129 
1142 string get_hardlink_dmr( hid_t h5obj_id, const string & oname) {
1143 
1144  BESDEBUG("h5", "dap4->get_hardlink_dmr():" << oname << endl);
1145 
1146  // Get the object info
1147  H5O_info_t obj_info;
1148  if (H5OGET_INFO(h5obj_id, &obj_info) <0) {
1149  throw InternalErr(__FILE__, __LINE__, "H5OGET_INFO() failed.");
1150  }
1151 
1152  // If the reference count is greater than 1,that means
1153  // hard links are found. return the original object name this
1154  // hard link points to.
1155 
1156  if (obj_info.rc >1) {
1157 
1158  string objno;
1159 
1160 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
1161  char *obj_tok_str = NULL;
1162  if(H5Otoken_to_str(h5obj_id, &(obj_info.token), &obj_tok_str) <0) {
1163  throw InternalErr(__FILE__, __LINE__, "H5Otoken_to_str failed.");
1164  }
1165  objno.assign(obj_tok_str,obj_tok_str+strlen(obj_tok_str));
1166  H5free_memory(obj_tok_str);
1167 
1168 #else
1169  ostringstream oss;
1170  oss << hex << obj_info.addr;
1171  objno = oss.str();
1172 #endif
1173 
1174  BESDEBUG("h5", "dap4->get_hardlink_dmr() objno=" << objno << endl);
1175 
1176  // Add this hard link to the map.
1177  // obj_paths is a global variable defined at the beginning of this file.
1178  // it is essentially a id to obj name map. See HDF5PathFinder.h.
1179  if (!obj_paths.add(objno, oname)) {
1180  return objno;
1181  }
1182  else {
1183  return "";
1184  }
1185  }
1186  else {
1187  return "";
1188  }
1189 
1190 }
A class for handling all types of array in HDF5 for the default option.
This class provides a way to map HDF5 byte to DAP Byte for the default option.
This file includes several helper functions for translating HDF5 to CF-compliant.
A class for mapping HDF5 32-bit float to DAP for the default option.
A class for mapping HDF5 64-bit float to DAP for the default option.
A class for HDF5 signed 16 bit integer type.
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
This class generates DAP URL type for the default option.
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1737
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1741
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1733
std::string get_name(std::string id)
bool add(std::string id, const std::string name)
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void read_objects(DAS &das, const string &varname, hid_t oid, int num_attr)
Definition: h5das.cc:295
void read_objects_base_type(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:257
void read_objects_structure(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:311
void map_h5_attrs_to_dap4(hid_t oid, D4Group *d4g, BaseType *d4b, Structure *d4s, int flag)
A function that map HDF5 attributes to DAP4.
Definition: h5dmr.cc:866
HDF5PathFinder obj_paths
A variable for remembering visited paths to break cyclic HDF5 groups.
Definition: h5dmr.cc:68
bool breadth_first(const hid_t file_id, hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
Definition: h5dmr.cc:321
void get_softlink(D4Group *par_grp, hid_t h5obj_id, const string &oname, int index, size_t val_size)
Definition: h5dmr.cc:1078
Data structure and retrieval processing header for the default option.
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:863
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:291
void get_dataset(hid_t pid, const string &dname, DS_t *dt_inst_ptr)
Definition: h5get.cc:452
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:90
The main header of the HDF5 OPeNDAP handler.
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:65
A structure for DDS generation.
Definition: hdf5_handler.h:71
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:89
hsize_t need
Space needed.
Definition: hdf5_handler.h:91
hid_t type
HDF5 data set id.
Definition: hdf5_handler.h:79
int size[DODS_MAX_RANK]
Size of each dimension.
Definition: hdf5_handler.h:85
int ndims
HDF5 data space id.
Definition: hdf5_handler.h:83
A structure for DAS generation.
Definition: hdf5_handler.h:94
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:96
int ndims
Number of dimensions.
Definition: hdf5_handler.h:100
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:104
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:106