bes  Updated for version 3.20.10
DmrppCommon.h
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of the BES
5 
6 // Copyright (c) 2016 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #ifndef _dmrpp_common_h
26 #define _dmrpp_common_h 1
27 
28 #include <string>
29 #include <vector>
30 #include <memory>
31 
32 #define PUGIXML_NO_XPATH
33 #define PUGIXML_HEADER_ONLY
34 #include <pugixml.hpp>
35 
36 namespace libdap {
37 class DMR;
38 class BaseType;
39 class D4BaseTypeFactory;
40 class D4Group;
41 class D4Attributes;
42 class D4EnumDef;
43 class D4Dimension;
44 class XMLWriter;
45 }
46 
47 namespace http {
48 class url;
49 }
50 
51 namespace pugi {
52 class xml_node;
53 }
54 
55 namespace dmrpp {
56 
57 class DMZ;
58 class Chunk;
59 class DmrppArray;
60 class SuperChunk;
61 
62 void join_threads(pthread_t threads[], unsigned int num_threads);
63 
76 class DmrppCommon {
77 
78  friend class DmrppCommonTest;
79  friend class DmrppParserTest;
80 
81 private:
82  bool d_compact = false;
83  std::string d_filters;
84  std::string d_byte_order;
85  std::vector<unsigned long long> d_chunk_dimension_sizes;
86  std::vector<std::shared_ptr<Chunk>> d_chunks;
87  bool d_twiddle_bytes = false;
88 
89  // These indicate that the chunks or attributes have been loaded into the
90  // variable when the DMR++ handler is using lazy-loading of this data.
91  bool d_chunks_loaded = false;
92  bool d_attributes_loaded = false;
93 
94  // Each instance of DmrppByte, ..., holds a shared pointer to the DMZ so that
95  // it can fetch more information from the XML if needed - this is how the lazy-load
96  // feature is implemented. The xml_node object is used to simplify finding where
97  // in the XML information about a variable is stored - to limit searching the
98  // document, the code caches the XML node.
99  std::shared_ptr<DMZ> d_dmz;
100  pugi::xml_node d_xml_node;
101 
102 protected:
103 #if 0
106  virtual std::vector<std::shared_ptr<Chunk>> get_chunks() {
107  return d_chunks;
108  }
109 #endif
110 
111  virtual char *read_atomic(const std::string &name);
112 
113  // This declaration allows code in the SuperChunky program to use the protected method.
114  // jhrg 10/25/21
115  friend void compute_super_chunks(dmrpp::DmrppArray *array, bool only_constrained, std::vector<dmrpp::SuperChunk *> &super_chunks);
116 
117 public:
118  static bool d_print_chunks;
119  static std::string d_dmrpp_ns;
120  static std::string d_ns_prefix;
121 
122  DmrppCommon() = default;
123 
124  DmrppCommon(std::shared_ptr<DMZ> dmz) : d_dmz(dmz) { }
125 
126  DmrppCommon(const DmrppCommon &) = default;
127 
128  virtual ~DmrppCommon()= default;
129 
131  virtual std::string get_filters() const {
132  return d_filters;
133  }
134 
135  void set_filter(const std::string &value);
136 
137  virtual bool is_filters_empty() const {
138  return d_filters.empty();
139  }
140 
142  virtual bool is_compact_layout() const {
143  return d_compact;
144  }
145 
147  void set_compact(bool value) {
148  d_compact = value;
149  }
150 
152  virtual bool twiddle_bytes() const { return d_twiddle_bytes; }
153 
154  // @brief Provide access to the DMZ instance bound to this variable
155  // virtual const std::shared_ptr<DMZ> &get_dmz() const { return d_dmz; }
156 
158  virtual bool get_chunks_loaded() const { return d_chunks_loaded; }
159  virtual void set_chunks_loaded(bool state) { d_chunks_loaded = state; }
160 
162  virtual bool get_attributes_loaded() const { return d_attributes_loaded; }
163  virtual void set_attributes_loaded(bool state) { d_attributes_loaded = state; }
164 
165  virtual const pugi::xml_node &get_xml_node() const { return d_xml_node; }
166  virtual void set_xml_node(pugi::xml_node node) { d_xml_node = node; }
167 
169  virtual const std::vector<std::shared_ptr<Chunk>> &get_immutable_chunks() const {
170  return d_chunks;
171  }
172 
175  virtual size_t get_chunks_size() const { return d_chunks.size(); }
176 
179  virtual const std::vector<unsigned long long> &get_chunk_dimension_sizes() const {
180  return d_chunk_dimension_sizes;
181  }
182 
185  virtual unsigned long long get_chunk_size_in_elements() const {
186  unsigned long long elements = 1;
187  for (auto d_chunk_dimension_size : d_chunk_dimension_sizes) {
188  elements *= d_chunk_dimension_size;
189  }
190 
191  return elements;
192  }
193 
194  void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space = "");
195 
196  void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space = "", const std::string &encoded = "");
197 
198  void print_dmrpp(libdap::XMLWriter &writer, bool constrained = false);
199 
200  // Replaced hsize_t with size_t. This eliminates a dependency on hdf5. jhrg 9/7/18
202  void set_chunk_dimension_sizes(const std::vector<size_t> &chunk_dims) {
203  // tried using copy(chunk_dims.begin(), chunk_dims.end(), d_chunk_dimension_sizes.begin())
204  // it didn't work, maybe because of the differing element types?
205  for (auto chunk_dim : chunk_dims) {
206  d_chunk_dimension_sizes.push_back(chunk_dim);
207  }
208  }
209 
210  // These two functions duplicate code in DMZ but provides access to the DMZ::load_chunks()
211  // method without having to cast a BaseType to a DmrppCommon in order to use it. jhrg 11/12/21
212  virtual void load_chunks(libdap::BaseType *btp);
213  virtual void load_attributes(libdap::BaseType *btp);
214 
215  virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string);
216 
217  virtual void ingest_compression_type(const std::string &compression_type_string);
218 
219  virtual void ingest_byte_order(const std::string &byte_order_string);
220  virtual std::string get_byte_order() const { return d_byte_order; }
221 
222  virtual unsigned long add_chunk(
223  std::shared_ptr<http::url> d_data_url,
224  const std::string &byte_order,
225  unsigned long long size,
226  unsigned long long offset,
227  const std::string &position_in_array);
228 
229  virtual unsigned long add_chunk(
230  std::shared_ptr<http::url> d_data_url,
231  const std::string &byte_order,
232  unsigned long long size,
233  unsigned long long offset,
234  const std::vector<unsigned long long> &position_in_array);
235 
236  virtual unsigned long add_chunk(
237  const std::string &byte_order,
238  unsigned long long size,
239  unsigned long long offset,
240  const std::string &position_in_array);
241 
242  virtual unsigned long add_chunk(
243  const std::string &byte_order,
244  unsigned long long size,
245  unsigned long long offset,
246  const std::vector<unsigned long long> &position_in_array);
247 
248  virtual void dump(std::ostream & strm) const;
249 };
250 
251 } // namespace dmrpp
252 
253 #endif // _dmrpp_common_h
254 
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
Size and offset information of data included in DMR++ files.
Definition: DmrppCommon.h:76
static std::string d_ns_prefix
The XML namespace prefix to use.
Definition: DmrppCommon.h:120
virtual bool twiddle_bytes() const
Returns true if this object utilizes shuffle compression.
Definition: DmrppCommon.h:152
void set_chunk_dimension_sizes(const std::vector< size_t > &chunk_dims)
Set the value of the chunk dimension sizes given a vector of HDF5 hsize_t.
Definition: DmrppCommon.h:202
static bool d_print_chunks
if true, print_dap4() prints chunk elements
Definition: DmrppCommon.h:118
virtual bool is_compact_layout() const
Returns true if this object utilizes COMPACT layout.
Definition: DmrppCommon.h:142
virtual void ingest_compression_type(const std::string &compression_type_string)
Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned i...
Definition: DmrppCommon.cc:171
virtual void load_attributes(libdap::BaseType *btp)
Load the attribute information for this variable.
Definition: DmrppCommon.cc:491
void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="")
Print the Compact base64-encoded information.
Definition: DmrppCommon.cc:404
virtual bool get_chunks_loaded() const
Have the chunks been loaded?
Definition: DmrppCommon.h:158
virtual size_t get_chunks_size() const
Use this when the number of chunks is needed.
Definition: DmrppCommon.h:175
static std::string d_dmrpp_ns
The DMR++ XML namespace.
Definition: DmrppCommon.h:119
void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space="")
Print the Chunk information.
Definition: DmrppCommon.cc:330
virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string)
Set the dimension sizes for a chunk.
Definition: DmrppCommon.cc:134
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Add a new chunk as defined by an h4:byteStream element.
Definition: DmrppCommon.cc:204
virtual const std::vector< unsigned long long > & get_chunk_dimension_sizes() const
The chunk dimension sizes held in a const vector.
Definition: DmrppCommon.h:179
void set_filter(const std::string &value)
Set the value of the filters property.
Definition: DmrppCommon.cc:108
virtual void ingest_byte_order(const std::string &byte_order_string)
Parses the text content of the XML element chunks:byteOrder.
Definition: DmrppCommon.cc:182
virtual const std::vector< std::shared_ptr< Chunk > > & get_immutable_chunks() const
A const reference to the vector of chunks.
Definition: DmrppCommon.h:169
void set_compact(bool value)
Set the value of the compact property.
Definition: DmrppCommon.h:147
void print_dmrpp(libdap::XMLWriter &writer, bool constrained=false)
Print the DMR++ response for the Scalar types.
Definition: DmrppCommon.cc:427
virtual void load_chunks(libdap::BaseType *btp)
Load chunk information for this variable.
Definition: DmrppCommon.cc:482
virtual unsigned long long get_chunk_size_in_elements() const
Get the number of elements in this chunk.
Definition: DmrppCommon.h:185
virtual std::string get_filters() const
Return the names of all the filters in the order they were applied.
Definition: DmrppCommon.h:131
virtual bool get_attributes_loaded() const
Have the attributes been loaded?
Definition: DmrppCommon.h:162
virtual char * read_atomic(const std::string &name)
read method for the atomic types
Definition: DmrppCommon.cc:311
utility class for the HTTP catalog module
Definition: AllowedHosts.cc:55