libdap++  Updated for version 3.13.1
DAP4StreamMarshaller.cc
Go to the documentation of this file.
1 // DAP4StreamMarshaller.cc
2 
3 // -*- mode: c++; c-basic-offset:4 -*-
4 
5 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
6 // Access Protocol.
7 
8 // Copyright (c) 2012 OPeNDAP, Inc.
9 // Author: James Gallagher <jgallagher@opendap.org>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 //
27 // Portions of this code are from Google's great protocol buffers library and
28 // are copyrighted as follows:
29 
30 // Protocol Buffers - Google's data interchange format
31 // Copyright 2008 Google Inc. All rights reserved.
32 // http://code.google.com/p/protobuf/
33 
34 #include "config.h"
35 
36 #include "DAP4StreamMarshaller.h"
37 
38 #include <stdint.h> // for the Google protobuf code
39 #include <byteswap.h>
40 
41 #include <iostream>
42 #include <sstream>
43 #include <iomanip>
44 
45 using namespace std;
46 
47 //#define DODS_DEBUG 1
48 
49 #include "dods-datatypes.h"
50 #include "util.h"
51 #include "debug.h"
52 
53 namespace libdap {
54 
55 static inline bool is_host_big_endian()
56 {
57 #ifdef COMPUTE_ENDIAN_AT_RUNTIME
58 
59  dods_int16 i = 0x0100;
60  char *c = reinterpret_cast<char*>(&i);
61  return *c;
62 
63 #else
64 
65 #ifdef WORDS_BIGENDIAN
66  return true;
67 #else
68  return false;
69 #endif
70 
71 #endif
72 }
73 
74 
75 // From the Google protobuf library
76 inline uint8_t* WriteVarint64ToArrayInline(uint64_t value, uint8_t* target) {
77  // Splitting into 32-bit pieces gives better performance on 32-bit
78  // processors.
79  uint32_t part0 = static_cast<uint32_t>(value );
80  uint32_t part1 = static_cast<uint32_t>(value >> 28);
81  uint32_t part2 = static_cast<uint32_t>(value >> 56);
82 
83  int size;
84 
85  // Here we can't really optimize for small numbers, since the value is
86  // split into three parts. Checking for numbers < 128, for instance,
87  // would require three comparisons, since you'd have to make sure part1
88  // and part2 are zero. However, if the caller is using 64-bit integers,
89  // it is likely that they expect the numbers to often be very large, so
90  // we probably don't want to optimize for small numbers anyway. Thus,
91  // we end up with a hard coded binary search tree...
92  if (part2 == 0) {
93  if (part1 == 0) {
94  if (part0 < (1 << 14)) {
95  if (part0 < (1 << 7)) {
96  size = 1; goto size1;
97  } else {
98  size = 2; goto size2;
99  }
100  } else {
101  if (part0 < (1 << 21)) {
102  size = 3; goto size3;
103  } else {
104  size = 4; goto size4;
105  }
106  }
107  } else {
108  if (part1 < (1 << 14)) {
109  if (part1 < (1 << 7)) {
110  size = 5; goto size5;
111  } else {
112  size = 6; goto size6;
113  }
114  } else {
115  if (part1 < (1 << 21)) {
116  size = 7; goto size7;
117  } else {
118  size = 8; goto size8;
119  }
120  }
121  }
122  } else {
123  if (part2 < (1 << 7)) {
124  size = 9; goto size9;
125  } else {
126  size = 10; goto size10;
127  }
128  }
129 
130  // GOOGLE_LOG(FATAL) << "Can't get here.";
131 
132  size10: target[9] = static_cast<uint8_t>((part2 >> 7) | 0x80);
133  size9 : target[8] = static_cast<uint8_t>((part2 ) | 0x80);
134  size8 : target[7] = static_cast<uint8_t>((part1 >> 21) | 0x80);
135  size7 : target[6] = static_cast<uint8_t>((part1 >> 14) | 0x80);
136  size6 : target[5] = static_cast<uint8_t>((part1 >> 7) | 0x80);
137  size5 : target[4] = static_cast<uint8_t>((part1 ) | 0x80);
138  size4 : target[3] = static_cast<uint8_t>((part0 >> 21) | 0x80);
139  size3 : target[2] = static_cast<uint8_t>((part0 >> 14) | 0x80);
140  size2 : target[1] = static_cast<uint8_t>((part0 >> 7) | 0x80);
141  size1 : target[0] = static_cast<uint8_t>((part0 ) | 0x80);
142 
143  target[size-1] &= 0x7F;
144  return target + size;
145 }
146 
154 DAP4StreamMarshaller::DAP4StreamMarshaller(ostream &out, bool write_data) :
155  d_out(out), d_ctx(0), d_write_data(write_data), d_checksum_ctx_valid(false)
156 {
157  // XDR is used if the call std::numeric_limits<double>::is_iec559()
158  // returns false indicating that the compiler is not using IEEE 754.
159  // If it is, we just write out the bytes. Why malloc()? Because
160  // xdr_destroy is going to call free() for us.
161  d_ieee754_buf = (char*)malloc(sizeof(dods_float64));
162  if (!d_ieee754_buf)
163  throw InternalErr(__FILE__, __LINE__, "Could not create DAP4StreamMarshaller");
164  xdrmem_create(&d_scalar_sink, d_ieee754_buf, sizeof(dods_float64), XDR_ENCODE);
165 
166  // This will cause exceptions to be thrown on i/o errors. The exception
167  // will be ostream::failure
168  out.exceptions(ostream::failbit | ostream::badbit);
169 
170  d_ctx = EVP_MD_CTX_create();
171 }
172 
174 {
175  // Free the buffer this contains. The xdr_destroy() macro does not
176  // free the XDR struct (which is fine since we did not dynamically
177  // allocate it).
178  xdr_destroy (&d_scalar_sink);
179 
180  EVP_MD_CTX_destroy(d_ctx);
181 }
182 
189 string
191 {
192  return (is_host_big_endian()) ? "big": "little";
193 }
194 
200 {
201  if (EVP_DigestInit_ex(d_ctx, EVP_md5(), 0) == 0)
202  throw Error("Failed to initialize checksum object.");
203 
204  d_checksum_ctx_valid = true;
205 }
206 
210 void DAP4StreamMarshaller::m_compute_checksum()
211 {
212  if (d_checksum_ctx_valid) {
213  // '...Final()' 'erases' the context so the next call without a reset
214  // returns a bogus value.
215  d_checksum_ctx_valid = false;
216 
217  // For MD5, the array md holds the 16 digits of the hash as values.
218  // The loop below turns that into a nice 32-digit hex number; see
219  // put_checksum() for a version that writes the 128-bit value without
220  // that conversion.
221  unsigned int md_len;
222  int status = EVP_DigestFinal_ex(d_ctx, &d_md[0], &md_len);
223  if (status == 0 || md_len != c_md5_length)
224  throw Error("Error computing the checksum (checksum computation).");
225  }
226 }
227 
234 {
235  if (d_checksum_ctx_valid) {
236  m_compute_checksum();
237  }
238 
239  ostringstream oss;
240  oss.setf(ios::hex, ios::basefield);
241  for (unsigned int i = 0; i < c_md5_length; ++i) {
242  oss << setfill('0') << setw(2) << (unsigned int) d_md[i];
243  }
244 
245  return oss.str();
246 }
247 
249 {
250  if (d_checksum_ctx_valid) {
251  m_compute_checksum();
252  }
253 
254  d_out.write(reinterpret_cast<char*>(&d_md[0]), c_md5_length);
255 }
256 
257 void DAP4StreamMarshaller::checksum_update(const void *data, unsigned long len)
258 {
259  if (!d_checksum_ctx_valid)
260  throw InternalErr(__FILE__, __LINE__, "Invalid checksum context (checksum update).");
261 
262  if (EVP_DigestUpdate(d_ctx, data, len) == 0) {
263  d_checksum_ctx_valid = false;
264  throw Error("Error computing the checksum (checksum update).");
265  }
266 }
267 
269 {
270  checksum_update(&val, sizeof(dods_byte));
271 
272  if (d_write_data) {
273  DBG( std::cerr << "put_byte: " << val << std::endl );
274 
275  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_byte));
276  }
277 }
278 
280 {
281  checksum_update(&val, sizeof(dods_int8));
282 
283  if (d_write_data) {
284  DBG( std::cerr << "put_int8: " << val << std::endl );
285 
286  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int8));
287  }
288 }
289 
291 {
292  checksum_update(&val, sizeof(dods_int16));
293 
294  if (d_write_data)
295  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int16));
296 }
297 
299 {
300  checksum_update(&val, sizeof(dods_int32));
301 
302  if (d_write_data)
303  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int32));
304 }
305 
307 {
308  checksum_update(&val, sizeof(dods_int64));
309 
310  if (d_write_data)
311  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int64));
312 }
313 
315 {
316  checksum_update(&val, sizeof(dods_float32));
317 
318  if (d_write_data) {
319  if (std::numeric_limits<float>::is_iec559 ) {
320  DBG2(cerr << "Native rep is ieee754." << endl);
321  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float32));
322  }
323  else {
324  if (!xdr_setpos(&d_scalar_sink, 0))
325  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
326 
327  if (!xdr_float(&d_scalar_sink, &val))
328  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
329 
330  if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float32))
331  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
332 
333  // If this is a little-endian host, twiddle the bytes
334  static bool twiddle_bytes = !is_host_big_endian();
335  if (twiddle_bytes) {
336  dods_int32 *i = reinterpret_cast<dods_int32*>(&d_ieee754_buf);
337  *i = bswap_32(*i);
338  }
339 
340  d_out.write(d_ieee754_buf, sizeof(dods_float32));
341  }
342  }
343 }
344 
346 {
347  checksum_update(&val, sizeof(dods_float64));
348 
349  if (d_write_data) {
350  if (std::numeric_limits<double>::is_iec559)
351  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float64));
352  else {
353  if (!xdr_setpos(&d_scalar_sink, 0))
354  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
355 
356  if (!xdr_double(&d_scalar_sink, &val))
357  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
358 
359  if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float64))
360  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
361 
362  // If this is a little-endian host, twiddle the bytes
363  static bool twiddle_bytes = !is_host_big_endian();
364  if (twiddle_bytes) {
365  dods_int64 *i = reinterpret_cast<dods_int64*>(&d_ieee754_buf);
366  *i = bswap_64(*i);
367  }
368 
369  d_out.write(d_ieee754_buf, sizeof(dods_float64));
370  }
371  }
372 }
373 
375 {
376  checksum_update(&val, sizeof(dods_uint16));
377 
378  if (d_write_data)
379  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint16));
380 }
381 
383 {
384  checksum_update(&val, sizeof(dods_uint32));
385 
386  if (d_write_data)
387  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint32));
388 }
389 
391 {
392  checksum_update(&val, sizeof(dods_uint64));
393 
394  if (d_write_data)
395  d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint64));
396 }
397 
398 
399 void DAP4StreamMarshaller::put_str(const string &val)
400 {
401  checksum_update(val.c_str(), val.length());
402 
403  if (d_write_data) {
404  put_length_prefix(val.length());
405  d_out.write(val.data(), val.length());
406  }
407 }
408 
409 void DAP4StreamMarshaller::put_url(const string &val)
410 {
411  put_str(val);
412 }
413 
414 void DAP4StreamMarshaller::put_opaque(char *val, unsigned int len)
415 {
416  checksum_update(val, len);
417 
418  if (d_write_data) {
419  put_length_prefix(len);
420  d_out.write(val, len);
421  }
422 }
423 
425 {
426  if (d_write_data) {
427  DBG2(cerr << "val: " << val << endl);
428 
429  vector<uint8_t> target(sizeof(dods_uint64) + 1, 0);
430  uint8_t* to_send = WriteVarint64ToArrayInline(val, &target[0]);
431  d_out.write(reinterpret_cast<char*>(&target[0]), to_send - &target[0]);
432 
433  DBG2(cerr << "varint: " << hex << *(uint64_t*)&target[0] << dec << endl);
434  }
435 }
436 
437 void DAP4StreamMarshaller::put_vector(char *val, unsigned int num)
438 {
439  checksum_update(val, num);
440 
441  d_out.write(val, num);
442 }
443 
454 void DAP4StreamMarshaller::put_varying_vector(char *val, unsigned int num)
455 {
456  put_opaque(val, num);
457 }
458 
464 void DAP4StreamMarshaller::m_serialize_reals(char *val, unsigned int num, int width, Type type)
465 {
466  dods_uint64 size = num * width;
467  // This is a leak!!! xdr_destroy does not free 'buf'.
468  //char *buf = (char*)malloc(size);
469  vector<char> buf(size);
470  XDR xdr;
471  xdrmem_create(&xdr, &buf[0], size, XDR_ENCODE);
472  try {
473  if(!xdr_array(&xdr, &val, (unsigned int *)&num, size, width, XDRUtils::xdr_coder(type)))
474  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
475 
476  if (xdr_getpos(&xdr) != size)
477  throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
478 
479  // If this is a little-endian host, twiddle the bytes
480  static bool twiddle_bytes = !is_host_big_endian();
481  if (twiddle_bytes) {
482  if (width == 4) {
483  dods_float32 *lbuf = reinterpret_cast<dods_float32*>(&buf[0]);
484  while (num--) {
485  dods_int32 *i = reinterpret_cast<dods_int32*>(lbuf++);
486  *i = bswap_32(*i);
487  }
488  }
489  else { // width == 8
490  dods_float64 *lbuf = reinterpret_cast<dods_float64*>(&buf[0]);
491  while (num--) {
492  dods_int64 *i = reinterpret_cast<dods_int64*>(lbuf++);
493  *i = bswap_64(*i);
494  }
495  }
496  }
497 
498  d_out.write(&buf[0], size);
499  }
500  catch (...) {
501  xdr_destroy(&xdr);
502  throw;
503  }
504  xdr_destroy(&xdr);
505 }
506 
507 void DAP4StreamMarshaller::put_vector(char *val, unsigned int num, int width, Type type)
508 {
509  checksum_update(val, num * width);
510 
511  if (d_write_data) {
512  if (type == dods_float32_c && !std::numeric_limits<float>::is_iec559) {
513  // If not using IEEE 754, use XDR to get it that way.
514  m_serialize_reals(val, num, 4, type);
515  }
516  else if (type == dods_float64_c && !std::numeric_limits<double>::is_iec559) {
517  m_serialize_reals(val, num, 8, type);
518  }
519  else {
520  d_out.write(val, num * width);
521  }
522  }
523 }
524 
536 void DAP4StreamMarshaller::put_varying_vector(char *val, unsigned int num, int width, Type type)
537 {
538  put_length_prefix(num);
539  put_vector(val, num, width, type);
540 }
541 
542 void DAP4StreamMarshaller::dump(ostream &strm) const
543 {
544  strm << DapIndent::LMarg << "DAP4StreamMarshaller::dump - (" << (void *) this << ")" << endl;
545 }
546 
547 } // namespace libdap
548 
virtual void put_uint16(dods_uint16 val)
uint8_t dods_byte
static const unsigned int c_md5_length
int64_t dods_int64
virtual void put_varying_vector(char *val, unsigned int num)
DINT64 dods_int64
virtual void put_float64(dods_float64 val)
virtual void put_float32(dods_float32 val)
virtual void put_vector(char *val, unsigned int num)
virtual void dump(ostream &strm) const
dump the contents of this object to the specified ostream
virtual void put_opaque(char *val, unsigned int len)
Type
Identifies the data type.
Definition: BaseType.h:137
uint16_t dods_uint16
#define DBG2(x)
Definition: debug.h:73
uint64_t dods_uint64
virtual string get_endian() const
A class for software fault reporting.
Definition: InternalErr.h:64
uint8_t * WriteVarint64ToArrayInline(uint64_t value, uint8_t *target)
#define DBG(x)
Definition: debug.h:58
virtual void put_int32(dods_int32 val)
double dods_float64
virtual void checksum_update(const void *data, unsigned long len)
virtual void put_int16(dods_int16 val)
uint32_t dods_uint32
virtual void put_int8(dods_int8 val)
virtual void put_byte(dods_byte val)
virtual void put_url(const string &val)
virtual void put_uint64(dods_uint64 val)
static ostream & LMarg(ostream &strm)
Definition: DapIndent.cc:80
int16_t dods_int16
virtual void put_uint32(dods_uint32 val)
virtual void put_str(const string &val)
virtual void put_int64(dods_int64 val)
DFLOAT64 dods_float64
static xdrproc_t xdr_coder(const Type &t)
Returns a function used to encode elements of an array.
Definition: XDRUtils.cc:145
A class for error processing.
Definition: Error.h:90
DINT16 dods_int16
virtual void put_length_prefix(dods_uint64 val)
int32_t dods_int32