HIPO  4.3.0
High Performance Output data format for experimental physics
record.cpp
Go to the documentation of this file.
1 //******************************************************************************
2 //* ██╗ ██╗██╗██████╗ ██████╗ ██╗ ██╗ ██████╗ *
3 //* ██║ ██║██║██╔══██╗██╔═══██╗ ██║ ██║ ██╔═████╗ *
4 //* ███████║██║██████╔╝██║ ██║ ███████║ ██║██╔██║ *
5 //* ██╔══██║██║██╔═══╝ ██║ ██║ ╚════██║ ████╔╝██║ *
6 //* ██║ ██║██║██║ ╚██████╔╝ ██║██╗╚██████╔╝ *
7 //* ╚═╝ ╚═╝╚═╝╚═╝ ╚═════╝ ╚═╝╚═╝ ╚═════╝ *
8 //************************ Jefferson National Lab (2017) ***********************
9 /*
10  * Copyright (c) 2017. Jefferson Lab (JLab). All rights reserved. Permission
11  * to use, copy, modify, and distribute this software and its documentation
12  * for educational, research, and not-for-profit purposes, without fee and
13  * without a signed licensing agreement.
14  *
15  * IN NO EVENT SHALL JLAB BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL
16  * INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING
17  * OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF JLAB HAS
18  * BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
19  *
20  * JLAB SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE. THE HIPO DATA FORMAT SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF
23  * ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". JLAB HAS NO OBLIGATION TO
24  * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
25  *
26  * This software was developed under the United States Government license.
27  * For more information contact author at gavalian@jlab.org
28  * Department of Experimental Nuclear Physics, Jefferson Lab.
29  */
30 
31  /*
32  * File: record.cpp
33  * Author: gavalian
34  *
35  * Created on April 11, 2017, 4:47 PM
36  */
37 
42 #include "record.h"
43 #include "constants.h"
44 //#include "hipoexceptions.h"
45 
46 #ifdef __LZ4__
47 #include <lz4.h>
48 #endif
49 
50 namespace hipo {
51 
52 
53  record::record()= default;
54 
55 
56  record::~record()= default;
57 
61  void record::readRecord(std::ifstream &stream, long position, int dataOffset){
62 
63  recordHeaderBuffer.resize(80);
64  stream.seekg(position,std::ios::beg);
65 
66  stream.read( (char *) &recordHeaderBuffer[0],80);
67  recordHeader.recordLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_RECORD_LENGTH_OFFSET]));
68  recordHeader.headerLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_HEADER_LENGTH_OFFSET]));
69  recordHeader.numberOfEvents = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_EVENT_COUNT_OFFSET]));
70  recordHeader.bitInfo = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_BIT_INFO_OFFSET]));
71  recordHeader.signatureString = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_MAGIC_NUMBER_OFFSET]));
72  recordHeader.recordDataLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_DATA_LENGTH_OFFSET]));
73  recordHeader.userHeaderLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_USER_HEADER_LEN_OFFSET]));
74  int compressedWord = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_COMP_WORD_OFFSET]));
75 
76  if(recordHeader.signatureString==HEADER_MAGIC) recordHeader.dataEndianness = 0;
77  if(recordHeader.signatureString==HEADER_MAGIC_BE) recordHeader.dataEndianness = 1;
78 
79  if(recordHeader.signatureString==HEADER_MAGIC_BE){
80  recordHeader.recordLength = __builtin_bswap32(recordHeader.recordLength);
81  recordHeader.headerLength = __builtin_bswap32(recordHeader.headerLength);
82  recordHeader.numberOfEvents = __builtin_bswap32(recordHeader.numberOfEvents);
83  recordHeader.recordDataLength = __builtin_bswap32(recordHeader.recordDataLength);
84  recordHeader.userHeaderLength = __builtin_bswap32(recordHeader.userHeaderLength);
85  recordHeader.bitInfo = __builtin_bswap32(recordHeader.bitInfo);
86  compressedWord = __builtin_bswap32(compressedWord);
87  }
88 
89  int compressedDataLengthPadding = (recordHeader.bitInfo>>BITINFO_PAD3_SHIFT)&BITINFO_PAD_MASK;
90  int headerLengthBytes = recordHeader.headerLength*4;
91  int dataBufferLengthBytes = recordHeader.recordLength * 4 - headerLengthBytes;
92 
93  recordHeader.userHeaderLengthPadding = (recordHeader.bitInfo>>BITINFO_PAD1_SHIFT)&BITINFO_PAD_MASK;
94  recordHeader.recordDataLengthCompressed = compressedWord&COMP_LENGTH_MASK;
95  recordHeader.compressionType = (compressedWord>>COMP_TYPE_SHIFT)&0x0000000F;
96  recordHeader.indexDataLength = 4*recordHeader.numberOfEvents;
97 
98  /*printf(" allocating buffer for record, size = %d, padding = %d length = %d type = %d nevents = %d data length = %d\n",
99  dataBufferLengthBytes,
100  compressedDataLengthPadding, recordHeader.recordDataLengthCompressed*4,
101  recordHeader.compressionType, recordHeader.numberOfEvents, recordHeader.recordDataLength);
102  */
103  //char *compressedBuffer = (char*) malloc(dataBufferLengthBytes);
104 
105  if(dataBufferLengthBytes>recordCompressedBuffer.size()){
106  int newSize = dataBufferLengthBytes + 5*1024;
107  //printf("---> resizing internal compressed buffer size to from %ld to %d\n",
108  // recordCompressedBuffer.size(), newSize);
109  recordCompressedBuffer.resize(newSize);
110  }
111  //dataBufferLengthBytes -= compressedDataLengthPadding;
112  long dataposition = position + headerLengthBytes;
113  //printf("position = %ld data position = %ld\n",position, dataposition);
114  stream.seekg(dataposition,std::ios::beg);
115  //stream.read( compressedBuffer, dataBufferLengthBytes);
116  stream.read( (&recordCompressedBuffer[0]), dataBufferLengthBytes);
117  //showBuffer(compressedBuffer, 10, 200);
118  //printf("position = %ld data position = %ld \n",position, dataposition);
119  int decompressedLength = recordHeader.indexDataLength +
120  recordHeader.userHeaderLength +
121  recordHeader.userHeaderLengthPadding +
122  recordHeader.recordDataLength;
123 
124  if(recordBuffer.size()<decompressedLength){
125  //printf(" resizing internal buffer from %lu to %d\n", recordBuffer.size(), recordHeader.recordDataLength);
126  recordBuffer.resize(decompressedLength+1024);
127  }
128  //readBenchmark.pause();
129  //for(int i = 0; i < recordBuffer.size(); i++) recordBuffer[i] = 0;
130  //printf("****************** BEFORE padding = %d\n", compressedDataLengthPadding);
131  //showBuffer(&recordBuffer[0], 10, 200);
132  //unzipBenchmark.resume();
133  if(recordHeader.compressionType==0){
134  // printf("compression type = 0 data length = %d\n",decompressedLength);
135  memcpy((&recordBuffer[0]),(&recordCompressedBuffer[0]),decompressedLength);
136  } else {
137  getUncompressed((&recordCompressedBuffer[0]) , (&recordBuffer[0]),
138  dataBufferLengthBytes-compressedDataLengthPadding,
139  decompressedLength);
140  }
141  //unzipBenchmark.pause();
142  //printf("******************\n");
143  //showBuffer(&recordBuffer[0], 10, 200);
144  //char *uncompressedBuffer = getUncompressed(compressedBuffer,dataBufferLengthBytes,recordHeader.recordDataLength);
145  //printf(" decompression size = %d error = %d\n", unc_result,
146  //unc_result - decompressedLength);
147  //free(compressedBuffer);
153  int eventPosition = 0;
154  for(int i = 0; i < recordHeader.numberOfEvents; i++){
155  auto *ptr = reinterpret_cast<int*>(&recordBuffer[i*4]);
156  int size = *ptr;
157  if(recordHeader.dataEndianness==1) size = __builtin_bswap32(size);
158  eventPosition += size;
159  *ptr = eventPosition;
160  }
161  //printf("final position = %d\n",eventPosition);
162  }
163 
164  bool record::readRecord(std::ifstream &stream, long position, int dataOffset, long inputSize){
165 
166  readBenchmark.resume();
167  if((position+80)>=inputSize) return false;
168 
169  recordHeaderBuffer.resize(80);
170  stream.seekg(position,std::ios::beg);
171 
172  stream.read( (char *) &recordHeaderBuffer[0],80);
173  recordHeader.recordLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_RECORD_LENGTH_OFFSET]));
174  recordHeader.headerLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_HEADER_LENGTH_OFFSET]));
175  recordHeader.numberOfEvents = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_EVENT_COUNT_OFFSET]));
176  recordHeader.bitInfo = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_BIT_INFO_OFFSET]));
177  recordHeader.signatureString = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_MAGIC_NUMBER_OFFSET]));
178  recordHeader.recordDataLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_DATA_LENGTH_OFFSET]));
179  recordHeader.userHeaderLength = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_USER_HEADER_LEN_OFFSET]));
180  int compressedWord = *(reinterpret_cast<int *>(&recordHeaderBuffer[RH_COMP_WORD_OFFSET]));
181 
182  if(recordHeader.signatureString==HEADER_MAGIC) recordHeader.dataEndianness = 0;
183  if(recordHeader.signatureString==HEADER_MAGIC_BE) recordHeader.dataEndianness = 1;
184 
185  if(recordHeader.signatureString==HEADER_MAGIC_BE){
186  recordHeader.recordLength = __builtin_bswap32(recordHeader.recordLength);
187  recordHeader.headerLength = __builtin_bswap32(recordHeader.headerLength);
188  recordHeader.numberOfEvents = __builtin_bswap32(recordHeader.numberOfEvents);
189  recordHeader.recordDataLength = __builtin_bswap32(recordHeader.recordDataLength);
190  recordHeader.userHeaderLength = __builtin_bswap32(recordHeader.userHeaderLength);
191  recordHeader.bitInfo = __builtin_bswap32(recordHeader.bitInfo);
192  compressedWord = __builtin_bswap32(compressedWord);
193  }
194 
195  int compressedDataLengthPadding = (recordHeader.bitInfo>>BITINFO_PAD3_SHIFT)&BITINFO_PAD_MASK;
196  int headerLengthBytes = recordHeader.headerLength*4;
197  int dataBufferLengthBytes = recordHeader.recordLength * 4 - headerLengthBytes;
198 
199  recordHeader.userHeaderLengthPadding = (recordHeader.bitInfo>>BITINFO_PAD1_SHIFT)&BITINFO_PAD_MASK;
200  recordHeader.recordDataLengthCompressed = compressedWord&COMP_LENGTH_MASK;
201  recordHeader.compressionType = (compressedWord>>COMP_TYPE_SHIFT)&0x0000000F;
202  recordHeader.indexDataLength = 4*recordHeader.numberOfEvents;
203 
204  /*printf(" allocating buffer for record, size = %d, padding = %d length = %d type = %d nevents = %d data length = %d\n",
205  dataBufferLengthBytes,
206  compressedDataLengthPadding, recordHeader.recordDataLengthCompressed*4,
207  recordHeader.compressionType, recordHeader.numberOfEvents, recordHeader.recordDataLength);
208  */
209  //char *compressedBuffer = (char*) malloc(dataBufferLengthBytes);
210 
211  if(dataBufferLengthBytes>recordCompressedBuffer.size()){
212  //-- resize the buffer extending it by 500kB. hopefully
213  //-- will be enough not to resize too many times.
214  int newSize = dataBufferLengthBytes + 500*1024;
215  // printout commented out --- by G.G.
216  /* printf("---> resizing internal compressed buffer size to from %ld to %d\n",
217  recordCompressedBuffer.size(), newSize); */
218  recordCompressedBuffer.resize(newSize);
219  }
220 
221  long dataposition = position + headerLengthBytes;
222  stream.seekg(dataposition,std::ios::beg);
223 
224  if(position+dataBufferLengthBytes+recordHeader.headerLength>inputSize){
225  printf("**** warning : record at position %ld is incomplete.",position);
226  return false;
227  }
228 
229  stream.read( (&recordCompressedBuffer[0]), dataBufferLengthBytes);
230 
231  int decompressedLength = recordHeader.indexDataLength +
232  recordHeader.userHeaderLength +
233  recordHeader.userHeaderLengthPadding +
234  recordHeader.recordDataLength;
235 
236  if(recordBuffer.size()<decompressedLength){
237  // Buffer resizes by 20% --- modified by me
238  int resize_fraction = (int) (decompressedLength*0.2);
239  //printf(" resizing internal buffer from %lu to %d\n", recordBuffer.size(), recordHeader.recordDataLength);
240  recordBuffer.resize(decompressedLength + resize_fraction);
241  }
242  //for(int i = 0; i < recordBuffer.size(); i++) recordBuffer[i] = 0;
243  //printf("****************** BEFORE padding = %d\n", compressedDataLengthPadding);
244  //showBuffer(&recordBuffer[0], 10, 200);
245  readBenchmark.pause();
246  unzipBenchmark.resume();
247  if(recordHeader.compressionType==0){
248  // printf("compression type = 0 data length = %d\n",decompressedLength);
249  memcpy((&recordBuffer[0]),(&recordCompressedBuffer[0]),decompressedLength);
250  } else {
251  getUncompressed((&recordCompressedBuffer[0]) , (&recordBuffer[0]),
252  dataBufferLengthBytes-compressedDataLengthPadding,
253  decompressedLength);
254  }
255  unzipBenchmark.pause();
256  //printf("******************\n");
257  //showBuffer(&recordBuffer[0], 10, 200);
258  //char *uncompressedBuffer = getUncompressed(compressedBuffer,dataBufferLengthBytes,recordHeader.recordDataLength);
259  //printf(" decompression size = %d error = %d\n", unc_result,
260  //unc_result - decompressedLength);
261  //free(compressedBuffer);
266  indexBenchmark.resume();
267  int eventPosition = 0;
268  for(int i = 0; i < recordHeader.numberOfEvents; i++){
269  auto *ptr = reinterpret_cast<int*>(&recordBuffer[i*4]);
270  int size = *ptr;
271  if(recordHeader.dataEndianness==1) size = __builtin_bswap32(size);
272  eventPosition += size;
273  *ptr = eventPosition;
274  }
275  indexBenchmark.pause();
276  return true;
277  }
278 
279  int record::getRecordSizeCompressed(){
280  return recordHeader.recordLength;
281  }
282 
283  void record::readRecord__(std::ifstream &stream, long position, long recordLength){
284 
285  stream.seekg(position,std::ios::beg);
286 
287  if(recordLength>recordCompressedBuffer.size()){
288  int newSize = recordLength + 5*1024;
289  //printf("---> resizing internal compressed buffer size to from %ld to %d\n",
290  //recordCompressedBuffer.size(), newSize);
291  //printf("---> after printout\n");
292  recordCompressedBuffer.resize(newSize);
293  }
294  //printf(" trying seeksg\n");
295  stream.seekg( position,std::ios::beg);
296  //printf(" trying read\n");
297  stream.read( (&recordCompressedBuffer[0]), recordLength);
298  //printf(" readin was successfull....\n");
299  //stream.read( (char *) &recordHeaderBuffer[0],80);
300  recordHeader.recordLength = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_RECORD_LENGTH_OFFSET]));
301  recordHeader.headerLength = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_HEADER_LENGTH_OFFSET]));
302  recordHeader.numberOfEvents = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_EVENT_COUNT_OFFSET]));
303  recordHeader.bitInfo = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_BIT_INFO_OFFSET]));
304  recordHeader.signatureString = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_MAGIC_NUMBER_OFFSET]));
305  recordHeader.recordDataLength = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_DATA_LENGTH_OFFSET]));
306  recordHeader.userHeaderLength = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_USER_HEADER_LEN_OFFSET]));
307  int compressedWord = *(reinterpret_cast<int *>(&recordCompressedBuffer[RH_COMP_WORD_OFFSET]));
308 
309  if(recordHeader.signatureString==HEADER_MAGIC) recordHeader.dataEndianness = 0;
310  if(recordHeader.signatureString==HEADER_MAGIC_BE) recordHeader.dataEndianness = 1;
311 
312  if(recordHeader.signatureString==HEADER_MAGIC_BE){
313  recordHeader.recordLength = __builtin_bswap32(recordHeader.recordLength);
314  recordHeader.headerLength = __builtin_bswap32(recordHeader.headerLength);
315  recordHeader.numberOfEvents = __builtin_bswap32(recordHeader.numberOfEvents);
316  recordHeader.recordDataLength = __builtin_bswap32(recordHeader.recordDataLength);
317  recordHeader.userHeaderLength = __builtin_bswap32(recordHeader.userHeaderLength);
318  recordHeader.bitInfo = __builtin_bswap32(recordHeader.bitInfo);
319  compressedWord = __builtin_bswap32(compressedWord);
320  }
321 
322  int compressedDataLengthPadding = (recordHeader.bitInfo>>BITINFO_PAD3_SHIFT)&BITINFO_PAD_MASK;
323  int headerLengthBytes = recordHeader.headerLength*4;
324  int dataBufferLengthBytes = recordHeader.recordLength * 4 - headerLengthBytes;
325 
326  recordHeader.userHeaderLengthPadding = (recordHeader.bitInfo>>BITINFO_PAD1_SHIFT)&BITINFO_PAD_MASK;
327  recordHeader.recordDataLengthCompressed = compressedWord&COMP_LENGTH_MASK;
328  recordHeader.compressionType = (compressedWord>>COMP_TYPE_SHIFT)&0x0000000F;
329  recordHeader.indexDataLength = 4*recordHeader.numberOfEvents;
330 
331  /*printf(" allocating buffer for record, size = %d, padding = %d length = %d type = %d nevents = %d data length = %d\n",
332  dataBufferLengthBytes,
333  compressedDataLengthPadding, recordHeader.recordDataLengthCompressed*4,
334  recordHeader.compressionType, recordHeader.numberOfEvents, recordHeader.recordDataLength);
335  */
336  //char *compressedBuffer = (char*) malloc(dataBufferLengthBytes);
337 
338 
339  //dataBufferLengthBytes -= compressedDataLengthPadding;
340  //printf(" record header intialized....\n");
341  long dataposition = position + headerLengthBytes;
342  //printf("position = %ld data position = %ld\n",position, dataposition);
343  //stream.seekg(dataposition,std::ios::beg);
344  //stream.read( compressedBuffer, dataBufferLengthBytes);
345  //stream.read( (&recordCompressedBuffer[0]), dataBufferLengthBytes);
346  //showBuffer(compressedBuffer, 10, 200);
347  //printf("position = %ld data position = %ld \n",position, dataposition);
348  int decompressedLength = recordHeader.indexDataLength +
349  recordHeader.userHeaderLength +
350  recordHeader.userHeaderLengthPadding +
351  recordHeader.recordDataLength;
352  //printf(" decompressed length = %d\n",decompressedLength);
353  if(recordBuffer.size()<decompressedLength){
354  //printf(" resizing internal buffer from %lu to %d\n", recordBuffer.size(), recordHeader.recordDataLength);
355  recordBuffer.resize(decompressedLength+1024);
356  }
357  //for(int i = 0; i < recordBuffer.size(); i++) recordBuffer[i] = 0;
358  //printf("****************** BEFORE padding = %d\n", compressedDataLengthPadding);
359  //showBuffer(&recordBuffer[0], 10, 200);
360  if(recordHeader.compressionType==0){
361  //printf("compression type = 0 data length = %d\n",decompressedLength);
362  memcpy((&recordBuffer[0]),(&recordCompressedBuffer[0]),decompressedLength);
363  } else {
364  //printf(" running deompression %d %d %d\n",dataBufferLengthBytes-compressedDataLengthPadding,decompressedLength,dataposition);
365  getUncompressed((&recordCompressedBuffer[RECORD_HEADER_SIZE]) , (&recordBuffer[0]),
366  dataBufferLengthBytes-compressedDataLengthPadding,
367  decompressedLength);
368  }
369  //printf("******************\n");
370  //showBuffer(&recordBuffer[0], 10, 200);
371  //char *uncompressedBuffer = getUncompressed(compressedBuffer,dataBufferLengthBytes,recordHeader.recordDataLength);
372  //printf(" decompression size = %d error = %d\n", unc_result,
373  //unc_result - decompressedLength);
374  //free(compressedBuffer);
379  //printf(" deompression ..... ok \n");
380  int eventPosition = dataposition;
381  for(int i = 0; i < recordHeader.numberOfEvents; i++){
382  auto *ptr = reinterpret_cast<int*>(&recordBuffer[i*4]);
383  int size = *ptr;
384  if(recordHeader.dataEndianness==1) size = __builtin_bswap32(size);
385  eventPosition += size;
386  *ptr = eventPosition;
387  }
388  //printf("final position = %d\n",eventPosition);
389  }
393  int record::getEventCount(){
394  return recordHeader.numberOfEvents;
395  }
401  void record::readEvent( std::vector<char> &vec, int index){
402 
403  }
404 
411  void record::getData(hipo::data &data, int index){
412  int first_position = 0;
413  if(index > 0){
414  first_position = *(reinterpret_cast<uint32_t *>(&recordBuffer[(index -1)*4]));
415  }
416  int last_position = *(reinterpret_cast<uint32_t *>(&recordBuffer[index*4]));
417  int offset = recordHeader.indexDataLength
418  + recordHeader.userHeaderLength
419  + recordHeader.userHeaderLengthPadding;
420  data.setDataPtr(&recordBuffer[first_position+offset]);
421  data.setDataSize(last_position-first_position);
422  data.setDataOffset(first_position + offset);
423  }
424 void record::read(hipo::bank &b, int event){
425  hipo::data event_data;
426  getData(event_data,event);
427  hipo::event::getStructure(event_data.getDataPtr(),b,b.getSchema().getGroup(),b.getSchema().getItem());
428 }
429 
430 void record::getColumn(hipo::data &data,const char* column, hipo::bank &bank, int event){
431  int order = bank.getSchema().getEntryOrder(column);
432  getColumn(data,order,bank,event);
433 }
434 
435 void record::getColumn(hipo::data &data,int column, hipo::bank &bank, int event){
436  hipo::data event_data;
437  getData(event_data,event);
438  std::pair<int,int> pos = hipo::event::getStructurePosition(event_data.getDataPtr(),
439  bank.getSchema().getGroup(), bank.getSchema().getItem());
440 
441  //printf(" returned position = %8d, %8d\n",pos.first,pos.second);
442 
443  if(pos.second<=0) {
444  data.setDataPtr(NULL);
445  data.setDataSize(0);
446  data.setDataType(0);
447  return;
448  }
449  //printf("----- hok ---- we are here now ----\n");
450  int size = *reinterpret_cast<const uint32_t *>(&event_data.getDataPtr()[pos.first+4]);
451  //printf("----- hok ---- one more step ----\n");
452  int bankRows = size/bank.getSchema().getRowLength();
453  //printf(" returned sizes = %8d, %8d\n",size,bankRows);
454  data.setDataSize(bankRows);
455  data.setDataType(bank.getSchema().getEntryType(column));
456 
457  //printf(" returned type = %d\n",data.getDataType());
458  int offset = bank.getSchema().getOffset(column,0,bankRows);
459  //printf("============ offset = %8d\n",offset);
460  data.setDataPtr(&(event_data.getDataPtr()[pos.first+offset+8]));
461 }
462 
463 void record::getEventsMap(std::vector<std::pair<int,int>> &emap){
464  emap.clear();
465  int offset = recordHeader.indexDataLength
466  + recordHeader.userHeaderLength
467  + recordHeader.userHeaderLengthPadding;
468  /*
469  printf(" offset for the record = %8d, %8d, %8d, %8d\n",offset,
470  recordHeader.indexDataLength
471  , recordHeader.userHeaderLength
472  , recordHeader.userHeaderLengthPadding
473  );*/
474 
475  int nevents = recordHeader.numberOfEvents;
476  for(int index = 0; index < nevents; index++){
477  int first_position = 0;
478  if(index > 0){
479  first_position = *(reinterpret_cast<uint32_t *>(&recordBuffer[(index -1)*4]));
480  }
481  int last_position = *(reinterpret_cast<uint32_t *>(&recordBuffer[index*4]));
482  emap.push_back(std::make_pair(first_position+offset,last_position+offset));
483  }
484 }
485  void record::readHipoEvent(hipo::event &event, int index){
486  hipo::data event_data;
487  getData(event_data,index);
488  event.init(event_data.getDataPtr(), event_data.getDataSize());
489  }
493  void record::showBuffer(const char *data, int wrapping, int maxsize)
494  {
495  for(int i = 0; i < maxsize; i++){
496  printf("%X ", 0x000000FF&((unsigned int) data[i]));
497  if( (i+1)%wrapping==0) printf("\n");
498  }
499  printf("\n");
500  }
507  int record::getUncompressed(const char *data, char *dest, int dataLength, int dataLengthUncompressed){
508  #ifdef __LZ4__
509  int result = LZ4_decompress_safe(data,dest,dataLength,dataLengthUncompressed);
510  //int result = LZ4_decompress_fast(data,dest,dataLengthUncompressed);
511  return result;
512  #endif
513 
514  #ifndef __LZ4__
515  printf("\n >>>>> LZ4 compression is not supported.");
516  printf("\n >>>>> check if libz4 is installed on your system.");
517  printf("\n >>>>> recompile the library with liblz4 installed.\n");
518  return NULL;
519  #endif
520 
521  }
526  char *record::getUncompressed(const char *data, int dataLength,
527  int dataLengthUncompressed){
528 
529  #ifdef __LZ4__
530  char *output = (char *) malloc(dataLengthUncompressed);
531  LZ4_decompress_safe(data,output,dataLength,dataLengthUncompressed);
532  return output;
533  //printf(" FIRST (%d) = %x %x %x %x\n",result);//,destUnCompressed[0],destUnCompressed[1],
534  //destUnCompressed[2],destUnCompressed[3]);
535  //LZ4_decompress_fast(buffer,destUnCompressed,decompressedLength);
536  //LZ4_uncompress(buffer,destUnCompressed,decompressedLength);
537  #endif
538 
539  #ifndef __LZ4__
540  printf("\n >>>>> LZ4 compression is not supported.");
541  printf("\n >>>>> check if libz4 is installed on your system.");
542  printf("\n >>>>> recompile the library with liblz4 installed.\n");
543  return nullptr;
544  #endif
545 
546  }
547 
548 
549 
550 dataframe::dataframe(int max_events, int max_size){
551  maxEvents = max_events; maxSize = max_size;
552  dataBuffer.resize(RECORD_HEADER_SIZE+maxSize);
553  reset();
554 }
555 
556 void dataframe::reset(){
557  *(reinterpret_cast<int *>(&dataBuffer[RH_RECORD_LENGTH_OFFSET])) = RECORD_HEADER_SIZE; // record length 1
558  *(reinterpret_cast<int *>(&dataBuffer[RH_RECORD_NUMBER_OFFSET])) = 1; // record number 2
559  *(reinterpret_cast<int *>(&dataBuffer[RH_HEADER_LENGTH_OFFSET])) = RECORD_HEADER_WORDS; // header length words 3
560  *(reinterpret_cast<int *>(&dataBuffer[RH_EVENT_COUNT_OFFSET])) = 0; // event count 4
561  *(reinterpret_cast<int *>(&dataBuffer[RH_INDEX_ARRAY_LEN_OFFSET]))= 0; // index array length 5
562  *(reinterpret_cast<int *>(&dataBuffer[RH_BIT_INFO_OFFSET])) = 5; // verion + bitinfo 6
563  *(reinterpret_cast<int *>(&dataBuffer[RH_USER_HEADER_LEN_OFFSET]))= 0; // user header length
564  *(reinterpret_cast<int *>(&dataBuffer[RH_MAGIC_NUMBER_OFFSET])) = HEADER_MAGIC;
565  *(reinterpret_cast<int *>(&dataBuffer[RH_DATA_LENGTH_OFFSET])) = 0; // uncompressed data length
566  *(reinterpret_cast<int *>(&dataBuffer[RH_COMP_WORD_OFFSET])) = 0; // compressed data length
567 }
568 
569 bool dataframe::addEvent(hipo::event &event){
570  int fsize = size();
571  if( (event.getSize() + fsize + RECORD_HEADER_SIZE ) > maxSize) return false;
572  int evCount = count();
573  if(evCount>=maxEvents) return false;
574  memcpy(&dataBuffer[fsize],&(event.getEventBuffer()[0]), event.getSize());
575  int datasize = *(reinterpret_cast<int *>(&dataBuffer[RH_DATA_LENGTH_OFFSET]));
576  *(reinterpret_cast<int *>(&dataBuffer[RH_EVENT_COUNT_OFFSET])) = evCount + 1;
577  *(reinterpret_cast<int *>(&dataBuffer[RH_DATA_LENGTH_OFFSET])) = datasize + event.getSize();
578  *(reinterpret_cast<int *>(&dataBuffer[RH_COMP_WORD_OFFSET])) = datasize + event.getSize();
579  *(reinterpret_cast<int *>(&dataBuffer[RH_RECORD_LENGTH_OFFSET])) = fsize + event.getSize();
580  return true;
581 }
582 
583 int dataframe::getEventAt(int pos, hipo::event &event){
584  int eventsize = *(reinterpret_cast<int *>(&dataBuffer[pos+4]));
585  event.init(&dataBuffer[pos],eventsize);
586  return pos+eventsize;
587 }
588 
589 void dataframe::init(const char *ptr){
590  int size = *(reinterpret_cast<const int *>(ptr));
591  if(size>dataBuffer.size()) dataBuffer.resize(size+RECORD_HEADER_SIZE);
592  memcpy(&dataBuffer[0],ptr,size);
593 }
594 
595 int dataframe::count(){ return *(reinterpret_cast<int *>(&dataBuffer[RH_EVENT_COUNT_OFFSET]));}
596 int dataframe::size(){ return (*(reinterpret_cast<int *>(&dataBuffer[RH_RECORD_LENGTH_OFFSET])));}
597 
598 void dataframe::summary(){
599  printf("## data frame summary:\n");
600  printf("frame size : %d\n",*(reinterpret_cast<int *>(&dataBuffer[RH_RECORD_LENGTH_OFFSET])));
601  printf("magic word : %X\n",*(reinterpret_cast<int *>(&dataBuffer[RH_MAGIC_NUMBER_OFFSET])));
602  printf("number of events : %d\n",*(reinterpret_cast<int *>(&dataBuffer[RH_EVENT_COUNT_OFFSET])));
603  printf("data length : %d\n",*(reinterpret_cast<int *>(&dataBuffer[RH_DATA_LENGTH_OFFSET])));
604 }
605 
606 }
hipo::bank
Represents a HIPO bank, a tabular data structure with rows and typed columns.
Definition: bank.h:352
hipo::bank::getSchema
schema & getSchema()
Definition: bank.h:443
hipo::benchmark::resume
void resume()
Starts or resumes the timer.
Definition: utils.cpp:196
hipo::benchmark::pause
void pause()
Pauses the timer and accumulates elapsed time.
Definition: utils.cpp:206
hipo::data
Lightweight wrapper for a raw data pointer with type, size, and offset metadata.
Definition: record.h:91
hipo::data::setDataSize
void setDataSize(int __size)
Set the data size in bytes.
Definition: record.h:108
hipo::data::setDataOffset
void setDataOffset(int __offset)
Set the byte offset into the source buffer.
Definition: record.h:110
hipo::data::setDataPtr
void setDataPtr(const char *__ptr)
Set the raw data pointer.
Definition: record.h:106
hipo::data::getDataPtr
const char * getDataPtr()
Return the raw data pointer.
Definition: record.h:120
hipo::data::setDataType
void setDataType(int __type)
Set the data type identifier.
Definition: record.h:114
hipo::data::getDataSize
int getDataSize()
Return the data size in bytes.
Definition: record.h:122
hipo::dataframe::addEvent
bool addEvent(hipo::event &event)
Append an event to the dataframe.
Definition: record.cpp:569
hipo::dataframe::reset
void reset()
Clear all events and reset the dataframe to an empty state.
Definition: record.cpp:556
hipo::dataframe::getEventAt
int getEventAt(int pos, hipo::event &event)
Retrieve the event at the given position.
Definition: record.cpp:583
hipo::dataframe::init
void init(const char *ptr)
Initialize the dataframe from an existing raw buffer.
Definition: record.cpp:589
hipo::dataframe::count
int count()
Return the number of events currently stored in the frame.
Definition: record.cpp:595
hipo::dataframe::dataframe
dataframe()
Construct an empty dataframe with default limits.
Definition: record.h:148
hipo::dataframe::summary
void summary()
Print a summary of the dataframe contents to standard output.
Definition: record.cpp:598
hipo::dataframe::size
int size()
Return the total size of the internal buffer in bytes.
Definition: record.cpp:596
hipo::event
Represents a HIPO event, a container for multiple structures/banks.
Definition: event.h:77
hipo::event::getEventBuffer
std::vector< char > & getEventBuffer()
Definition: event.cpp:329
hipo::event::getStructure
void getStructure(hipo::structure &str, int group, int item)
Extract a structure from the event by group and item identifiers.
Definition: event.cpp:72
hipo::event::getStructurePosition
std::pair< int, int > getStructurePosition(int group, int item)
Find the position and length of a structure in the event buffer.
Definition: event.cpp:256
hipo::event::getSize
int getSize()
Definition: event.cpp:315
hipo::record::readEvent
void readEvent(std::vector< char > &vec, int index)
Read raw event bytes into a vector.
Definition: record.cpp:401
hipo::record::record
record()
Construct an empty record.
hipo::record::getEventCount
int getEventCount()
Return the number of events stored in this record.
Definition: record.cpp:393
hipo::record::getEventsMap
void getEventsMap(std::vector< std::pair< int, int >> &emap)
Build a map of event positions (offset, length pairs).
Definition: record.cpp:463
hipo::record::read
void read(hipo::bank &b, int event)
Read a specific bank from a given event in this record.
Definition: record.cpp:424
hipo::record::getRecordSizeCompressed
int getRecordSizeCompressed()
Return the compressed size of this record in bytes.
Definition: record.cpp:279
hipo::record::readRecord
void readRecord(std::ifstream &stream, long position, int dataOffset)
Read a record from a file stream at the specified position.
Definition: record.cpp:61
hipo::record::readHipoEvent
void readHipoEvent(hipo::event &event, int index)
Read an event and populate a hipo::event object.
Definition: record.cpp:485
hipo::record::~record
~record()
Destructor.
hipo::record::getData
void getData(hipo::data &data, int index)
Get a lightweight data handle for a specific event.
Definition: record.cpp:411
hipo::record::getColumn
void getColumn(hipo::data &data, int column, hipo::bank &bank, int event)
Extract a single column from a bank for a given event.
Definition: record.cpp:435
hipo::record::readRecord__
void readRecord__(std::ifstream &stream, long position, long recordLength)
Read a record using its total length (internal variant).
Definition: record.cpp:283
hipo::schema::getEntryOrder
int getEntryOrder(const char *name) const
Get the column index for a given column name.
Definition: dictionary.h:239
hipo::schema::getOffset
int getOffset(int item, int order, int rows) const
Compute the byte offset for a column value by column index.
Definition: dictionary.h:171
hipo::schema::getRowLength
int getRowLength() const noexcept
Get the size of a single row in bytes.
Definition: dictionary.h:141
hipo::schema::getItem
int getItem()
Get the item identifier.
Definition: dictionary.h:131
hipo::schema::getGroup
int getGroup()
Get the group identifier.
Definition: dictionary.h:128
hipo::schema::getEntryType
int getEntryType(int item) const noexcept
Get the type identifier of a column by index.
Definition: dictionary.h:193
hipo
Definition: bank.cpp:47
hipo::COMP_LENGTH_MASK
constexpr uint32_t COMP_LENGTH_MASK
Definition: constants.h:120
hipo::HEADER_MAGIC_BE
constexpr uint32_t HEADER_MAGIC_BE
Definition: constants.h:9
hipo::BITINFO_PAD3_SHIFT
constexpr int BITINFO_PAD3_SHIFT
Definition: constants.h:111
hipo::BITINFO_PAD_MASK
constexpr uint32_t BITINFO_PAD_MASK
Definition: constants.h:112
hipo::BITINFO_PAD1_SHIFT
constexpr int BITINFO_PAD1_SHIFT
Definition: constants.h:109
hipo::RECORD_HEADER_SIZE
constexpr int RECORD_HEADER_SIZE
Definition: constants.h:15
hipo::RECORD_HEADER_WORDS
constexpr int RECORD_HEADER_WORDS
Definition: constants.h:13
hipo::RH_INDEX_ARRAY_LEN_OFFSET
constexpr int RH_INDEX_ARRAY_LEN_OFFSET
Definition: constants.h:38
hipo::RH_EVENT_COUNT_OFFSET
constexpr int RH_EVENT_COUNT_OFFSET
Definition: constants.h:37
hipo::RH_BIT_INFO_OFFSET
constexpr int RH_BIT_INFO_OFFSET
Definition: constants.h:39
hipo::RH_USER_HEADER_LEN_OFFSET
constexpr int RH_USER_HEADER_LEN_OFFSET
Definition: constants.h:40
hipo::RH_RECORD_NUMBER_OFFSET
constexpr int RH_RECORD_NUMBER_OFFSET
Definition: constants.h:35
hipo::COMP_TYPE_SHIFT
constexpr int COMP_TYPE_SHIFT
Definition: constants.h:119
hipo::RH_HEADER_LENGTH_OFFSET
constexpr int RH_HEADER_LENGTH_OFFSET
Definition: constants.h:36
hipo::RH_MAGIC_NUMBER_OFFSET
constexpr int RH_MAGIC_NUMBER_OFFSET
Definition: constants.h:41
hipo::RH_RECORD_LENGTH_OFFSET
constexpr int RH_RECORD_LENGTH_OFFSET
Definition: constants.h:34
hipo::HEADER_MAGIC
constexpr uint32_t HEADER_MAGIC
Definition: constants.h:8
hipo::RH_COMP_WORD_OFFSET
constexpr int RH_COMP_WORD_OFFSET
Definition: constants.h:43
hipo::RH_DATA_LENGTH_OFFSET
constexpr int RH_DATA_LENGTH_OFFSET
Definition: constants.h:42
record.h
HIPO record reading and event extraction.
hipo::recordHeader_t::dataEndianness
int dataEndianness
Endianness of the stored data.
Definition: record.h:81
hipo::recordHeader_t::bitInfo
int bitInfo
Bit-packed information field.
Definition: record.h:78
hipo::recordHeader_t::userHeaderLengthPadding
int userHeaderLengthPadding
Padding added to the user header length.
Definition: record.h:77
hipo::recordHeader_t::userHeaderLength
int userHeaderLength
User header length in bytes.
Definition: record.h:76
hipo::recordHeader_t::compressionType
int compressionType
Compression algorithm identifier.
Definition: record.h:79
hipo::recordHeader_t::indexDataLength
int indexDataLength
Length of the index buffer in bytes.
Definition: record.h:75
hipo::recordHeader_t::recordDataLength
int recordDataLength
Length of the uncompressed data buffer.
Definition: record.h:71
hipo::recordHeader_t::headerLength
int headerLength
Length of the record header buffer.
Definition: record.h:74
hipo::recordHeader_t::numberOfEvents
int numberOfEvents
Number of events (data buckets) in the data buffer.
Definition: record.h:73
hipo::recordHeader_t::signatureString
int signatureString
Identifier string "HREC" (int = 0x43455248).
Definition: record.h:69
hipo::recordHeader_t::recordLength
int recordLength
Total length of the record, including the index array.
Definition: record.h:70
hipo::recordHeader_t::recordDataLengthCompressed
int recordDataLengthCompressed
Length of the compressed data buffer.
Definition: record.h:72