reader.cpp

Go to the documentation of this file.

//******************************************************************************
//*       ██╗  ██╗██╗██████╗  ██████╗     ██╗  ██╗    ██████╗                  *
//*       ██║  ██║██║██╔══██╗██╔═══██╗    ██║  ██║   ██╔═████╗                 *
//*       ███████║██║██████╔╝██║   ██║    ███████║   ██║██╔██║                 *
//*       ██╔══██║██║██╔═══╝ ██║   ██║    ╚════██║   ████╔╝██║                 *
//*       ██║  ██║██║██║     ╚██████╔╝         ██║██╗╚██████╔╝                 *
//*       ╚═╝  ╚═╝╚═╝╚═╝      ╚═════╝          ╚═╝╚═╝ ╚═════╝                  *
//************************ Jefferson National Lab (2017) ***********************
/*
 *   Copyright (c) 2017.  Jefferson Lab (JLab). All rights reserved. Permission
 *   to use, copy, modify, and distribute  this software and its documentation
 *   for educational, research, and not-for-profit purposes, without fee and
 *   without a signed licensing agreement.
 *
 *   IN NO EVENT SHALL JLAB BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL
 *   INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING
 *   OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF JLAB HAS
 *   BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *   JLAB SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 *   THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 *   PURPOSE. THE HIPO DATA FORMAT SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF
 *   ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". JLAB HAS NO OBLIGATION TO
 *   PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 *
 *   This software was developed under the United States Government license.
 *   For more information contact author at gavalian@jlab.org
 *   Department of Experimental Nuclear Physics, Jefferson Lab.
 */
 /*
  * File:   reader.cpp
  * Author: gavalian
  *
  * Additions by: dglazier
  * Created on April 11, 2017, 2:07 PM
  */
 
#include "reader.h"
#include "constants.h"
#include "hipoexceptions.h"
#include "record.h"
 
#include <cstdlib>
namespace hipo {
  reader::reader(){
    printWarning();
    //hipoutils.printLogo();
  }
 
  reader::~reader(){
    if(inputStream.is_open()==true){
      inputStream.close();
    }
  }
 
  void  reader::about(){
    hipoutils.printLogo();
  }
  /*
   * Open file, if file stream is open, it is closed first.
   * At open time verification of file structure is performed.
   * If the signature does not match EVIO/HIPO template, the
   * file will be closed and warning message is printed.
   */
  void reader::open(const char *filename){
 
    if(inputStream.is_open()==true){
      inputStream.close();
 
    }
 
    inputStream.open(filename, std::ios::binary);
    inputStream.seekg(0,std::ios_base::end);
    inputStreamSize = inputStream.tellg();
    inputStream.seekg(0,std::ios_base::beg);
    if(inputStream.is_open()==false){
      printf("[ERROR] something went wrong with openning file : %s\n",
            filename);
      return;
    }
    readHeader();
    readIndex();
}
 
 void reader::readHeader(){
 
    std::vector<char>               headerBuffer;
    headerBuffer.resize(80);
    inputStream.read(&headerBuffer[0],80);
 
    header.uniqueid     = *(reinterpret_cast<int *>(&headerBuffer[FH_UNIQUE_WORD_OFFSET]));
    header.filenumber   = *(reinterpret_cast<int *>(&headerBuffer[FH_FILE_NUMBER_OFFSET]));
    header.headerLength = *(reinterpret_cast<int *>(&headerBuffer[FH_HEADER_LENGTH_OFFSET]));
    header.recordCount  = *(reinterpret_cast<int *>(&headerBuffer[FH_RECORD_COUNT_OFFSET]));
 
    header.indexArrayLength = *(reinterpret_cast<int *>(&headerBuffer[FH_INDEX_ARRAY_LEN_OFFSET]));
    int word_8 = *(reinterpret_cast<int *>(&headerBuffer[FH_BIT_INFO_OFFSET]));
 
    header.userHeaderLength = *(reinterpret_cast<int *>(&headerBuffer[FH_USER_HEADER_LEN_OFFSET]));
    header.magicNumber = *(reinterpret_cast<int *>(&headerBuffer[FH_MAGIC_NUMBER_OFFSET]));
    header.userRegister = *(reinterpret_cast<long *>(&headerBuffer[FH_USER_REGISTER_OFFSET]));
    header.trailerPosition = *(reinterpret_cast<long *>(&headerBuffer[FH_TRAILER_POS_OFFSET]));
    // If magic word is reversed, then the file was written in BIG_ENDIAN
    // format, the bytes have to be swapped
    if(header.magicNumber==HEADER_MAGIC_BE){
       printf(" THIS FILE IS BIG ENDIAN: SWAPPING BYTES\n");
       header.uniqueid  = __builtin_bswap32(header.uniqueid);
       header.filenumber = __builtin_bswap32(header.filenumber);
       header.headerLength = __builtin_bswap32(header.headerLength);
       header.recordCount  = __builtin_bswap32(header.recordCount);
       header.userHeaderLength = __builtin_bswap32(header.userHeaderLength);
       header.indexArrayLength = __builtin_bswap32(header.indexArrayLength);
       word_8 = __builtin_bswap32(word_8);
       header.userRegister = __builtin_bswap64(header.userRegister);
       header.trailerPosition = __builtin_bswap64(header.trailerPosition);
    }
 
    header.version = word_8&BITINFO_VERSION_MASK;
    header.bitInfo = (word_8>>BITINFO_VERSION_BITS)&STRUCT_SIZE_MASK;
    header.firstRecordPosition = 4*header.headerLength + header.userHeaderLength;
    if(_verbose){
      printf("----------------------------------------\n");
      printf("**** reader:: header version   : %d \n",header.version);
      printf("**** reader:: header length    : %d \n",header.headerLength*4);
      printf("**** reader:: first record pos : %lu\n",header.firstRecordPosition);
      printf("**** reader:: trailer position : %lu\n",header.trailerPosition);
      printf("**** reader:: file size        : %lu\n",inputStreamSize);
      printf("----------------------------------------\n");
    }
    //int *signature = reinterpret_cast<int *>(&headerBuffer[0]);
    //printf("signature = %X\n",(unsigned int) *signature);
    //std::cout << "signature = " << std::ios::hex << (*signature) << '\n';
}
 
void  reader::readIndex(){
 
    inputRecord.readRecord(inputStream,header.trailerPosition,0);
    if(_verbose)printf("*** reader:: trailer record event count : %d\n",inputRecord.getEventCount());
 
    //a catch for broken files
    //can use hiporeader.getNRecords() to see if succesfull (=-1 if not)
    if(inputRecord.getEventCount()==0) return;//dglazier
 
    hipo::event event;
    inputRecord.readHipoEvent(event,0);
    if(_verbose)event.show();
    hipo::structure base;
    event.getStructure(base,FILE_INDEX_GROUP,FILE_INDEX_ITEM);
    if(_verbose)base.show();
    readerEventIndex.clear();
    int rows = base.getSize()/32;
 
    //printf(" number of rows = %d\n",rows);
    for(int i = 0; i < rows; i++){
       long position = base.getLongAt( i*8);
       int  entries  = base.getIntAt ( rows*12 + i*4);
       long uid1     = base.getLongAt( rows*16 + i*8);
       if(tagsToRead.size()==0){
          readerEventIndex.addSize(entries);
          readerEventIndex.addPosition(position);
       } else {
          bool accept = false;
          for(long jr : tagsToRead){
            if(jr==uid1) accept = true;
          }
          if(accept==true){
            readerEventIndex.addSize(entries);
            readerEventIndex.addPosition(position);
          }
        }
    }
    readerEventIndex.rewind();
    //printf("**** reader:: header version   : %d \n",readerEventIndex.getMaxEvents());
    if(_verbose)printf("**** reader::  # of events     : %d \n",readerEventIndex.getMaxEvents());
}
bool  reader::hasNext(){ return readerEventIndex.canAdvance();}
 
/*
 * Reads next event from the file, the event pointer is automatically
 * advanced to the next event.
 */
bool  reader::next(hipo::event &dataevent){
    if(readerEventIndex.canAdvance()==false) return false;
    int recordNumber = readerEventIndex.getRecordNumber();
    readerEventIndex.advance();
    int recordToBeRead = readerEventIndex.getRecordNumber();
    if(recordToBeRead!=recordNumber){
      long position = readerEventIndex.getPosition(recordToBeRead);
      inputRecord.readRecord(inputStream,position,0);
      /*printf(" record changed from %d to %d at event %d total event # %d\n",
        recordNumber, recordToBeRead,readerEventIndex.getEventNumber(),
        readerEventIndex.getMaxEvents());*/
    }
    int eventNumberInRecord = readerEventIndex.getRecordEventNumber();
    inputRecord.readHipoEvent(dataevent,eventNumberInRecord);
    return true;
}
/*
 * Reads current event from the record without advancing the current event
 * position. Multiple reads can be performed on the same event.
 */
void  reader::read(hipo::event &dataevent){
  int eventNumberInRecord = readerEventIndex.getRecordEventNumber();
  inputRecord.readHipoEvent(dataevent,eventNumberInRecord);
}
/*
 * Reads the structure from the current event without copying the event
 * buffer.
 */
void  reader::getStructure(hipo::structure &structure,int group, int item){
  hipo::data data;
  int eventNumberInRecord = readerEventIndex.getRecordEventNumber();
  inputRecord.getData(data,eventNumberInRecord);
  event::getStructure(data.getDataPtr(),structure,group,item);
}
/*
 * Reads the structure from the current event without copying the event
 * buffer and without copying the structure buffer. This should be used
 * very carefully, since it sets the pointer of the structure directly
 * to the buffer owned by the record. If new record is loaded this
 * structure will be pointing to the wrong part of the buffer.
 */
void  reader::getStructureNoCopy(hipo::structure &structure,int group, int item){
  hipo::data data;
  int eventNumberInRecord = readerEventIndex.getRecordEventNumber();
  inputRecord.getData(data,eventNumberInRecord);
  event::getStructureNoCopy(data.getDataPtr(),structure,group,item);
}
 
bool  reader::next(std::vector<hipo::bank> &list){
  bool status = next();
  if(status==false) return false;
  read(event);
  //printf("---------- reader::next() event with next()\n");
  //event.show();
  for(size_t k = 0; k < list.size(); k++){
    event.read(list[k]);
  }
  return true;
}
 
std::vector<hipo::bank> reader::getBanks(std::vector<std::string> names){
  std::vector<hipo::bank> list;
  readDictionary(factory);
  for(size_t k = 0; k < names.size(); k++){
    hipo::bank b(factory.getSchema(names[k].c_str()),48);
     list.push_back(b);
  }
  return list;
}
 
  void reader::readUserConfig(std::map<std::string,std::string> &mapConfig){
  
  if(inputStream.is_open()==false){
    throw hipo::file_error("hipo::reader::readUserConfig: file is not open");
  }
  long position = header.headerLength*4;
  hipo::record  dictRecord;
  dictRecord.readRecord(inputStream,position,0);
  int nevents = dictRecord.getEventCount();
  hipo::structure sKey;
  hipo::structure sConfig;
  hipo::event  event;
  for(int i = 0; i < nevents; i++){
    dictRecord.readHipoEvent(event,i);
    event.getStructure(sKey,CONFIG_GROUP,CONFIG_KEY_ITEM);
    event.getStructure(sConfig,CONFIG_GROUP,CONFIG_STRING_ITEM);
    if(sKey.getSize()>0){
      mapConfig[std::string(sKey.getStringAt(0).c_str())] = std::string(sConfig.getStringAt(0).c_str());
    }
    //printf("schema : %s\n",schemaStructure.getStringAt(0).c_str());
    //dict.parse(schemaStructure.getStringAt(0).c_str());
  }
}
  
/*
 * Reads the dictionary for the file. C++ API is reading packed dictionary
 * format from node (120,2), parses each schema and creates schema dictionary.
 */
void  reader::readDictionary(hipo::dictionary &dict){
  if(inputStream.is_open()==false){
    throw hipo::file_error("hipo::reader::readDictionary: file is not open");
  }
  long position = header.headerLength*4;
  hipo::record  dictRecord;
  dictRecord.readRecord(inputStream,position,0);
  int nevents = dictRecord.getEventCount();
  hipo::structure schemaStructure;
  hipo::event  event;
  for(int i = 0; i < nevents; i++){
    dictRecord.readHipoEvent(event,i);
    event.getStructure(schemaStructure,DICT_GROUP,DICT_ITEM);
    //printf("schema : %s\n",schemaStructure.getStringAt(0).c_str());
    dict.parse(schemaStructure.getStringAt(0).c_str());
  }
}
bool  reader::next(){
    if(readerEventIndex.canAdvance()==false) return false;
    int recordNumber = readerEventIndex.getRecordNumber();
    readerEventIndex.advance();
    int recordToBeRead = readerEventIndex.getRecordNumber();
    if(recordToBeRead!=recordNumber){
      long position = readerEventIndex.getPosition(recordToBeRead);
      inputRecord.readRecord(inputStream,position,0);
      /*printf(" record changed from %d to %d at event %d total event # %d\n",
        recordNumber, recordToBeRead,readerEventIndex.getEventNumber(),
        readerEventIndex.getMaxEvents());*/
    }
    return true;
}
/*
 * Moves the pointer of the event to the event number provided. If the event
 * number is within the same record no reading is done, otherwise the appropriate
 * record is loaded and event indicies are updated.
 */
bool reader::gotoEvent(int eventNumber){
  int recordNumber = readerEventIndex.getRecordNumber();
 
  //goto event in index if exists, if not return
  if(readerEventIndex.gotoEvent(eventNumber)==false){
    printf("[WARNING] hipo::reader::gotoEvent event %d greater than max events = %d, will stay at current event\n",
       eventNumber, readerEventIndex.getMaxEvents());
    return false;
  }
 
  int recordToBeRead = readerEventIndex.getRecordNumber();
  
  if(recordToBeRead!=recordNumber){
    //readerEventIndex.show(); - commented out by gg, this was for debugging
    long position = readerEventIndex.getPosition(recordToBeRead);
    inputRecord.readRecord(inputStream,position,0);
    /*printf(" record changed from %d to %d at event %d total event # %d\n",
      recordNumber, recordToBeRead,readerEventIndex.getEventNumber(),
      readerEventIndex.getMaxEvents());*/
  }
  return true;
}
/*
 * Moves the current event pointer to the new record # irec. The readerIndex
 * is updated so current event is the first event in loaded record.
 */
bool reader::gotoRecord(int irec){
  bool status = readerEventIndex.gotoRecord(irec);
  if(status==false) return false;
  long position = readerEventIndex.getPosition(irec);
  inputRecord.readRecord(inputStream,position,0);
  return true;
}
 
//dglazier
// Added by Derek - for some reason
// Use gotoRecord instead, since it checks first if record exists.
bool  reader::loadRecord(int irec){
    long position = readerEventIndex.getPosition(irec);
    inputRecord.readRecord(inputStream,position,0);
    return readerEventIndex.loadRecord(irec);
}
 
  bool  reader::loadRecord(hipo::record &record, int irec){
    long position = readerEventIndex.getPosition(irec);
    record.readRecord(inputStream,position,0);
    return true;
}
 
std::vector<int>   reader::getInt(  const char *bank, const char *column, int max ){
   std::vector<int> rowvec;
   std::vector<hipo::bank> b = getBanks({bank});
   int item = b[0].getSchema().getEntryOrder(column);
   int counter = 0;
   while(next(b)==true){
        for(int row = 0 ; row < b[0].getRows();row++){
            rowvec.push_back(b[0].getInt(item,row));
            counter++;
        }
        if(max>0&&counter>max) break;
   }
   return rowvec;
}
 
std::vector<float> reader::getFloat(const char *bank, const char *column, int max ){
  std::vector<float> rowvec;
   std::vector<hipo::bank> b = getBanks({bank});
   int item = b[0].getSchema().getEntryOrder(column);
   int counter = 0;
   while(next(b)==true){
        for(int row = 0 ; row < b[0].getRows();row++){
            rowvec.push_back(b[0].getFloat(item,row));
            counter++;
        }
        if(max>0&&counter>max) break;
   }
   return rowvec;
}
 
//dglazier
bool  reader::nextInRecord(){
  if(readerEventIndex.canAdvanceInRecord()==false) return false;
  readerEventIndex.advance();
  return true;
}
void reader::printWarning(){
    #ifndef __LZ4__
      std::cout << "******************************************************" << std::endl;
      std::cout << "* WARNING:                                           *" << std::endl;
      std::cout << "*   This library war compiled without LZ4 support.   *" << std::endl;
      std::cout << "*   Reading and writing compressed buffers will not  *" << std::endl;
      std::cout << "*   work. However un-compressed file I/O will work.  *" << std::endl;
      std::cout << "******************************************************" << std::endl;
    #endif
  }
}
 
//*****************************************************************************
//* Implementation of reader index class to keep track of events and records
//*****************************************************************************
//--
namespace hipo {
void readerIndex::addSize(int size){
  if(recordEvents.size()==0){
    recordEvents.push_back(0);
    recordEvents.push_back(size);
  } else {
    int cz = recordEvents[recordEvents.size()-1] + size;
    recordEvents.push_back(cz);
  }
}
bool readerIndex::canAdvance(){
  return (currentEvent<getMaxEvents()-1);
}
bool readerIndex::advance(){
  if(recordEvents.size()==0) return false;
  if(currentEvent+1<recordEvents[currentRecord+1]){
    currentEvent++;
    currentRecordEvent++;
    return true;
  }
  if(recordEvents.size() < currentRecord + 2 + 1){
    printf("advance(): Warning, reached the limit of events.\n");
    return false;
  }
  currentEvent++;
  currentRecord++;
  currentRecordEvent = 0;
  return true;
}
/*
 * Moves the event pointer to give event number. The record number and
 * relative event number in the record are recalculated.
 */
bool readerIndex::gotoEvent(int eventNumber){
  //check if event exists
  if(eventNumber>=getMaxEvents()) return false;
  // The proper record number is found by binary search through records array
  //------ 
  // G.Gavalian (July 25/2022)
  // Changed the eventNumber -> eventNumber+1 in binary search. Now it changes the
  // record number properly and starts with record event number from 0
  // I think this was a bug.
  
    std::vector<int>::iterator l_bound =
      std::lower_bound(recordEvents.begin(), recordEvents.end(), eventNumber+1);
 
  
    long  position = (l_bound - recordEvents.begin()) - 1;
    //printf("event # %d , lbound = %ld\n",eventNumber,position);
    currentRecord       = position;
    currentRecordEvent  = eventNumber - recordEvents[currentRecord];
    currentEvent        = eventNumber;
    return true;
}
 
 
  void readerIndex::show(){
    for(size_t i = 0; i < recordEvents.size(); i++){
      printf("record = %8zu, %8d\n",i,recordEvents[i]);
    }
  }
int readerIndex::getMaxEvents(){
  if(recordEvents.size()==0) return 0;
  return recordEvents[recordEvents.size()-1];
}
/*
 * Sets the event pointer to the first event of provided record.
 * Originally implemented by Derek as loadRecord(), renamed for
 * consistency but left the old one not to break clas12root code.
 */
bool readerIndex::gotoRecord(int irec){
  if(irec==0){
    currentEvent=-1; currentRecord=0; currentRecordEvent = -1;
    return true;
  }
  if(irec+1>(int)recordEvents.size())
    return false;
  currentEvent = recordEvents[irec]-1; currentRecord=irec;
  currentRecordEvent = -1;
  return true;
}
//dglazier
bool readerIndex::loadRecord(int irec){
    if(irec==0){
      currentEvent=-1;
      currentRecord=0;
      currentRecordEvent = -1;
      return true;
    }
    if(irec+1>(int)recordEvents.size())
      return false;
 
    currentEvent = recordEvents[irec]-1;
    currentRecord=irec;
    currentRecordEvent = -1;
    return true;
}
//dglazier
bool readerIndex::canAdvanceInRecord(){
  return (currentEvent<recordEvents[currentRecord+1]-1);
}
 
}