QwRootFile.cc

Go to the documentation of this file.

/*!
 * \file   QwRootFile.cc
 * \brief  Implementation for ROOT file and tree management wrapper classes
 */
 
#include "QwRootFile.h"
#include "QwRunCondition.h"
#include "TH1.h"
 
#include <unistd.h>
#include <cstdio>
 
#include <filesystem>
namespace fs = std::filesystem;
 
std::string QwRootFile::fDefaultRootFileDir = ".";
std::string QwRootFile::fDefaultRootFileStem = "Qweak_";
 
const Long64_t QwRootFile::kMaxTreeSize = 100000000000LL;
const Int_t QwRootFile::kMaxMapFileSize = 0x3fffffff; // 1 GiB
 
const TString QwRootTree::kUnitsName = "ppm/D:ppb/D:um/D:mm/D:mV_uA/D:V_uA/D";
Double_t QwRootTree::kUnitsValue[] = { 1e-6, 1e-9, 1e-3, 1 , 1e-3, 1};
 
/**
 * Constructor with relative filename
 */
QwRootFile::QwRootFile(const TString& run_label)
  : fRootFile(0), fMakePermanent(0),
    fMapFile(0), fEnableMapFile(kFALSE),
    fUpdateInterval(-1)
#ifdef HAS_RNTUPLE_SUPPORT
    , fEnableRNTuples(kFALSE)
#endif // HAS_RNTUPLE_SUPPORT
{
  // Process the configuration options
  ProcessOptions(gQwOptions);
 
#ifdef QW_ENABLE_MAPFILE
  // Check for the memory-mapped file flag
  if (fEnableMapFile) {
 
    TString mapfilename = "/dev/shm/";
 
    mapfilename += "/QwMemMapFile.map";
 
    fMapFile = TMapFile::Create(mapfilename,"UPDATE", kMaxMapFileSize, "RealTime Producer File");
 
    if (not fMapFile) {
      QwError << "Memory-mapped file " << mapfilename
              << " could not be opened!" << QwLog::endl;
      return;
    }
 
    QwMessage << "================== RealTime Producer Memory Map File =================" << QwLog::endl;
    fMapFile->Print();
    QwMessage << "======================================================================" << QwLog::endl;
  } else
#endif
  {
 
    TString rootfilename = fRootFileDir;
    TString hostname = gSystem -> HostName();
 
    // Use a probably-unique temporary file name.
    pid_t pid = getpid();
 
    fPermanentName = rootfilename
      + Form("/%s%s.root", fRootFileStem.Data(), run_label.Data());
    if (fUseTemporaryFile){
      rootfilename += Form("/%s%s.%s.%d.root",
                           fRootFileStem.Data(), run_label.Data(),
                           hostname.Data(), pid);
      // Delete permanent file if it exists to prevent accumulation across segments
      if (gSystem->AccessPathName(fPermanentName.Data()) == 0) {
        QwVerbose << "Removing existing permanent file: " << fPermanentName << QwLog::endl;
        gSystem->Unlink(fPermanentName.Data());
      }
      // CRITICAL: Also delete the temporary file if it exists!
      // RECREATE mode doesn't properly clear files that contain RNTuples,
      // so we must manually delete before opening
      if (gSystem->AccessPathName(rootfilename.Data()) == 0) {
        QwVerbose << "Removing existing temporary file before RECREATE: " << rootfilename << QwLog::endl;
        gSystem->Unlink(rootfilename.Data());
      }
    } else {
      rootfilename = fPermanentName;
      // Delete permanent file if it exists to ensure RECREATE truly starts fresh
      // This is especially important for RNTuple files where RECREATE doesn't properly clear
      if (gSystem->AccessPathName(rootfilename.Data()) == 0) {
        QwMessage << "File exists before RECREATE, deleting: " << rootfilename << QwLog::endl;
        int unlink_result = gSystem->Unlink(rootfilename.Data());
        if (unlink_result == 0) {
          QwMessage << "Successfully deleted file" << QwLog::endl;
        } else {
          QwError << "Failed to delete file! Error code: " << unlink_result << QwLog::endl;
        }
      } else {
        QwMessage << "File does not exist before RECREATE: " << rootfilename << QwLog::endl;
      }
    }
    QwMessage << "Opening file with RECREATE mode: " << rootfilename << QwLog::endl;
    QwMessage << "QwRootFile constructor called for: " << rootfilename << QwLog::endl;
    // Use TFile::Open instead of `new TFile(...)` so the ROOT plug-in manager
    // can dispatch remote URLs (e.g. root://host/path) to TNetXNGFile etc.
    // The TFile(name, opt, title) constructor refuses remote paths and returns
    // a half-built object, which then segfaults at first use.
    fRootFile = TFile::Open(rootfilename.Data(), "RECREATE", "myfile1");
    if (!fRootFile || fRootFile->IsZombie()) {
      QwError << "ROOT file " << rootfilename
              << " could not be opened!" << QwLog::endl;
      delete fRootFile;
      fRootFile = nullptr;
      return;
    } else {
      QwMessage << "Opened "<< (fUseTemporaryFile?"temporary ":"")
                <<"rootfile " << rootfilename << QwLog::endl;
    }
 
    TString run_condition_name = Form("condition_%s", run_label.Data());
    TList *run_cond_list = (TList*) fRootFile -> FindObjectAny(run_condition_name);
    if (not run_cond_list) {
      QwRunCondition run_condition(
          gQwOptions.GetArgc(),
          gQwOptions.GetArgv(),
          run_condition_name
      );
 
      fRootFile -> WriteObject(
          run_condition.Get(),
          run_condition.GetName()
      );
    }
 
    fRootFile->SetCompressionAlgorithm(fCompressionAlgorithm);
    fRootFile->SetCompressionLevel(fCompressionLevel);
  }
}
 
 
/**
 * Destructor
 */
QwRootFile::~QwRootFile()
{
  // Keep the file on disk if any trees or histograms have been filled.
  // Also respect any other requests to keep the file around.
  if (!fMakePermanent) fMakePermanent = HasAnyFilled();
 
  // Close the map file
  if (fMapFile) {
    fMapFile->Close();
    // TMapFiles may not be deleted
    fMapFile = 0;
  }
 
  // Close the ROOT file.
  // Rename if permanence is requested, remove otherwise
  if (fRootFile) {
    TString rootfilename = fRootFile->GetName();
 
    fRootFile->Close();
    delete fRootFile;
    fRootFile = 0;
 
    int err;
    const char* action;
    if (fUseTemporaryFile){
      if (fMakePermanent) {
        // Delete existing permanent file first to avoid accumulation
        if (gSystem->AccessPathName(fPermanentName.Data()) == 0) {
          remove(fPermanentName.Data());
        }
        action = " rename ";
        err = rename( rootfilename.Data(), fPermanentName.Data() );
      } else {
        action = " remove ";
        err = remove( rootfilename.Data() );
      }
      // It'd be proper to "extern int errno" and strerror() here,
      // but that doesn't seem very C++-ish.
      if (err) {
        QwWarning << "Couldn't" << action << rootfilename << QwLog::endl;
      } else {
        QwMessage << "Was able to" << action << rootfilename << QwLog::endl;
        QwMessage << "Root file is " << fPermanentName << QwLog::endl;
      }
    }
  }
 
  // Delete Qweak ROOT trees
  std::map< const std::string, std::vector<QwRootTree*> >::iterator map_iter;
  std::vector<QwRootTree*>::iterator vec_iter;
  for (map_iter = fTreeByName.begin(); map_iter != fTreeByName.end(); map_iter++) {
    for (vec_iter = map_iter->second.begin(); vec_iter != map_iter->second.end(); vec_iter++) {
      delete *vec_iter;
    }
  }
}
 
/**
 * Defines configuration options using QwOptions functionality.
 * @param options Options object
 */
void QwRootFile::DefineOptions(QwOptions &options)
{
  // Define the ROOT files directory
  options.AddOptions("Default options")
    ("rootfiles", po::value<std::string>()->default_value(fDefaultRootFileDir),
     "directory of the output ROOT files");
 
  // Define the ROOT filename stem
  options.AddOptions("Default options")
    ("rootfile-stem", po::value<std::string>()->default_value(fDefaultRootFileStem),
     "stem of the output ROOT filename");
 
  // Define the memory map option
  options.AddOptions()
    ("enable-mapfile", po::value<bool>()->default_bool_value(false),
     "enable output to memory-mapped file\n(likely requires circular-buffer too)");
  options.AddOptions()
    ("write-temporary-rootfiles", po::value<bool>()->default_bool_value(true),
     "When writing ROOT files, use the PID to create a temporary filename");
 
  // Define the histogram and tree options
  options.AddOptions("ROOT output options")
    ("disable-tree", po::value<std::vector<std::string>>()->composing(),
     "disable output to tree regex");
  options.AddOptions("ROOT output options")
    ("disable-trees", po::value<bool>()->default_bool_value(false),
     "disable output to all trees");
  options.AddOptions("ROOT output options")
    ("disable-histos", po::value<bool>()->default_bool_value(false),
     "disable output to all histograms");
 
  // Define the helicity window versus helicity pattern options
  options.AddOptions("ROOT output options")
    ("disable-mps-tree", po::value<bool>()->default_bool_value(false),
     "disable helicity window output");
  options.AddOptions("ROOT output options")
    ("disable-pair-tree", po::value<bool>()->default_bool_value(false),
     "disable helicity pairs output");
  options.AddOptions("ROOT output options")
    ("disable-hel-tree", po::value<bool>()->default_bool_value(false),
     "disable helicity pattern output");
  options.AddOptions("ROOT output options")
    ("disable-burst-tree", po::value<bool>()->default_bool_value(false),
     "disable burst tree");
  options.AddOptions("ROOT output options")
    ("disable-slow-tree", po::value<bool>()->default_bool_value(false),
     "disable slow control tree");
 
#ifdef HAS_RNTUPLE_SUPPORT
  // Define the RNTuple options
  options.AddOptions("ROOT output options")
    ("enable-rntuples", po::value<bool>()->default_bool_value(false),
     "enable RNTuple output");
#endif // HAS_RNTUPLE_SUPPORT
 
  // Define the tree output prescaling options
  options.AddOptions("ROOT output options")
    ("num-mps-accepted-events", po::value<int>()->default_value(0),
     "number of accepted consecutive MPS events");
  options.AddOptions("ROOT output options")
    ("num-mps-discarded-events", po::value<int>()->default_value(0),
     "number of discarded consecutive MPS events");
  options.AddOptions("ROOT output options")
    ("num-hel-accepted-events", po::value<int>()->default_value(0),
     "number of accepted consecutive pattern events");
  options.AddOptions("ROOT output options")
    ("num-hel-discarded-events", po::value<int>()->default_value(0),
     "number of discarded consecutive pattern events");
  options.AddOptions("ROOT output options")
    ("mapfile-update-interval", po::value<int>()->default_value(-1),
     "Events between a map file update");
 
  // Define the autoflush and autosave option (default values by ROOT)
  options.AddOptions("ROOT performance options")
    ("autoflush", po::value<int>()->default_value(0),
     "TTree autoflush");
  options.AddOptions("ROOT performance options")
    ("autosave", po::value<int>()->default_value(300000000),
     "TTree autosave");
  options.AddOptions("ROOT performance options")
    ("basket-size", po::value<int>()->default_value(16000),
     "TTree basket size");
  options.AddOptions("ROOT performance options")
    ("circular-buffer", po::value<int>()->default_value(0),
     "TTree circular buffer");
  options.AddOptions("ROOT performance options")
    ("compression-algorithm", po::value<int>()->default_value(1),
     "TFile compression algorithm (1=ZLIB, 2=LZMA, 4=LZ4, 5=ZSTD, default=1 ZLIB)");
  options.AddOptions("ROOT performance options")
    ("compression-level", po::value<int>()->default_value(1),
     "TFile compression level (default = 1, no compression = 0)");
  options.AddOptions("ROOT performance options")
    ("rntuple-compression-algorithm", po::value<int>()->default_value(4),
     "RNTuple compression algorithm (1=ZLIB, 2=LZMA, 4=LZ4, 5=ZSTD, default=4 LZ4)");
  options.AddOptions("ROOT performance options")
    ("rntuple-compression-level", po::value<int>()->default_value(0),
     "RNTuple compression level (0-12, default=0 for maximum performance)");
}
 
 
/**
 * Parse the configuration options and store in class fields
 * @param options Options object
 */
void QwRootFile::ProcessOptions(QwOptions &options)
{
  // Option 'rootfiles' to specify ROOT files dir
  fRootFileDir = TString(options.GetValue<std::string>("rootfiles"));
  fs::path tmppath(fRootFileDir.Data());
  if( ! fs::exists(tmppath) || ! fs::is_directory(tmppath)) {
    QwError << "ERROR:  The rootfile directory path, " << fRootFileDir
            << ", does not exist.  Exiting."
            << QwLog::endl;
    exit(2);
  }
 
  // Option 'root-stem' to specify ROOT file stem
  fRootFileStem = TString(options.GetValue<std::string>("rootfile-stem"));
 
  // Option 'mapfile' to enable memory-mapped ROOT file
  fEnableMapFile = options.GetValue<bool>("enable-mapfile");
#ifndef QW_ENABLE_MAPFILE
  if( fEnableMapFile ) {
    QwMessage << QwLog::endl;
    QwWarning << "QwRootFile::ProcessOptions:  "
              << "The 'enable-mapfile' flag is not supported by the ROOT "
                 "version with which this app is built. Disabling it."
              << QwLog::endl;
    fEnableMapFile = false;
  }
#endif
  fUseTemporaryFile = options.GetValue<bool>("write-temporary-rootfiles");
 
#ifdef HAS_RNTUPLE_SUPPORT
  // Option 'enable-rntuples' to enable RNTuple output
  fEnableRNTuples = options.GetValue<bool>("enable-rntuples");
  // RNTuples require a TFile (RNTupleWriter::Append takes a TDirectory&);
  // they cannot be hosted by a TMapFile.  If both flags are requested,
  // disable RNTuples and warn loudly so the run does not crash later in
  // QwRootNTuple::InitializeWriter with a null TFile*.
  if (fEnableMapFile && fEnableRNTuples) {
    QwMessage << QwLog::endl;
    QwWarning << "QwRootFile::ProcessOptions:  "
              << "RNTuple output is not supported alongside --enable-mapfile "
                 "(TMapFile is not a TDirectory). Disabling RNTuples."
              << QwLog::endl;
    fEnableRNTuples = false;
  }
#endif // HAS_RNTUPLE_SUPPORT
 
  // Options 'disable-trees' and 'disable-histos' for disabling
  // tree and histogram output
  auto v = options.GetValueVector<std::string>("disable-tree");
  std::for_each(v.begin(), v.end(), [&](const std::string& s){ this->DisableTree(s); });
  if (options.GetValue<bool>("disable-trees"))  DisableTree(".*");
  if (options.GetValue<bool>("disable-histos")) DisableHisto(".*");
 
  // Read --circular-buffer up front so the mapfile-mode logic below can use
  // it (the original ProcessOptions read it further down, but we now need
  // it during the mapfile-tree decision).
  fCircularBufferSize = options.GetValue<int>("circular-buffer");
 
  // TMapFile-mode tree publishing.
  //
  // TTrees can be published into the mapfile: TDirectoryFile::Append
  // auto-calls TMapFile::Add() when the directory's mother is a TMapFile,
  // so a TTree created while gDirectory == fMapFile->GetDirectory()
  // becomes a TMapRec on its own.  TMapFile::Update() then re-streams the
  // tree (header + baskets) into the 1 GiB mmap on every interval.
  //
  // The hazard is that an unbounded TTree's serialized size grows
  // monotonically and CustomReAlloc2 aborts as soon as it overruns the
  // mmap.  TTree::SetCircular(N) caps the in-memory entry count, which
  // caps the serialized size.  Require a non-zero circular buffer in
  // mapfile mode; if the user did not pass one, force a safe default
  // and warn loudly rather than silently dropping all trees.
  if (fEnableMapFile && fCircularBufferSize == 0) {
    const UInt_t kMapFileCircularDefault = 100;
    QwMessage << QwLog::endl;
    QwWarning << "QwRootFile::ProcessOptions:  "
              << "--enable-mapfile requires a bounded TTree to avoid "
                 "overrunning the " << (kMaxMapFileSize >> 20)
              << " MiB mmap region.  "
                 "Forcing --circular-buffer=" << kMapFileCircularDefault
              << " (pass --circular-buffer=N to override; pass "
                 "--disable-trees to suppress tree output entirely)."
              << QwLog::endl;
    fCircularBufferSize = kMapFileCircularDefault;
  }
 
#ifdef HAS_RNTUPLE_SUPPORT
  // TTree and RNTuple writers share per-channel state (fTreeArrayIndex,
  // fTreeArrayNumEntries, fDataToSave, b* flags). When both are active,
  // the second Construct*AndVector() call clobbers the first writer's
  // layout, and subsequent Fill*Vector() then walks a vector whose entry
  // types no longer match what was pushed (e.g. SetValue throws
  // "entry type 'D' cannot store unsigned int value 'block2'").
  // Until per-writer layout state is added, make the two writers
  // mutually exclusive: keep RNTuples and silence TTrees.
  if (fEnableRNTuples) {
    QwMessage << QwLog::endl;
    QwMessage << "QwRootFile::ProcessOptions:  "
              << "--enable-rntuples is set; disabling tree output "
                 "(channels share layout state between TTree and RNTuple "
                 "writers, so the two cannot be produced in the same run)."
              << QwLog::endl;
    DisableTree(".*");
  }
#endif // HAS_RNTUPLE_SUPPORT
 
  // Options 'disable-mps' and 'disable-hel' for disabling
  // helicity window and helicity pattern output
  if (options.GetValue<bool>("disable-mps-tree"))  DisableTree("^evt$");
  if (options.GetValue<bool>("disable-pair-tree"))  DisableTree("^pr$");
  if (options.GetValue<bool>("disable-hel-tree"))  DisableTree("^mul$");
  if (options.GetValue<bool>("disable-burst-tree"))  DisableTree("^burst$");
  if (options.GetValue<bool>("disable-slow-tree")) DisableTree("^slow$");
 
  // Options 'num-accepted-events' and 'num-discarded-events' for
  // prescaling of the tree output
  fNumMpsEventsToSave = options.GetValue<int>("num-mps-accepted-events");
  fNumMpsEventsToSkip = options.GetValue<int>("num-mps-discarded-events");
  fNumHelEventsToSave = options.GetValue<int>("num-mps-accepted-events");
  fNumHelEventsToSkip = options.GetValue<int>("num-mps-discarded-events");
 
  // Update interval for the map file
  fUpdateInterval = options.GetValue<int>("mapfile-update-interval");
  fCompressionAlgorithm = options.GetValue<int>("compression-algorithm");
  fCompressionLevel = options.GetValue<int>("compression-level");
  fRNTupleCompressionAlgorithm = options.GetValue<int>("rntuple-compression-algorithm");
  fRNTupleCompressionLevel = options.GetValue<int>("rntuple-compression-level");
  fBasketSize = options.GetValue<int>("basket-size");
 
  // Autoflush and autosave
  fAutoFlush = options.GetValue<int>("autoflush");
  if ((ROOT_VERSION_CODE < ROOT_VERSION(5,26,00)) && fAutoFlush != -30000000){
    QwMessage << QwLog::endl;
    QwWarning << "QwRootFile::ProcessOptions:  "
              << "The 'autoflush' flag is not supported by ROOT version "
              << ROOT_RELEASE
              << QwLog::endl;
  }
  fAutoSave  = options.GetValue<int>("autosave");
  return;
}
 
/**
 * Determine whether the rootfile object has any non-empty trees or
 * histograms.
 */
Bool_t QwRootFile::HasAnyFilled(void) {
  return this->HasAnyFilled(fRootFile);
}
Bool_t QwRootFile::HasAnyFilled(TDirectory* d) {
  if (!d) {
 
    return false;
  }
 
  // First check if any in-memory trees have been filled
  for (auto& pair : fTreeByName) {
    for (auto& tree : pair.second) {
      if (tree && tree->GetTree()) {
        Long64_t entries = tree->GetTree()->GetEntries();
        if (entries > 0) {
 
          return true;
        }
      }
    }
  }
 
#ifdef HAS_RNTUPLE_SUPPORT
  // Then check if any RNTuples have been filled
  for (auto& pair : fNTupleByName) {
    for (auto& ntuple : pair.second) {
      if (ntuple && ntuple->fCurrentEvent > 0) {
 
        return true;
      }
    }
  }
#endif // HAS_RNTUPLE_SUPPORT
 
  TList* l = d->GetListOfKeys();
 
 
  for( int i=0; i < l->GetEntries(); ++i) {
    const char* name = l->At(i)->GetName();
    TObject* obj = d->FindObjectAny(name);
 
 
 
    // Objects which can't be found don't count.
    if (!obj) {
 
      continue;
    }
 
    // Lists of parameter files, map files, and job conditions don't count.
    if ( TString(name).Contains("parameter_file") ) {
 
      continue;
    }
    if ( TString(name).Contains("mapfile") ) {
      continue;
    }
    if ( TString(name).Contains("_condition") ) {
      continue;
    }
    //  The EPICS tree doesn't count
    if ( TString(name).Contains("slow") ) {
      continue;
    }
 
    // Recursively check subdirectories.
    if (obj->IsA()->InheritsFrom( "TDirectory" )) {
      if (this->HasAnyFilled( (TDirectory*)obj )) return true;
    }
 
    if (obj->IsA()->InheritsFrom( "TTree" )) {
      Long64_t entries = ((TTree*) obj)->GetEntries();
      if ( entries ) return true;
    }
 
    if (obj->IsA()->InheritsFrom( "TH1" )) {
      Double_t entries = ((TH1*) obj)->GetEntries();
      if ( entries ) return true;
    }
  }
  return false;
}