Skip to content

Commit

Permalink
DPL: read metadata from parent files
Browse files Browse the repository at this point in the history 
If the metadata is not found in the main file and if there is a
list of parent files, try those as well.
  • Loading branch information
@ktf
ktf committed Nov 7, 2024
1 parent f422114 commit e253485
Showing 1 changed file with 76 additions and 26 deletions.
102 changes: 76 additions & 26 deletions Framework/AnalysisSupport/src/Plugin.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <TObjString.h>
#include <TString.h>
#include <fmt/format.h>
#include <memory>

O2_DECLARE_DYNAMIC_LOG(analysis_support);

Expand Down Expand Up @@ -65,7 +66,7 @@ struct RunSummary : o2::framework::ServicePlugin {
}
};

std::vector<std::string> getListOfTables(TFile* f)
std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
{
std::vector<std::string> r;
TList* keyList = f->GetListOfKeys();
Expand All @@ -83,6 +84,32 @@ std::vector<std::string> getListOfTables(TFile* f)
}
return r;
}
auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
{
// Get the metadata, if any
auto m = (TMap*)currentFile->Get("metaData");
if (!m) {
return {};
}
std::vector<ConfigParamSpec> results;
auto it = m->MakeIterator();

// Serialise metadata into a ; separated string with : separating key and value
bool first = true;
while (auto obj = it->Next()) {
if (first) {
LOGP(info, "Metadata for file \"{}\":", currentFile->GetName());
first = false;
}
auto objString = (TObjString*)m->GetValue(obj);
LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
std::string key = "aod-metadata-" + std::string(obj->GetName());
char const* value = strdup(objString->String());
results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
}

return results;
}

struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
ConfigDiscovery* create() override
Expand All @@ -94,8 +121,6 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
if (filename.empty()) {
return {};
}
std::vector<ConfigParamSpec> results;
TFile* currentFile = nullptr;
if (filename.at(0) == '@') {
filename.erase(0, 1);
// read the text file and set filename to the contents of the first line
Expand All @@ -110,39 +135,64 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
TGrid::Connect("alien://");
}
LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
currentFile = TFile::Open(filename.c_str());
if (!currentFile) {
std::unique_ptr<TFile> currentFile{TFile::Open(filename.c_str())};
if (currentFile.get() == nullptr) {
LOGP(fatal, "Couldn't open file \"{}\"!", filename);
}
std::vector<ConfigParamSpec> results = readMetadata(currentFile);
// Found metadata already in the main file.
if (!results.empty()) {
auto tables = getListOfTables(currentFile);
if (tables.empty() == false) {
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
}
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
return results;
}

// Get the metadata, if any
auto m = (TMap*)currentFile->Get("metaData");
if (!m) {
// Lets try in parent files
auto parentFiles = (TMap*)currentFile->Get("parentFiles");
if (!parentFiles) {
LOGP(info, "No metadata found in file \"{}\"", filename);
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
return results;
}
auto it = m->MakeIterator();

// Serialise metadata into a ; separated string with : separating key and value
bool first = true;
while (auto obj = it->Next()) {
if (first) {
LOGP(info, "Metadata for file \"{}\":", filename);
first = false;
for (auto* p : *parentFiles) {
std::string parentFilename = ((TPair*)p)->Value()->GetName();
// Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
// a workflow option (because the metadata itself is potentially changing the topology).
if (registry.isSet("aod-parent-base-path-replacement")) {
auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
auto pos = parentFileReplacement.find(';');
if (pos == std::string::npos) {
throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
}
auto from = parentFileReplacement.substr(0, pos);
auto to = parentFileReplacement.substr(pos + 1);
pos = parentFilename.find(from);
if (pos != std::string::npos) {
parentFilename.replace(pos, from.length(), to);
}
}
auto objString = (TObjString*)m->GetValue(obj);
LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
std::string key = "aod-metadata-" + std::string(obj->GetName());
char const* value = strdup(objString->String());
results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
}

auto tables = getListOfTables(currentFile);
if (tables.empty() == false) {
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
if (parentFile.get() == nullptr) {
LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
}
results = readMetadata(parentFile);
// Found metadata already in the main file.
if (!results.empty()) {
auto tables = getListOfTables(parentFile);
if (tables.empty() == false) {
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
}
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
return results;
}
LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
break;
}
currentFile->Close();
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
return results;
}};
}
Expand Down

0 comments on commit e253485

Please # to comment.