From f07a6f3213a7cf439445e5eadbf31a97d1ce8acc Mon Sep 17 00:00:00 2001 From: Simon Haegler Date: Sat, 27 Jan 2018 17:55:52 +0100 Subject: [PATCH 1/4] #58: reload RPKs if file modification time changes --- src/palladio/PRTContext.cpp | 65 +++++++++++++++++++++++++++++-------- src/palladio/PRTContext.h | 6 +++- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/palladio/PRTContext.cpp b/src/palladio/PRTContext.cpp index 9a316262..df1d7693 100644 --- a/src/palladio/PRTContext.cpp +++ b/src/palladio/PRTContext.cpp @@ -18,6 +18,11 @@ #include "LogHandler.h" #include +#include +#include +#ifndef WIN32 +# include +#endif namespace { @@ -92,6 +97,17 @@ uint32_t getNumCores() { return n > 0 ? n : 1; } +const timespec NO_MODIFICATION_TIME{0, 0}; + +timespec getFileModificationTime(const boost::filesystem::path& p) { + struct stat result; + if (stat(p.c_str(), &result) == 0) { + return result.st_mtim; + } + else + return NO_MODIFICATION_TIME; +} + } // namespace @@ -159,17 +175,40 @@ PRTContext::~PRTContext() { } const ResolveMapUPtr& PRTContext::getResolveMap(const boost::filesystem::path& rpk) { - if (rpk.empty()) - return mResolveMapNone; - - auto it = mResolveMapCache.find(rpk); - if (it == mResolveMapCache.end()) { - prt::Status status = prt::STATUS_UNSPECIFIED_ERROR; - const auto rpkURI = toFileURI(rpk); - ResolveMapUPtr rm(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status)); - if (status != prt::STATUS_OK) - return mResolveMapNone; - it = mResolveMapCache.emplace(rpk, std::move(rm)).first; - } - return it->second; + if (rpk.empty()) + return mResolveMapNone; + + const auto timeStamp = getFileModificationTime(rpk); + LOG_DBG << "rpk: current timestamp: " << timeStamp.tv_sec << "s " << timeStamp.tv_nsec << "ns"; + + bool reload = false; + auto it = mResolveMapCache.find(rpk); + if (it != mResolveMapCache.end()) { + LOG_DBG << "rpk: cache timestamp: " << it->second.mTimeStamp.tv_sec << "s " << it->second.mTimeStamp.tv_nsec << "ns"; + if (it->second.mTimeStamp.tv_sec != timeStamp.tv_sec || it->second.mTimeStamp.tv_nsec != timeStamp.tv_nsec) { + mResolveMapCache.erase(it); + const auto cnt = boost::filesystem::remove_all(mRPKUnpackPath / rpk.leaf()); + mPRTCache->flushAll(); // TODO: or do we need to explicitely remove cgbs from cache? + LOG_DBG << "RPK change detected, refreshing unpack cache: " << rpk << "( removed " << cnt << " files)"; + reload = true; + } + } + else + reload = true; + + if (reload) { + const auto rpkURI = toFileURI(rpk); + + ResolveMapCacheEntry rmce; + rmce.mTimeStamp = timeStamp; + + prt::Status status = prt::STATUS_UNSPECIFIED_ERROR; + rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status)); + if (status != prt::STATUS_OK) + return mResolveMapNone; + + it = mResolveMapCache.emplace(rpk, std::move(rmce)).first; + } + + return it->second.mResolveMap; } diff --git a/src/palladio/PRTContext.h b/src/palladio/PRTContext.h index 2928b159..34d7a2e1 100644 --- a/src/palladio/PRTContext.h +++ b/src/palladio/PRTContext.h @@ -50,7 +50,11 @@ struct PRTContext final { boost::filesystem::path mRPKUnpackPath; const uint32_t mCores; - using ResolveMapCache = std::map; + struct ResolveMapCacheEntry { + ResolveMapUPtr mResolveMap; + timespec mTimeStamp; + }; + using ResolveMapCache = std::map; ResolveMapCache mResolveMapCache; const ResolveMapUPtr mResolveMapNone; }; From 9f20fabdbb61370df783365dbbccd7ccfbdb2d2b Mon Sep 17 00:00:00 2001 From: Simon Haegler Date: Sat, 3 Mar 2018 18:39:04 +0100 Subject: [PATCH 2/4] #58: abort cooking if all shapes fail to generate --- src/palladio/ModelConverter.cpp | 9 +- src/palladio/ModelConverter.h | 3 +- src/palladio/SOPGenerate.cpp | 150 ++++++++++++++++++-------------- 3 files changed, 91 insertions(+), 71 deletions(-) diff --git a/src/palladio/ModelConverter.cpp b/src/palladio/ModelConverter.cpp index 5f0568ac..d7239a5f 100644 --- a/src/palladio/ModelConverter.cpp +++ b/src/palladio/ModelConverter.cpp @@ -186,8 +186,8 @@ GA_Offset createPrimitives(GU_Detail* mDetail, GenerateNodeParams::GroupCreation } // namespace ModelConversion -ModelConverter::ModelConverter(GU_Detail* detail, GenerateNodeParams::GroupCreation gc, UT_AutoInterrupt* autoInterrupt) -: mDetail(detail), mGroupCreation(gc), mAutoInterrupt(autoInterrupt) { } +ModelConverter::ModelConverter(GU_Detail* detail, GenerateNodeParams::GroupCreation gc, std::vector& statuses, UT_AutoInterrupt* autoInterrupt) +: mDetail(detail), mGroupCreation(gc), mStatuses(statuses), mAutoInterrupt(autoInterrupt) { } void ModelConverter::add(const wchar_t* name, const double* vtx, size_t vtxSize, @@ -254,7 +254,8 @@ void ModelConverter::add(const wchar_t* name, } prt::Status ModelConverter::generateError(size_t isIndex, prt::Status status, const wchar_t* message) { - LOG_ERR << message; + LOG_WRN << message; // generate error for one shape is not yet a reason to abort cooking + mStatuses[isIndex] = status; return prt::STATUS_OK; } @@ -264,7 +265,7 @@ prt::Status ModelConverter::assetError(size_t isIndex, prt::CGAErrorLevel level, } prt::Status ModelConverter::cgaError(size_t isIndex, int32_t shapeID, prt::CGAErrorLevel level, int32_t methodId, int32_t pc, const wchar_t* message) { - LOG_ERR << message; + LOG_WRN << message; return prt::STATUS_OK; } diff --git a/src/palladio/ModelConverter.h b/src/palladio/ModelConverter.h index 15d9145b..d012e4af 100644 --- a/src/palladio/ModelConverter.h +++ b/src/palladio/ModelConverter.h @@ -65,7 +65,7 @@ void setUVs( class ModelConverter : public HoudiniCallbacks { public: - explicit ModelConverter(GU_Detail* gdp, GenerateNodeParams::GroupCreation gc, UT_AutoInterrupt* autoInterrupt = nullptr); + explicit ModelConverter(GU_Detail* gdp, GenerateNodeParams::GroupCreation gc, std::vector& statuses, UT_AutoInterrupt* autoInterrupt = nullptr); protected: void add( @@ -104,6 +104,7 @@ class ModelConverter : public HoudiniCallbacks { private: GU_Detail* mDetail; GenerateNodeParams::GroupCreation mGroupCreation; + std::vector& mStatuses; UT_AutoInterrupt* mAutoInterrupt; std::map mShapeAttributeBuilders; }; diff --git a/src/palladio/SOPGenerate.cpp b/src/palladio/SOPGenerate.cpp index 49f85b80..8227f946 100644 --- a/src/palladio/SOPGenerate.cpp +++ b/src/palladio/SOPGenerate.cpp @@ -25,6 +25,7 @@ #include "UT/UT_Interrupt.h" #include +#include namespace { @@ -92,85 +93,102 @@ OP_ERROR SOPGenerate::cookMySop(OP_Context& context) { duplicateSource(0, context); - if (error() < UT_ERROR_ABORT && cookInputGroups(context) < UT_ERROR_ABORT) { - UT_AutoInterrupt progress("Generating CityEngine geometry..."); + if (error() >= UT_ERROR_ABORT || cookInputGroups(context) >= UT_ERROR_ABORT) + return error(); - const auto groupCreation = GenerateNodeParams::getGroupCreation(this, context.getTime()); - ShapeData shapeData(groupCreation, toUTF16FromOSNarrow(getName().toStdString())); - ShapeGenerator shapeGen; - shapeGen.get(gdp, DEFAULT_PRIMITIVE_CLASSIFIER, shapeData, mPRTCtx); + UT_AutoInterrupt progress("Generating CityEngine geometry..."); - const InitialShapeNOPtrVector& is = shapeData.getInitialShapes(); - if (is.empty()) { - LOG_ERR << getName() << ": could not extract any initial shapes from detail!"; - return UT_ERROR_ABORT; - } + const auto groupCreation = GenerateNodeParams::getGroupCreation(this, context.getTime()); + ShapeData shapeData(groupCreation, toUTF16FromOSNarrow(getName().toStdString())); + + ShapeGenerator shapeGen; + shapeGen.get(gdp, DEFAULT_PRIMITIVE_CLASSIFIER, shapeData, mPRTCtx); - if (!progress.wasInterrupted()) { - gdp->clearAndDestroy(); - { - WA("generate"); + const InitialShapeNOPtrVector& is = shapeData.getInitialShapes(); + if (is.empty()) { + LOG_ERR << getName() << ": could not extract any initial shapes from detail!"; + return UT_ERROR_ABORT; + } - // establish threads - const size_t nThreads = std::min(mPRTCtx->mCores, is.size()); - const size_t isRangeSize = std::ceil(is.size() / nThreads); + // establish threads + const size_t nThreads = std::min(mPRTCtx->mCores, is.size()); + const size_t isRangeSize = std::ceil(is.size() / nThreads); + + // prepare generate status receivers + std::vector initialShapeStatus(is.size(), prt::STATUS_OK); + std::vector batchStatus(nThreads, prt::STATUS_UNSPECIFIED_ERROR); + + if (!progress.wasInterrupted()) { + gdp->clearAndDestroy(); + { + WA("generate"); + + // prt requires one callback instance per generate call + std::vector hg(nThreads); + std::generate(hg.begin(), hg.end(), [this, &groupCreation, &initialShapeStatus, &progress]() -> ModelConverterUPtr { + return ModelConverterUPtr(new ModelConverter(gdp, groupCreation, initialShapeStatus, &progress)); + }); + + LOG_INF << getName() << ": calling generate: #initial shapes = " << is.size() << ", #threads = " + << nThreads << ", initial shapes per thread = " << isRangeSize; + + // kick-off generate threads + std::vector> futures; + futures.reserve(nThreads); + for (int8_t ti = 0; ti < nThreads; ti++) { + auto f = std::async(std::launch::async, [this, &hg, ti, &nThreads, &isRangeSize, &is, &batchStatus] { + const size_t isStartPos = ti * isRangeSize; + const size_t isPastEndPos = (ti < nThreads - 1) ? (ti + 1) * isRangeSize : is.size(); + const size_t isActualRangeSize = isPastEndPos - isStartPos; + const auto isRangeStart = &is[isStartPos]; + + LOG_DBG << "thread " << ti << ": #is = " << isActualRangeSize; + + OcclusionSetUPtr occlSet; + std::vector occlHandles; + if (ENABLE_OCCLUSION) { + occlSet.reset(prt::OcclusionSet::create()); + occlHandles.resize(isActualRangeSize); + prt::generateOccluders(isRangeStart, isActualRangeSize, occlHandles.data(), nullptr, 0, + nullptr, hg[ti].get(), mPRTCtx->mPRTCache.get(), occlSet.get(), + mGenerateOptions.get()); + } + + batchStatus[ti] = prt::generate(isRangeStart, isActualRangeSize, occlHandles.data(), + mAllEncoders.data(), mAllEncoders.size(), + mAllEncoderOptions.data(), hg[ti].get(), + mPRTCtx->mPRTCache.get(), occlSet.get(), + mGenerateOptions.get()); + + if (ENABLE_OCCLUSION) + occlSet->dispose(occlHandles.data(), occlHandles.size()); + + if (batchStatus[ti] != prt::STATUS_OK) { + LOG_WRN << "prt::generate() failed with status: '" + << prt::getStatusDescription(batchStatus[ti]) << "' (" << batchStatus[ti] << ")"; + } - // prt requires one callback instance per generate call - std::vector hg(nThreads); - std::generate(hg.begin(), hg.end(), [this, &groupCreation, &progress]() -> ModelConverterUPtr { - return ModelConverterUPtr(new ModelConverter(gdp, groupCreation, &progress)); }); - - LOG_INF << getName() << ": calling generate: #initial shapes = " << is.size() << ", #threads = " - << nThreads << ", initial shapes per thread = " << isRangeSize; - - // kick-off generate threads - std::vector> futures; - futures.reserve(nThreads); - for (int8_t ti = 0; ti < nThreads; ti++) { - auto f = std::async(std::launch::async, [this, &hg, ti, &nThreads, &isRangeSize, &is] { - const size_t isStartPos = ti * isRangeSize; - const size_t isPastEndPos = (ti < nThreads - 1) ? (ti + 1) * isRangeSize : is.size(); - const size_t isActualRangeSize = isPastEndPos - isStartPos; - const auto isRangeStart = &is[isStartPos]; - - LOG_DBG << "thread " << ti << ": #is = " << isActualRangeSize; - - OcclusionSetUPtr occlSet; - std::vector occlHandles; - if (ENABLE_OCCLUSION) { - occlSet.reset(prt::OcclusionSet::create()); - occlHandles.resize(isActualRangeSize); - prt::generateOccluders(isRangeStart, isActualRangeSize, occlHandles.data(), nullptr, 0, - nullptr, hg[ti].get(), mPRTCtx->mPRTCache.get(), occlSet.get(), - mGenerateOptions.get()); - } - - prt::Status stat = prt::generate(isRangeStart, isActualRangeSize, occlHandles.data(), - mAllEncoders.data(), mAllEncoders.size(), - mAllEncoderOptions.data(), hg[ti].get(), - mPRTCtx->mPRTCache.get(), occlSet.get(), - mGenerateOptions.get()); - if (stat != prt::STATUS_OK) { - LOG_ERR << "prt::generate() failed with status: '" - << prt::getStatusDescription(stat) << "' (" << stat << ")"; - } - - if (ENABLE_OCCLUSION) - occlSet->dispose(occlHandles.data(), occlHandles.size()); - }); - futures.emplace_back(std::move(f)); - } - std::for_each(futures.begin(), futures.end(), [](std::future& f) { f.wait(); }); + futures.emplace_back(std::move(f)); } - select(); + std::for_each(futures.begin(), futures.end(), [](std::future& f) { f.wait(); }); } + select(); } + WA_PRINT_TIMINGS + unlockInputs(); - WA_PRINT_TIMINGS + // generate status check: if all shapes fail, we abort cooking (failure of individual shapes is sometimes expected) + const size_t isSuccesses = std::count(initialShapeStatus.begin(), initialShapeStatus.end(), prt::STATUS_OK); + if (isSuccesses == 0) { + LOG_ERR << getName() << ": All initial shapes failed to generate, cooking aborted."; + addError(UT_ERROR_ABORT, "All initial shapes failed to generate."); + return UT_ERROR_ABORT; + } + // TODO: evaluate batchStatus as well... return error(); } From f325176eca1e674a9bf268d649c70d3cc009fa44 Mon Sep 17 00:00:00 2001 From: Simon Haegler Date: Sat, 3 Mar 2018 18:40:52 +0100 Subject: [PATCH 3/4] #58: introduce ResolveMapCache and trigger force cook on RPK change * allowing the pldAssign params to have values other than in the dropdowns (easier to debug the values when the RPK changes) * the RPK change detection can be triggered by toggling a pldAssign parameter or by clicking on the parameter dropdowns of start rule, style or rule file. --- src/palladio/CMakeLists.txt | 1 + src/palladio/NodeParameter.cpp | 6 --- src/palladio/NodeParameter.h | 9 ++-- src/palladio/PRTContext.cpp | 90 +++++++++++++------------------- src/palladio/PRTContext.h | 11 +--- src/palladio/PalladioMain.cpp | 8 +-- src/palladio/PalladioMain.h | 4 ++ src/palladio/ResolveMapCache.cpp | 73 ++++++++++++++++++++++++++ src/palladio/ResolveMapCache.h | 37 +++++++++++++ src/palladio/SOPAssign.h | 1 + 10 files changed, 161 insertions(+), 79 deletions(-) create mode 100644 src/palladio/PalladioMain.h create mode 100644 src/palladio/ResolveMapCache.cpp create mode 100644 src/palladio/ResolveMapCache.h diff --git a/src/palladio/CMakeLists.txt b/src/palladio/CMakeLists.txt index d356c47e..ab13656c 100644 --- a/src/palladio/CMakeLists.txt +++ b/src/palladio/CMakeLists.txt @@ -17,6 +17,7 @@ add_library(${PROJECT_NAME} SHARED ShapeGenerator.cpp NodeParameter.cpp PRTContext.cpp + ResolveMapCache.cpp SOPAssign.cpp SOPGenerate.cpp PrimitivePartition.cpp diff --git a/src/palladio/NodeParameter.cpp b/src/palladio/NodeParameter.cpp index 9d3963bc..c09b3b45 100644 --- a/src/palladio/NodeParameter.cpp +++ b/src/palladio/NodeParameter.cpp @@ -84,12 +84,6 @@ int updateRPK(void* data, int, fpreal32 time, const PRM_Template*) { node->evalString(utNextRPKStr, AssignNodeParams::RPK.getToken(), 0, time); const boost::filesystem::path nextRPK(utNextRPKStr.toStdString()); - // -- early exit if rpk path is not valid - if (!boost::filesystem::exists(nextRPK)) { - LOG_WRN << "non-existent rpk"; - return NOT_CHANGED; - } - const ResolveMapUPtr& resolveMap = prtCtx->getResolveMap(nextRPK); if (!resolveMap ) { LOG_WRN << "invalid resolve map"; diff --git a/src/palladio/NodeParameter.h b/src/palladio/NodeParameter.h index 135b40ce..d9dca023 100644 --- a/src/palladio/NodeParameter.h +++ b/src/palladio/NodeParameter.h @@ -70,8 +70,7 @@ const std::string RULE_FILE_HELP = "Sets value for primitive attribute '" + PLD_ void buildRuleFileMenu(void *data, PRM_Name *theMenu, int theMaxSize, const PRM_SpareData *, const PRM_Parm *); -static PRM_ChoiceList ruleFileMenu(static_cast(PRM_CHOICELIST_EXCLUSIVE | PRM_CHOICELIST_REPLACE), - &buildRuleFileMenu); +static PRM_ChoiceList ruleFileMenu(static_cast(PRM_CHOICELIST_REPLACE), &buildRuleFileMenu); auto getRuleFile = [](const OP_Node* node, fpreal t) -> std::wstring { UT_String s; @@ -91,8 +90,7 @@ const std::string STYLE_HELP = "Sets value for primitive attribute '" + PLD_STYL void buildStyleMenu(void *data, PRM_Name *theMenu, int theMaxSize, const PRM_SpareData *, const PRM_Parm *); -static PRM_ChoiceList styleMenu(static_cast(PRM_CHOICELIST_EXCLUSIVE | PRM_CHOICELIST_REPLACE), - &buildStyleMenu); +static PRM_ChoiceList styleMenu(static_cast(PRM_CHOICELIST_REPLACE), &buildStyleMenu); auto getStyle = [](const OP_Node* node, fpreal t) -> std::wstring { UT_String s; @@ -112,8 +110,7 @@ const std::string START_RULE_HELP = "Sets value for primitive attribute '" + PLD void buildStartRuleMenu(void *data, PRM_Name *theMenu, int theMaxSize, const PRM_SpareData *, const PRM_Parm *); -static PRM_ChoiceList startRuleMenu(static_cast(PRM_CHOICELIST_EXCLUSIVE | PRM_CHOICELIST_REPLACE), - &buildStartRuleMenu); +static PRM_ChoiceList startRuleMenu(static_cast(PRM_CHOICELIST_REPLACE), &buildStartRuleMenu); auto getStartRule = [](const OP_Node* node, fpreal t) -> std::wstring { UT_String s; diff --git a/src/palladio/PRTContext.cpp b/src/palladio/PRTContext.cpp index df1d7693..3d11495f 100644 --- a/src/palladio/PRTContext.cpp +++ b/src/palladio/PRTContext.cpp @@ -15,14 +15,15 @@ */ #include "PRTContext.h" +#include "PalladioMain.h" #include "LogHandler.h" +#include "SOPAssign.h" + +#include "OP/OP_Node.h" +#include "OP/OP_Director.h" +#include "OP/OP_Network.h" #include -#include -#include -#ifndef WIN32 -# include -#endif namespace { @@ -97,15 +98,24 @@ uint32_t getNumCores() { return n > 0 ? n : 1; } -const timespec NO_MODIFICATION_TIME{0, 0}; +/** + * schedule recook of all assign nodes with matching rpk + */ +void scheduleRecook(const boost::filesystem::path& rpk) { + auto visit = [](OP_Node& n, void* data) -> bool { + if (n.getOperator()->getName().equal(OP_PLD_ASSIGN)) { + auto rpk = reinterpret_cast(data); + SOPAssign& sa = static_cast(n); + if (sa.getRPK() == *rpk) { + LOG_DBG << "forcing recook of: " << n.getName() << ", " << n.getOpType() << ", " << n.getOperator()->getName(); + sa.forceRecook(); + } + } + return false; + }; -timespec getFileModificationTime(const boost::filesystem::path& p) { - struct stat result; - if (stat(p.c_str(), &result) == 0) { - return result.st_mtim; - } - else - return NO_MODIFICATION_TIME; + OP_Network* objMgr = OPgetDirector()->getManager("obj"); + objMgr->traverseChildren(visit, const_cast(reinterpret_cast(&rpk)), true); } } // namespace @@ -115,8 +125,8 @@ PRTContext::PRTContext(const std::vector& addExtDirs) : mLogHandler(new logging::LogHandler(PLD_LOG_PREFIX)), mLicHandle{nullptr}, mPRTCache{prt::CacheObject::create(prt::CacheObject::CACHE_TYPE_DEFAULT)}, - mRPKUnpackPath{boost::filesystem::temp_directory_path() / (PLD_TMP_PREFIX + std::to_string(::getpid()))}, - mCores{getNumCores()} + mCores{getNumCores()}, + mResolveMapCache{new ResolveMapCache(boost::filesystem::temp_directory_path() / (PLD_TMP_PREFIX + std::to_string(::getpid())))} { const prt::LogLevel logLevel = getLogLevel(); prt::setLogLevel(logLevel); @@ -162,53 +172,23 @@ PRTContext::PRTContext(const std::vector& addExtDirs) } PRTContext::~PRTContext() { - mPRTCache.reset(); + mResolveMapCache.reset(); + LOG_INF << "Released RPK Cache"; + + mPRTCache.reset(); // calling reset manually to ensure order LOG_INF << "Released PRT cache"; - mLicHandle.reset(); + mLicHandle.reset(); // same here LOG_INF << "Shutdown PRT & returned license"; - boost::filesystem::remove_all(mRPKUnpackPath); - LOG_INF << "Removed RPK unpack directory"; - prt::removeLogHandler(mLogHandler.get()); } const ResolveMapUPtr& PRTContext::getResolveMap(const boost::filesystem::path& rpk) { - if (rpk.empty()) - return mResolveMapNone; - - const auto timeStamp = getFileModificationTime(rpk); - LOG_DBG << "rpk: current timestamp: " << timeStamp.tv_sec << "s " << timeStamp.tv_nsec << "ns"; - - bool reload = false; - auto it = mResolveMapCache.find(rpk); - if (it != mResolveMapCache.end()) { - LOG_DBG << "rpk: cache timestamp: " << it->second.mTimeStamp.tv_sec << "s " << it->second.mTimeStamp.tv_nsec << "ns"; - if (it->second.mTimeStamp.tv_sec != timeStamp.tv_sec || it->second.mTimeStamp.tv_nsec != timeStamp.tv_nsec) { - mResolveMapCache.erase(it); - const auto cnt = boost::filesystem::remove_all(mRPKUnpackPath / rpk.leaf()); - mPRTCache->flushAll(); // TODO: or do we need to explicitely remove cgbs from cache? - LOG_DBG << "RPK change detected, refreshing unpack cache: " << rpk << "( removed " << cnt << " files)"; - reload = true; - } + auto lookupResult = mResolveMapCache->get(rpk); + if (lookupResult.second == ResolveMapCache::CacheStatus::MISS) { + mPRTCache->flushAll(); + scheduleRecook(rpk); } - else - reload = true; - - if (reload) { - const auto rpkURI = toFileURI(rpk); - - ResolveMapCacheEntry rmce; - rmce.mTimeStamp = timeStamp; - - prt::Status status = prt::STATUS_UNSPECIFIED_ERROR; - rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status)); - if (status != prt::STATUS_OK) - return mResolveMapNone; - - it = mResolveMapCache.emplace(rpk, std::move(rmce)).first; - } - - return it->second.mResolveMap; + return lookupResult.first; } diff --git a/src/palladio/PRTContext.h b/src/palladio/PRTContext.h index 34d7a2e1..dd606f64 100644 --- a/src/palladio/PRTContext.h +++ b/src/palladio/PRTContext.h @@ -16,6 +16,7 @@ #pragma once +#include "ResolveMapCache.h" #include "Utils.h" #include "prt/Object.h" @@ -47,16 +48,8 @@ struct PRTContext final { logging::LogHandlerPtr mLogHandler; ObjectUPtr mLicHandle; CacheObjectUPtr mPRTCache; - boost::filesystem::path mRPKUnpackPath; const uint32_t mCores; - - struct ResolveMapCacheEntry { - ResolveMapUPtr mResolveMap; - timespec mTimeStamp; - }; - using ResolveMapCache = std::map; - ResolveMapCache mResolveMapCache; - const ResolveMapUPtr mResolveMapNone; + ResolveMapCacheUPtr mResolveMapCache; }; using PRTContextUPtr = std::unique_ptr; diff --git a/src/palladio/PalladioMain.cpp b/src/palladio/PalladioMain.cpp index ce451c05..2dc86082 100644 --- a/src/palladio/PalladioMain.cpp +++ b/src/palladio/PalladioMain.cpp @@ -14,9 +14,11 @@ * limitations under the License. */ +#include "PalladioMain.h" + +#include "PRTContext.h" #include "SOPAssign.h" #include "SOPGenerate.h" -#include "PRTContext.h" #include "NodeParameter.h" #include "OP/OP_OperatorTable.h" @@ -47,7 +49,7 @@ void newSopOperator(OP_OperatorTable *table) { auto createSOPAssign = [](OP_Network *net, const char *name, OP_Operator *op) -> OP_Node* { return new SOPAssign(prtCtx, net, name, op); }; - table->addOperator(new OP_Operator("pldAssign", "pldAssign", createSOPAssign, + table->addOperator(new OP_Operator(OP_PLD_ASSIGN, OP_PLD_ASSIGN, createSOPAssign, AssignNodeParams::PARAM_TEMPLATES, 1, 1, nullptr, OP_FLAG_GENERATOR )); @@ -55,7 +57,7 @@ void newSopOperator(OP_OperatorTable *table) { auto createSOPGenerate = [](OP_Network *net, const char *name, OP_Operator *op) -> OP_Node* { return new SOPGenerate(prtCtx, net, name, op); }; - table->addOperator(new OP_Operator("pldGenerate", "pldGenerate", createSOPGenerate, + table->addOperator(new OP_Operator(OP_PLD_GENERATE, OP_PLD_GENERATE, createSOPGenerate, GenerateNodeParams::PARAM_TEMPLATES, 1, 1, nullptr, OP_FLAG_GENERATOR )); } diff --git a/src/palladio/PalladioMain.h b/src/palladio/PalladioMain.h new file mode 100644 index 00000000..7b16a3a1 --- /dev/null +++ b/src/palladio/PalladioMain.h @@ -0,0 +1,4 @@ +#pragma once + +constexpr const char* OP_PLD_ASSIGN = "pldAssign"; +constexpr const char* OP_PLD_GENERATE = "pldGenerate"; diff --git a/src/palladio/ResolveMapCache.cpp b/src/palladio/ResolveMapCache.cpp new file mode 100644 index 00000000..75ca74ff --- /dev/null +++ b/src/palladio/ResolveMapCache.cpp @@ -0,0 +1,73 @@ +// +// Created by shaegler on 3/3/18. +// + +#include "ResolveMapCache.h" +#include "LogHandler.h" + + +namespace { + +const ResolveMapUPtr RESOLVE_MAP_NONE; +const timespec NO_MODIFICATION_TIME{0, 0}; + +timespec getFileModificationTime(const boost::filesystem::path& p) { + struct stat result; + if (stat(p.c_str(), &result) == 0) { + return result.st_mtim; + } + else + return NO_MODIFICATION_TIME; +} + +} // namespace + + +ResolveMapCache::~ResolveMapCache() { + boost::filesystem::remove_all(mRPKUnpackPath); + LOG_INF << "Removed RPK unpack directory"; +} + +ResolveMapCache::LookupResult ResolveMapCache::get(const boost::filesystem::path& rpk) { + if (rpk.empty()) + return { RESOLVE_MAP_NONE, CacheStatus::MISS }; + + if (!boost::filesystem::exists(rpk)) + return { RESOLVE_MAP_NONE, CacheStatus::MISS }; + + const auto timeStamp = getFileModificationTime(rpk); + LOG_DBG << "rpk: current timestamp: " << timeStamp.tv_sec << "s " << timeStamp.tv_nsec << "ns"; + + CacheStatus cs = CacheStatus::HIT; + auto it = mCache.find(rpk); + if (it != mCache.end()) { + LOG_DBG << "rpk: cache timestamp: " << it->second.mTimeStamp.tv_sec << "s " << it->second.mTimeStamp.tv_nsec << "ns"; + if (it->second.mTimeStamp.tv_sec != timeStamp.tv_sec || it->second.mTimeStamp.tv_nsec != timeStamp.tv_nsec) { + mCache.erase(it); + const auto cnt = boost::filesystem::remove_all(mRPKUnpackPath / rpk.leaf()); + LOG_INF << "RPK change detected, forcing reload and clearing cache for " << rpk << " (removed " << cnt << " files)"; + cs = CacheStatus::MISS; + } + } + else + cs = CacheStatus::MISS; + + if (cs == CacheStatus::MISS) { + const auto rpkURI = toFileURI(rpk); + + ResolveMapCacheEntry rmce; + rmce.mTimeStamp = timeStamp; + + prt::Status status = prt::STATUS_UNSPECIFIED_ERROR; + rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status)); + if (status != prt::STATUS_OK) + return { RESOLVE_MAP_NONE, CacheStatus::MISS }; + + it = mCache.emplace(rpk, std::move(rmce)).first; + LOG_INF << "Upacked RPK " << rpk << " to " << mRPKUnpackPath; + } + + return { it->second.mResolveMap, cs }; + +} + diff --git a/src/palladio/ResolveMapCache.h b/src/palladio/ResolveMapCache.h new file mode 100644 index 00000000..33eec05d --- /dev/null +++ b/src/palladio/ResolveMapCache.h @@ -0,0 +1,37 @@ +#pragma once + +#include "Utils.h" + +#include "boost/filesystem.hpp" + +#include +#include +#ifndef WIN32 +# include +#endif + + +class ResolveMapCache { +public: + ResolveMapCache(const boost::filesystem::path& unpackPath) : mRPKUnpackPath{unpackPath} { } + ResolveMapCache(const ResolveMapCache&) = delete; + ResolveMapCache(ResolveMapCache&&) = delete; + ResolveMapCache& operator=(ResolveMapCache&) = delete; + ~ResolveMapCache(); + + enum class CacheStatus { HIT, MISS }; + using LookupResult = std::pair; + LookupResult get(const boost::filesystem::path& rpk); + +private: + struct ResolveMapCacheEntry { + ResolveMapUPtr mResolveMap; + timespec mTimeStamp; + }; + using Cache = std::map; + Cache mCache; + + const boost::filesystem::path mRPKUnpackPath; +}; + +using ResolveMapCacheUPtr = std::unique_ptr; \ No newline at end of file diff --git a/src/palladio/SOPAssign.h b/src/palladio/SOPAssign.h index 2abb7238..6fa58d4f 100644 --- a/src/palladio/SOPAssign.h +++ b/src/palladio/SOPAssign.h @@ -28,6 +28,7 @@ class SOPAssign : public SOP_Node { ~SOPAssign() override = default; const PRTContextUPtr& getPRTCtx() const { return mPRTCtx; } + const boost::filesystem::path& getRPK() const { return mShapeConverter->mRPK; } void opChanged(OP_EventType reason, void* data = nullptr) override; From 62b4dc2a5d750b7b416b6d5768bbc19d68f6b842 Mon Sep 17 00:00:00 2001 From: Simon Haegler Date: Sat, 3 Mar 2018 22:42:42 +0100 Subject: [PATCH 4/4] #58: add explict RPK reload button --- src/palladio/NodeParameter.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/palladio/NodeParameter.h b/src/palladio/NodeParameter.h index d9dca023..f2385ca5 100644 --- a/src/palladio/NodeParameter.h +++ b/src/palladio/NodeParameter.h @@ -64,6 +64,10 @@ auto getRPK = [](const OP_Node* node, fpreal t) -> boost::filesystem::path { }; +// -- RPK RELOADER +static PRM_Name RPK_RELOAD("rpkReload", "Reload Rule Package"); + + // -- RULE FILE (cgb) static PRM_Name RULE_FILE("ruleFile", "Rule File"); const std::string RULE_FILE_HELP = "Sets value for primitive attribute '" + PLD_RULE_FILE.toStdString() + "'"; @@ -126,11 +130,12 @@ auto setStartRule = [](OP_Node* node, const std::wstring& s, fpreal t) { // -- ASSIGN NODE PARAMS static PRM_Template PARAM_TEMPLATES[] = { - PRM_Template(PRM_STRING, 1, &PRIM_CLS, &PRIM_CLS_DEFAULT, nullptr, nullptr, PRM_Callback(), nullptr, 1, PRIM_CLS_HELP.c_str()), - PRM_Template(PRM_FILE, 1, &RPK, &RPK_DEFAULT, nullptr, nullptr, rpkCallback, &PRM_SpareData::fileChooserModeRead, 1, RPK_HELP.c_str()), - PRM_Template(PRM_STRING, 1, &RULE_FILE, PRMoneDefaults, &ruleFileMenu, nullptr, PRM_Callback(), nullptr, 1, RULE_FILE_HELP.c_str()), - PRM_Template(PRM_STRING, 1, &STYLE, PRMoneDefaults, &styleMenu, nullptr, PRM_Callback(), nullptr, 1, STYLE_HELP.c_str()), - PRM_Template(PRM_STRING, 1, &START_RULE, PRMoneDefaults, &startRuleMenu, nullptr, PRM_Callback(), nullptr, 1, START_RULE_HELP.c_str()), + PRM_Template(PRM_STRING, 1, &PRIM_CLS, &PRIM_CLS_DEFAULT, nullptr, nullptr, PRM_Callback(), nullptr, 1, PRIM_CLS_HELP.c_str()), + PRM_Template(PRM_FILE, 1, &RPK, &RPK_DEFAULT, nullptr, nullptr, rpkCallback, &PRM_SpareData::fileChooserModeRead, 1, RPK_HELP.c_str()), + PRM_Template(PRM_CALLBACK, 1, &RPK_RELOAD, PRMoneDefaults, nullptr, nullptr, rpkCallback), + PRM_Template(PRM_STRING, 1, &RULE_FILE, PRMoneDefaults, &ruleFileMenu, nullptr, PRM_Callback(), nullptr, 1, RULE_FILE_HELP.c_str()), + PRM_Template(PRM_STRING, 1, &STYLE, PRMoneDefaults, &styleMenu, nullptr, PRM_Callback(), nullptr, 1, STYLE_HELP.c_str()), + PRM_Template(PRM_STRING, 1, &START_RULE, PRMoneDefaults, &startRuleMenu, nullptr, PRM_Callback(), nullptr, 1, START_RULE_HELP.c_str()), PRM_Template() };