fix: add plugin clear state callback

Some plugins maintain their own state which is currently persisted
even if its never needed anymore

For example lambda/ecs plugins cache AWS responses during exec report.
After exec report is sent, there is no more need to keep that state
as cloud metadata is not reported during heartbeats however its memory
is still used.

With the clear state callback each plugin can manage its own cache
as necessary not to waste memory.

src/chalk_common.nim

@@ -71,20 +71,21 @@ type
                                      ## need to inspect twice when we build + push.
     resourceType*:  set[ResourceType]
 
+  PluginClearCb*       = proc (a: Plugin) {.cdecl.}
   ChalkTimeHostCb*     = proc (a: Plugin): ChalkDict {.cdecl.}
   ChalkTimeArtifactCb* = proc (a: Plugin, b: ChalkObj): ChalkDict {.cdecl.}
-  RunTimeArtifactCb*   = proc (a: Plugin, b: ChalkObj, c: bool):
-                             ChalkDict {.cdecl.}
+  RunTimeArtifactCb*   = proc (a: Plugin, b: ChalkObj, c: bool): ChalkDict {.cdecl.}
   RunTimeHostCb*       = proc (a: Plugin, b: seq[ChalkObj]): ChalkDict {.cdecl.}
   ScanCb*              = proc (a: Plugin, b: string): Option[ChalkObj] {.cdecl.}
   UnchalkedHashCb*     = proc (a: Plugin, b: ChalkObj): Option[string] {.cdecl.}
   EndingHashCb*        = proc (a: Plugin, b: ChalkObj): Option[string] {.cdecl.}
   ChalkIdCb*           = proc (a: Plugin, b: ChalkObj): string {.cdecl.}
-  HandleWriteCb*       = proc (a: Plugin, b: ChalkObj,
-                               c: Option[string]) {.cdecl.}
+  HandleWriteCb*       = proc (a: Plugin, b: ChalkObj, c: Option[string]) {.cdecl.}
+
   Plugin* = ref object
     name*:                     string
     enabled*:                  bool
+    clearState*:               PluginClearCb
     getChalkTimeHostInfo*:     ChalkTimeHostCb
     getChalkTimeArtifactInfo*: ChalkTimeArtifactCb
     getRunTimeArtifactInfo*:   RunTimeArtifactCb
@@ -474,12 +475,13 @@ var
   doingTestRun*           = false
   nativeCodecsOnly*       = false
   passedHelpFlag*         = false
-  con4mRuntime*:          ConfigStack
-  commandName*:           string
-  gitExeLocation*:        string = ""
-  sshKeyscanExeLocation*: string = ""
-  dockerInvocation*:      DockerInvocation
-  externalActions*:       seq[seq[string]] = @[]
+  installedPlugins*       = Table[string, Plugin]()
+  externalActions*        = newSeq[seq[string]]()
+  commandName*            = ""
+  gitExeLocation*         = ""
+  sshKeyscanExeLocation*  = ""
+  con4mRuntime*:          ConfigStack # can be nil
+  dockerInvocation*:      DockerInvocation # ca be nil
 
 template dumpExOnDebug*() =
   when not defined(release):

src/commands/cmd_exec.nim

@@ -143,7 +143,6 @@ proc getParentExitStatus(trueParentPid: Pid): bool =
   return false
 
 proc doHeartbeatReport(chalkOpt: Option[ChalkObj]) =
-  clearReportingState()
   initCollection()
   if chalkOpt.isSome():
     let chalk = chalkOpt.get()
@@ -166,10 +165,12 @@ template doHeartbeat(chalkOpt: Option[ChalkObj], pid: Pid, fn: untyped) =
     sleepInterval = int(inMicroSec / 1000)
 
   setCommandName("heartbeat")
+  clearReportingState()
 
   while true:
     sleep(sleepInterval)
     chalkOpt.doHeartbeatReport()
+    clearReportingState()
     if fn(pid):
       break
 

src/plugin_api.nim

@@ -544,9 +544,7 @@ proc defaultCodecWrite*(s:     Plugin,
   if not chalk.replaceFileContents(contents):
     chalk.opFailed = true
 
-var
-  installedPlugins: Table[string, Plugin]
-  codecs:           seq[Plugin] = @[]
+var codecs: seq[Plugin] = @[]
 
 template isCodec*(plugin: Plugin): bool = attrGet[bool]("plugin." & plugin.name & ".codec")
 
@@ -598,13 +596,15 @@ proc getAllCodecs*(): seq[Plugin] =
 
 proc newPlugin*(
   name:           string,
+  clearCallback:  PluginClearCb       = PluginClearCb(nil),
   ctHostCallback: ChalkTimeHostCb     = ChalkTimeHostCb(nil),
   ctArtCallback:  ChalkTimeArtifactCb = ChalkTimeArtifactCb(nil),
   rtArtCallback:  RunTimeArtifactCb   = RunTimeArtifactCb(nil),
   rtHostCallback: RunTimeHostCb       = RunTimeHostCb(nil),
   cache:          RootRef             = RootRef(nil)):
     Plugin {.discardable, cdecl.} =
   result = Plugin(name:                     name,
+                  clearState:               clearCallback,
                   getChalkTimeHostInfo:     ctHostCallback,
                   getChalkTimeArtifactInfo: ctArtCallback,
                   getRunTimeArtifactInfo:   rtArtCallback,

src/plugins/awsEcs.nim

@@ -17,11 +17,14 @@ let
 
 # returns ecs metadata as a json blob
 var
-  ecsMetadata: ChalkDict = ChalkDict()
+  ecsMetadata = ChalkDict()
   ecsUrl = cloudMetadataUrl4
 if ecsUrl == "":
   ecsUrl = cloudMetadataUrl3
 
+proc clearCallback(self: Plugin) {.cdecl.} =
+  ecsMetadata = ChalkDict()
+
 proc requestECSMetadata(path: string): Option[Box] =
   let url = ecsUrl & path
   var body = ""
@@ -45,7 +48,7 @@ proc requestECSMetadata(path: string): Option[Box] =
 proc readECSMetadata*(): ChalkDict =
   # This can be called from outside if anything needs to query the JSON.
   # For now, we just return the whole blob.
-  once:
+  if len(ecsMetadata) == 0:
     if ecsUrl == "":
       trace("ecs: metadata env var is not defined: no AWS info available")
       return ecsMetadata
@@ -93,5 +96,6 @@ proc ecsGetRunTimeHostInfo*(self: Plugin, objs: seq[ChalkObj]):
 
 proc loadAwsEcs*() =
   newPlugin("aws_ecs",
+            clearCallback  = PluginClearCb(clearCallback),
             ctHostCallback = ChalkTimeHostCb(ecsGetChalkTimeHostInfo),
             rtHostCallback = RunTimeHostCb(ecsGetRunTimeHostInfo))

src/plugins/awsLambda.nim

@@ -10,12 +10,15 @@
 import pkg/[nimutils/stsclient]
 import ".."/[config, plugin_api]
 
-var lambdaMetadata: ChalkDict = ChalkDict()
+var lambdaMetadata = ChalkDict()
+
+proc clearCallback(self: Plugin) {.cdecl.} =
+  lambdaMetadata = ChalkDict()
 
 proc collectLambdaMetadata(): ChalkDict =
   # This can be called from outside if anything needs to query the JSON.
   # For now, we just return the whole blob.
-  once:
+  if len(lambdaMetadata) == 0:
     let
       region          = os.getEnv("AWS_REGION", os.getEnv("AWS_DEFAULT_REGION"))
       functionName    = os.getEnv("AWS_LAMBDA_FUNCTION_NAME")
@@ -94,4 +97,5 @@ proc lambdaCallback*(self: Plugin, objs: seq[ChalkObj]):
 
 proc loadAwsLambda*() =
   newPlugin("aws_lambda",
+            clearCallback  = PluginClearCb(clearCallback),
             rtHostCallback = RunTimeHostCb(lambdaCallback))

src/plugins/codecMacOs.nim

@@ -12,7 +12,7 @@
 import std/base64
 import ".."/[config, chalkjson, util, plugin_api]
 
-var prefix = """
+let prefix = """
 #!/bin/bash
 
 BASE_NAME=$(basename -- "${BASH_SOURCE[0]}")
@@ -38,7 +38,7 @@ fi
 (base64 -d)  < /bin/cat << CHALK_DADFEDABBADABBEDBAD_END > ${CMDLOC}
 """
 
-var postfixLines = [
+let postfixLines = [
   "CHALK_DADFEDABBADABBEDBAD_END",
   "chmod +x ${CMDLOC}",
   "exec ${CMDLOC} ${@}"

src/plugins/codecZip.nim

@@ -21,8 +21,8 @@ type
     tmpDir:        string
 
 var
-    zipDirs:  seq[string]
-    chalkIds: seq[Box]
+  zipDirs:  seq[string]
+  chalkIds: seq[Box]
 
 template pushZipDir(s: string)   = zipDirs.add(s)
 template popZipDir()             = discard zipDirs.pop()

src/plugins/externalTool.nim

@@ -13,16 +13,20 @@ import ".."/[config, plugin_api, util]
 
 type
   AlreadyRanError = object of CatchableError
-  PIInfo = ref object
+  PIInfo          = ref object
     name: string
 
+var alreadyRan = initHashSet[string]()
+
+proc clearCallback(self: Plugin) {.cdecl.} =
+  alreadyRan = initHashSet[string]()
+
 proc ensureRunCallback[T](cb: CallbackObj, args: seq[Box]): T =
   let value = runCallback(cb, args)
   if value.isNone():
     raise newException(ValueError, "missing implemenetation of " & $(cb))
   return unpack[T](value.get())
 
-var alreadyRan = initHashSet[string]()
 proc runOneTool(info: PIInfo, path: string): ChalkDict =
   let key = info.name & ":" & path
   if key in alreadyRan:
@@ -130,5 +134,6 @@ proc toolGetChalkTimeArtifactInfo(self: Plugin, obj: ChalkObj):
 
 proc loadExternalTool*() =
   newPlugin("tool",
+            clearCallback  = PluginClearCb(clearCallback),
             ctHostCallback = ChalkTimeHostCb(toolGetChalkTimeHostInfo),
             ctArtCallback  = ChalkTimeArtifactCb(toolGetChalkTimeArtifactInfo))

src/plugins/procfs.nim

@@ -29,6 +29,9 @@ type
     udpSockInfoCache: Option[ProcTable]
     psInfoCache:      Option[ProcFdSet]
 
+proc clearCallback(self: Plugin) {.cdecl.} =
+  self.internalState = RootRef(ProcFsCache())
+
 const PATH_MAX = 4096 # PROC_PIDPATHINFO_MAXSIZE on mac
 let
   clockSpeed = float(sysconf(SC_CLK_TCK))
@@ -694,6 +697,7 @@ proc procfsGetRunTimeArtifactInfo(self: Plugin, obj: ChalkObj, ins: bool):
 proc loadProcFs*() =
   when hostOs == "linux":
     newPlugin("procfs",
+              clearCallback  = PluginClearCb(clearCallback),
               rtHostCallback = RunTimeHostCb(procfsGetRunTimeHostInfo),
               rtArtCallback  = RunTimeArtifactCb(procfsGetRunTimeArtifactInfo),
               cache          = RootRef(ProcFsCache()))

src/plugins/system.nim

@@ -166,10 +166,14 @@ proc sysGetRunTimeArtifactInfo*(self: Plugin, obj: ChalkObj, insert: bool):
     result["_OP_ARTIFACT_REPORT_KEYS"] = pack(reportKeys)
 
 var
-  instant   = epochTime()
-  timestamp = instant.fromUnixFloat()
-  envdict:          Con4mDict[string, string]
-  cachedSearchPath: seq[string] = @[]
+  instant          = epochTime()
+  timestamp        = instant.fromUnixFloat()
+  envdict          = Con4mDict[string, string]()
+  cachedSearchPath = newSeq[string]()
+
+proc clearCallback(self: Plugin) {.cdecl.} =
+  envdict          = Con4mDict[string, string]()
+  cachedSearchPath = @[]
 
 when defined(posix):
   let uinfo = uname()
@@ -181,7 +185,7 @@ template getOffset(): string   = timestamp.format("zzz")
 template getDateTime(): string = getDate() & "T" & getTime() & getOffset()
 
 proc getEnvDict(): Box =
-  once:
+  if len(envdict) == 0:
     envdict = Con4mDict[string, string]()
     let
       always = attrGet[seq[string]]("env_always_show")
@@ -326,6 +330,7 @@ proc metsysGetRunTimeHostInfo(self: Plugin, objs: seq[ChalkObj]):
 
 proc loadSystem*() =
   newPlugin("system",
+            clearCallback  = PluginClearCb(clearCallback),
             ctHostCallback = ChalkTimeHostCb(sysGetChalkTimeHostInfo),
             ctArtCallback  = ChalkTimeArtifactCb(sysGetChalkTimeArtifactInfo),
             rtArtCallback  = RunTimeArtifactCb(sysGetRunTimeArtifactInfo),

src/plugins/vctlGit.nim

@@ -272,6 +272,9 @@ type
     origin:     Option[string]
     vcsDirs:    OrderedTable[string, RepoInfo]
 
+proc clearCallback(self: Plugin) {.cdecl.} =
+  self.internalState = RootRef(GitInfo())
+
 proc isAnnotated(self: GitTag): bool =
   return self.tagCommitId != ""
 
@@ -793,8 +796,8 @@ proc isInRepo(obj: ChalkObj, repo: string): bool =
   return false
 
 proc gitInit(self: Plugin) =
-  once:
-    let cache = GitInfo(self.internalState)
+  let cache = GitInfo(self.internalState)
+  if len(cache.vcsDirs) == 0:
     for path in getContextDirectories():
       cache.findAndLoad(path.resolvePath())
 
@@ -837,6 +840,7 @@ proc gitGetRunTimeHostInfo(self: Plugin, chalks: seq[ChalkObj]):
 
 proc loadVctlGit*() =
   newPlugin("vctl_git",
+            clearCallback  = PluginClearCb(clearCallback),
             ctArtCallback  = ChalkTimeArtifactCb(gitGetChalkTimeArtifactInfo),
             rtHostCallback = RunTimeHostCb(gitGetRunTimeHostInfo),
             cache          = RootRef(GitInfo()))

src/run_management.nim

@@ -82,6 +82,11 @@ proc clearReportingState*() =
   systemErrors    = @[]
   failedKeys      = ChalkDict()
   externalActions = @[]
+  for name, plugin in installedPlugins.pairs():
+    if plugin.clearState == nil:
+      continue
+    let cb = plugin.clearState
+    cb(plugin)
 
 proc pushCollectionCtx*(): CollectionCtx =
   result = CollectionCtx()