Introduces an experimental declarative parser into nimble

config.nims

@@ -6,3 +6,4 @@ switch("define", "ssl")
 switch("path", "vendor" / "zippy" / "src")
 switch("path", "vendor" / "sat" / "src")
 switch("path", "vendor" / "checksums" / "src")
+switch("define", "zippyNoSimd")

src/nimble.nim

@@ -19,7 +19,7 @@ import nimblepkg/packageinfotypes, nimblepkg/packageinfo, nimblepkg/version,
        nimblepkg/nimbledatafile, nimblepkg/packagemetadatafile,
        nimblepkg/displaymessages, nimblepkg/sha1hashes, nimblepkg/syncfile,
        nimblepkg/deps, nimblepkg/nimblesat, nimblepkg/forge_aliases, nimblepkg/nimenv,
-       nimblepkg/downloadnim
+       nimblepkg/downloadnim, nimblepkg/declarativeparser
 
 const
   nimblePathsFileName* = "nimble.paths"
@@ -32,7 +32,7 @@ const
 proc initPkgList(pkgInfo: PackageInfo, options: Options): seq[PackageInfo] =
   let
     installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
-    developPkgs = processDevelopDependencies(pkgInfo, options)
+    developPkgs = processDevelopDependencies(pkgInfo, options)  
   result = concat(installedPkgs, developPkgs)
 
 proc install(packages: seq[PkgTuple], options: Options,
@@ -83,20 +83,27 @@ proc addReverseDeps(solvedPkgs: seq[SolvedPackage], allPkgsInfo: seq[PackageInfo
 proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): HashSet[PackageInfo] = 
   if satProccesedPackages.len > 0:
     return satProccesedPackages
-  var solvedPkgs = newSeq[SolvedPackage]()
-  var pkgsToInstall: seq[(string, Version)] = @[]
   var rootPkgInfo = rootPkgInfo
   rootPkgInfo.requires &= options.extraRequires
-  var pkgList = initPkgList(rootPkgInfo, options).mapIt(it.toFullInfo(options))
+  var rootMin = rootPkgInfo.getMinimalInfo(options)
+  rootMin.isRoot = true
+
+  var pkgList = initPkgList(rootPkgInfo, options)#.mapIt(it.toFullInfo(options))
+  if options.useDeclarativeParser:
+    pkgList = pkgList.mapIt(it.toRequiresInfo())
+  else:
+    pkgList = pkgList.mapIt(it.toFullInfo(options))
+
+  var state = initSATState(rootMin, pkgList, options)
   var allPkgsInfo: seq[PackageInfo] = pkgList & rootPkgInfo
   #Remove from the pkglist the packages that exists in lock file and has a different vcsRevision
   var upgradeVersions = initTable[string, VersionRange]()
-  var isUpgrading = options.action.typ == actionUpgrade
+  let isUpgrading = options.action.typ == actionUpgrade  
   if isUpgrading:
     for pkg in options.action.packages:
       upgradeVersions[pkg.name] = pkg.ver
     pkgList = pkgList.filterIt(it.basicInfo.name notin upgradeVersions)
-
+  
   var toRemoveFromLocked = newSeq[PackageInfo]()
   if rootPkgInfo.lockedDeps.hasKey(""):
     for name, lockedPkg in rootPkgInfo.lockedDeps[""]:
@@ -107,14 +114,15 @@ proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): Has
               toRemoveFromLocked.add pkg
 
   var systemNimCompatible = options.nimBin.isSome
-  result = solveLocalPackages(rootPkgInfo, pkgList, solvedPkgs, systemNimCompatible,  options)
-  if solvedPkgs.len > 0: 
-    displaySatisfiedMsg(solvedPkgs, pkgsToInstall, options)
-    addReverseDeps(solvedPkgs, allPkgsInfo, options)
+  solveLocalPackages(rootPkgInfo, state, systemNimCompatible,  options)
+  result = state.solutionAsPackageInfo(options).toHashSet
+  if state.foundSolution(): 
+    displaySatisfiedMsg(state.solvedPkgs, state.pkgToInstall, options)
+    addReverseDeps(state.solvedPkgs, allPkgsInfo, options)
     for pkg in allPkgsInfo:
       if pkg.basicInfo.name.isNim and systemNimCompatible:
         continue #Dont add nim from the solution as we will use system nim
-      result.incl pkg
+      result.incl pkg #TODO likely this is only adding root. But review later
     for nonLocked in toRemoveFromLocked:
       #only remove if the vcsRevision is different
       for pkg in result:
@@ -124,15 +132,17 @@ proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): Has
       result.toSeq
       .deleteStaleDependencies(rootPkgInfo, options)
       .toHashSet
+
     satProccesedPackages = result
     return result
-
-  var output = ""
-  result = solvePackages(rootPkgInfo, pkgList, pkgsToInstall, options, output, solvedPkgs)
-  displaySatisfiedMsg(solvedPkgs, pkgsToInstall, options)
-  displayUsingSpecialVersionWarning(solvedPkgs, options)
-  var solved = solvedPkgs.len > 0 #A pgk can be solved and still dont return a set of PackageInfo
-  for (name, ver) in pkgsToInstall:
+
+  state = initSATState(rootMin, pkgList, options)
+  solvePackages(rootPkgInfo, state, options)
+  result = state.solutionAsPackageInfo(options).toHashSet
+  displaySatisfiedMsg(state.solvedPkgs, state.pkgToInstall, options)
+  displayUsingSpecialVersionWarning(state.solvedPkgs, options)
+  var solved = state.foundSolution() #A pgk can be solved and still dont return a set of PackageInfo  
+  for (name, ver) in state.pkgToInstall:
     var versionRange = ver.toVersionRange
     if name in upgradeVersions:
       versionRange = upgradeVersions[name]
@@ -148,10 +158,11 @@ proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): Has
           pkg.metaData.specialVersions)
       else:
         result.incl pkg
+      state.pkgInfoInstalled.incl pkg
 
   for pkg in result:
     allPkgsInfo.add pkg
-  addReverseDeps(solvedPkgs, allPkgsInfo, options)
+  addReverseDeps(state.solvedPkgs, allPkgsInfo, options)
 
   for nonLocked in toRemoveFromLocked:
     result.excl nonLocked
@@ -160,7 +171,7 @@ proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, options: Options): Has
   satProccesedPackages = result
 
   if not solved:
-    display("Error", output, Error, priority = HighPriority)
+    display("Error", state.output, Error, priority = HighPriority)
     raise nimbleError("Unsatisfiable dependencies")
 
 proc processFreeDependencies(pkgInfo: PackageInfo,

src/nimblepkg/declarativeparser.nim

@@ -0,0 +1,182 @@
+## Utility API for Nim package managers.
+## (c) 2021 Andreas Rumpf
+
+import std/strutils
+
+import compiler/[ast, idents, msgs, syntaxes, options, pathutils, lineinfos]
+import version, packageinfotypes
+
+type NimbleFileInfo* = object
+  requires*: seq[string]
+  srcDir*: string
+  version*: string
+  tasks*: seq[(string, string)]
+  hasInstallHooks*: bool
+  hasErrors*: bool
+
+proc eqIdent(a, b: string): bool {.inline.} =
+  cmpIgnoreCase(a, b) == 0 and a[0] == b[0]
+
+proc extract(n: PNode, conf: ConfigRef, result: var NimbleFileInfo) =
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    for child in n:
+      extract(child, conf, result)
+  of nkCallKinds:
+    if n[0].kind == nkIdent:
+      case n[0].ident.s
+      of "requires":
+        for i in 1 ..< n.len:
+          var ch = n[i]
+          while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0:
+            ch = ch.lastSon
+          if ch.kind in {nkStrLit .. nkTripleStrLit}:
+            result.requires.add ch.strVal
+          else:
+            localError(conf, ch.info, "'requires' takes string literals")
+            result.hasErrors = true
+      of "task":
+        if n.len >= 3 and n[1].kind == nkIdent and
+            n[2].kind in {nkStrLit .. nkTripleStrLit}:
+          result.tasks.add((n[1].ident.s, n[2].strVal))
+      of "before", "after":
+        #[
+          before install do:
+            exec "git submodule update --init"
+            var make = "make"
+            when defined(windows):
+              make = "mingw32-make"
+            exec make
+        ]#
+        if n.len >= 3 and n[1].kind == nkIdent and n[1].ident.s == "install":
+          result.hasInstallHooks = true
+      else:
+        discard
+  of nkAsgn, nkFastAsgn:
+    if n[0].kind == nkIdent and eqIdent(n[0].ident.s, "srcDir"):
+      if n[1].kind in {nkStrLit .. nkTripleStrLit}:
+        result.srcDir = n[1].strVal
+      else:
+        localError(conf, n[1].info, "assignments to 'srcDir' must be string literals")
+        result.hasErrors = true
+    elif n[0].kind == nkIdent and eqIdent(n[0].ident.s, "version"):
+      if n[1].kind in {nkStrLit .. nkTripleStrLit}:
+        result.version = n[1].strVal
+      else:
+        localError(conf, n[1].info, "assignments to 'version' must be string literals")
+        result.hasErrors = true
+  else:
+    discard
+
+proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo =
+  ## Extract the `requires` information from a Nimble file. This does **not**
+  ## evaluate the Nimble file. Errors are produced on stderr/stdout and are
+  ## formatted as the Nim compiler does it. The parser uses the Nim compiler
+  ## as an API. The result can be empty, this is not an error, only parsing
+  ## errors are reported.
+  var conf = newConfigRef()
+  conf.foreignPackageNotes = {}
+  conf.notes = {}
+  conf.mainPackageNotes = {}
+  conf.errorMax = high(int)
+  conf.structuredErrorHook = proc(
+      config: ConfigRef, info: TLineInfo, msg: string, severity: Severity
+  ) {.gcsafe.} =
+    localError(config, info, warnUser, msg)
+
+  let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
+  var parser: Parser
+  if setupParser(parser, fileIdx, newIdentCache(), conf):
+    extract(parseAll(parser), conf, result)
+    closeParser(parser)
+  result.hasErrors = result.hasErrors or conf.errorCounter > 0
+
+type PluginInfo* = object
+  builderPatterns*: seq[(string, string)]
+
+proc extractPlugin(
+    nimscriptFile: string, n: PNode, conf: ConfigRef, result: var PluginInfo
+) =
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    for child in n:
+      extractPlugin(nimscriptFile, child, conf, result)
+  of nkCallKinds:
+    if n[0].kind == nkIdent:
+      case n[0].ident.s
+      of "builder":
+        if n.len >= 3 and n[1].kind in {nkStrLit .. nkTripleStrLit}:
+          result.builderPatterns.add((n[1].strVal, nimscriptFile))
+      else:
+        discard
+  else:
+    discard
+
+proc extractPluginInfo*(nimscriptFile: string, info: var PluginInfo) =
+  var conf = newConfigRef()
+  conf.foreignPackageNotes = {}
+  conf.notes = {}
+  conf.mainPackageNotes = {}
+
+  let fileIdx = fileInfoIdx(conf, AbsoluteFile nimscriptFile)
+  var parser: Parser
+  if setupParser(parser, fileIdx, newIdentCache(), conf):
+    extractPlugin(nimscriptFile, parseAll(parser), conf, info)
+    closeParser(parser)
+
+const Operators* = {'<', '>', '=', '&', '@', '!', '^'}
+
+proc token(s: string, idx: int, lit: var string): int =
+  var i = idx
+  if i >= s.len:
+    return i
+  while s[i] in Whitespace:
+    inc(i)
+  case s[i]
+  of Letters, '#':
+    lit.add s[i]
+    inc i
+    while i < s.len and s[i] notin (Whitespace + {'@', '#'}):
+      lit.add s[i]
+      inc i
+  of '0' .. '9':
+    while i < s.len and s[i] in {'0' .. '9', '.'}:
+      lit.add s[i]
+      inc i
+  of '"':
+    inc i
+    while i < s.len and s[i] != '"':
+      lit.add s[i]
+      inc i
+    inc i
+  of Operators:
+    while i < s.len and s[i] in Operators:
+      lit.add s[i]
+      inc i
+  else:
+    lit.add s[i]
+    inc i
+  result = i
+
+iterator tokenizeRequires*(s: string): string =
+  var start = 0
+  var tok = ""
+  while start < s.len:
+    tok.setLen 0
+    start = token(s, start, tok)
+    yield tok
+
+proc getRequires*(nimbleFileInfo: NimbleFileInfo): seq[PkgTuple] =
+  for require in nimbleFileInfo.requires:
+    result.add(parseRequires(require))
+
+proc toRequiresInfo*(pkgInfo: PackageInfo): PackageInfo =
+  let nimbleFileInfo = extractRequiresInfo(pkgInfo.myPath)
+  result = pkgInfo
+  result.requires = getRequires(nimbleFileInfo)
+  result.infoKind = pikRequires
+
+when isMainModule:
+  for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):
+    echo x
+