From 77c8795e5a45fa3fb78d52f28ad4b205d1fc7ebc Mon Sep 17 00:00:00 2001 From: Bilal2453 Date: Mon, 21 Feb 2022 23:20:48 +0300 Subject: [PATCH 1/5] implement installing over git --- libs/calculate-deps.lua | 211 ++++++++++++++++++++++++++++------------ libs/pkg.lua | 48 ++++++++- libs/rdb.lua | 9 +- 3 files changed, 204 insertions(+), 64 deletions(-) diff --git a/libs/calculate-deps.lua b/libs/calculate-deps.lua index 23346b92..e0d32101 100644 --- a/libs/calculate-deps.lua +++ b/libs/calculate-deps.lua @@ -16,81 +16,173 @@ limitations under the License. --]] -local normalize = require('semver').normalize local gte = require('semver').gte local log = require('log').log +local exec = require('exec') local queryDb = require('pkg').queryDb local colorize = require('pretty-print').colorize +local queryGit = require('pkg').queryGit +local normalize = require('semver').normalize -return function (db, deps, newDeps) - - local addDep, processDeps - - function processDeps(dependencies) - if not dependencies then return end - for alias, dep in pairs(dependencies) do - local name, version = dep:match("^([^@]+)@?(.*)$") - if #version == 0 then - version = nil - end - if type(alias) == "number" then - alias = name:match("([^/]+)$") - end - if not name:find("/") then - error("Package names must include owner/name at a minimum") - end - if version then - local ok - ok, version = pcall(normalize, version) - if not ok then - error("Invalid dependency version: " .. dep) - end - end - addDep(alias, name, version) +local gitSchemes = { + "^https?://", -- over http/s protocol + "^ssh://", -- over ssh protocol + "^git://", -- over git protocol + "^ftps?://", -- over ftp/s protocol + "^[^:]+:", -- over ssh protocol +} + +local processDeps +local db, deps + +local function isGit(dep) + for i = 1, #gitSchemes do + if dep:match(gitSchemes[i]) then + return true end end + return false +end + +local function resolveDep(alias, dep) + -- match for author/name@version + local name, version = dep:match("^([^@]+)@?(.*)$") + -- resolve alias name, in case it's a number (an array index) + if type(alias) == "number" then + alias = name:match("([^/]+)$") + end + + -- make sure owner is provided + if not name:find("/") then -- FIXME: this does match on `author/` or `/package` + error("Package names must include owner/name at a minimum") + end - function addDep(alias, name, version) - local meta = deps[alias] - if meta then - if name ~= meta.name then - local message = string.format("%s %s ~= %s", - alias, meta.name, name) - log("alias conflict", message, "failure") - return - end - if version then - if not gte(meta.version, version) then - local message = string.format("%s %s ~= %s", - alias, meta.version, version) - log("version conflict", message, "failure") - return - end - end + -- resolve version + if #version ~= 0 then + local ok + ok, version = pcall(normalize, version) + if not ok then + error("Invalid dependency version: " .. dep) + end + else + version = nil + end + + -- check for already installed packages + local meta = deps[alias] + if meta then + -- is there an alias conflict? + if name ~= meta.name then + local message = string.format("%s %s ~= %s", + alias, meta.name, name) + log("alias conflict", message, "failure") + -- is there a version conflict? + elseif version and not gte(meta.version, version) then + local message = string.format("%s %s ~= %s", + alias, meta.version, version) + log("version conflict", message, "failure") + -- re-process package dependencies if everything is ok else - local author, pname = name:match("^([^/]+)/(.*)$") - local match, hash = db.match(author, pname, version) - - if not match then - error("No such " - .. (version and "version" or "package") .. ": " - .. name - .. (version and '@' .. version or '')) - end - local kind - meta, kind, hash = assert(queryDb(db, hash)) - meta.db = db - meta.hash = hash - meta.kind = kind - deps[alias] = meta + processDeps(meta.dependencies) end + return + end - processDeps(meta.dependencies) + -- extract author and package names from "author/package" + -- and match against the local db for the resources + -- if not available locally, and an upstream is set, match the upstream db + local author, pname = name:match("^([^/]+)/(.*)$") + local match, hash = db.match(author, pname, version) + + -- no such package has been found locally nor upstream + if not match then + error("No such " + .. (version and "version" or "package") .. ": " + .. name + .. (version and '@' .. version or '')) + end + + -- query package metadata, and mark it for installation + local kind + meta, kind, hash = assert(queryDb(db, hash)) + meta.db = db + meta.hash = hash + meta.kind = kind + deps[alias] = meta + + -- handle the dependencies of the module + processDeps(meta.dependencies) +end +-- TODO: implement git protocol over https, to be used in case `git` cli isn't available +-- TODO: implement someway to specify a branch/tag when fetching +-- TODO: implement handling git submodules +local function resolveGitDep(url) + -- fetch the repo tree, don't include any tags + log("fetching", colorize("highlight", url)) + local _, stderr, code = exec("git", "fetch", "--no-tags", "--depth=1", "--recurse-submodules", url) + + -- was the fetch successful? + if code ~= 0 then + if stderr:match("^ENOENT") then + error("Cannot find git. Please make sure git is installed and available.") + else + error(stderr:gsub("\n$", "")) + end + end + + -- load the fetched module tree + local raw = db.storage.read("FETCH_HEAD") + local hash = raw:match("^(.-)\t\t.-\n$") + assert(hash and #hash ~= 0, "Attempt to retrive FETCH_HEAD") + hash = db.loadAs("commit", hash).tree + + -- query module's metadata, and match author/name + local meta, kind + meta, kind, hash = assert(queryGit(db, hash)) + local author, name = meta.name:match("^([^/]+)/(.*)$") + + -- check for installed packages and their version + local oldMeta = deps[name] + if oldMeta and not gte(oldMeta.version, meta.version) then + local message = string.format("%s %s ~= %s", + name, oldMeta.version, meta.version) + log("version conflict", message, "failure") + return + end + + -- create a ref/tags/author/name/version pointing to module's tree + db.write(author, name, meta.version, hash) + + -- mark the dep for installation + meta.db = db + meta.hash = hash + meta.kind = kind + deps[name] = meta + + -- handle the dependencies of the module + processDeps(meta.dependencies) +end + +function processDeps(dependencies) + if not dependencies then return end + -- iterate through dependencies and resolve each entry + for alias, dep in pairs(dependencies) do + if isGit(dep) then + resolveGitDep(dep) + else + resolveDep(alias, dep) + end end +end +return function (gitDb, depsMap, newDeps) + -- assign gitDb and depsMap to upvalue to be visible everywhere + -- then start processing newDeps + db, deps = gitDb, depsMap processDeps(newDeps) + -- collect all deps names and log them local names = {} for k in pairs(deps) do names[#names + 1] = k @@ -103,6 +195,5 @@ return function (db, deps, newDeps) colorize("highlight", name), meta.path or meta.version)) end - return deps end diff --git a/libs/pkg.lua b/libs/pkg.lua index 8081bfe8..9017962c 100644 --- a/libs/pkg.lua +++ b/libs/pkg.lua @@ -22,9 +22,10 @@ Package Metadata Commands These commands work with packages metadata. -pkg.query(fs, path) -> meta, path - Query an on-disk path for package info. -pkg.queryDb(db, path) -> meta, kind - Query an in-db hash for package info. -pky.normalize(meta) -> author, tag, version - Extract and normalize pkg info +pkg.query(fs, path) -> meta, path - Query an on-disk path for package info. +pkg.queryDb(db, path) -> meta, kind, hash - Query an in-db hash for package info. +plg.queryGit(db, path) -> meta, kind, hash - Query an in-db hash fetched with `git fetch` for package info. +pky.normalize(meta) -> author, tag, version - Extract and normalize pkg info ]] local isFile = require('git').modes.isFile @@ -167,6 +168,46 @@ local function queryDb(db, hash) return meta, kind, hash end +local function queryGit(db, hash) + local method = db.offlineLoadAny or db.load -- is rdb loaded? + local kind, value = method(hash) + if not kind then + error("Attempt to load the fetched tree") + elseif kind ~= "tree" then + error("Illegal kind: " .. kind) + end + + local tree = listToMap(value) + local path = "tree:" .. hash + local entry = tree["package.lua"] + if entry then + path = path .. "/package.lua" + elseif tree["init.lua"] then + entry = tree["init.lua"] + path = path .. "/init.lua" + else + -- check if the tree only contains a single lua file, and treat it as a package. + -- since in most git hosting services you won't have blob-pointing tag, + -- this has to make some assumption (or otherwise not support it) + -- in this case, it makes the assumption that a single-file package's repo + -- only has a single lua file + for name, meta in pairs(tree) do + if name:sub(-4) == ".lua" and isFile(meta.mode) then + if entry then -- it contains more than a single lua file + return nil, "ENOENT: No package.lua or init.lua in tree:" .. hash + end + entry = tree[name] + path = "blob:" .. entry.hash + kind = "blob" + hash = entry.hash + end + end + end + + local meta = evalModule(db.loadAs("blob", entry.hash), path) + return meta, kind, hash +end + local function normalize(meta) local author, tag = meta.name:match("^([^/]+)/(.*)$") return author, tag, semver.normalize(meta.version) @@ -176,5 +217,6 @@ end return { query = query, queryDb = queryDb, + queryGit = queryGit, normalize = normalize, } diff --git a/libs/rdb.lua b/libs/rdb.lua index d390e241..40c22159 100644 --- a/libs/rdb.lua +++ b/libs/rdb.lua @@ -24,7 +24,8 @@ local httpCodec = require('http-codec') local websocketCodec = require('websocket-codec') local makeRemote = require('codec').makeRemote local deframe = require('git').deframe -local decodeTag = require('git').decoders.tag +local decoders = require('git').decoders +local decodeTag = decoders.tag local verifySignature = require('verify-signature') local function connectRemote(url, timeout) @@ -141,6 +142,12 @@ return function(db, url, timeout) return assert(db.offlineLoad(hash)) end + function db.offlineLoadAny(hash) + local raw = assert(db.offlineLoad(hash), "no such hash") + local kind, value = deframe(raw) + return kind, decoders[kind](value) + end + function db.fetch(list) local refs = {} repeat From c9a3f79e61daa1b5c8e8c510ac4d0f34053e609d Mon Sep 17 00:00:00 2001 From: Bilal2453 Date: Tue, 22 Feb 2022 02:02:29 +0300 Subject: [PATCH 2/5] stop recursing into submodules --- libs/calculate-deps.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/calculate-deps.lua b/libs/calculate-deps.lua index e0d32101..60b3215b 100644 --- a/libs/calculate-deps.lua +++ b/libs/calculate-deps.lua @@ -116,11 +116,11 @@ end -- TODO: implement git protocol over https, to be used in case `git` cli isn't available -- TODO: implement someway to specify a branch/tag when fetching --- TODO: implement handling git submodules +-- TODO: implement handling git submodules, or shall we not? local function resolveGitDep(url) -- fetch the repo tree, don't include any tags log("fetching", colorize("highlight", url)) - local _, stderr, code = exec("git", "fetch", "--no-tags", "--depth=1", "--recurse-submodules", url) + local _, stderr, code = exec("git", "fetch", "--no-tags", "--depth=1", url) -- was the fetch successful? if code ~= 0 then From 78661f75adfa901c41cdf7a4caa1ec524fd44389 Mon Sep 17 00:00:00 2001 From: Bilal2453 Date: Tue, 22 Feb 2022 15:12:05 +0300 Subject: [PATCH 3/5] some style improvments --- libs/calculate-deps.lua | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/calculate-deps.lua b/libs/calculate-deps.lua index 60b3215b..9d39b462 100644 --- a/libs/calculate-deps.lua +++ b/libs/calculate-deps.lua @@ -24,7 +24,10 @@ local colorize = require('pretty-print').colorize local queryGit = require('pkg').queryGit local normalize = require('semver').normalize -local gitSchemes = { +local processDeps +local db, deps + +local GIT_SCHEMES = { "^https?://", -- over http/s protocol "^ssh://", -- over ssh protocol "^git://", -- over git protocol @@ -32,12 +35,9 @@ local gitSchemes = { "^[^:]+:", -- over ssh protocol } -local processDeps -local db, deps - local function isGit(dep) - for i = 1, #gitSchemes do - if dep:match(gitSchemes[i]) then + for i = 1, #GIT_SCHEMES do + if dep:match(GIT_SCHEMES[i]) then return true end end @@ -127,14 +127,14 @@ local function resolveGitDep(url) if stderr:match("^ENOENT") then error("Cannot find git. Please make sure git is installed and available.") else - error(stderr:gsub("\n$", "")) + error((stderr:gsub("\n$", ""))) end end -- load the fetched module tree local raw = db.storage.read("FETCH_HEAD") local hash = raw:match("^(.-)\t\t.-\n$") - assert(hash and #hash ~= 0, "Attempt to retrive FETCH_HEAD") + assert(hash and #hash ~= 0, "Unable to retrieve FETCH_HEAD\n" .. raw) hash = db.loadAs("commit", hash).tree -- query module's metadata, and match author/name From 88b825df8708fd6bafc9dd63ffa88f9a359014da Mon Sep 17 00:00:00 2001 From: Bilal2453 Date: Tue, 22 Feb 2022 18:51:59 +0300 Subject: [PATCH 4/5] properly set lit's git repo - using config.database for git fetch - fix an assert error message --- libs/calculate-deps.lua | 12 +++++++----- libs/core.lua | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/libs/calculate-deps.lua b/libs/calculate-deps.lua index 9d39b462..328d09b4 100644 --- a/libs/calculate-deps.lua +++ b/libs/calculate-deps.lua @@ -25,7 +25,7 @@ local queryGit = require('pkg').queryGit local normalize = require('semver').normalize local processDeps -local db, deps +local db, deps, configs local GIT_SCHEMES = { "^https?://", -- over http/s protocol @@ -120,7 +120,8 @@ end local function resolveGitDep(url) -- fetch the repo tree, don't include any tags log("fetching", colorize("highlight", url)) - local _, stderr, code = exec("git", "fetch", "--no-tags", "--depth=1", url) + local _, stderr, code = exec("git", "--git-dir=" .. configs.database, + "fetch", "--no-tags", "--depth=1", url) -- was the fetch successful? if code ~= 0 then @@ -139,7 +140,8 @@ local function resolveGitDep(url) -- query module's metadata, and match author/name local meta, kind - meta, kind, hash = assert(queryGit(db, hash)) + meta, kind, hash = queryGit(db, hash) + assert(meta, "Unable to find a valid package") local author, name = meta.name:match("^([^/]+)/(.*)$") -- check for installed packages and their version @@ -176,10 +178,10 @@ function processDeps(dependencies) end end -return function (gitDb, depsMap, newDeps) +return function (gitDb, depsMap, newDeps, core) -- assign gitDb and depsMap to upvalue to be visible everywhere -- then start processing newDeps - db, deps = gitDb, depsMap + db, deps, configs = gitDb, depsMap, core.config processDeps(newDeps) -- collect all deps names and log them diff --git a/libs/core.lua b/libs/core.lua index 7fecd129..2cfa5ca0 100644 --- a/libs/core.lua +++ b/libs/core.lua @@ -533,7 +533,7 @@ local function makeCore(config) function core.installList(path, newDeps) local deps = getInstalled(gfs, path) - calculateDeps(core.db, deps, newDeps) + calculateDeps(core.db, deps, newDeps, core) installDepsFs(core.db, gfs, path, deps, true) return deps end From d78089f4c10fabf0fb339204b2d19e3664ba572c Mon Sep 17 00:00:00 2001 From: Bilal2453 Date: Wed, 22 Jun 2022 19:14:03 +0300 Subject: [PATCH 5/5] fix not passing core to some calls --- libs/calculate-deps.lua | 16 ++++++++-------- libs/core.lua | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libs/calculate-deps.lua b/libs/calculate-deps.lua index 328d09b4..f7961c96 100644 --- a/libs/calculate-deps.lua +++ b/libs/calculate-deps.lua @@ -28,11 +28,11 @@ local processDeps local db, deps, configs local GIT_SCHEMES = { - "^https?://", -- over http/s protocol - "^ssh://", -- over ssh protocol - "^git://", -- over git protocol - "^ftps?://", -- over ftp/s protocol - "^[^:]+:", -- over ssh protocol + "^https?://", -- over http/s + "^ssh://", -- over ssh + "^git://", -- over git + "^ftps?://", -- over ftp/s + "^[^:]+:", -- over ssh } local function isGit(dep) @@ -178,10 +178,10 @@ function processDeps(dependencies) end end -return function (gitDb, depsMap, newDeps, core) - -- assign gitDb and depsMap to upvalue to be visible everywhere +return function (core, depsMap, newDeps) + -- assign gitDb and depsMap as upvalues to be visible everywhere -- then start processing newDeps - db, deps, configs = gitDb, depsMap, core.config + db, deps, configs = core.db, depsMap, core.config processDeps(newDeps) -- collect all deps names and log them diff --git a/libs/core.lua b/libs/core.lua index 2cfa5ca0..818552bc 100644 --- a/libs/core.lua +++ b/libs/core.lua @@ -148,7 +148,7 @@ local function makeCore(config) end if meta.dependencies and kind == "tree" then local deps = {} - calculateDeps(core.db, deps, meta.dependencies) + calculateDeps(core, deps, meta.dependencies) meta.snapshot = installDeps(core.db, hash, deps, false) log("snapshot hash", meta.snapshot) end @@ -424,7 +424,7 @@ local function makeCore(config) local kind, hash = assert(import(core.db, zfs, source, rules, true)) assert(kind == "tree", "Only tree packages are supported for now") local deps = getInstalled(zfs, source) - calculateDeps(core.db, deps, meta.dependencies) + calculateDeps(core, deps, meta.dependencies) hash = installDeps(core.db, hash, deps, true) return makeZip(hash, target, luvi_source) end @@ -491,7 +491,7 @@ local function makeCore(config) end local deps = {} - calculateDeps(core.db, deps, meta.dependencies) + calculateDeps(core, deps, meta.dependencies) local tagObj = db.loadAs("tag", hash) if tagObj.type ~= "tree" then error("Only tags pointing to trees are currently supported for make") @@ -533,7 +533,7 @@ local function makeCore(config) function core.installList(path, newDeps) local deps = getInstalled(gfs, path) - calculateDeps(core.db, deps, newDeps, core) + calculateDeps(core, deps, newDeps) installDepsFs(core.db, gfs, path, deps, true) return deps end