housekeeping.go

package main

import (
	"context"
	log "github.com/sirupsen/logrus"
	"os"
	"os/exec"
	"path/filepath"
	"somegit.dev/ALHP/ALHP.GO/ent"
	"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
	"strings"
	"sync"
	"time"
)

func housekeeping(ctx context.Context, repo, march string, wg *sync.WaitGroup) error {
	defer wg.Done()
	fullRepo := repo + "-" + march
	log.Debugf("[%s] start housekeeping", fullRepo)
	packages, err := Glob(filepath.Join(conf.Basedir.Repo, fullRepo, "/**/*.pkg.tar.zst"))
	if err != nil {
		return err
	}

	log.Debugf("[HK/%s] removing orphans, signature check", fullRepo)
	for _, path := range packages {
		mPackage := Package(path)

		dbPkg, err := mPackage.DBPackage(ctx, db)
		if ent.IsNotFound(err) {
			log.Infof("[HK] removing orphan %s->%s", fullRepo, filepath.Base(path))
			pkg := &ProtoPackage{
				FullRepo: *mPackage.FullRepo(),
				PkgFiles: []string{path},
				March:    *mPackage.MArch(),
			}
			buildManager.repoPurge[pkg.FullRepo] <- []*ProtoPackage{pkg}
			continue
		} else if err != nil {
			log.Warningf("[HK] error fetching %s->%q from db: %v", fullRepo, path, err)
			continue
		}

		pkg := &ProtoPackage{
			Pkgbase:   dbPkg.Pkgbase,
			Repo:      mPackage.Repo(),
			FullRepo:  *mPackage.FullRepo(),
			DBPackage: dbPkg,
			March:     *mPackage.MArch(),
			Arch:      *mPackage.Arch(),
		}

		// check if package is still part of repo
		dbs, err := alpmHandle.SyncDBs()
		if err != nil {
			return err
		}
		buildManager.alpmMutex.Lock()
		pkgResolved, err := dbs.FindSatisfier(mPackage.Name())
		buildManager.alpmMutex.Unlock()
		if err != nil ||
			pkgResolved.DB().Name() != pkg.DBPackage.Repository.String() ||
			pkgResolved.DB().Name() != pkg.Repo.String() ||
			pkgResolved.Architecture() != pkg.Arch ||
			pkgResolved.Name() != mPackage.Name() ||
			MatchGlobList(pkg.Pkgbase, conf.Blacklist.Packages) {
			switch {
			case err != nil:
				log.Infof("[HK] %s->%s not included in repo (resolve error: %v)", pkg.FullRepo, mPackage.Name(), err)
			case pkgResolved.DB().Name() != pkg.DBPackage.Repository.String():
				log.Infof("[HK] %s->%s not included in repo (repo mismatch: repo:%s != db:%s)", pkg.FullRepo,
					mPackage.Name(), pkgResolved.DB().Name(), pkg.DBPackage.Repository.String())
			case pkgResolved.DB().Name() != pkg.Repo.String():
				log.Infof("[HK] %s->%s not included in repo (repo mismatch: repo:%s != pkg:%s)", pkg.FullRepo,
					mPackage.Name(), pkgResolved.DB().Name(), pkg.Repo.String())
			case pkgResolved.Architecture() != pkg.Arch:
				log.Infof("[HK] %s->%s not included in repo (arch mismatch: repo:%s != pkg:%s)", pkg.FullRepo,
					mPackage.Name(), pkgResolved.Architecture(), pkg.Arch)
			case pkgResolved.Name() != mPackage.Name():
				log.Infof("[HK] %s->%s not included in repo (name mismatch: repo:%s != pkg:%s)", pkg.FullRepo,
					mPackage.Name(), pkgResolved.Name(), mPackage.Name())
			case MatchGlobList(pkg.Pkgbase, conf.Blacklist.Packages):
				log.Infof("[HK] %s->%s not included in repo (blacklisted pkgbase %s)", pkg.FullRepo, mPackage.Name(), pkg.Pkgbase)
			}

			// package not found on mirror/db -> not part of any repo anymore
			err = pkg.findPkgFiles()
			if err != nil {
				log.Errorf("[HK] %s->%s unable to get pkg-files: %v", pkg.FullRepo, mPackage.Name(), err)
				continue
			}
			err = db.DBPackage.DeleteOne(pkg.DBPackage).Exec(ctx)
			pkg.DBPackage = nil
			buildManager.repoPurge[pkg.FullRepo] <- []*ProtoPackage{pkg}
			if err != nil {
				return err
			}
			continue
		}

		if pkg.DBPackage.LastVerified.Before(pkg.DBPackage.BuildTimeStart) {
			err := pkg.DBPackage.Update().SetLastVerified(time.Now().UTC()).Exec(ctx)
			if err != nil {
				return err
			}
			// check if pkg signature is valid
			valid, err := mPackage.HasValidSignature()
			if err != nil {
				return err
			}
			if !valid {
				log.Infof("[HK] %s->%s invalid package signature", pkg.FullRepo, pkg.Pkgbase)
				buildManager.repoPurge[pkg.FullRepo] <- []*ProtoPackage{pkg}
				continue
			}
		}

		// compare db-version with repo version
		repoVer, err := pkg.repoVersion()
		if err == nil && repoVer != dbPkg.RepoVersion {
			log.Infof("[HK] %s->%s update repoVersion %s->%s", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer)
			pkg.DBPackage, err = pkg.DBPackage.Update().SetRepoVersion(repoVer).ClearTagRev().Save(context.Background())
			if err != nil {
				return err
			}
		}
	}

	// check all packages from db for existence
	dbPackages, err := db.DBPackage.Query().Where(
		dbpackage.And(
			dbpackage.RepositoryEQ(dbpackage.Repository(repo)),
			dbpackage.March(march),
		)).All(context.Background())
	if err != nil {
		return err
	}

	log.Debugf("[HK/%s] checking %d packages from database", fullRepo, len(dbPackages))

	for _, dbPkg := range dbPackages {
		pkg := &ProtoPackage{
			Pkgbase:   dbPkg.Pkgbase,
			Repo:      dbPkg.Repository,
			March:     dbPkg.March,
			FullRepo:  dbPkg.Repository.String() + "-" + dbPkg.March,
			DBPackage: dbPkg,
		}

		if !pkg.isAvailable(ctx, alpmHandle) {
			log.Infof("[HK] %s->%s not found on mirror, removing", pkg.FullRepo, pkg.Pkgbase)
			err = db.DBPackage.DeleteOne(dbPkg).Exec(context.Background())
			if err != nil {
				log.Errorf("[HK] error deleting package %s->%s: %v", pkg.FullRepo, dbPkg.Pkgbase, err)
			}
			continue
		}

		switch {
		case dbPkg.Status == dbpackage.StatusLatest && dbPkg.RepoVersion != "":
			// check lastVersionBuild
			if dbPkg.LastVersionBuild != dbPkg.RepoVersion {
				log.Infof("[HK] %s->%s updating lastVersionBuild %s -> %s", fullRepo, dbPkg.Pkgbase, dbPkg.LastVersionBuild, dbPkg.RepoVersion)
				nDBPkg, err := dbPkg.Update().SetLastVersionBuild(dbPkg.RepoVersion).Save(ctx)
				if err != nil {
					log.Warningf("[HK] error updating lastVersionBuild for %s->%s: %v", fullRepo, dbPkg.Pkgbase, err)
				} else {
					dbPkg = nDBPkg
				}
			}

			var existingSplits []string
			var missingSplits []string
			for _, splitPkg := range dbPkg.Packages {
				pkgFile := filepath.Join(conf.Basedir.Repo, fullRepo, "os", conf.Arch,
					splitPkg+"-"+dbPkg.RepoVersion+"-"+conf.Arch+".pkg.tar.zst")
				_, err = os.Stat(pkgFile)
				switch {
				case os.IsNotExist(err):
					missingSplits = append(missingSplits, splitPkg)
				case err != nil:
					log.Warningf("[HK] error reading package-file %s: %v", splitPkg, err)
				default:
					existingSplits = append(existingSplits, pkgFile)
				}
			}
			if len(missingSplits) > 0 {
				log.Infof("[HK] %s->%s missing split-package(s): %s", fullRepo, dbPkg.Pkgbase, missingSplits)
				pkg.DBPackage, err = pkg.DBPackage.Update().
					ClearRepoVersion().
					ClearTagRev().
					SetStatus(dbpackage.StatusQueued).
					Save(ctx)
				if err != nil {
					return err
				}

				pkg := &ProtoPackage{
					FullRepo:  fullRepo,
					PkgFiles:  existingSplits,
					March:     march,
					DBPackage: dbPkg,
				}
				buildManager.repoPurge[fullRepo] <- []*ProtoPackage{pkg}
			}

			rawState, err := os.ReadFile(filepath.Join(conf.Basedir.Work, stateDir, dbPkg.Repository.String()+"-"+conf.Arch, dbPkg.Pkgbase))
			if err != nil {
				log.Infof("[HK] state not found for %s->%s: %v, removing package", fullRepo, dbPkg.Pkgbase, err)
				pkg := &ProtoPackage{
					FullRepo:  fullRepo,
					PkgFiles:  existingSplits,
					March:     march,
					DBPackage: dbPkg,
				}
				buildManager.repoPurge[fullRepo] <- []*ProtoPackage{pkg}
				continue
			}

			state, err := parseState(string(rawState))
			if err != nil {
				log.Warningf("[HK] error parsing state file for %s->%s: %v", fullRepo, dbPkg.Pkgbase, err)
				continue
			}

			if dbPkg.TagRev != nil && state.TagRev == *dbPkg.TagRev && state.PkgVer != dbPkg.Version {
				log.Infof("[HK] reseting package %s->%s with mismatched state information (%s!=%s)",
					fullRepo, dbPkg.Pkgbase, state.PkgVer, dbPkg.Version)
				err = dbPkg.Update().SetStatus(dbpackage.StatusQueued).ClearTagRev().Exec(ctx)
				if err != nil {
					return err
				}
			}
		case dbPkg.Status == dbpackage.StatusLatest && dbPkg.RepoVersion == "":
			log.Infof("[HK] reseting missing package %s->%s with no repo version", fullRepo, dbPkg.Pkgbase)
			err = dbPkg.Update().SetStatus(dbpackage.StatusQueued).ClearTagRev().ClearRepoVersion().Exec(ctx)
			if err != nil {
				return err
			}
		case dbPkg.Status == dbpackage.StatusSkipped && dbPkg.RepoVersion != "" && !strings.HasPrefix(dbPkg.SkipReason, "delayed"):
			log.Infof("[HK] delete skipped package %s->%s", fullRepo, dbPkg.Pkgbase)
			pkg := &ProtoPackage{
				FullRepo:  fullRepo,
				March:     march,
				DBPackage: dbPkg,
			}
			buildManager.repoPurge[fullRepo] <- []*ProtoPackage{pkg}
		case dbPkg.Status == dbpackage.StatusSkipped && dbPkg.SkipReason == "blacklisted" && !MatchGlobList(pkg.Pkgbase, conf.Blacklist.Packages):
			log.Infof("[HK] requeue previously blacklisted package %s->%s", fullRepo, dbPkg.Pkgbase)
			err = dbPkg.Update().SetStatus(dbpackage.StatusQueued).ClearSkipReason().ClearTagRev().Exec(context.Background())
			if err != nil {
				return err
			}
		case dbPkg.Status == dbpackage.StatusFailed && dbPkg.RepoVersion != "":
			log.Infof("[HK] package %s->%s failed but still present in repo, removing", fullRepo, dbPkg.Pkgbase)
			pkg := &ProtoPackage{
				FullRepo:  fullRepo,
				March:     march,
				DBPackage: dbPkg,
			}
			buildManager.repoPurge[fullRepo] <- []*ProtoPackage{pkg}
		}
	}

	log.Debugf("[HK/%s] all tasks finished", fullRepo)
	return nil
}

func logHK(ctx context.Context) error {
	// check if package for log exists and if error can be fixed by rebuild
	logFiles, err := Glob(filepath.Join(conf.Basedir.Repo, logDir, "/**/*.log"))
	if err != nil {
		return err
	}

	for _, logFile := range logFiles {
		pathSplit := strings.Split(logFile, string(filepath.Separator))
		extSplit := strings.Split(filepath.Base(logFile), ".")
		pkgbase := strings.Join(extSplit[:len(extSplit)-1], ".")
		march := pathSplit[len(pathSplit)-2]

		pkg := ProtoPackage{
			Pkgbase: pkgbase,
			March:   march,
		}

		if exists, err := pkg.exists(); err != nil {
			return err
		} else if !exists {
			_ = os.Remove(logFile)
			continue
		}

		pkgSkipped, err := db.DBPackage.Query().Where(
			dbpackage.Pkgbase(pkg.Pkgbase),
			dbpackage.March(pkg.March),
			dbpackage.StatusEQ(dbpackage.StatusSkipped),
		).Exist(ctx)
		if err != nil {
			return err
		}

		if pkgSkipped {
			_ = os.Remove(logFile)
			continue
		}

		logContent, err := os.ReadFile(logFile)
		if err != nil {
			return err
		}
		sLogContent := string(logContent)

		if rePortError.MatchString(sLogContent) || reSigError.MatchString(sLogContent) || reDownloadError.MatchString(sLogContent) ||
			reDownloadError2.MatchString(sLogContent) {
			rows, err := db.DBPackage.Update().Where(dbpackage.Pkgbase(pkg.Pkgbase), dbpackage.March(pkg.March),
				dbpackage.StatusEQ(dbpackage.StatusFailed)).ClearTagRev().SetStatus(dbpackage.StatusQueued).Save(ctx)
			if err != nil {
				return err
			}

			if rows > 0 {
				log.Infof("[HK/%s/%s] fixable build-error detected, requeueing package (%d)", pkg.March, pkg.Pkgbase, rows)
			}
		} else if reLdError.MatchString(sLogContent) || reRustLTOError.MatchString(sLogContent) {
			rows, err := db.DBPackage.Update().Where(
				dbpackage.Pkgbase(pkg.Pkgbase),
				dbpackage.March(pkg.March),
				dbpackage.StatusEQ(dbpackage.StatusFailed),
				dbpackage.LtoNotIn(dbpackage.LtoAutoDisabled, dbpackage.LtoDisabled),
			).ClearTagRev().SetStatus(dbpackage.StatusQueued).SetLto(dbpackage.LtoAutoDisabled).Save(ctx)
			if err != nil {
				return err
			}

			if rows > 0 {
				log.Infof("[HK/%s/%s] fixable build-error detected (linker-error), requeueing package (%d)", pkg.March, pkg.Pkgbase, rows)
			}
		}
	}
	return nil
}

func debugHK() {
	for _, march := range conf.March {
		if _, err := os.Stat(filepath.Join(conf.Basedir.Debug, march)); err == nil {
			log.Debugf("[DHK/%s] start cleanup debug packages", march)
			cleanCmd := exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Debug, march), "-k", "1")
			res, err := cleanCmd.CombinedOutput()
			if err != nil {
				log.Warningf("[DHK/%s] cleanup debug packages failed: %v (%s)", march, err, string(res))
			}
		}
	}
}