cmd-fetch

#!/usr/bin/env bash
set -euo pipefail

dn=$(dirname "$0")
# shellcheck source=src/cmdlib.sh
. "${dn}"/cmdlib.sh

FILE=cache/pkgcache-repo
if [ -d "${FILE}" ]
then
        pkgcachesize=$(sudo du --bytes --max-depth 0 "${FILE}" \
                       | awk '{print $1; exit}')
        pkglimit=$((1024 * 1024 * 1024 * 5))
        if [[ "${pkgcachesize}" -gt "${pkglimit}" ]]
        then
                sudo cosa prune --pkgcache
        fi
fi

print_help() {
    cat 1>&2 <<'EOF'
Usage: coreos-assembler fetch --help
       coreos-assembler fetch [OPTIONS]...

  Fetch and import the latest packages.

  The following options are supported:

  --force                  Redownload packages even if nothing changed since last build
  --strict                 Only download locked packages when using lockfiles
  --update-lockfile        Update base lockfile to latest packages
  --write-lockfile-to=FILE Write updated base lockfile to separate file
  --with-cosa-overrides    Don't ignore cosa overrides in `overrides/rpm`
  --autolock=VERSION       If no base lockfile used, create one from any arch build of `VERSION`

EOF
}

AUTOLOCK_VERSION=
UPDATE_LOCKFILE=
OUTPUT_LOCKFILE=
IGNORE_COSA_OVERRIDES_ARG=--ignore-cosa-overrides
DRY_RUN=
FORCE_ARG=
STRICT=
rc=0
options=$(getopt --options h --longoptions help,update-lockfile,dry-run,with-cosa-overrides,write-lockfile-to:,strict,force,autolock: -- "$@") || rc=$?
[ $rc -eq 0 ] || {
    print_help
    exit 1
}
eval set -- "$options"
while true; do
    case "$1" in
        -h | --help)
            print_help
            exit 0
            ;;
        --update-lockfile)
            UPDATE_LOCKFILE=1
            ;;
        --write-lockfile-to)
            shift;
            UPDATE_LOCKFILE=1
            OUTPUT_LOCKFILE=$1
            ;;
        --with-cosa-overrides)
            IGNORE_COSA_OVERRIDES_ARG=
            ;;
        --dry-run)
            DRY_RUN=1
            ;;
        --strict)
            STRICT=1
            ;;
        --force)
            FORCE_ARG=--force-nocache
            ;;
        --autolock)
            shift;
            AUTOLOCK_VERSION=$1
            ;;
        --)
            shift
            break
            ;;
        *)
            fatal "$0: unrecognized option: $1"
            ;;
    esac
    shift
done

if [ $# -ne 0 ]; then
    print_help
    fatal "ERROR: Too many arguments"
fi

prepare_build

lock_args=
extra_args=

# Apply autolock from another build for this version if no base lockfile
# exists. Do this before so that overrides come after.
if  [ -n "${AUTOLOCK_VERSION}" ]; then
    if [ -f "${manifest_lock}" ]; then
        fatal "ERROR: Requested --autolock, but base lockfile found"
    fi
    autolockfile=$(generate_autolock "${AUTOLOCK_VERSION}")
    if [ -z "${autolockfile}" ]; then
        fatal "ERROR: Requested --autolock, but no generated lockfile found"
    fi
    lock_args+=" --ex-lockfile=${autolockfile}"
fi

if [ -n "${UPDATE_LOCKFILE}" ]; then
    # Put this under tmprepo so it gets automatically chown'ed if needed
    extra_args+=" --ex-write-lockfile-to=${tmprepo}/tmp/manifest-lock.json"
    # Include the overrides in the resulting lockfile here; otherwise, we
    # might not even be able to get a depsolve solely from the non-lockfile
    # repos.
    for lock in "${manifest_lock_overrides}" "${manifest_lock_arch_overrides}"; do
      if [ -f "${lock}" ]; then
          lock_args+=" --ex-lockfile=${lock}"
      fi
    done
else
    for lock in "${manifest_lock}" "${manifest_lock_overrides}" "${manifest_lock_arch_overrides}"; do
        if [ -f "${lock}" ]; then
            lock_args+=" --ex-lockfile=${lock}"
        fi
    done
fi

if [ -n "${DRY_RUN}" ]; then
    extra_args+=" --dry-run"
fi

if [ -n "${STRICT}" ]; then
    if [ -n "${AUTOLOCK_VERSION}" ]; then
        # They're theoretically independent, but in practice it's unlikely
        # that an autolockfile will include all the packages needed to satisfy
        # --strict. So let's just catch it early.
        fatal "ERROR: Can't use --autolock and --strict together"
    fi
    extra_args+=" --ex-lockfile-strict"
fi

# By default, we ignore cosa overrides since they're temporary. With
# --with-cosa-overrides, we don't ignore them (and thus don't need to fetch any
# overridden packages). Disable SC2086 since we don't want an extra arg in the
# empty case.
# shellcheck disable=SC2086
prepare_compose_overlays ${IGNORE_COSA_OVERRIDES_ARG}

# shellcheck disable=SC2086
runcompose_tree --download-only ${lock_args} ${extra_args} ${FORCE_ARG}
# This stamp file signifies we successfully fetched once; it's
# validated in cmd-build.
touch "${fetch_stamp}"

if [ -n "${UPDATE_LOCKFILE}" ]; then
    # Write out to the lockfile specified by the user or to the
    # existing manifest lockfile if none was specified by the user
    outfile=${OUTPUT_LOCKFILE:-${manifest_lock}}
    strip_out_lockfile_digests "${tmprepo}/tmp/manifest-lock.json"
    # cd back to workdir in case OUTPUT_LOCKFILE is relative
    (cd "${workdir}" && mv -f "${tmprepo}/tmp/manifest-lock.json" "${outfile}")
    echo "Wrote out lockfile ${outfile}"
fi