mk-agents-shar

#!/bin/bash
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

#
# Copyright 2020 Joyent, Inc.
# Copyright 2024 MNX Cloud, Inc.
#

#
# Make an agents shar (agents-$BUILDSTAMP.sh) package.
#
# Examples:
#
#    # Build name "release-20110714", includes all agents by default.
#    # Source dir (-d) is a local path or Manta path.
#    bin/mk-agents-shar -b release-20110714 -d /Joyent_Dev/public/builds
#
#    # Using built agent packages from a local "bits" dir.
#    bin/mk-agents-shar -b release-20110714 -d ../../bits
#
#    If $ENGBLD_DEST_OUT_PATH is set in the environment, that will be
#    used unless a -d option is passed.
#

if [ "$TRACE" != "" ]; then
    export PS4='${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
    set -o xtrace
fi
set -o errexit
set -o pipefail



#---- config/globals

TOP=$(cd $(dirname $0)/; pwd)

# Put the Manta tools that 'make deps' installed for us first, ensuring
# we have a compatible version of node.
export PATH="${TOP}/node_modules/manta/bin:${PATH}"


# --- Manta config

if [[ -z "$MANTA_KEY_ID" ]]; then
    export MANTA_KEY_ID=$(ssh-keygen -l -f ~/.ssh/id_rsa.pub | awk '{print $2}' | tr -d '\n')
fi
if [[ -z "$MANTA_USER" ]]; then
    export MANTA_USER="Joyent_Dev";
fi
export MANTA_URL=https://us-central.manta.mnx.io
export MANTA_BASE="/Joyent_Dev/public/builds"

[[ -n ${MANTA_KEY_ID} ]] || fatal "missing MANTA_KEY_ID"
[[ -n ${MANTA_USER} ]] || fatal "missing MANTA_USER"
[[ -n ${MANTA_URL} ]] || fatal "missing MANTA_URL"
[[ -n ${MANTA_BASE} ]] || fatal "missing MANTA_BASE"

if [[ -n "$ENGBLD_DEST_OUT_PATH" ]]; then
	DEFAULT_SOURCE_DIR=${ENGBLD_DEST_OUT_PATH}
else
	DEFAULT_SOURCE_DIR=${MANTA_BASE}
fi
ALL_AGENTS=$(cat agents.json | json -a | xargs)

BUILD_DIR=build

if [[ -z "$TIMESTAMP" ]]; then
    TIMESTAMP=$(date -u "+%Y%m%dT%H%M%SZ")
fi
# Need GNU awk for multi-char arg to "-F".
AWK=$( (which gawk 2>/dev/null | grep -v "^no ") || which awk)
GITDESCRIBE=g$(git describe --all --long --dirty | ${AWK} -F'-g' '{print $NF}')

SHAR=`which shar`

#---- functions

function fatal {
    echo "$(basename $0): error: $1"
    exit 1
}

function usage() {
    if [[ -n "$1" ]]; then
        echo "error: $1"
        echo ""
    fi
    echo "Usage:"
    echo "  mk-agents-shar [OPTIONS] [AGENTS...]"
    echo ""
    echo "Options:"
    echo "  -h      print this help and exit"
    echo "  -b      build name(s), e.g. 'master', 'release-20110714'"
    echo "          Typically the name of the repo branch. Note this can"
    echo "          be multiple space-separated build names (e.g. "
    echo "          'feature1 master'). Each name is attempted to find an "
    echo "          agent package for each agent."
    echo "  -B      branch name to use for the generated shar"
    echo "  -d      source dir, a local path or URL"
    echo "          (default is <$DEFAULT_SOURCE_DIR> (uses \$ENGBLD_DEST_OUT_PATH if set, otherwise $MANTA_BASE)"
    echo "  -o      output directory to which to write results, default is 'build'"
    echo ""
    echo "Build an agents shar. Without any given 'AGENTS' strings all agents"
    echo "are included."
}


# Fetch the latest agent package matching the given name and buildname.
#
# Here "latest" is just a lexographical sort, so we are relying on agent
# package names to be sortable (currently that is acheived by using a
# ISO timestamp in the package names).
function fetch_agent_package() {
    local name=$1         # e.g. "cn-agent/cn-agent" or "config-agent/config-agent-pkg"
    local build_names=$2  # e.g. "release-20110901" or "master"
                          #     or "feature1 master" to try a feature1 branch
                          #     with master fallback if not available
    local from_dir=$3     # See example below.
    local to_dir=$4
    local success=
    local dir=$(dirname $name)

    set +o errexit
    for one_build_name in $build_names; do
        local base_pattern="^$(basename $name)-$one_build_name-.*"

        #
        # Assume bits are first in "Manta layout" and look for them if
        # the local directory does not exist. This looks like e.g.
        # <public/builds>/cn-agent/master-20180822T140522Z-g378f995/cn-agent/cn-agent-master-20180821T140729Z-g792ebd2.tgz
        #
        if [[ ! -d ${from_dir} ]]; then
            # E.g. "cn-agent/cn-agent-master-20181121T204600Z-g1df1666.tgz"
            local scope=${name%%/*}    # "a/b/c" -> "a"
            local latest_dir=$(mget -q ${from_dir}/${scope}/${one_build_name}-latest)

            if [[ -z ${latest_dir} ]]; then
                echo "WARN: could not find '${one_build_name}-latest' in Manta"
            else
                local latest_name=$(mls ${latest_dir}/${dir} \
                    | grep "${base_pattern}" \
                    | sort | tail -1
                )
                if [[ -z "${latest_name}" ]]; then
                    echo "WARN: could not find '${base_pattern}' in '${url_dir}'"
                else
                    echo "Downloading '${latest_dir}/${dir}/${latest_name}' to '${to_dir}'."
                    mget -q -o ${to_dir}/${latest_name} ${latest_dir}/${dir}/${latest_name}
                    if [[ $? == 0 ]]; then
                        success=1
                        break
                    fi
                fi
            fi
        else
            #
            # We have two local directory layouts to choose from, for compatibility
            # reasons:
            # 1. Mountain gorilla layout:
            #
            # Local bits dir 'from_dir' example:
            #   /home/jill/joy/mountain-gorilla/bits
            # where name='cn-agent/cn-agent' is at:
            #   /home/jill/joy/mountain-gorilla/bits/cn-agent/cn-agent-$branch-*
            # where name='agents/metadata/metadata' is at:
            #   /home/jill/joy/mountain-gorilla/bits/agents/metadata/metadata-$branch-*
            #
            # 2. Manta layout:
            #    (see above)

            # Try the first layout
            local latest_name=$(ls -1 ${from_dir}/${dir}/ | grep "${base_pattern}" | sort | tail -1)
            local latest_path="${from_dir}/${dir}/${latest_name}"
            if [[ -z "${latest_name}" ]]; then
                # Try the second layout
                scope=${name%%/*}
                latest_name=$(ls -H1 ${from_dir}/${scope}/${one_build_name}-latest/${scope} | grep "${base_pattern}" | sort | tail -1)
                if [[ -n "$latest_name" ]]; then
                    latest_dir=${from_dir}/${scope}/${one_build_name}-latest/${scope}
                fi
                latest_path=${latest_dir}/${latest_name}
            fi

            if [[ -z "${latest_name}" ]]; then
                echo "WARN: could not find '${base_pattern}' in '${from_dir}/${dir}'"
                echo "      also tried ${from_dir}/${scope}/${one_build_name}-latest"
            else
                echo "Copying '${latest_path}' to '${to_dir}'."
                cp ${latest_path} ${to_dir}
                success=1
                break
            fi
        fi
    done
    set -o errexit

    if [[ -z "$success" ]]; then
        fatal "could not find '$name' agent package with build name '$build_names' in '$from_dir'"
    fi
}

function mk_shar_manifest() {
    local release_manifest=$1

    local name=agentsshar
    local stage_dir=/tmp
    local stamp=${BRANCH}-${TIMESTAMP}-${GITDESCRIBE}
    local release_file
    release_file=$(basename $(ls ${TOP}/build/agents-*.sh | head -n 1))

    mkdir -p ${stage_dir}/${name}
    uuid -v4 > ${stage_dir}/${name}/image_uuid
    cat ${TOP}/manifest.tmpl | sed \
            -e "s/UUID/$(cat ${stage_dir}/${name}/image_uuid)/" \
            -e "s/NAME/$(json name < ${TOP}/package.json)/" \
            -e "s/VERSION/$(json version < ${TOP}/package.json)/" \
            -e "s/BUILDSTAMP/${stamp}/g" \
            -e "s/SIZE/$(stat --printf="%s" ${TOP}/build/${release_file})/" \
            -e "s/SHA/$(openssl sha1 ${TOP}/build/${release_file} \
                | cut -d ' ' -f2)/" \
            > ${release_manifest}
}


function mk_agents_shar() {
    local agents="$1"
    local build_names="$2"
    local source_dir="$3"
    local output_dir="$4"
    echo "Make '$build_names' agents shar (timestamp $TIMESTAMP) from '${source_dir}'."

    # Get all the agents files and boilerplate into the build dir.
    local build_dir=$BUILD_DIR/agents
    rm -rf $build_dir
    mkdir -p $build_dir
    cp install.sh $build_dir
    for agent in $agents; do
        fetch_agent_package "$agent" "$build_names" "$source_dir" "$build_dir"
    done

    # Create the shar.
    local tmp=/var/tmp
    local sh_filename=agents-$BRANCH-$TIMESTAMP-$GITDESCRIBE.sh
    cd $BUILD_DIR

    (cat <<__EOF__
#!/bin/sh

if [[ -n \$AGENTSSHAR_UNPACK_DIR ]]; then
    mkdir -p \$AGENTSSHAR_UNPACK_DIR
    cd \$AGENTSSHAR_UNPACK_DIR
elif [[ \`pwd\` != '$tmp' ]]; then
    cd $tmp
fi

# Clean up possible previous runs
rm -fr agents

__EOF__
)> $sh_filename

    (${SHAR} -D -n "Joyent" agents | grep -v '^exit 0'; cat <<EOF

set -o errexit
set -o pipefail

if [[ \$AGENTSSHAR_UNPACK_ONLY == "true" ]]; then
    exit 0
fi

if [[ -f agents/install.sh ]]; then
    (cd agents && /bin/bash ./install.sh)
fi

# Delete agents directory.
rm -fr agents

exit 0
EOF
)>> $sh_filename

    # Create the md5sums file.
    local md5sum_filename=agents-$BRANCH-$TIMESTAMP-$GITDESCRIBE.md5sum
    openssl dgst -md5 $sh_filename | cut -d' ' -f 2 > $md5sum_filename

    local manifest_filename=agents-$BRANCH-$TIMESTAMP-$GITDESCRIBE.manifest
    mk_shar_manifest $manifest_filename

    # Copy bits to output dir, if necessary.
    cd $TOP
    if [[ "$output_dir" != "$BUILD_DIR" ]]; then
        cp $BUILD_DIR/$sh_filename $output_dir/
        cp $BUILD_DIR/$md5sum_filename $output_dir/
        cp $BUILD_DIR/$manifest_filename $output_dir/
    fi
    echo "Created '$output_dir/$sh_filename'."
    echo "Created '$output_dir/$md5sum_filename'."
    echo "Created '$output_dir/$manifest_filename'."
}



#---- mainline

# Process arguments.
source_dir=$DEFAULT_SOURCE_DIR
output_dir=$BUILD_DIR
while getopts "hb:B:d:o:" c; do
    case "$c" in
    h)
        usage
        exit 0
        ;;
    b)
        # Remove any PRs from build_names. Whatever matching PR found for
        # various agents will almost certainly not match the build we're making
        # here and could very well be dangerous.
        build_names=${OPTARG//PR-*/master}
        ;;
    B)
        BRANCH=$OPTARG
        ;;
    d)
        source_dir=$OPTARG
        ;;
    o)
        output_dir=$OPTARG
        ;;
    *)
        usage "illegal option -- $OPTARG"
        exit 1
        ;;
    esac
done
shift $((OPTIND - 1))

if [[ -z "$build_names" ]]; then
    echo "ERROR: build_names (-b) is required"
    usage
    exit 1
fi

if [[ -z "$BRANCH" ]]; then
    echo "ERROR: \$BRANCH should be set in the environment, or passed with -B"
    usage
    exit 1
fi

agents="$*"
if [[ -z "$agents" ]]; then
    agents="$ALL_AGENTS"
fi

mk_agents_shar "$agents" "$build_names" "$source_dir" "$output_dir"
exit $?