Cirrus: Run checks directly on the host

In order to meet achievable deadlines converting from Travis to Cirrus
CI, one significant artifact was carried forward (instead of fixing):

Depending on a `--privileged` container to execute all/most automated
checks/tests.

Prior attempts to remove this aspect resulted in several test failures.
Fixing the problems was viewed as more time-consuming than simply
preserving this runtime environment.

Time has passed, and the code has since moved on.  This commit removes
the legacy need to execute most operations in a `--privileged`
container, instead running them directly on the host.  This should
greatly simplify debugging of code and automation issues while not
precluding (future) additional container-based testing.

In short, we should verify and test the project "works" on the host
first.  Then, secondarily (if needed) confirm the same inside a
container environment.

Signed-off-by: Chris Evich <cevich@redhat.com>

.cirrus.yml

@@ -41,9 +41,6 @@ env:
     UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX}"
     PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX}"
 
-    # Equivilent to image produced by 'make build-container'
-    SKOPEO_CI_CONTAINER_FQIN: "quay.io/skopeo/ci:${DEST_BRANCH}"
-
 
 # Default timeout for each task
 timeout_in: 45m
@@ -57,8 +54,8 @@ validate_task:
     # under Cirrus-CI, due to challenges obtaining the starting commit ID.
     # Only do validation for PRs.
     only_if: $CIRRUS_PR != ''
-    container:  &build_container
-        image: "${SKOPEO_CI_CONTAINER_FQIN}"
+    container:
+        image: '${FEDORA_CONTAINER_FQIN}'
         cpu: 4
         memory: 8
     script: make validate-local

Dockerfile

@@ -1,56 +1,23 @@
 FROM registry.fedoraproject.org/fedora:latest
 
-RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-md2man \
-	# storage deps
-	btrfs-progs-devel \
-	device-mapper-devel \
-	# gpgme bindings deps
-	libassuan-devel gpgme-devel \
-	gnupg \
-	# htpasswd for system tests
-	httpd-tools \
-	# OpenShift deps
-	which tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \
-        bats jq podman runc \
-	golint \
-	openssl \
-	&& dnf clean all
-
-# Install two versions of the registry. The first is an older version that
-# only supports schema1 manifests. The second is a newer version that supports
-# both. This allows integration-cli tests to cover push/pull with both schema1
-# and schema2 manifests.
-RUN set -x \
-	&& export GO111MODULE=off \
-	&& REGISTRY_COMMIT_SCHEMA1=ec87e9b6971d831f0eff752ddb54fb64693e51cd \
-	&& REGISTRY_COMMIT=47a064d4195a9b56133891bbb13620c3ac83a827 \
-	&& export GOPATH="$(mktemp -d)" \
-	&& git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \
-	&& (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \
-	&& GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \
-		go build -o /usr/local/bin/registry-v2 github.com/docker/distribution/cmd/registry \
-	&& (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1") \
-	&& GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \
-		go build -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry \
-	&& rm -rf "$GOPATH"
+ENV GOPATH="/var/tmp/go"
+ENV GOSRC="$GOPATH/src/github.com/containers/skopeo"
+ENV PATH="$GOPATH/bin:$GOSRC/bin:/usr/share/gocode/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin"
 
-RUN set -x \
-	&& export GO111MODULE=off \
-	&& export GOPATH=$(mktemp -d) \
-	&& git clone --depth 1 -b v1.5.0-alpha.3 git://github.com/openshift/origin "$GOPATH/src/github.com/openshift/origin" \
-	# The sed edits out a "go < 1.5" check which works incorrectly with go ≥ 1.10. \
-	&& sed -i -e 's/\[\[ "\${go_version\[2]}" < "go1.5" ]]/false/' "$GOPATH/src/github.com/openshift/origin/hack/common.sh" \
-	&& (cd "$GOPATH/src/github.com/openshift/origin" && make clean build && make all WHAT=cmd/dockerregistry) \
-	&& cp -a "$GOPATH/src/github.com/openshift/origin/_output/local/bin/linux"/*/* /usr/local/bin \
-	&& cp "$GOPATH/src/github.com/openshift/origin/images/dockerregistry/config.yml" /atomic-registry-config.yml \
-	&& rm -rf "$GOPATH" \
-	&& mkdir /registry
-
-ENV GOPATH /usr/share/gocode:/go
-ENV PATH $GOPATH/bin:/usr/share/gocode/bin:$PATH
-ENV container_magic 85531765-346b-4316-bdb8-358e4cca9e5d
-RUN go version
-WORKDIR /go/src/github.com/containers/skopeo
-COPY . /go/src/github.com/containers/skopeo
+RUN dnf -y update && \
+	dnf -y install \
+		make git golang golang-github-cpuguy83-md2man \
+		btrfs-progs-devel device-mapper-devel \
+		libassuan-devel gpgme-devel gnupg \
+		httpd-tools which tar wget hostname util-linux \
+		bsdtar socat ethtool device-mapper iptables \
+		tree findutils nmap-ncat e2fsprogs xfsprogs lsof \
+		docker iproute bats jq podman runc golint \
+		openssl \
+	&& dnf -y upgrade \
+	&& dnf erase -y skopeo \
+	&& dnf clean all
 
-#ENTRYPOINT ["hack/dind"]
+COPY . $GOSRC
+WORKDIR $GOSRC
+RUN bash ./hack/test_env_setup.sh

contrib/cirrus/runner.sh

@@ -25,48 +25,39 @@ OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 
 export "PATH=$PATH:$GOPATH/bin"
 
-podmanmake() {
-    req_env_vars GOPATH SKOPEO_PATH SKOPEO_CI_CONTAINER_FQIN
-    warn "Accumulated technical-debt requires execution inside a --privileged container.  This is very likely hiding bugs!"
-    showrun podman run -it --rm --privileged \
-        -e GOPATH=$GOPATH \
-        -v $GOPATH:$GOPATH:Z \
-        -w $SKOPEO_PATH \
-        $SKOPEO_CI_CONTAINER_FQIN \
-            make "$@"
-}
-
 _run_setup() {
     if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
         # This is required as part of the standard Fedora VM setup
         growpart /dev/sda 1
         resize2fs /dev/sda1
 
-        # VM's come with the distro. skopeo pre-installed
+        # VM's come with the distro. skopeo package pre-installed
         dnf erase -y skopeo
+
+        bash ./hack/test_env_setup.sh
     else
         die "Unknown/unsupported distro. $OS_REL_VER"
     fi
 }
 
 _run_vendor() {
-    podmanmake vendor BUILDTAGS="$BUILDTAGS"
+    make vendor BUILDTAGS="$BUILDTAGS"
 }
 
 _run_build() {
-    podmanmake bin/skopeo BUILDTAGS="$BUILDTAGS"
+    make bin/skopeo BUILDTAGS="$BUILDTAGS"
 }
 
 _run_validate() {
-    podmanmake validate-local BUILDTAGS="$BUILDTAGS"
+    make validate-local BUILDTAGS="$BUILDTAGS"
 }
 
 _run_unit() {
-    podmanmake test-unit-local BUILDTAGS="$BUILDTAGS"
+    make test-unit-local BUILDTAGS="$BUILDTAGS"
 }
 
 _run_integration() {
-    podmanmake test-integration-local BUILDTAGS="$BUILDTAGS"
+    make test-integration-local BUILDTAGS="$BUILDTAGS"
 }
 
 _run_system() {

hack/make.sh

@@ -23,22 +23,9 @@ export SKOPEO_PKG='github.com/containers/skopeo'
 export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export MAKEDIR="$SCRIPTDIR/make"
 
-# We're a nice, sexy, little shell script, and people might try to run us;
-# but really, they shouldn't. We want to be in a container!
-# The magic value is defined inside our Dockerfile.
-if [[ "$container_magic" != "85531765-346b-4316-bdb8-358e4cca9e5d" ]]; then
-	{
-		echo "# WARNING! I don't seem to be running in a Docker container."
-		echo "# The result of this command might be an incorrect build, and will not be"
-		echo "# officially supported."
-		echo "#"
-		echo "# Try this instead: make all"
-		echo "#"
-	} >&2
-else
-    echo "# I appear to be running inside my designated container image, good!"
-    export SKOPEO_CONTAINER_TESTS=1
-fi
+# Override this to "0" to disable tests which may fail w/o
+# having applied hack/test_env_setup.sh
+export SKOPEO_CONTAINER_TESTS=${SKOPEO_CONTAINER_TESTS:-1}
 
 echo
 

hack/test_env_setup.sh

@@ -0,0 +1,60 @@
+
+
+# This script is used by both Dockerfile and CI, in order to configure
+# Fedora environment to execute the unit and integration tests.
+# It should never ever ever be used under any other circumstances
+# (nor set as executable).
+
+# Install three registry server versions. The first is an older version that
+# only supports schema1 manifests. The second is a newer version that supports
+# both.  The third is an ancient version from OpenShift Origin.
+REG_REPO="https://github.com/docker/distribution.git"
+REG_COMMIT="47a064d4195a9b56133891bbb13620c3ac83a827"
+REG_COMMIT_SCHEMA1="ec87e9b6971d831f0eff752ddb54fb64693e51cd"
+REG_GOSRC="$GOPATH/src/github.com/docker/distribution"
+OSO_REPO="https://github.com/openshift/origin.git"
+OSO_TAG="v1.5.0-alpha.3"
+OSO_GOSRC="$GOPATH/src/github.com/openshift/origin"
+
+# Workaround unnecessary swap-enabling shenanagains in openshift-origin build
+export OS_BUILD_SWAP_DISABLE=1
+
+# This golang code pre-dates support of go modules
+export GO111MODULE=off
+
+if [[ -z "$GOPATH" ]]; then
+    echo "Error: \$GOPATH must be non-empty"
+    exit 1
+fi
+
+# Make debugging easier
+set -x
+
+git clone "$REG_REPO" "$REG_GOSRC"
+cd "$REG_GOSRC"
+
+git config --global advice.detachedHead false
+
+# Don't pollute the environment
+(
+    # This is required to be set like this by the build system
+    GOPATH="$PWD/Godeps/_workspace:$GOPATH"
+    git checkout -q "$REG_COMMIT"
+    go build -o /usr/local/bin/registry-v2 \
+        github.com/docker/distribution/cmd/registry
+
+    git checkout -q "$REG_COMMIT_SCHEMA1"
+    go build -o /usr/local/bin/registry-v2-schema1 \
+        github.com/docker/distribution/cmd/registry
+)
+
+git clone --depth 1 -b "$OSO_TAG" "$OSO_REPO" "$OSO_GOSRC"
+cd "$OSO_GOSRC"
+
+# Edit out a "go < 1.5" check which works incorrectly with go ≥ 1.10.
+sed -i -e 's/\[\[ "\${go_version\[2]}" < "go1.5" ]]/false/' ./hack/common.sh
+
+make all WHAT=cmd/dockerregistry
+cp -a ./_output/local/bin/linux/*/* /usr/local/bin/
+cp ./images/dockerregistry/config.yml /atomic-registry-config.yml
+mkdir /registry