Skip to content

Commit

Permalink
chore(build): add build strategy in workflow (#6)
Browse files Browse the repository at this point in the history
* feat(build): add centos8 and ubuntu18 os

* chore(build): add build strategy in workflow

* chore(build): add fedora33 fedora34 build dockerfile

* chore(build): fix centos8 yum repo metadata error
  • Loading branch information
muzi502 authored Sep 2, 2021
1 parent 051fe8c commit 41fbc55
Show file tree
Hide file tree
Showing 12 changed files with 271 additions and 61 deletions.
166 changes: 121 additions & 45 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,41 @@
name: Build os-packages image
on:
push:
branches: [main, release-*]
paths-ignore:
- 'docs/**'
tag:
- 'v*'
release:
branches: [main, release-*]

branch: [main, release-*, master]
workflow_dispatch:
env:
IMAGE_REGISTRY: "ghcr.io"
REGISTRY_USER: "${{ github.repository_owner }}"
REGISTRY_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
OS_PACKAGES_IMAGE_REPO: "ghcr.io/${{ github.repository_owner }}/os-packages"
IMAGE_REPO: "ghcr.io/${{ github.repository_owner }}"

jobs:
build:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
include:
- name: ubuntu-bionic
image_name: os-packages-ubuntu1804
dockerfile: build/Dockerfile.os.ubuntu1804
- name: ubuntu-focal
image_name: os-packages-ubuntu2004
dockerfile: build/Dockerfile.os.ubuntu2004
- name: centos-7
image_name: os-packages-centos7
dockerfile: build/Dockerfile.os.centos7
- name: centos-8
image_name: os-packages-centos8
dockerfile: build/Dockerfile.os.centos8
- name: debian-buster
image_name: os-packages-debian10
dockerfile: build/Dockerfile.os.debian10
- name: debian-stretch
image_name: os-packages-debian9
dockerfile: build/Dockerfile.os.debian9
steps:
- name: Checkout
uses: actions/checkout@v2
Expand All @@ -32,14 +50,6 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Log in to GitHub Docker Registry
uses: docker/#-action@v1
with:
Expand All @@ -51,44 +61,110 @@ jobs:
shell: bash
run: |
git describe --tags --always | sed 's/^/IMAGE_TAG=/' >> $GITHUB_ENV
git branch --show-current | sed 's/^/BRANCH_NAME=/' >> $GITHUB_ENV
git branch --show-current | sed 's/master/latest/;s/main/latest/;s/^/IMAGE_TAG_BY_BRANCH=/' >> $GITHUB_ENV
source $GITHUB_ENV
make merge-dockerfile
cat build/* .github/workflows/build.yaml packages.yaml \
| md5sum | tr -d '\ -' | sed 's/^/OS_PACKAGES_MD5=md5-/' >> $GITHUB_ENV
if skopeo inspect docker://${OS_PACKAGES_IMAGE_REPO}:${BRANCH_NAME} > mainfest.json; then
jq -r '.Labels.MD5' mainfest.json | sed 's/^/LATEST_OS_PACKAGES_MD5=/' >> $GITHUB_ENV
else
echo 'LATEST_OS_PACKAGES_MD5=null' >> $GITHUB_ENV
fi
- name: Replace Dockerfile if MD5 not update
if: ${{ env.OS_PACKAGES_MD5 == env.LATEST_OS_PACKAGES_MD5 }}
run: |
echo "FROM ${{ env.OS_PACKAGES_IMAGE_REPO }}:${{ env.OS_PACKAGES_MD5 }}" > build/Dockerfile.all
- name: Build and push os-package images
uses: docker/build-push-action@v2
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
file: build/Dockerfile.all
file: ${{ matrix.dockerfile }}
platforms: linux/amd64,linux/arm64
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
labels: MD5=${{ env.OS_PACKAGES_MD5 }}
tags: |
${{ env.OS_PACKAGES_IMAGE_REPO }}:${{ env.IMAGE_TAG }}
${{ env.OS_PACKAGES_IMAGE_REPO }}:${{ env.OS_PACKAGES_MD5 }}
${{ env.OS_PACKAGES_IMAGE_REPO }}:${{ env.BRANCH_NAME }}
${{ env.OS_PACKAGES_IMAGE_REPO }}:${{ env.IMAGE_TAG_BY_BRANCH }}
${{ env.IMAGE_REPO }}/${{ matrix.image_name }}:${{ env.IMAGE_TAG }}
- name: Gen new Dockerfile
shell: bash
run: |
echo -e "FROM scratch\nCOPY --from=${{ env.IMAGE_REPO }}/${{ matrix.image_name }}:${{ env.IMAGE_TAG }} / /" > Dockerfile
- name: Build kubeplay image to local
uses: docker/build-push-action@v2
with:
context: .
file: Dockerfile
platforms: linux/amd64,linux/arm64
outputs: type=local,dest=./

- name: Prepare for upload package
shell: bash
run: |
mv linux_amd64/resources resources
tar -I pigz -cf resources-${{ matrix.image_name }}-${IMAGE_TAG}-amd64.tar.gz resources --remove-files
mv linux_arm64/resources resources
tar -I pigz -cf resources-${{ matrix.image_name }}-${IMAGE_TAG}-arm64.tar.gz resources --remove-files
sha256sum resources-${{ matrix.image_name }}-${IMAGE_TAG}-{amd64,arm64}.tar.gz > resources-${{ matrix.image_name }}-${IMAGE_TAG}.sha256sum.txt
- name: Release and upload packages
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
files: |
resources-${{ matrix.image_name }}-${{ env.IMAGE_TAG }}.sha256sum.txt
resources-${{ matrix.image_name }}-${{ env.IMAGE_TAG }}-amd64.tar.gz
resources-${{ matrix.image_name }}-${{ env.IMAGE_TAG }}-arm64.tar.gz
upload:
needs: [build]
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
with:
# fetch all git repo tag for define image tag
fetch-depth: 0

- name: Move new cache
- name: Set up QEMU
uses: docker/setup-qemu-action@v1

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Log in to GitHub Docker Registry
uses: docker/#-action@v1
with:
registry: ${{ env.IMAGE_REGISTRY }}
username: ${{ env.REGISTRY_USER }}
password: ${{ env.REGISTRY_TOKEN }}

- name: Prepare for build images
shell: bash
run: |
rm -rf /tmp/.buildx-cache
mkdir -p /tmp/.buildx-cache-new
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
git describe --tags --always | sed 's/^/IMAGE_TAG=/' >> $GITHUB_ENV
source $GITHUB_ENV
echo "FROM scratch" > Dockerfile
echo "COPY --from=${{ env.IMAGE_REPO }}/os-packages-ubuntu1804:${IMAGE_TAG} / /" >> Dockerfile
echo "COPY --from=${{ env.IMAGE_REPO }}/os-packages-ubuntu2004:${IMAGE_TAG} / /" >> Dockerfile
echo "COPY --from=${{ env.IMAGE_REPO }}/os-packages-centos7:${IMAGE_TAG} / /" >> Dockerfile
echo "COPY --from=${{ env.IMAGE_REPO }}/os-packages-centos8:${IMAGE_TAG} / /" >> Dockerfile
echo "COPY --from=${{ env.IMAGE_REPO }}/os-packages-debian9:${IMAGE_TAG} / /" >> Dockerfile
echo "COPY --from=${{ env.IMAGE_REPO }}/os-packages-debian10:${IMAGE_TAG} / /" >> Dockerfile
# debug action
# - uses: mxschmitt/action-tmate@v3
- name: Build os-packages images to local
uses: docker/build-push-action@v2
with:
context: .
file: Dockerfile
platforms: linux/amd64,linux/arm64
outputs: type=local,dest=./

- name: Prepare for upload package
shell: bash
run: |
mv linux_amd64/resources resources
tar -I pigz -cf resources-os-packages-all-${IMAGE_TAG}-amd64.tar.gz resources --remove-files
mv linux_arm64/resources resources
tar -I pigz -cf resources-os-packages-all-${IMAGE_TAG}-arm64.tar.gz resources --remove-files
sha256sum resources-os-packages-all-${IMAGE_TAG}-{amd64,arm64}.tar.gz > resources-os-packages-all-${IMAGE_TAG}.sha256sum.txt
- name: Release and upload packages
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
files: |
resources-os-packages-all-${{ env.IMAGE_TAG }}.sha256sum.txt
resources-os-packages-all-${{ env.IMAGE_TAG }}-amd64.tar.gz
resources-os-packages-all-${{ env.IMAGE_TAG }}-arm64.tar.gz
11 changes: 5 additions & 6 deletions build/Dockerfile.os.centos7
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
FROM centos:7.9.2009 as os-centos7
ARG OS_VERSION=7
ARG DOCKER_MIRROR_URL="https://download.docker.com"
ARG BUILD_TOOLS="yum-utils createrepo epel-release wget"

RUN yum install -q -y ${BUILD_TOOLS} \
&& yum-config-manager --add-repo ${DOCKER_MIRROR_URL}/linux/centos/docker-ce.repo \
&& yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo \
&& yum makecache

WORKDIR /centos/$OS_VERSION/os
COPY packages.yaml .
COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
RUN yq eval '.common[],.yum[],.centos7[],.kubespray.common[],.kubespray.yum[]' packages.yaml > packages.list \
&& sort -u packages.list | xargs repotrack --urls | sort -u > packages.urls
RUN yq eval '.common[],.yum[],.centos7[],.kubespray.common[],.kubespray.yum[]' packages.yaml > packages.list

RUN ARCH=$(uname -m) \
&& wget -q -x -P ${ARCH} -i packages.urls \
&& sort -u packages.list | xargs repotrack -p ${ARCH} \
&& createrepo -d ${ARCH}

FROM scratch
COPY --from=os-centos7 /centos /centos
COPY --from=os-centos7 /centos /resources/nginx/centos
COPY repos/CentOS-All-in-One.repo /resources/nginx/repos/CentOS-7-All-in-One.repo
24 changes: 24 additions & 0 deletions build/Dockerfile.os.centos8
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
FROM centos:8 as os-centos8
ARG OS_VERSION=8
ARG BUILD_TOOLS="yum-utils createrepo epel-release wget"

RUN ARCH=$(uname -m) \
&& dnf install -q -y ${BUILD_TOOLS} \
&& dnf install -q -y http://mirror.centos.org/centos/8-stream/AppStream/${ARCH}/os/Packages/modulemd-tools-0.7-4.el8.noarch.rpm \
&& yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo \
&& dnf makecache

WORKDIR /centos/$OS_VERSION/os
COPY packages.yaml .
COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
RUN yq eval '.common[],.yum[],.centos8[],.kubespray.common[],.kubespray.yum[]' packages.yaml > packages.list

RUN ARCH=$(uname -m) \
&& sort -u packages.list | xargs repotrack --destdir ${ARCH} \
&& createrepo_c ${ARCH} \
&& repo2module -s stable ${ARCH} ${ARCH}/modules.yaml \
&& modifyrepo_c --mdtype=modules ${ARCH}/modules.yaml ${ARCH}/repodata

FROM scratch
COPY --from=os-centos8 /centos /resources/nginx/centos
COPY repos/CentOS-All-in-One.repo /resources/nginx/repos/CentOS-8-All-in-One.repo
7 changes: 3 additions & 4 deletions build/Dockerfile.os.debian10
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
FROM debian:buster-slim as os-buster
ARG TARGETARCH
ARG OS_VERSION=buster
ARG DOCKER_MIRROR_URL="https://download.docker.com"
ARG DEP_PACKAGES="apt-transport-https ca-certificates curl wget gnupg dpkg-dev"

RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
&& apt update -qq \
&& apt install -y -qq --no-install-recommends $DEP_PACKAGES \
&& curl -fsSL "${DOCKER_MIRROR_URL}/linux/debian/gpg" | apt-key add -qq - \
&& echo "deb [arch=${ARCH}] ${DOCKER_MIRROR_URL}/linux/debian ${OS_VERSION} stable" > /etc/apt/sources.list.d/docker.list \
&& curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \
&& echo "deb [arch=${ARCH}] https://download.docker.com/linux/debian ${OS_VERSION} stable" > /etc/apt/sources.list.d/docker.list \
&& apt update -qq

WORKDIR /debian/${TARGETARCH}
Expand All @@ -24,4 +23,4 @@ RUN wget -q -x -P ${OS_VERSION} -i packages.urls \
&& dpkg-scanpackages ${OS_VERSION} | gzip -9c > ${OS_VERSION}/Packages.gz

FROM scratch
COPY --from=os-buster /debian /debian
COPY --from=os-buster /debian /resources/nginx/debian
26 changes: 26 additions & 0 deletions build/Dockerfile.os.debian9
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
FROM debian:stretch-slim as os-stretch
ARG TARGETARCH
ARG OS_VERSION=stretch
ARG DEP_PACKAGES="apt-transport-https ca-certificates curl wget gnupg dpkg-dev"

RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
&& apt update -qq \
&& apt install -y -qq --no-install-recommends $DEP_PACKAGES \
&& curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \
&& echo "deb [arch=${ARCH}] https://download.docker.com/linux/debian ${OS_VERSION} stable" > /etc/apt/sources.list.d/docker.list \
&& apt update -qq

WORKDIR /debian/${TARGETARCH}
COPY packages.yaml .

COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
RUN yq eval '.common[],.apt[],.kubespray.common[],.kubespray.apt[],.debian9[]' packages.yaml > packages.list \
&& dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 >> packages.list \
&& sort -u packages.list | xargs apt-get install --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls

RUN wget -q -x -P ${OS_VERSION} -i packages.urls \
&& dpkg-scanpackages ${OS_VERSION} | gzip -9c > ${OS_VERSION}/Packages.gz

FROM scratch
COPY --from=os-stretch /debian /resources/nginx/debian
20 changes: 20 additions & 0 deletions build/Dockerfile.os.fedora33
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
FROM fedora:33 as os-fedora33
ARG OS_VERSION=33
ARG BUILD_TOOLS="yum-utils createrepo findutils"

RUN dnf install -q -y ${BUILD_TOOLS}
RUN yum-config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
RUN dnf makecache -q

WORKDIR /fedora/$OS_VERSION/os
COPY packages.yaml .
COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
RUN yq eval '.common[],.yum[],.fedora[],.fedora33[],.kubespray.common[],.kubespray.yum[]' packages.yaml > packages.list

RUN ARCH=$(uname -m) \
&& sort -u packages.list | xargs repotrack --destdir ${ARCH} \
&& createrepo -d ${ARCH}

FROM scratch
COPY --from=os-fedora33 /fedora /resources/nginx/fedora
COPY repos/Fedora-All-in-One.repo /resources/nginx/repos/Fedora-33-All-in-One.repo
20 changes: 20 additions & 0 deletions build/Dockerfile.os.fedora34
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
FROM fedora:34 as os-fedora34
ARG OS_VERSION=34
ARG BUILD_TOOLS="yum-utils createrepo findutils"

RUN dnf install -q -y ${BUILD_TOOLS} \
&& yum-config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \
&& dnf makecache -q

WORKDIR /fedora/$OS_VERSION/os
COPY packages.yaml .
COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
RUN yq eval '.common[],.yum[],.fedora[],.fedora34[],.kubespray.common[],.kubespray.yum[]' packages.yaml > packages.list

RUN ARCH=$(uname -m) \
&& sort -u packages.list | xargs repotrack --destdir ${ARCH} \
&& createrepo -d ${ARCH}

FROM scratch
COPY --from=os-fedora34 /fedora /resources/nginx/fedora
COPY repos/Fedora-All-in-One.repo /resources/nginx/repos/Fedora-34-All-in-One.repo
25 changes: 25 additions & 0 deletions build/Dockerfile.os.ubuntu1804
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
FROM ubuntu:bionic as os-bionic
ARG TARGETARCH
ARG OS_VERSION=bionic
ARG DEP_PACKAGES="apt-transport-https ca-certificates curl wget gnupg dpkg-dev"

RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
&& apt update -qq \
&& apt install -y --no-install-recommends $DEP_PACKAGES \
&& curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \
&& echo "deb [arch=$ARCH] https://download.docker.com/linux/ubuntu ${OS_VERSION} stable" > /etc/apt/sources.list.d/docker.list\
&& apt update -qq

WORKDIR /ubuntu/${TARGETARCH}
COPY packages.yaml .

COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
RUN yq eval '.common[],.apt[],.kubespray.common[],.kubespray.apt[],.ubuntu[]' packages.yaml > packages.list \
&& dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 >> packages.list \
&& sort -u packages.list | xargs apt-get install --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls

RUN wget -q -x -P ${OS_VERSION} -i packages.urls \
&& dpkg-scanpackages ${OS_VERSION} | gzip -9c > ${OS_VERSION}/Packages.gz

FROM scratch
COPY --from=os-bionic /ubuntu /resources/nginx/ubuntu
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
FROM ubuntu:focal as os-focal
ARG TARGETARCH
ARG OS_VERSION=focal
ARG DOCKER_MIRROR_URL="https://download.docker.com"
ARG DEP_PACKAGES="apt-transport-https ca-certificates curl wget gnupg dpkg-dev"

RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
&& apt update -qq \
&& apt install -y --no-install-recommends $DEP_PACKAGES \
&& curl -fsSL "${DOCKER_MIRROR_URL}/linux/debian/gpg" | apt-key add -qq - \
&& echo "deb [arch=$ARCH] ${DOCKER_MIRROR_URL}/linux/ubuntu ${OS_VERSION} stable" > /etc/apt/sources.list.d/docker.list\
&& curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \
&& echo "deb [arch=$ARCH] https://download.docker.com/linux/ubuntu ${OS_VERSION} stable" > /etc/apt/sources.list.d/docker.list\
&& apt update -qq

WORKDIR /ubuntu/${TARGETARCH}
Expand All @@ -23,4 +22,4 @@ RUN wget -q -x -P ${OS_VERSION} -i packages.urls \
&& dpkg-scanpackages ${OS_VERSION} | gzip -9c > ${OS_VERSION}/Packages.gz

FROM scratch
COPY --from=os-focal /ubuntu /ubuntu
COPY --from=os-focal /ubuntu /resources/nginx/ubuntu
Loading

0 comments on commit 41fbc55

Please # to comment.