Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

feat: options to configure local registry #113

Merged
merged 4 commits into from
Dec 18, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
@@ -169,3 +169,93 @@ jobs:
run: |
kubectl cluster-info
kubectl get nodes

test-without-registry:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

- name: Create kind cluster without registry
uses: ./
with:
registry: false

- name: Test
run: |
kubectl cluster-info
kubectl get storageclass standard

if [[ -n "$(docker ps --filter "name=kind-registry" --format "{{.ID}}")" ]]; then
echo "Registry is present"
exit 1
fi

test-with-registry:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

- name: Create kind cluster with registry
id: kind
uses: ./
with:
registry: true
registry_name: custom-registry
registry_port: 5001

- name: Test
env:
LOCAL_REGISTRY: ${{ steps.kind.outputs.LOCAL_REGISTRY }}
run: |
kubectl cluster-info
kubectl get storageclass standard

if [[ -z "$(docker ps --filter "name=custom-registry" --format "{{.ID}}")" ]]; then
echo "Registry is not present"
exit 1
fi

docker pull busybox
docker tag busybox $LOCAL_REGISTRY/localbusybox
docker push $LOCAL_REGISTRY/localbusybox

kubectl create job test --image=$LOCAL_REGISTRY/localbusybox
kubectl wait --for=condition=complete --timeout=30s job/test

test-with-registry-and-delete-enabled:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

- name: Create kind cluster with registry and delete enabled
id: kind
uses: ./
with:
registry: true
registry_name: custom-registry
registry_port: 5001
registry_enable_delete: true

- name: Test
env:
LOCAL_REGISTRY: ${{ steps.kind.outputs.LOCAL_REGISTRY }}
run: |
kubectl cluster-info
kubectl get storageclass standard

if [[ -z "$(docker ps --filter "name=custom-registry" --format "{{.ID}}")" ]]; then
echo "Registry is not present"
exit 1
fi

docker pull busybox
docker tag busybox $LOCAL_REGISTRY/localbusybox

DIGEST=$(docker push $LOCAL_REGISTRY/localbusybox | grep -oE 'sha256:\w+')

curl -X DELETE $LOCAL_REGISTRY/v2/localbusybox/manifests/$DIGEST
[[ "$(curl -Ls $LOCAL_REGISTRY/v2/localbusybox/tags/list | jq .tags)" == null ]]

42 changes: 42 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -22,6 +22,11 @@ For more information on inputs, see the [API Documentation](https://developer.gi
- `wait`: The duration to wait for the control plane to become ready (default: `60s`)
- `verbosity`: info log verbosity, higher value produces more output
- `kubectl_version`: The kubectl version to use (default: v1.30.4)
- `registry`: Whether to configure an insecure local registry (default: false)
- `registry_image`: The registry image to use (default: registry:2)
- `registry_name`: The registry name to use (default: kind-registry)
- `registry_port`: The local port used to bind the registry (default: 5000)
- `registry_enable_delete`: Enable delete operations on the registry (default: false)
- `install_only`: Skips cluster creation, only install kind (default: false)
- `ignore_failed_clean`: Whether to ignore the post delete cluster action failing (default: false)

@@ -45,6 +50,43 @@ jobs:
This uses [@helm/kind-action](https://github.com/helm/kind-action) GitHub Action to spin up a [kind](https://kind.sigs.k8s.io/) Kubernetes cluster on every Pull Request.
See [@helm/chart-testing-action](https://github.com/helm/chart-testing-action) for a more practical example.

### Configuring Local Registry

Create a workflow (eg: `.github/workflows/create-cluster-with-registry.yml`):


```yaml
name: Create Cluster with Registry

on: pull_request

jobs:
create-cluster-with-registry:
runs-on: ubuntu-latest
steps:
- name: Kubernetes KinD Cluster
id: kind
uses: helm/kind-action@v1
with:
registry: true
registry_name: my-registry
registry_port: 5001
registry_enable_delete: true
```

This will configure the cluster with an insecure local registry at `my-registry:5001` on both the host and within cluster. Subsequent steps can refer to the registry address with the output of the kind setup step (i.e. `${{ steps.kind.outputs.LOCAL_REGISTRY }}`).

**Note**: If `config` option is used, you must manually configure the cluster nodes with registry config dir enabled at `/etc/containerd/certs.d`. For example:

```yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
```

## Code of conduct

Participation in the Helm community is governed by the [Code of Conduct](CODE_OF_CONDUCT.md).
20 changes: 20 additions & 0 deletions action.yml
Original file line number Diff line number Diff line change
@@ -34,6 +34,26 @@ inputs:
description: "The kubectl version to use (default: v1.31.4)"
required: false
default: "v1.31.4"
registry:
description: "Whether to configure an insecure local registry (default: false)"
required: false
default: "false"
registry_image:
description: "The registry image to use (default: registry:2)"
required: false
default: "registry:2"
registry_name:
description: "The registry name to use (default: kind-registry)"
required: false
default: "kind-registry"
registry_port:
description: "The local port used to bind the registry (default: 5000)"
required: false
default: "5000"
registry_enable_delete:
description: "Enable delete operations on the registry (default: false)"
required: false
default: "false"
install_only:
description: "Skips cluster creation, only install kind (default: false)"
required: false
12 changes: 5 additions & 7 deletions cleanup.sh
Original file line number Diff line number Diff line change
@@ -19,16 +19,14 @@ set -o nounset
set -o pipefail

DEFAULT_CLUSTER_NAME=chart-testing
DEFAULT_REGISTRY_NAME=kind-registry

main() {
args=()

if [[ -n "${INPUT_CLUSTER_NAME:-}" ]]; then
args+=(--name "${INPUT_CLUSTER_NAME}")
else
args+=(--name "${DEFAULT_CLUSTER_NAME}")
fi
args=(--name "${INPUT_CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME}")
registry_args=("${INPUT_REGISTRY_NAME:-$DEFAULT_REGISTRY_NAME}")

docker rm -f "${registry_args[@]}" || "${INPUT_IGNORE_FAILED_CLEAN}"

kind delete cluster "${args[@]}" || "${INPUT_IGNORE_FAILED_CLEAN}"
}

34 changes: 34 additions & 0 deletions kind.sh
Original file line number Diff line number Diff line change
@@ -36,6 +36,7 @@ Usage: $(basename "$0") <options>
-l, --verbosity info log verbosity, higher value produces more output
-k, --kubectl-version The kubectl version to use (default: $DEFAULT_KUBECTL_VERSION)
-o, --install-only Skips cluster creation, only install kind (default: false)
--with-registry Enables registry config dir for the cluster (default: false)

EOF
}
@@ -50,6 +51,8 @@ main() {
local verbosity=
local kubectl_version="${DEFAULT_KUBECTL_VERSION}"
local install_only=false
local with_registry=false
local config_with_registry_path="/etc/kind-registry/config.yaml"

parse_command_line "$@"

@@ -187,6 +190,14 @@ parse_command_line() {
install_only=true
fi
;;
--with-registry)
if [[ -n "${2:-}" ]]; then
with_registry="$2"
shift
else
with_registry=true
fi
;;
*)
break
;;
@@ -220,6 +231,20 @@ install_kubectl() {
chmod +x "${kubectl_dir}/kubectl"
}

create_config_with_registry() {
sudo mkdir -p $(dirname "$config_with_registry_path")
cat <<EOF | sudo tee "$config_with_registry_path"
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"

EOF
sudo chmod a+r "$config_with_registry_path"
}

create_kind_cluster() {
echo 'Creating kind cluster...'
local args=(create cluster "--name=${cluster_name}" "--wait=${wait}")
@@ -240,6 +265,15 @@ create_kind_cluster() {
args+=("--verbosity=${verbosity}")
fi

if [[ "${with_registry}" == true ]]; then
if [[ -n "${config}" ]]; then
echo 'WARNING: when using the "config" option, you need to manually configure the registry in the provided configurations'
else
create_config_with_registry
args+=(--config "$config_with_registry_path")
fi
fi

"${kind_dir}/kind" "${args[@]}"
}

28 changes: 27 additions & 1 deletion main.sh
Original file line number Diff line number Diff line change
@@ -21,7 +21,8 @@ set -o pipefail
SCRIPT_DIR=$(dirname -- "$(readlink -f "${BASH_SOURCE[0]}" || realpath "${BASH_SOURCE[0]}")")

main() {
args=()
local args=()
local registry_args=()

if [[ -n "${INPUT_VERSION:-}" ]]; then
args+=(--version "${INPUT_VERSION}")
@@ -41,6 +42,7 @@ main() {

if [[ -n "${INPUT_CLUSTER_NAME:-}" ]]; then
args+=(--cluster-name "${INPUT_CLUSTER_NAME}")
registry_args+=(--cluster-name "${INPUT_CLUSTER_NAME}")
fi

if [[ -n "${INPUT_WAIT:-}" ]]; then
@@ -59,7 +61,31 @@ main() {
args+=(--install-only)
fi

if [[ -n "${INPUT_REGISTRY:-}" ]]; then
args+=(--with-registry "${INPUT_REGISTRY}")
fi

if [[ -n "${INPUT_REGISTRY_IMAGE:-}" ]]; then
registry_args+=(--registry-image "${INPUT_REGISTRY_IMAGE}")
fi

if [[ -n "${INPUT_REGISTRY_NAME:-}" ]]; then
registry_args+=(--registry-name "${INPUT_REGISTRY_NAME}")
fi

if [[ -n "${INPUT_REGISTRY_PORT:-}" ]]; then
registry_args+=(--registry-port "${INPUT_REGISTRY_PORT}")
fi

if [[ -n "${INPUT_REGISTRY_ENABLE_DELETE:-}" ]]; then
registry_args+=(--enable-delete "${INPUT_REGISTRY_ENABLE_DELETE}")
fi

"${SCRIPT_DIR}/kind.sh" ${args[@]+"${args[@]}"}

if [[ "${INPUT_REGISTRY:-}" == true ]]; then
"${SCRIPT_DIR}/registry.sh" "${registry_args[@]}"
fi
}

main
173 changes: 173 additions & 0 deletions registry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
#!/usr/bin/env bash

# Copyright The Helm Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

DEFAULT_REGISTRY_IMAGE=registry:2
DEFAULT_REGISTRY_NAME=kind-registry
DEFAULT_REGISTRY_PORT=5000
DEFAULT_CLUSTER_NAME=chart-testing

show_help() {
cat <<EOF
Usage: $(basename "$0") <options>
-h, --help Display help
-i, --registry-image The registry image to use (default: $DEFAULT_REGISTRY_IMAGE)
-n, --registry-name The registry name to use (default: $DEFAULT_REGISTRY_NAME)
-p, --registry-port The local port used to bind the registry (default: $DEFAULT_REGISTRY_PORT)
-d, --enable-delete Enable delete operations on the registry (default: false)
--cluster-name The name of the cluster to configure with the registry (default: $DEFAULT_CLUSTER_NAME)
EOF
}

main() {
local registry_name="$DEFAULT_REGISTRY_NAME"
local registry_image="$DEFAULT_REGISTRY_IMAGE"
local registry_port="$DEFAULT_REGISTRY_PORT"
local enable_delete=false
local cluster_name="${DEFAULT_CLUSTER_NAME}"

parse_command_line "$@"

create_registry
connect_registry
config_registry_for_nodes
document_registry
}

parse_command_line() {
while :; do
case "${1:-}" in
-h|--help)
show_help
exit
;;
-n|--registry-name)
if [[ -n "${2:-}" ]]; then
registry_name="$2"
shift
else
echo "ERROR: '-n|--registry-name' cannot be empty." >&2
show_help
exit 1
fi
;;
-i|--registry-image)
if [[ -n "${2:-}" ]]; then
registry_image="$2"
shift
else
echo "ERROR: '-i|--registry-image' cannot be empty." >&2
show_help
exit 1
fi
;;
-p|--registry-port)
if [[ -n "${2:-}" ]]; then
registry_port="$2"
shift
else
echo "ERROR: '-p|--registry-port' cannot be empty." >&2
show_help
exit 1
fi
;;
-d|--enable-delete)
if [[ -n "${2:-}" ]]; then
enable_delete="$2"
shift
else
enable_delete=true
fi
;;
--cluster-name)
if [[ -n "${2:-}" ]]; then
cluster_name="$2"
shift
else
echo "ERROR: '--cluster-name' cannot be empty." >&2
show_help
exit 1
fi
;;
*)
break
;;
esac

shift
done
}

create_registry() {
echo "Creating local registry..."

docker run -d --restart=always \
--name "${registry_name}" \
--network bridge \
-p "${registry_port}:5000" \
-e REGISTRY_STORAGE_DELETE_ENABLED="$enable_delete" \
$registry_image

# Local registry is available at $registry_name:$registry_port
echo "127.0.0.1 $registry_name" | sudo tee -a /etc/hosts

# Write registry address to output for subsequent steps
echo "LOCAL_REGISTRY=$registry_name:$registry_port" >> "$GITHUB_OUTPUT"
}

connect_registry() {
echo "Connecting local registry to cluster network..."

docker network connect kind "$registry_name"
}

config_registry_for_nodes() {
# Reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md
REGISTRY_DIR="/etc/containerd/certs.d/${registry_name}:${registry_port}"

for node in $(kind get nodes -n "${cluster_name}"); do
docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
[host."http://${registry_name}:5000"]
EOF
done
}

document_registry() {
# Reference: https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
echo "Documenting local registry..."

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
name: local-registry-hosting
namespace: kube-public
data:
localRegistryHosting.v1: |
host: "${registry_name}:${registry_port}"
hostFromClusterNetwork: "${registry_name}:${registry_port}"
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
EOF
}

main "$@"