Skip to content

Commit

Permalink
Introduced JIT as a secret parameter to EphemeralRunner (#62)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nikola-jokic
nikola-jokic authored Jan 11, 2023
1 parent b086df0 commit 37fd1fd
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 71 deletions.
131 changes: 60 additions & 71 deletions controllers/ephemeralrunner_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,9 @@ import (

"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
"github.com/actions-runner-controller/actions-runner-controller/github/actions"
"github.com/actions-runner-controller/actions-runner-controller/hash"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -48,9 +46,10 @@ const (
// EphemeralRunnerReconciler reconciles a EphemeralRunner object
type EphemeralRunnerReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
ActionsClient *actions.MultiClient
Log logr.Logger
Scheme *runtime.Scheme
ActionsClient *actions.MultiClient
resourceBuilder resourceBuilder
}

// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=ephemeralrunners,verbs=get;list;watch;create;update;patch;delete
Expand Down Expand Up @@ -118,6 +117,16 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
return ctrl.Result{}, nil
}

secret := new(corev1.Secret)
if err := r.Get(ctx, req.NamespacedName, secret); err != nil {
if !kerrors.IsNotFound(err) {
log.Error(err, "Failed to fetch secret")
return ctrl.Result{}, err
}
// create secret if not created
return r.createSecret(ctx, ephemeralRunner)
}

pod := new(corev1.Pod)
if err := r.Get(ctx, req.NamespacedName, pod); err != nil {
switch {
Expand All @@ -134,7 +143,7 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ

default:
// Pod was not found. Create if the pod has never been created
return r.createPod(ctx, ephemeralRunner, log)
return r.createPod(ctx, ephemeralRunner, secret, log)
}
}

Expand Down Expand Up @@ -190,21 +199,40 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}

func (r *EphemeralRunnerReconciler) cleanupResources(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner) (deleted bool, err error) {
pod := new(corev1.Pod)
if err := r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunner.Namespace, Name: ephemeralRunner.Name}, pod); err != nil {
if kerrors.IsNotFound(err) {
return true, nil
secret := new(corev1.Secret)
err = r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunner.Namespace, Name: ephemeralRunner.Name}, secret)
switch {
case err == nil:
if !secret.ObjectMeta.DeletionTimestamp.IsZero() {
return false, nil
}
if err := r.Delete(ctx, secret); err != nil {
return false, fmt.Errorf("failed to delete secret: %v", err)
}
return false, nil
case err != nil && !kerrors.IsNotFound(err):
return false, err
}
if !pod.ObjectMeta.DeletionTimestamp.IsZero() {

// secret not found, everything is cleaned up

pod := new(corev1.Pod)
err = r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunner.Namespace, Name: ephemeralRunner.Name}, pod)
switch {
case err == nil:
if !pod.ObjectMeta.DeletionTimestamp.IsZero() {
return false, nil
}
if err := r.Delete(ctx, pod); err != nil {
return false, fmt.Errorf("failed to delete pod: %v", err)
}
return false, nil
case err != nil && !kerrors.IsNotFound(err):
return false, err
}

if err := r.Delete(ctx, pod); err != nil {
return false, fmt.Errorf("failed to delete pod: %v", err)
}
return false, nil
// pod is deleted
return true, nil
}

func (r *EphemeralRunnerReconciler) markAsFailed(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner) error {
Expand Down Expand Up @@ -267,9 +295,9 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
return nil
}

func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, log logr.Logger) (ctrl.Result, error) {
func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, log logr.Logger) (ctrl.Result, error) {
log.Info("Start creating new pod for ephemeral runner")
newPod := newEphemeralRunnerPod(ctx, runner)
newPod := r.resourceBuilder.newEphemeralRunnerPod(ctx, runner, secret)

if err := ctrl.SetControllerReference(runner, newPod, r.Scheme); err != nil {
log.Error(err, "Failed to set controller reference to a new pod")
Expand All @@ -292,6 +320,20 @@ func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alp
return ctrl.Result{}, nil
}

func (r *EphemeralRunnerReconciler) createSecret(ctx context.Context, runner *v1alpha1.EphemeralRunner) (ctrl.Result, error) {
jitSecret := r.resourceBuilder.newEphemeralRunnerJitSecret(runner)

if err := ctrl.SetControllerReference(runner, jitSecret, r.Scheme); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to set controller reference: %v", err)
}

if err := r.Create(ctx, jitSecret); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to create jit secret: %v", err)
}

return ctrl.Result{}, nil
}

func (r *EphemeralRunnerReconciler) updateStatusFromPod(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, pod *corev1.Pod) error {
// Fetch the latest state of the ephemeral runner.
// On frequent updates, Update() may fail due to the cache being stale.
Expand All @@ -315,60 +357,6 @@ func (r *EphemeralRunnerReconciler) updateStatusFromPod(ctx context.Context, eph
return nil
}

func newEphemeralRunnerPod(ctx context.Context, runner *v1alpha1.EphemeralRunner) *corev1.Pod {
var newPod corev1.Pod

labels := map[string]string{}
annotations := map[string]string{}

for k, v := range runner.ObjectMeta.Labels {
labels[k] = v
}
for k, v := range runner.Spec.PodTemplateSpec.Labels {
labels[k] = v
}

for k, v := range runner.ObjectMeta.Annotations {
annotations[k] = v
}
for k, v := range runner.Spec.PodTemplateSpec.Annotations {
annotations[k] = v
}

labels[LabelKeyPodTemplateHash] = hash.FNVHashStringObjects(
filterLabels(labels, LabelKeyRunnerTemplateHash),
annotations,
runner.Spec,
runner.Status.RunnerJITConfig,
)

labels["actions-ephemeral-runner"] = string(corev1.ConditionTrue)

objectMeta := metav1.ObjectMeta{
Name: runner.ObjectMeta.Name,
Namespace: runner.ObjectMeta.Namespace,
Labels: labels,
Annotations: annotations,
}

newPod.ObjectMeta = objectMeta
newPod.Spec = runner.Spec.PodTemplateSpec.Spec
newPod.Spec.Containers = make([]corev1.Container, 0, len(runner.Spec.PodTemplateSpec.Spec.Containers))

for _, c := range runner.Spec.PodTemplateSpec.Spec.Containers {
if c.Name == EphemeralRunnerContainerName {
c.Env = append(c.Env, corev1.EnvVar{
Name: EnvVarRunnerJITConfig,
Value: runner.Status.RunnerJITConfig,
})
}

newPod.Spec.Containers = append(newPod.Spec.Containers, c)
}

return &newPod
}

func (r *EphemeralRunnerReconciler) actionsClientFor(ctx context.Context, runner *v1alpha1.EphemeralRunner) (*actions.Client, error) {
secret := new(corev1.Secret)
if err := r.Get(ctx, types.NamespacedName{Namespace: runner.Namespace, Name: runner.Spec.GitHubConfigSecret}, secret); err != nil {
Expand Down Expand Up @@ -419,6 +407,7 @@ func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.EphemeralRunner{}).
Owns(&corev1.Pod{}).
Owns(&corev1.Secret{}).
Named("ephemeral-runner-controller").
Complete(r)
}
Expand Down
76 changes: 76 additions & 0 deletions controllers/resourcebuilder.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package controllers

import (
"context"
"fmt"
"math"
"strconv"
Expand All @@ -12,6 +13,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const jitTokenKey = "jitToken"

type resourceBuilder struct {
}

Expand Down Expand Up @@ -318,6 +321,79 @@ func (b *resourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.Epheme
}
}

func (b *resourceBuilder) newEphemeralRunnerPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret) *corev1.Pod {
var newPod corev1.Pod

labels := map[string]string{}
annotations := map[string]string{}

for k, v := range runner.ObjectMeta.Labels {
labels[k] = v
}
for k, v := range runner.Spec.PodTemplateSpec.Labels {
labels[k] = v
}

for k, v := range runner.ObjectMeta.Annotations {
annotations[k] = v
}
for k, v := range runner.Spec.PodTemplateSpec.Annotations {
annotations[k] = v
}

labels[LabelKeyPodTemplateHash] = hash.FNVHashStringObjects(
filterLabels(labels, LabelKeyRunnerTemplateHash),
annotations,
runner.Spec,
runner.Status.RunnerJITConfig,
)

labels["actions-ephemeral-runner"] = string(corev1.ConditionTrue)

objectMeta := metav1.ObjectMeta{
Name: runner.ObjectMeta.Name,
Namespace: runner.ObjectMeta.Namespace,
Labels: labels,
Annotations: annotations,
}

newPod.ObjectMeta = objectMeta
newPod.Spec = runner.Spec.PodTemplateSpec.Spec
newPod.Spec.Containers = make([]corev1.Container, 0, len(runner.Spec.PodTemplateSpec.Spec.Containers))

for _, c := range runner.Spec.PodTemplateSpec.Spec.Containers {
if c.Name == EphemeralRunnerContainerName {
c.Env = append(c.Env, corev1.EnvVar{
Name: EnvVarRunnerJITConfig,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret.Name,
},
Key: jitTokenKey,
},
},
})
}

newPod.Spec.Containers = append(newPod.Spec.Containers, c)
}

return &newPod
}

func (b *resourceBuilder) newEphemeralRunnerJitSecret(ephemeralRunner *v1alpha1.EphemeralRunner) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: ephemeralRunner.Name,
Namespace: ephemeralRunner.Namespace,
},
Data: map[string][]byte{
jitTokenKey: []byte(ephemeralRunner.Status.RunnerJITConfig),
},
}
}

func scaleSetListenerName(autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet) string {
namespaceHash := hash.FNVHashString(autoscalingRunnerSet.Name)[:8]
return fmt.Sprintf("%v-%v-listener", autoscalingRunnerSet.Name, namespaceHash)
Expand Down

0 comments on commit 37fd1fd

Please # to comment.