Merge pull request #102 from devtron-labs/release-candidate-v0.26.0

sync: Release candidate v0.26.0

chart-sync/go.mod

@@ -5,7 +5,7 @@ go 1.22.4
 toolchain go1.22.6
 
 replace (
-	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
+	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13
 	helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42
 )
 

chart-sync/go.sum

@@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b h1:0Mua8RfGFNDbaAprezc6NM5TnQdNbqo+qMVtbERx6Yg=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13 h1:0nYnqC8SuDbXJY9vfC6Wg4xMgsmCi2s+d57SpoRfJ84=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
 github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY=
 github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=

chart-sync/vendor/modules.txt

@@ -93,7 +93,7 @@ github.com/containerd/platforms
 # github.com/davecgh/go-spew v1.1.1
 ## explicit
 github.com/davecgh/go-spew/spew
-# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
+# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13
 ## explicit; go 1.21
 github.com/devtron-labs/common-lib/helmLib/registry
 github.com/devtron-labs/common-lib/utils/http
@@ -785,4 +785,4 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.3.0
 ## explicit; go 1.12
 sigs.k8s.io/yaml
-# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
+# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13

ci-runner/executor/adaptor/ImageScanAdaptor.go

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package adaptor
+
+import (
+	"github.com/devtron-labs/ci-runner/helper"
+	"github.com/devtron-labs/common-lib/constants"
+	"github.com/devtron-labs/common-lib/imageScan/bean"
+)
+
+func GetImageScanEvent(dest, digest string, commonWorkflowRequest *helper.CommonWorkflowRequest) *helper.ScanEvent {
+	if commonWorkflowRequest == nil {
+		return &helper.ScanEvent{}
+	}
+	return &helper.ScanEvent{
+		ImageScanEvent: bean.ImageScanEvent{
+			Image:            dest,
+			ImageDigest:      digest,
+			PipelineId:       commonWorkflowRequest.PipelineId,
+			UserId:           commonWorkflowRequest.TriggeredBy,
+			DockerRegistryId: commonWorkflowRequest.DockerRegistryId,
+			DockerConnection: commonWorkflowRequest.DockerConnection,
+			DockerCert:       commonWorkflowRequest.DockerCert,
+			SourceType:       constants.SourceTypeImage,
+			SourceSubType:    constants.SourceSubTypeCi,
+		},
+		ImageScanMaxRetries: commonWorkflowRequest.ImageScanMaxRetries,
+		ImageScanRetryDelay: commonWorkflowRequest.ImageScanRetryDelay,
+	}
+}

ci-runner/executor/stage/bean/bean.go

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bean
+
+const (
+	ExternalCiArtifact = "externalCiArtifact"
+	ImageDigest        = "imageDigest"
+	UseAppDockerConfig = "useAppDockerConfig"
+	CiProjectDetails   = "ciProjectDetails"
+)

ci-runner/executor/stage/ciStages.go

@@ -22,7 +22,9 @@ import (
 	"errors"
 	"fmt"
 	"github.com/devtron-labs/ci-runner/executor"
+	adaptor2 "github.com/devtron-labs/ci-runner/executor/adaptor"
 	cicxt "github.com/devtron-labs/ci-runner/executor/context"
+	bean2 "github.com/devtron-labs/ci-runner/executor/stage/bean"
 	util2 "github.com/devtron-labs/ci-runner/executor/util"
 	"github.com/devtron-labs/ci-runner/helper"
 	"github.com/devtron-labs/ci-runner/helper/adaptor"
@@ -144,6 +146,7 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
 	return
 }
 
+// TODO: take as tech debt and break this function into parts for better code readability
 func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, err error) {
 
 	metrics := &helper.CIMetrics{}
@@ -281,6 +284,18 @@ func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper.
 	if err != nil {
 		return artifactUploaded, err
 	}
+	if scriptEnvs.RuntimeEnv[bean2.ExternalCiArtifact] != "" {
+		runtimeImage, runtimeDigest, err := impl.handleRuntimeParametersForCiJob(scriptEnvs.RuntimeEnv, ciCdRequest)
+		if err != nil {
+			log.Println(util.DEVTRON, "error in handling runtime parameters for ci job and getting runtime image and digest")
+			return artifactUploaded, err
+		}
+		if len(runtimeImage) > 0 {
+			dest = runtimeImage
+			digest = runtimeDigest
+		}
+	}
+
 	// scan only if ci scan enabled
 	if helper.IsEventTypeEligibleToScanImage(ciCdRequest.Type) &&
 		ciCdRequest.CommonWorkflowRequest.ScanEnabled {
@@ -292,51 +307,7 @@ func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper.
 
 	log.Println(util.DEVTRON, " event")
 	metrics.TotalDuration = time.Since(metrics.TotalStartTime).Seconds()
-	// When externalCiArtifact is provided (run time Env at time of build) then this image will be used further in the pipeline
-	// imageDigest and ciProjectDetails are optional fields
-	if scriptEnvs.RuntimeEnv["externalCiArtifact"] != "" {
-		log.Println(util.DEVTRON, "external ci artifact found! exiting now with success event")
-		dest = scriptEnvs.RuntimeEnv["externalCiArtifact"]
-		digest = scriptEnvs.RuntimeEnv["imageDigest"]
-		if len(digest) == 0 {
-			var useAppDockerConfigForPrivateRegistries bool
-			var err error
-			useAppDockerConfig, ok := ciCdRequest.CommonWorkflowRequest.RuntimeEnvironmentVariables["useAppDockerConfig"]
-			if ok && len(useAppDockerConfig) > 0 {
-				useAppDockerConfigForPrivateRegistries, err = strconv.ParseBool(useAppDockerConfig)
-				if err != nil {
-					fmt.Println(fmt.Sprintf("Error in parsing useAppDockerConfig runtime param to bool from string useAppDockerConfigForPrivateRegistries:- %s, err:", useAppDockerConfig), err)
-				}
-			}
-			var dockerAuthConfig *bean.DockerAuthConfig
-			if useAppDockerConfigForPrivateRegistries {
-				dockerAuthConfig = impl.dockerHelper.GetDockerAuthConfigForPrivateRegistries(ciCdRequest.CommonWorkflowRequest)
-			}
-			startTime := time.Now()
-			//user has not provided imageDigest in that case fetch from docker.
-			imgDigest, err := impl.dockerHelper.ExtractDigestFromImage(dest, ciCdRequest.CommonWorkflowRequest.UseDockerApiToGetDigest, dockerAuthConfig)
-			if err != nil {
-				fmt.Println(fmt.Sprintf("Error in extracting digest from image %s, err:", dest), err)
-				return artifactUploaded, err
-			}
-			log.Println(fmt.Sprintf("time since extract digest from image process:- %s", time.Since(startTime).String()))
-			digest = imgDigest
-		}
-		var tempDetails []*helper.CiProjectDetailsMin
-		err := json.Unmarshal([]byte(scriptEnvs.RuntimeEnv["ciProjectDetails"]), &tempDetails)
-		if err != nil {
-			fmt.Println("Error unmarshalling ciProjectDetails JSON:", err)
-			fmt.Println("ignoring the error and continuing without saving ciProjectDetails")
-		}
 
-		if len(tempDetails) > 0 && len(ciCdRequest.CommonWorkflowRequest.CiProjectDetails) > 0 {
-			detail := tempDetails[0]
-			ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitHash = detail.CommitHash
-			ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Message = detail.Message
-			ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Author = detail.Author
-			ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitTime = detail.CommitTime
-		}
-	}
 	event := adaptor.NewCiCompleteEvent(ciCdRequest.CommonWorkflowRequest).WithMetrics(*metrics).
 		WithDockerImage(dest).WithDigest(digest).WithIsArtifactUploaded(artifactUploaded).
 		WithImageDetailsFromCR(resultsFromPlugin).WithPluginArtifacts(pluginArtifacts)
@@ -467,18 +438,8 @@ func (impl *CiStage) runPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent, script
 func runImageScanning(dest string, digest string, ciCdRequest *helper.CiCdTriggerEvent, metrics *helper.CIMetrics, artifactUploaded bool) error {
 	imageScanningStage := func() error {
 		log.Println("Image Scanning Started for digest", digest)
-		scanEvent := &helper.ScanEvent{
-			Image:               dest,
-			ImageDigest:         digest,
-			PipelineId:          ciCdRequest.CommonWorkflowRequest.PipelineId,
-			UserId:              ciCdRequest.CommonWorkflowRequest.TriggeredBy,
-			DockerRegistryId:    ciCdRequest.CommonWorkflowRequest.DockerRegistryId,
-			DockerConnection:    ciCdRequest.CommonWorkflowRequest.DockerConnection,
-			DockerCert:          ciCdRequest.CommonWorkflowRequest.DockerCert,
-			ImageScanMaxRetries: ciCdRequest.CommonWorkflowRequest.ImageScanMaxRetries,
-			ImageScanRetryDelay: ciCdRequest.CommonWorkflowRequest.ImageScanRetryDelay,
-		}
-		err := helper.SendEventToClairUtility(scanEvent)
+		scanEvent := adaptor2.GetImageScanEvent(dest, digest, ciCdRequest.CommonWorkflowRequest)
+		err := helper.ExecuteImageScanningViaRest(scanEvent)
 		if err != nil {
 			log.Println("error in running Image Scan", "err", err)
 			return helper.NewCiStageError(err).
@@ -639,3 +600,59 @@ func (impl *CiStage) AddExtraEnvVariableFromRuntimeParamsToCiCdEvent(ciRequest *
 	}
 	return ciRequest.RuntimeEnvironmentVariables, nil
 }
+
+// When externalCiArtifact is provided (run time Env at time of build) then this image will be used further in the pipeline
+// imageDigest and ciProjectDetails are optional fields
+func (impl *CiStage) handleRuntimeParametersForCiJob(runtimeEnv map[string]string, ciCdRequest *helper.CiCdTriggerEvent) (string, string, error) {
+	log.Println(util.DEVTRON, "external ci artifact found! exiting now with success event")
+	dest := runtimeEnv[bean2.ExternalCiArtifact]
+	digest := runtimeEnv[bean2.ImageDigest]
+	var err error
+	if len(digest) == 0 {
+		digest, err = impl.extractDigestForCiJob(ciCdRequest.CommonWorkflowRequest, dest)
+		if err != nil {
+			log.Println(util.DEVTRON, " extract digest for ci job error", "dest", dest, "err", err)
+			return dest, digest, err
+		}
+	}
+	var tempDetails []*helper.CiProjectDetailsMin
+	err = json.Unmarshal([]byte(runtimeEnv[bean2.CiProjectDetails]), &tempDetails)
+	if err != nil {
+		fmt.Println("Error unmarshalling ciProjectDetails JSON:", err)
+		fmt.Println("ignoring the error and continuing without saving ciProjectDetails")
+	}
+	if len(tempDetails) > 0 && len(ciCdRequest.CommonWorkflowRequest.CiProjectDetails) > 0 {
+		detail := tempDetails[0]
+		ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitHash = detail.CommitHash
+		ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Message = detail.Message
+		ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Author = detail.Author
+		ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitTime = detail.CommitTime
+	}
+	return dest, digest, nil
+}
+
+func (impl *CiStage) extractDigestForCiJob(workflowRequest *helper.CommonWorkflowRequest, image string) (string, error) {
+	var useAppDockerConfigForPrivateRegistries bool
+	var err error
+	useAppDockerConfig, ok := workflowRequest.RuntimeEnvironmentVariables[bean2.UseAppDockerConfig]
+	if ok && len(useAppDockerConfig) > 0 {
+		useAppDockerConfigForPrivateRegistries, err = strconv.ParseBool(useAppDockerConfig)
+		if err != nil {
+			fmt.Println(fmt.Sprintf("Error in parsing useAppDockerConfig runtime param to bool from string useAppDockerConfigForPrivateRegistries:- %s, err:", useAppDockerConfig), err)
+			// would use default val of useAppDockerConfigForPrivateRegistries i.e false in case error arises
+		}
+	}
+	var dockerAuthConfig *bean.DockerAuthConfig
+	if useAppDockerConfigForPrivateRegistries {
+		dockerAuthConfig = impl.dockerHelper.GetDockerAuthConfigForPrivateRegistries(workflowRequest)
+	}
+	startTime := time.Now()
+	//user has not provided imageDigest in that case fetch from docker.
+	imgDigest, err := impl.dockerHelper.ExtractDigestFromImage(image, workflowRequest.UseDockerApiToGetDigest, dockerAuthConfig)
+	if err != nil {
+		fmt.Println(fmt.Sprintf("Error in extracting digest from image %s, err:", image), err)
+		return "", err
+	}
+	log.Println(fmt.Sprintf("time since extract digest from image process:- %s", time.Since(startTime).String()))
+	return imgDigest, nil
+}

ci-runner/go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 toolchain go1.21.8
 
-replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
+replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible

ci-runner/go.sum

@@ -57,8 +57,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b h1:0Mua8RfGFNDbaAprezc6NM5TnQdNbqo+qMVtbERx6Yg=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13 h1:0nYnqC8SuDbXJY9vfC6Wg4xMgsmCi2s+d57SpoRfJ84=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=

ci-runner/helper/EventHelper.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	bean2 "github.com/devtron-labs/common-lib/imageScan/bean"
 	"github.com/devtron-labs/common-lib/utils/remoteConnection/bean"
 	"log"
 	"net/http"
@@ -597,7 +598,7 @@ func PublishEventsOnRest(jsonBody []byte, topic string, cdRequest *ExtEnvRequest
 	return nil
 }
 
-func SendEventToClairUtility(event *ScanEvent) error {
+func ExecuteImageScanningViaRest(event *ScanEvent) error {
 	jsonBody, err := json.Marshal(event)
 	if err != nil {
 		log.Println(util.DEVTRON, "err", err)
@@ -642,22 +643,9 @@ func SendEventToClairUtility(event *ScanEvent) error {
 }
 
 type ScanEvent struct {
-	Image               string `json:"image"`
-	ImageDigest         string `json:"imageDigest"`
-	AppId               int    `json:"appId"`
-	EnvId               int    `json:"envId"`
-	PipelineId          int    `json:"pipelineId"`
-	CiArtifactId        int    `json:"ciArtifactId"`
-	UserId              int    `json:"userId"`
-	AccessKey           string `json:"accessKey"`
-	SecretKey           string `json:"secretKey"`
-	Token               string `json:"token"`
-	AwsRegion           string `json:"awsRegion"`
-	DockerRegistryId    string `json:"dockerRegistryId"`
-	DockerConnection    string `json:"dockerConnection"`
-	DockerCert          string `json:"dockerCert"`
-	ImageScanMaxRetries int    `json:"imageScanMaxRetries,omitempty"`
-	ImageScanRetryDelay int    `json:"imageScanRetryDelay,omitempty"`
+	bean2.ImageScanEvent
+	ImageScanMaxRetries int `json:"imageScanMaxRetries,omitempty"`
+	ImageScanRetryDelay int `json:"imageScanRetryDelay,omitempty"`
 }
 
 func (dockerBuildConfig *DockerBuildConfig) GetProvenanceFlag() string {