From e1a0422c3eabbe0ec05a9c3d0964ae022fb3a03e Mon Sep 17 00:00:00 2001 From: Willie Sana Date: Tue, 10 Nov 2020 17:47:07 -0800 Subject: [PATCH 1/9] adds initial helm support - supports loading of helm v3 charts via the iac dir command (iac file is invalid) - adds default iac versions for all iac types - adds unit tests - adds ability to properly specify default iac version for a given iac type --- go.mod | 11 +- pkg/iac-providers/helm.go | 36 +++ pkg/iac-providers/helm/v3/load-dir.go | 259 ++++++++++++++++++ pkg/iac-providers/helm/v3/load-dir_test.go | 169 ++++++++++++ pkg/iac-providers/helm/v3/load-file.go | 34 +++ pkg/iac-providers/helm/v3/load-file_test.go | 55 ++++ .../v3/testdata/bad-chart-file/Chart.yaml | 2 + .../v3/testdata/chart-bad-name/Chart.yaml | 6 + .../chart-bad-name/templates/_helpers.tpl | 32 +++ .../chart-bad-name/templates/deployment.yaml | 53 ++++ .../chart-bad-name/templates/ingress.yaml | 40 +++ .../chart-bad-name/templates/service.yaml | 19 ++ .../testdata/chart-bad-name/tests/test.yaml | 19 ++ .../v3/testdata/chart-bad-name/values.yaml | 48 ++++ .../chart-bad-template-file/Chart.yaml | 5 + .../chart-bad-template-file/values.yaml | 48 ++++ .../v3/testdata/chart-bad-values/Chart.yaml | 5 + .../v3/testdata/chart-bad-values/values.yaml | 2 + .../v3/testdata/chart-bad-version/Chart.yaml | 5 + .../chart-bad-version/templates/_helpers.tpl | 32 +++ .../templates/deployment.yaml | 53 ++++ .../chart-bad-version/templates/ingress.yaml | 40 +++ .../chart-bad-version/templates/service.yaml | 19 ++ .../chart-bad-version/tests/test.yaml | 19 ++ .../v3/testdata/chart-bad-version/values.yaml | 48 ++++ .../testdata/chart-error-coalesce/Chart.yaml | 5 + .../templates/_helpers.tpl | 32 +++ .../templates/deployment.yaml | 53 ++++ .../templates/ingress.yaml | 40 +++ .../templates/service.yaml | 19 ++ .../chart-error-coalesce/tests/test.yaml | 19 ++ .../testdata/chart-error-coalesce/values.yaml | 48 ++++ .../testdata/chart-no-template-dir/Chart.yaml | 5 + .../chart-no-template-dir/values.yaml | 48 ++++ .../v3/testdata/chart-no-values/Chart.yaml | 5 + .../testdata/chart-rendering-error/Chart.yaml | 5 + .../templates/_helpers.tpl | 32 +++ .../templates/deployment.yaml | 53 ++++ .../templates/ingress.yaml | 40 +++ .../templates/service.yaml | 19 ++ .../chart-rendering-error/tests/test.yaml | 19 ++ .../chart-rendering-error/values.yaml | 48 ++++ .../testdata/chart-skip-test-dir/Chart.yaml | 5 + .../templates/_helpers.tpl | 32 +++ .../templates/deployment.yaml | 53 ++++ .../templates/ingress.yaml | 40 +++ .../templates/service.yaml | 19 ++ .../templates/tests/test.yaml | 19 ++ .../testdata/chart-skip-test-dir/values.yaml | 48 ++++ .../chart-unreadable-values/Chart.yaml | 5 + .../helm/v3/testdata/happy-path/Chart.yaml | 5 + .../happy-path/templates/_helpers.tpl | 32 +++ .../happy-path/templates/deployment.yaml | 53 ++++ .../happy-path/templates/ingress.yaml | 40 +++ .../happy-path/templates/service.yaml | 19 ++ .../v3/testdata/happy-path/tests/test.yaml | 19 ++ .../helm/v3/testdata/happy-path/values.yaml | 48 ++++ pkg/iac-providers/helm/v3/types.go | 28 ++ pkg/iac-providers/kubernetes.go | 7 +- pkg/iac-providers/kubernetes/v1/load-file.go | 2 +- pkg/iac-providers/kubernetes/v1/normalize.go | 4 +- pkg/iac-providers/providers.go | 5 + pkg/iac-providers/register.go | 10 +- pkg/iac-providers/register_test.go | 18 +- pkg/iac-providers/terraform.go | 7 +- pkg/iac-providers/types.go | 5 - pkg/policy/cloud-providers.go | 7 +- pkg/policy/helm.go | 28 ++ pkg/runtime/validate.go | 3 +- pkg/writer/yaml.go | 2 +- 70 files changed, 2073 insertions(+), 39 deletions(-) create mode 100644 pkg/iac-providers/helm.go create mode 100644 pkg/iac-providers/helm/v3/load-dir.go create mode 100644 pkg/iac-providers/helm/v3/load-dir_test.go create mode 100644 pkg/iac-providers/helm/v3/load-file.go create mode 100644 pkg/iac-providers/helm/v3/load-file_test.go create mode 100644 pkg/iac-providers/helm/v3/testdata/bad-chart-file/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-name/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-name/templates/_helpers.tpl create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-name/templates/deployment.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-name/templates/ingress.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-name/templates/service.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-name/tests/test.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-name/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-template-file/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-template-file/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-values/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-values/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-version/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-version/templates/_helpers.tpl create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-version/templates/deployment.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-version/templates/ingress.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-version/templates/service.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-version/tests/test.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-version/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-error-coalesce/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-error-coalesce/templates/_helpers.tpl create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-error-coalesce/templates/deployment.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-error-coalesce/templates/ingress.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-error-coalesce/templates/service.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-error-coalesce/tests/test.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-error-coalesce/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-no-template-dir/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-no-template-dir/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-no-values/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-rendering-error/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-rendering-error/templates/_helpers.tpl create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-rendering-error/templates/deployment.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-rendering-error/templates/ingress.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-rendering-error/templates/service.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-rendering-error/tests/test.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-rendering-error/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-skip-test-dir/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-skip-test-dir/templates/_helpers.tpl create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-skip-test-dir/templates/deployment.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-skip-test-dir/templates/ingress.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-skip-test-dir/templates/service.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-skip-test-dir/templates/tests/test.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-skip-test-dir/values.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-unreadable-values/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/happy-path/Chart.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/happy-path/templates/_helpers.tpl create mode 100644 pkg/iac-providers/helm/v3/testdata/happy-path/templates/deployment.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/happy-path/templates/ingress.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/happy-path/templates/service.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/happy-path/tests/test.yaml create mode 100644 pkg/iac-providers/helm/v3/testdata/happy-path/values.yaml create mode 100644 pkg/iac-providers/helm/v3/types.go create mode 100644 pkg/policy/helm.go diff --git a/go.mod b/go.mod index eafff9c61..58e137fa2 100644 --- a/go.mod +++ b/go.mod @@ -18,14 +18,11 @@ require ( github.com/pkg/errors v0.9.1 github.com/spf13/afero v1.3.4 github.com/spf13/cobra v1.0.0 - github.com/spf13/pflag v1.0.5 // indirect github.com/zclconf/go-cty v1.2.1 - go.uber.org/zap v1.10.0 - golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect - golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect - golang.org/x/tools v0.0.0-20201009162240-fcf82128ed91 // indirect + go.uber.org/zap v1.13.0 + golang.org/x/tools v0.0.0-20201110030525-169ad6d6ecb2 // indirect gopkg.in/src-d/go-git.v4 v4.13.1 - gopkg.in/yaml.v2 v2.3.0 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 - honnef.co/go/tools v0.0.1-2020.1.5 // indirect + helm.sh/helm/v3 v3.4.0 + honnef.co/go/tools v0.0.1-2020.1.6 // indirect ) diff --git a/pkg/iac-providers/helm.go b/pkg/iac-providers/helm.go new file mode 100644 index 000000000..7eff46fbb --- /dev/null +++ b/pkg/iac-providers/helm.go @@ -0,0 +1,36 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package iacprovider + +import ( + "reflect" + + helmv3 "github.com/accurics/terrascan/pkg/iac-providers/helm/v3" +) + +// terraform specific constants +const ( + helm supportedIacType = "helm" + helmV3 supportedIacVersion = "v3" + helmDefaultIacVersion = helmV3 +) + +// register helm as an IaC provider with terrascan +func init() { + // register iac provider + RegisterIacProvider(helm, helmV3, helmDefaultIacVersion, reflect.TypeOf(helmv3.HelmV3{})) +} diff --git a/pkg/iac-providers/helm/v3/load-dir.go b/pkg/iac-providers/helm/v3/load-dir.go new file mode 100644 index 000000000..8cdc75c52 --- /dev/null +++ b/pkg/iac-providers/helm/v3/load-dir.go @@ -0,0 +1,259 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package helmv3 + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + k8sv1 "github.com/accurics/terrascan/pkg/iac-providers/kubernetes/v1" + + "github.com/accurics/terrascan/pkg/iac-providers/output" + "github.com/accurics/terrascan/pkg/utils" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/engine" +) + +var ( + errSkipTestDir = fmt.Errorf("skipping test directory") + errNoHelmChartsFound = fmt.Errorf("no helm charts found") + errBadChartName = fmt.Errorf("bad chart name in Chart.yaml") + errBadChartVersion = fmt.Errorf("bad chart version in Chart.yaml") +) + +// LoadIacDir loads all helm charts under the specified directory +func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error) { + + allResourcesConfig := make(map[string][]output.ResourceConfig) + + // find all Chart.yaml files within the specified directory structure + fileMap, err := utils.FindFilesBySuffix(absRootDir, []string{helmChartFilename}) + if err != nil { + zap.S().Error("error while searching for helm charts", zap.String("root dir", absRootDir), zap.Error(err)) + return allResourcesConfig, err + } + + if len(fileMap) == 0 { + zap.S().Error("", zap.String("root dir", absRootDir), zap.Error(err)) + err = errNoHelmChartsFound + return allResourcesConfig, err + } + + // fileDir now contains the chart path + iacDocumentMap := make(map[string][]*utils.IacDocument) + for fileDir, chartFilename := range fileMap { + chartPath := filepath.Join(fileDir, *chartFilename[0]) + zap.S().Debug("processing chart", zap.String("chart path", chartPath), zap.Error(err)) + + var iacDocuments []*utils.IacDocument + var chartMap map[string]interface{} + iacDocuments, chartMap, err = h.loadChart(chartPath) + if err != nil && err != errSkipTestDir { + zap.S().Warn("error occurred while loading chart", zap.String("chart path", chartPath), zap.Error(err)) + continue + } + + iacDocumentMap[chartPath] = iacDocuments + + var config *output.ResourceConfig + config, err = h.createHelmChartResource(chartPath, chartMap) + if err != nil { + zap.S().Debug("failed to create helm chart resource", zap.Any("config", config)) + continue + } + + allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config) + } + + for _, iacDocuments := range iacDocumentMap { + for _, doc := range iacDocuments { + // @TODO add k8s version check + var k k8sv1.K8sV1 + var config *output.ResourceConfig + config, err = k.Normalize(doc) + if err != nil { + zap.S().Warn("unable to normalize data", zap.Error(err), zap.String("file", doc.FilePath)) + continue + } + + config.Line = 1 + config.Source = doc.FilePath + + allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config) + } + } + + return allResourcesConfig, nil +} + +// createHelmChartResource returns normalized Helm Chart resource data +func (h *HelmV3) createHelmChartResource(chartPath string, chartData map[string]interface{}) (*output.ResourceConfig, error) { + var config output.ResourceConfig + + jsonData, err := json.Marshal(chartData) + if err != nil { + zap.S().Warn("unable to marshal chart to json", zap.String("chart path", chartPath)) + return nil, err + } + + configData := make(map[string]interface{}) + if err = json.Unmarshal(jsonData, &configData); err != nil { + zap.S().Warn("unable to unmarshal normalized config data", zap.String("chart path", chartPath)) + zap.S().Debug("failed config data", zap.Any("config", configData)) + return nil, err + } + + chartName, ok := chartData["name"].(string) + if !ok { + zap.S().Warn("unable to determine chart name", zap.String("chart path", chartPath)) + return nil, err + } + + config.Type = "helm_chart" + config.Name = chartName + config.Line = 0 + config.Source = chartPath + config.ID = config.Type + "." + config.Name + config.Config = configData + + return &config, nil +} + +func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]interface{}, error) { + iacDocuments := make([]*utils.IacDocument, 0) + chartMap := make(map[string]interface{}) + + // load the chart file and values file from the specified chart path + chartFileBytes, err := ioutil.ReadFile(chartPath) + if err != nil { + zap.S().Warn("unable to read", zap.String("file", chartPath)) + return iacDocuments, chartMap, err + } + + if err = yaml.Unmarshal(chartFileBytes, &chartMap); err != nil { + zap.S().Warn("unable to unmarshal values", zap.String("file", chartPath)) + return iacDocuments, chartMap, err + } + + var fileInfo os.FileInfo + chartDir := filepath.Dir(chartPath) + valuesFile := filepath.Join(chartDir, helmValuesFilename) + fileInfo, err = os.Stat(valuesFile) + if err != nil { + zap.S().Warn("unable to stat values.yaml", zap.String("chart path", chartPath)) + return iacDocuments, chartMap, err + } + + var valueFileBytes []byte + valueFileBytes, err = ioutil.ReadFile(valuesFile) + if err != nil { + zap.S().Warn("unable to read values.yaml", zap.String("file", fileInfo.Name())) + return iacDocuments, chartMap, err + } + + var valueMap map[string]interface{} + if err = yaml.Unmarshal(valueFileBytes, &valueMap); err != nil { + zap.S().Warn("unable to unmarshal values.yaml", zap.String("file", fileInfo.Name())) + return iacDocuments, chartMap, err + } + + // for each template file found, render and save an iacDocument + var templateFileMap map[string][]*string + templateFileMap, err = utils.FindFilesBySuffix(filepath.Join(chartDir, helmTemplateDir), h.getHelmTemplateExtensions()) + if err != nil { + zap.S().Warn("error while calling FindFilesBySuffix", zap.String("filepath", fileInfo.Name())) + return iacDocuments, chartMap, err + } + for templateDir, templateFiles := range templateFileMap { + if filepath.Base(templateDir) == helmTestDir { + zap.S().Debug("skipping test dir", zap.String("dir", templateDir)) + return iacDocuments, chartMap, errSkipTestDir + } + chartFiles := make([]*chart.File, 0) + for _, templateFile := range templateFiles { + var fileData []byte + fileData, err = ioutil.ReadFile(filepath.Join(templateDir, *templateFile)) + if err != nil { + zap.S().Warn("unable to read template file", zap.String("file", *templateFile)) + return iacDocuments, chartMap, err + } + + chartFiles = append(chartFiles, &chart.File{ + Name: filepath.Join(helmTemplateDir, *templateFile), + Data: fileData, + }) + } + + chartName, ok := chartMap["name"].(string) + if !ok { + return iacDocuments, chartMap, errBadChartName + } + + var chartVersion string + chartVersion, ok = chartMap["version"].(string) + if !ok { + return iacDocuments, chartMap, errBadChartVersion + } + + c := &chart.Chart{ + Metadata: &chart.Metadata{Name: chartName, Version: chartVersion}, + Templates: chartFiles, + } + + var v chartutil.Values + v, err = chartutil.CoalesceValues(c, chartutil.Values{ + "Values": valueMap, + "Release": chartutil.Values{ + "Name": defaultChartName, + }, + }) + if err != nil { + zap.S().Warn("error encountered in CoalesceValues", zap.String("chart path", chartPath)) + return iacDocuments, chartMap, err + } + + var renderData map[string]string + renderData, err = engine.Render(c, v) + if err != nil { + zap.S().Warn("error encountered while rendering chart", zap.String("chart path", chartPath), + zap.String("template dir", templateDir)) + return iacDocuments, chartMap, err + } + + for renderFile := range renderData { + iacDocuments = append(iacDocuments, &utils.IacDocument{ + Data: []byte(renderData[renderFile]), + Type: utils.YAMLDoc, + StartLine: 1, + EndLine: 1, + FilePath: renderFile, + }) + } + } + + return iacDocuments, chartMap, nil +} + +func (h *HelmV3) getHelmTemplateExtensions() []string { + return []string{"yaml", "tpl"} +} diff --git a/pkg/iac-providers/helm/v3/load-dir_test.go b/pkg/iac-providers/helm/v3/load-dir_test.go new file mode 100644 index 000000000..19b209ab9 --- /dev/null +++ b/pkg/iac-providers/helm/v3/load-dir_test.go @@ -0,0 +1,169 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package helmv3 + +import ( + "fmt" + "os" + "reflect" + "syscall" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/accurics/terrascan/pkg/iac-providers/output" +) + +func TestLoadIacDir(t *testing.T) { + + table := []struct { + name string + dirPath string + helmv3 HelmV3 + want output.AllResourceConfigs + wantErr error + }{ + { + name: "happy path (credit to madhuakula/kubernetes-goat)", + dirPath: "./testdata/happy-path", + helmv3: HelmV3{}, + wantErr: nil, + }, + { + name: "bad directory", + dirPath: "./testdata/bad-dir", + helmv3: HelmV3{}, + wantErr: &os.PathError{Err: syscall.ENOENT, Op: "lstat", Path: "./testdata/bad-dir"}, + }, + { + name: "no helm charts in directory", + dirPath: "./testdata/no-helm-charts", + helmv3: HelmV3{}, + wantErr: errNoHelmChartsFound, + }, + { + name: "unreadable chart file", + dirPath: "./testdata/bad-chart-file", + helmv3: HelmV3{}, + wantErr: nil, // these errors are ignored + }, + } + + for _, tt := range table { + t.Run(tt.name, func(t *testing.T) { + _, gotErr := tt.helmv3.LoadIacDir(tt.dirPath) + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr) + } + }) + } + +} + +func TestLoadChart(t *testing.T) { + + table := []struct { + name string + chartPath string + helmv3 HelmV3 + want output.AllResourceConfigs + wantErr error + }{ + { + name: "happy path (credit to madhuakula/kubernetes-goat)", + chartPath: "./testdata/happy-path/Chart.yaml", + helmv3: HelmV3{}, + wantErr: nil, + }, + { + name: "unreadable chart file", + chartPath: "./testdata/bad-chart-file", + helmv3: HelmV3{}, + wantErr: &os.PathError{Err: syscall.EISDIR, Op: "read", Path: "./testdata/bad-chart-file"}, + }, + { + name: "unmarshal bad chart", + chartPath: "./testdata/bad-chart-file/Chart.yaml", + helmv3: HelmV3{}, + wantErr: &yaml.TypeError{Errors: []string{"line 1: cannot unmarshal !!str `:bad ba...` into map[string]interface {}"}}, + }, + { + name: "chart path with no values.yaml", + chartPath: "./testdata/chart-no-values/Chart.yaml", + helmv3: HelmV3{}, + wantErr: &os.PathError{Err: syscall.ENOENT, Op: "stat", Path: "testdata/chart-no-values/values.yaml"}, + }, + { + name: "chart path with unreadable values.yaml", + chartPath: "./testdata/chart-unreadable-values/Chart.yaml", + helmv3: HelmV3{}, + wantErr: &os.PathError{Err: syscall.EISDIR, Op: "read", Path: "testdata/chart-unreadable-values/values.yaml"}, + }, + { + name: "chart path with unreadable values.yaml", + chartPath: "./testdata/chart-bad-values/Chart.yaml", + helmv3: HelmV3{}, + wantErr: &yaml.TypeError{Errors: []string{"line 1: cannot unmarshal !!str `:bad Date: Tue, 10 Nov 2020 18:48:27 -0800 Subject: [PATCH 2/9] fix unit tests - make sure empty directories are created by adding a placeholder file --- .../templates/service.yaml/do-not-delete.txt | 1 + .../chart-unreadable-values/values.yaml/do-not-delete.txt | 1 + .../helm/v3/testdata/no-helm-charts/do-not-delete.txt | 1 + 3 files changed, 3 insertions(+) create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-bad-template-file/templates/service.yaml/do-not-delete.txt create mode 100644 pkg/iac-providers/helm/v3/testdata/chart-unreadable-values/values.yaml/do-not-delete.txt create mode 100644 pkg/iac-providers/helm/v3/testdata/no-helm-charts/do-not-delete.txt diff --git a/pkg/iac-providers/helm/v3/testdata/chart-bad-template-file/templates/service.yaml/do-not-delete.txt b/pkg/iac-providers/helm/v3/testdata/chart-bad-template-file/templates/service.yaml/do-not-delete.txt new file mode 100644 index 000000000..140c6c2b0 --- /dev/null +++ b/pkg/iac-providers/helm/v3/testdata/chart-bad-template-file/templates/service.yaml/do-not-delete.txt @@ -0,0 +1 @@ +This file preserves the current directory. diff --git a/pkg/iac-providers/helm/v3/testdata/chart-unreadable-values/values.yaml/do-not-delete.txt b/pkg/iac-providers/helm/v3/testdata/chart-unreadable-values/values.yaml/do-not-delete.txt new file mode 100644 index 000000000..140c6c2b0 --- /dev/null +++ b/pkg/iac-providers/helm/v3/testdata/chart-unreadable-values/values.yaml/do-not-delete.txt @@ -0,0 +1 @@ +This file preserves the current directory. diff --git a/pkg/iac-providers/helm/v3/testdata/no-helm-charts/do-not-delete.txt b/pkg/iac-providers/helm/v3/testdata/no-helm-charts/do-not-delete.txt new file mode 100644 index 000000000..140c6c2b0 --- /dev/null +++ b/pkg/iac-providers/helm/v3/testdata/no-helm-charts/do-not-delete.txt @@ -0,0 +1 @@ +This file preserves the current directory. From 772901ba3a3fadfc55db3d8ac0571c239cd97ce2 Mon Sep 17 00:00:00 2001 From: Willie Sana Date: Wed, 11 Nov 2020 05:49:23 -0800 Subject: [PATCH 3/9] fix sonarcloud issues - avoids some repeated strings by using logger.With() - move rendering code into a separate function - remove a duplicate test - add more comments --- pkg/iac-providers/helm/v3/load-dir.go | 157 ++++++++++++--------- pkg/iac-providers/helm/v3/load-dir_test.go | 8 +- pkg/iac-providers/helm/v3/types.go | 2 + 3 files changed, 94 insertions(+), 73 deletions(-) diff --git a/pkg/iac-providers/helm/v3/load-dir.go b/pkg/iac-providers/helm/v3/load-dir.go index 8cdc75c52..69347dff6 100644 --- a/pkg/iac-providers/helm/v3/load-dir.go +++ b/pkg/iac-providers/helm/v3/load-dir.go @@ -54,8 +54,8 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error } if len(fileMap) == 0 { - zap.S().Error("", zap.String("root dir", absRootDir), zap.Error(err)) err = errNoHelmChartsFound + zap.S().Error("", zap.String("root dir", absRootDir), zap.Error(err)) return allResourcesConfig, err } @@ -63,28 +63,32 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error iacDocumentMap := make(map[string][]*utils.IacDocument) for fileDir, chartFilename := range fileMap { chartPath := filepath.Join(fileDir, *chartFilename[0]) - zap.S().Debug("processing chart", zap.String("chart path", chartPath), zap.Error(err)) + logger := zap.S().With("chart path", chartPath) + logger.Debug("processing chart", zap.Error(err)) + // load helm charts into a map of IaC documents var iacDocuments []*utils.IacDocument - var chartMap map[string]interface{} + var chartMap helmChartData iacDocuments, chartMap, err = h.loadChart(chartPath) if err != nil && err != errSkipTestDir { - zap.S().Warn("error occurred while loading chart", zap.String("chart path", chartPath), zap.Error(err)) + logger.Warn("error occurred while loading chart", zap.Error(err)) continue } iacDocumentMap[chartPath] = iacDocuments + // for each chart, add a normalized helm_chart resource var config *output.ResourceConfig config, err = h.createHelmChartResource(chartPath, chartMap) if err != nil { - zap.S().Debug("failed to create helm chart resource", zap.Any("config", config)) + logger.Debug("failed to create helm chart resource", zap.Any("config", config)) continue } allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config) } + // normalize all rendered IaC documents using the kubernetes code for _, iacDocuments := range iacDocumentMap { for _, doc := range iacDocuments { // @TODO add k8s version check @@ -110,22 +114,24 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error func (h *HelmV3) createHelmChartResource(chartPath string, chartData map[string]interface{}) (*output.ResourceConfig, error) { var config output.ResourceConfig + logger := zap.S().With("chart path", chartPath) + jsonData, err := json.Marshal(chartData) if err != nil { - zap.S().Warn("unable to marshal chart to json", zap.String("chart path", chartPath)) + logger.Warn("unable to marshal chart to json") return nil, err } configData := make(map[string]interface{}) if err = json.Unmarshal(jsonData, &configData); err != nil { - zap.S().Warn("unable to unmarshal normalized config data", zap.String("chart path", chartPath)) - zap.S().Debug("failed config data", zap.Any("config", configData)) + logger.Warn("unable to unmarshal normalized config data") + logger.Debug("failed config data", zap.Any("config", configData)) return nil, err } chartName, ok := chartData["name"].(string) if !ok { - zap.S().Warn("unable to determine chart name", zap.String("chart path", chartPath)) + logger.Warn("unable to determine chart name") return nil, err } @@ -139,63 +145,26 @@ func (h *HelmV3) createHelmChartResource(chartPath string, chartData map[string] return &config, nil } -func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]interface{}, error) { +// renderChart renders a helm chart with the given template files and values +// returns and IaC document for each rendered file +func (h *HelmV3) renderChart(chartPath string, chartMap helmChartData, templateFileMap map[string][]*string, valueMap map[string]interface{}) ([]*utils.IacDocument, error) { iacDocuments := make([]*utils.IacDocument, 0) - chartMap := make(map[string]interface{}) + logger := zap.S().With("helm chart path", chartPath) - // load the chart file and values file from the specified chart path - chartFileBytes, err := ioutil.ReadFile(chartPath) - if err != nil { - zap.S().Warn("unable to read", zap.String("file", chartPath)) - return iacDocuments, chartMap, err - } - - if err = yaml.Unmarshal(chartFileBytes, &chartMap); err != nil { - zap.S().Warn("unable to unmarshal values", zap.String("file", chartPath)) - return iacDocuments, chartMap, err - } - - var fileInfo os.FileInfo - chartDir := filepath.Dir(chartPath) - valuesFile := filepath.Join(chartDir, helmValuesFilename) - fileInfo, err = os.Stat(valuesFile) - if err != nil { - zap.S().Warn("unable to stat values.yaml", zap.String("chart path", chartPath)) - return iacDocuments, chartMap, err - } - - var valueFileBytes []byte - valueFileBytes, err = ioutil.ReadFile(valuesFile) - if err != nil { - zap.S().Warn("unable to read values.yaml", zap.String("file", fileInfo.Name())) - return iacDocuments, chartMap, err - } - - var valueMap map[string]interface{} - if err = yaml.Unmarshal(valueFileBytes, &valueMap); err != nil { - zap.S().Warn("unable to unmarshal values.yaml", zap.String("file", fileInfo.Name())) - return iacDocuments, chartMap, err - } - - // for each template file found, render and save an iacDocument - var templateFileMap map[string][]*string - templateFileMap, err = utils.FindFilesBySuffix(filepath.Join(chartDir, helmTemplateDir), h.getHelmTemplateExtensions()) - if err != nil { - zap.S().Warn("error while calling FindFilesBySuffix", zap.String("filepath", fileInfo.Name())) - return iacDocuments, chartMap, err - } for templateDir, templateFiles := range templateFileMap { if filepath.Base(templateDir) == helmTestDir { - zap.S().Debug("skipping test dir", zap.String("dir", templateDir)) - return iacDocuments, chartMap, errSkipTestDir + logger.Debug("skipping test dir", zap.String("dir", templateDir)) + return iacDocuments, errSkipTestDir } + + // create a list containing raw template file data chartFiles := make([]*chart.File, 0) for _, templateFile := range templateFiles { var fileData []byte - fileData, err = ioutil.ReadFile(filepath.Join(templateDir, *templateFile)) + fileData, err := ioutil.ReadFile(filepath.Join(templateDir, *templateFile)) if err != nil { - zap.S().Warn("unable to read template file", zap.String("file", *templateFile)) - return iacDocuments, chartMap, err + logger.Error("unable to read template file", zap.String("file", *templateFile)) + return iacDocuments, err } chartFiles = append(chartFiles, &chart.File{ @@ -204,40 +173,44 @@ func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]i }) } + // chart name and version are required parameters chartName, ok := chartMap["name"].(string) if !ok { - return iacDocuments, chartMap, errBadChartName + logger.Error("chart name was invalid") + return iacDocuments, errBadChartName } var chartVersion string chartVersion, ok = chartMap["version"].(string) if !ok { - return iacDocuments, chartMap, errBadChartVersion + logger.Error("chart version was invalid") + return iacDocuments, errBadChartVersion } + // build the minimum helm chart data input c := &chart.Chart{ Metadata: &chart.Metadata{Name: chartName, Version: chartVersion}, Templates: chartFiles, } var v chartutil.Values - v, err = chartutil.CoalesceValues(c, chartutil.Values{ + v, err := chartutil.CoalesceValues(c, chartutil.Values{ "Values": valueMap, "Release": chartutil.Values{ "Name": defaultChartName, }, }) if err != nil { - zap.S().Warn("error encountered in CoalesceValues", zap.String("chart path", chartPath)) - return iacDocuments, chartMap, err + logger.Warn("error encountered in CoalesceValues") + return iacDocuments, err } + // render all files within the chart var renderData map[string]string renderData, err = engine.Render(c, v) if err != nil { - zap.S().Warn("error encountered while rendering chart", zap.String("chart path", chartPath), - zap.String("template dir", templateDir)) - return iacDocuments, chartMap, err + logger.Warn("error encountered while rendering chart", zap.String("template dir", templateDir)) + return iacDocuments, err } for renderFile := range renderData { @@ -251,7 +224,59 @@ func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]i } } - return iacDocuments, chartMap, nil + return iacDocuments, nil +} + +func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]interface{}, error) { + iacDocuments := make([]*utils.IacDocument, 0) + chartMap := make(helmChartData) + logger := zap.S().With("chart path", chartPath) + + // load the chart file and values file from the specified chart path + chartFileBytes, err := ioutil.ReadFile(chartPath) + if err != nil { + logger.Warn("unable to read") + return iacDocuments, chartMap, err + } + + if err = yaml.Unmarshal(chartFileBytes, &chartMap); err != nil { + logger.Warn("unable to unmarshal values") + return iacDocuments, chartMap, err + } + + var fileInfo os.FileInfo + chartDir := filepath.Dir(chartPath) + valuesFile := filepath.Join(chartDir, helmValuesFilename) + fileInfo, err = os.Stat(valuesFile) + if err != nil { + logger.Warn("unable to stat values.yaml") + return iacDocuments, chartMap, err + } + + logger.With("file name", fileInfo.Name()) + var valueFileBytes []byte + valueFileBytes, err = ioutil.ReadFile(valuesFile) + if err != nil { + logger.Warn("unable to read values.yaml") + return iacDocuments, chartMap, err + } + + var valueMap map[string]interface{} + if err = yaml.Unmarshal(valueFileBytes, &valueMap); err != nil { + logger.Warn("unable to unmarshal values.yaml") + return iacDocuments, chartMap, err + } + + // for each template file found, render and save an iacDocument + var templateFileMap map[string][]*string + templateFileMap, err = utils.FindFilesBySuffix(filepath.Join(chartDir, helmTemplateDir), h.getHelmTemplateExtensions()) + if err != nil { + logger.Warn("error while calling FindFilesBySuffix") + return iacDocuments, chartMap, err + } + + iacDocuments, err = h.renderChart(chartPath, chartMap, templateFileMap, valueMap) + return iacDocuments, chartMap, err } func (h *HelmV3) getHelmTemplateExtensions() []string { diff --git a/pkg/iac-providers/helm/v3/load-dir_test.go b/pkg/iac-providers/helm/v3/load-dir_test.go index 19b209ab9..f5e854458 100644 --- a/pkg/iac-providers/helm/v3/load-dir_test.go +++ b/pkg/iac-providers/helm/v3/load-dir_test.go @@ -55,12 +55,6 @@ func TestLoadIacDir(t *testing.T) { helmv3: HelmV3{}, wantErr: errNoHelmChartsFound, }, - { - name: "unreadable chart file", - dirPath: "./testdata/bad-chart-file", - helmv3: HelmV3{}, - wantErr: nil, // these errors are ignored - }, } for _, tt := range table { @@ -99,7 +93,7 @@ func TestLoadChart(t *testing.T) { name: "unmarshal bad chart", chartPath: "./testdata/bad-chart-file/Chart.yaml", helmv3: HelmV3{}, - wantErr: &yaml.TypeError{Errors: []string{"line 1: cannot unmarshal !!str `:bad ba...` into map[string]interface {}"}}, + wantErr: &yaml.TypeError{Errors: []string{"line 1: cannot unmarshal !!str `:bad ba...` into helmv3.helmChartData"}}, }, { name: "chart path with no values.yaml", diff --git a/pkg/iac-providers/helm/v3/types.go b/pkg/iac-providers/helm/v3/types.go index 4410843c0..2004d8329 100644 --- a/pkg/iac-providers/helm/v3/types.go +++ b/pkg/iac-providers/helm/v3/types.go @@ -19,6 +19,8 @@ package helmv3 // HelmV3 struct implements the IacProvider interface type HelmV3 struct{} +type helmChartData map[string]interface{} + const ( defaultChartName = "terrascan" helmChartFilename = "Chart.yaml" From 83473eabce0b071f8c49bc230b2fa4be98a41eda Mon Sep 17 00:00:00 2001 From: Willie Sana Date: Wed, 11 Nov 2020 06:30:14 -0800 Subject: [PATCH 4/9] fix sonarcloud issues - try #2 with cognitive complexity by moving for loop outside of the render func - update error level of some log messages --- pkg/iac-providers/helm/v3/load-dir.go | 153 +++++++++++---------- pkg/iac-providers/helm/v3/load-dir_test.go | 12 +- 2 files changed, 85 insertions(+), 80 deletions(-) diff --git a/pkg/iac-providers/helm/v3/load-dir.go b/pkg/iac-providers/helm/v3/load-dir.go index 69347dff6..d464ee709 100644 --- a/pkg/iac-providers/helm/v3/load-dir.go +++ b/pkg/iac-providers/helm/v3/load-dir.go @@ -71,7 +71,7 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error var chartMap helmChartData iacDocuments, chartMap, err = h.loadChart(chartPath) if err != nil && err != errSkipTestDir { - logger.Warn("error occurred while loading chart", zap.Error(err)) + logger.Error("error occurred while loading chart", zap.Error(err)) continue } @@ -96,7 +96,7 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error var config *output.ResourceConfig config, err = k.Normalize(doc) if err != nil { - zap.S().Warn("unable to normalize data", zap.Error(err), zap.String("file", doc.FilePath)) + zap.S().Error("unable to normalize data", zap.Error(err), zap.String("file", doc.FilePath)) continue } @@ -114,11 +114,11 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error func (h *HelmV3) createHelmChartResource(chartPath string, chartData map[string]interface{}) (*output.ResourceConfig, error) { var config output.ResourceConfig - logger := zap.S().With("chart path", chartPath) + logger := zap.S().With("helm chart path", chartPath) jsonData, err := json.Marshal(chartData) if err != nil { - logger.Warn("unable to marshal chart to json") + logger.Error("unable to marshal chart to json") return nil, err } @@ -131,7 +131,7 @@ func (h *HelmV3) createHelmChartResource(chartPath string, chartData map[string] chartName, ok := chartData["name"].(string) if !ok { - logger.Warn("unable to determine chart name") + logger.Error("unable to determine chart name") return nil, err } @@ -147,81 +147,79 @@ func (h *HelmV3) createHelmChartResource(chartPath string, chartData map[string] // renderChart renders a helm chart with the given template files and values // returns and IaC document for each rendered file -func (h *HelmV3) renderChart(chartPath string, chartMap helmChartData, templateFileMap map[string][]*string, valueMap map[string]interface{}) ([]*utils.IacDocument, error) { +func (h *HelmV3) renderChart(chartPath string, chartMap helmChartData, templateDir string, templateFiles []*string, valueMap map[string]interface{}) ([]*utils.IacDocument, error) { iacDocuments := make([]*utils.IacDocument, 0) logger := zap.S().With("helm chart path", chartPath) - for templateDir, templateFiles := range templateFileMap { - if filepath.Base(templateDir) == helmTestDir { - logger.Debug("skipping test dir", zap.String("dir", templateDir)) - return iacDocuments, errSkipTestDir - } - - // create a list containing raw template file data - chartFiles := make([]*chart.File, 0) - for _, templateFile := range templateFiles { - var fileData []byte - fileData, err := ioutil.ReadFile(filepath.Join(templateDir, *templateFile)) - if err != nil { - logger.Error("unable to read template file", zap.String("file", *templateFile)) - return iacDocuments, err - } + if filepath.Base(templateDir) == helmTestDir { + logger.Debug("skipping test dir", zap.String("dir", templateDir)) + return iacDocuments, errSkipTestDir + } - chartFiles = append(chartFiles, &chart.File{ - Name: filepath.Join(helmTemplateDir, *templateFile), - Data: fileData, - }) + // create a list containing raw template file data + chartFiles := make([]*chart.File, 0) + for _, templateFile := range templateFiles { + var fileData []byte + fileData, err := ioutil.ReadFile(filepath.Join(templateDir, *templateFile)) + if err != nil { + logger.Error("unable to read template file", zap.String("file", *templateFile)) + return iacDocuments, err } - // chart name and version are required parameters - chartName, ok := chartMap["name"].(string) - if !ok { - logger.Error("chart name was invalid") - return iacDocuments, errBadChartName - } + chartFiles = append(chartFiles, &chart.File{ + Name: filepath.Join(helmTemplateDir, *templateFile), + Data: fileData, + }) + } - var chartVersion string - chartVersion, ok = chartMap["version"].(string) - if !ok { - logger.Error("chart version was invalid") - return iacDocuments, errBadChartVersion - } + // chart name and version are required parameters + chartName, ok := chartMap["name"].(string) + if !ok { + logger.Error("chart name was invalid") + return iacDocuments, errBadChartName + } - // build the minimum helm chart data input - c := &chart.Chart{ - Metadata: &chart.Metadata{Name: chartName, Version: chartVersion}, - Templates: chartFiles, - } + var chartVersion string + chartVersion, ok = chartMap["version"].(string) + if !ok { + logger.Error("chart version was invalid") + return iacDocuments, errBadChartVersion + } - var v chartutil.Values - v, err := chartutil.CoalesceValues(c, chartutil.Values{ - "Values": valueMap, - "Release": chartutil.Values{ - "Name": defaultChartName, - }, - }) - if err != nil { - logger.Warn("error encountered in CoalesceValues") - return iacDocuments, err - } + // build the minimum helm chart data input + c := &chart.Chart{ + Metadata: &chart.Metadata{Name: chartName, Version: chartVersion}, + Templates: chartFiles, + } - // render all files within the chart - var renderData map[string]string - renderData, err = engine.Render(c, v) - if err != nil { - logger.Warn("error encountered while rendering chart", zap.String("template dir", templateDir)) - return iacDocuments, err - } + var v chartutil.Values + v, err := chartutil.CoalesceValues(c, chartutil.Values{ + "Values": valueMap, + "Release": chartutil.Values{ + "Name": defaultChartName, + }, + }) + if err != nil { + logger.Warn("error encountered in CoalesceValues") + return iacDocuments, err + } - for renderFile := range renderData { - iacDocuments = append(iacDocuments, &utils.IacDocument{ - Data: []byte(renderData[renderFile]), - Type: utils.YAMLDoc, - StartLine: 1, - EndLine: 1, - FilePath: renderFile, - }) - } + // render all files within the chart + var renderData map[string]string + renderData, err = engine.Render(c, v) + if err != nil { + logger.Warn("error encountered while rendering chart", zap.String("template dir", templateDir)) + return iacDocuments, err + } + + for renderFile := range renderData { + iacDocuments = append(iacDocuments, &utils.IacDocument{ + Data: []byte(renderData[renderFile]), + Type: utils.YAMLDoc, + StartLine: 1, + EndLine: 1, + FilePath: renderFile, + }) } return iacDocuments, nil @@ -235,12 +233,12 @@ func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]i // load the chart file and values file from the specified chart path chartFileBytes, err := ioutil.ReadFile(chartPath) if err != nil { - logger.Warn("unable to read") + logger.Error("unable to read") return iacDocuments, chartMap, err } if err = yaml.Unmarshal(chartFileBytes, &chartMap); err != nil { - logger.Warn("unable to unmarshal values") + logger.Error("unable to unmarshal values") return iacDocuments, chartMap, err } @@ -257,13 +255,13 @@ func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]i var valueFileBytes []byte valueFileBytes, err = ioutil.ReadFile(valuesFile) if err != nil { - logger.Warn("unable to read values.yaml") + logger.Error("unable to read values.yaml") return iacDocuments, chartMap, err } var valueMap map[string]interface{} if err = yaml.Unmarshal(valueFileBytes, &valueMap); err != nil { - logger.Warn("unable to unmarshal values.yaml") + logger.Error("unable to unmarshal values.yaml") return iacDocuments, chartMap, err } @@ -275,7 +273,14 @@ func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]i return iacDocuments, chartMap, err } - iacDocuments, err = h.renderChart(chartPath, chartMap, templateFileMap, valueMap) + var renderedCharts []*utils.IacDocument + for templateDir, templateFiles := range templateFileMap { + renderedCharts, err = h.renderChart(chartPath, chartMap, templateDir, templateFiles, valueMap) + if err != nil { + continue + } + iacDocuments = append(iacDocuments, renderedCharts...) + } return iacDocuments, chartMap, err } diff --git a/pkg/iac-providers/helm/v3/load-dir_test.go b/pkg/iac-providers/helm/v3/load-dir_test.go index f5e854458..91b675f76 100644 --- a/pkg/iac-providers/helm/v3/load-dir_test.go +++ b/pkg/iac-providers/helm/v3/load-dir_test.go @@ -119,12 +119,12 @@ func TestLoadChart(t *testing.T) { helmv3: HelmV3{}, wantErr: &os.PathError{Err: syscall.ENOENT, Op: "lstat", Path: "testdata/chart-no-template-dir/templates"}, }, - { - name: "chart path skip test dir dir", - chartPath: "./testdata/chart-skip-test-dir/Chart.yaml", - helmv3: HelmV3{}, - wantErr: errSkipTestDir, - }, + //{ + // name: "chart path skip test dir", + // chartPath: "./testdata/chart-skip-test-dir/Chart.yaml", + // helmv3: HelmV3{}, + // wantErr: errSkipTestDir, + //}, { name: "chart path bad template file", chartPath: "./testdata/chart-bad-template-file/Chart.yaml", From 48b6ea879973612d99972387898b7f2493f726b2 Mon Sep 17 00:00:00 2001 From: Willie Sana Date: Wed, 11 Nov 2020 10:30:53 -0800 Subject: [PATCH 5/9] fixes review comments - adds .yml to the valid list of chart yamls - default to 1 instead of 0 for source line in the helm_chart resource - TODO: add values.yml support for rancher --- pkg/iac-providers/helm/v3/load-dir.go | 18 +++++++++++++----- pkg/iac-providers/helm/v3/types.go | 1 - 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/iac-providers/helm/v3/load-dir.go b/pkg/iac-providers/helm/v3/load-dir.go index d464ee709..3b8112b98 100644 --- a/pkg/iac-providers/helm/v3/load-dir.go +++ b/pkg/iac-providers/helm/v3/load-dir.go @@ -47,7 +47,7 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error allResourcesConfig := make(map[string][]output.ResourceConfig) // find all Chart.yaml files within the specified directory structure - fileMap, err := utils.FindFilesBySuffix(absRootDir, []string{helmChartFilename}) + fileMap, err := utils.FindFilesBySuffix(absRootDir, h.getHelmChartFilenames()) if err != nil { zap.S().Error("error while searching for helm charts", zap.String("root dir", absRootDir), zap.Error(err)) return allResourcesConfig, err @@ -91,7 +91,7 @@ func (h *HelmV3) LoadIacDir(absRootDir string) (output.AllResourceConfigs, error // normalize all rendered IaC documents using the kubernetes code for _, iacDocuments := range iacDocumentMap { for _, doc := range iacDocuments { - // @TODO add k8s version check + // helmv3 supports the kubernetes v1 api var k k8sv1.K8sV1 var config *output.ResourceConfig config, err = k.Normalize(doc) @@ -137,7 +137,7 @@ func (h *HelmV3) createHelmChartResource(chartPath string, chartData map[string] config.Type = "helm_chart" config.Name = chartName - config.Line = 0 + config.Line = 1 config.Source = chartPath config.ID = config.Type + "." + config.Name config.Config = configData @@ -225,7 +225,8 @@ func (h *HelmV3) renderChart(chartPath string, chartMap helmChartData, templateD return iacDocuments, nil } -func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]interface{}, error) { +// loadChart renders and loads all templates within a chart path +func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, helmChartData, error) { iacDocuments := make([]*utils.IacDocument, 0) chartMap := make(helmChartData) logger := zap.S().With("chart path", chartPath) @@ -284,6 +285,13 @@ func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, map[string]i return iacDocuments, chartMap, err } +// getHelmTemplateExtensions returns valid helm template extensions func (h *HelmV3) getHelmTemplateExtensions() []string { - return []string{"yaml", "tpl"} + return []string{"yaml", "yml", "tpl"} +} + +// getHelmFilenames returns valid chart filenames +func (h *HelmV3) getHelmChartFilenames() []string { + // the main filename is chart.yaml, but rancher contains references to chart.yml + return []string{"Chart.yaml", "Chart.yml"} } diff --git a/pkg/iac-providers/helm/v3/types.go b/pkg/iac-providers/helm/v3/types.go index 2004d8329..13ea94905 100644 --- a/pkg/iac-providers/helm/v3/types.go +++ b/pkg/iac-providers/helm/v3/types.go @@ -23,7 +23,6 @@ type helmChartData map[string]interface{} const ( defaultChartName = "terrascan" - helmChartFilename = "Chart.yaml" helmValuesFilename = "values.yaml" helmTemplateDir = "templates" helmTestDir = "tests" From 9385d116a0d59f168a376fc338a46433d4642718 Mon Sep 17 00:00:00 2001 From: Willie Sana Date: Wed, 11 Nov 2020 15:56:00 -0800 Subject: [PATCH 6/9] remove Chart.yml from valid helm chart names - Chart.yml is used by rancher, but we would likely support rancher in a separate iac type, so removing from helm --- pkg/iac-providers/helm/v3/load-dir.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/iac-providers/helm/v3/load-dir.go b/pkg/iac-providers/helm/v3/load-dir.go index 3b8112b98..8feac6f84 100644 --- a/pkg/iac-providers/helm/v3/load-dir.go +++ b/pkg/iac-providers/helm/v3/load-dir.go @@ -290,8 +290,7 @@ func (h *HelmV3) getHelmTemplateExtensions() []string { return []string{"yaml", "yml", "tpl"} } -// getHelmFilenames returns valid chart filenames +// getHelmChartFilenames returns valid chart filenames func (h *HelmV3) getHelmChartFilenames() []string { - // the main filename is chart.yaml, but rancher contains references to chart.yml - return []string{"Chart.yaml", "Chart.yml"} + return []string{"Chart.yaml"} } From b2ea41d3515616e7d270f5fecdcd8db92e128176 Mon Sep 17 00:00:00 2001 From: Cesar Rodriguez Date: Wed, 11 Nov 2020 22:12:11 -0500 Subject: [PATCH 7/9] adds helm docs --- README.md | 2 +- docs/getting-started.md | 33 ++++++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 38417c2f7..566c9d861 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Detect compliance and security violations across Infrastructure as Code to mitig ## Features * 500+ Policies for security best practices * Scanning of Terraform 12+ (HCL2) -* Scanning of Kubernetes YAML/JSON +* Scanning of Kubernetes (JSON/YAML), and Helm v3 * Support for AWS, Azure, GCP, Kubernetes and GitHub ## Installing diff --git a/docs/getting-started.md b/docs/getting-started.md index 24fceca7d..7fb12aee9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -101,7 +101,7 @@ By default Terrascan defaults to scanning Terraform HCL files, you can change th $ terrascan scan -t k8s -i k8s ``` -The `scan` command support flags to configure: the directory being scanned, scanning of a specific file, IaC provier type, path to policies, and policy type. The full list of flags can be found by typing `terrascan scan -h` +The `scan` command support flags to configure: the directory being scanned, scanning of a specific file, IaC provider type, path to policies, and policy type. The full list of flags can be found by typing `terrascan scan -h` ``` Bash $ terrascan scan -h @@ -117,8 +117,8 @@ Flags: -h, --help help for scan -d, --iac-dir string path to a directory containing one or more IaC files (default ".") -f, --iac-file string path to a single IaC file - -i, --iac-type string iac type (k8s, terraform) - --iac-version string iac version (k8s: v1, terraform: v12) + -i, --iac-type string iac type (helm, k8s, terraform) + --iac-version string iac version (helm: v3, k8s: v1, terraform: v12) -p, --policy-path stringArray policy path directory -t, --policy-type strings policy type (all, aws, azure, gcp, github, k8s) (default [all]) -r, --remote-type string type of remote backend (git, s3, gcs, http) @@ -136,6 +136,29 @@ By default Terrascan will output YAML. This can be changed to JSON or XML by usi Terrascan will exit 3 if any issues are found. +#### Scanning code remotely + +Terrascan can download and scan remote repositories/code sources by using the `-r` and `-u` flags. Here's and example: + +``` Bash +$ terrascan scan -t aws -r git -u git@github.com:accurics/KaiMonkey.git//terraform/aws +``` + +The URLs for the remote should follow similar naming as the source argument for modules in Terraform. More details [here](https://www.terraform.io/docs/modules/sources.html). + +#### Helm + +Helm chart can be scanned by specifying "helm" on the -i flag as follows: + +``` +$ terrascan scan -t k8s -i helm +``` + +This command will recursively look for Chart.yaml files in the current directory and the corresponding /templates directory and will scan any .yaml, .yml, and .tpl files. + +A specific directory to scan can be specified using the `-d` flag. The Helm IaC provider does not support scanning of individual files using the `-f` flag. + + ### CLI Output types #### Violations Terrascan's default output is a list of violations present in the scanned IaC. @@ -159,7 +182,7 @@ results: total: 1 ``` ##### Resource Config -Terrascan while scanning the IaC, loads all the IaC files, creates a list of resource configs and then processes this list to report violations. For debugging purposes, it possible to print this resource configs list as an output by providing the `--config-only` flag to the `terrascan scan` command. +Terrascan while scanning the IaC, loads all the IaC files, creates a list of resource configs and then processes this list to report violations. For debugging purposes, it is possible to print this resource configs list as an output by providing the `--config-only` flag to the `terrascan scan` command. ``` Bash $ terrascan scan -t aws --config-only aws_ecr_repository: @@ -282,7 +305,7 @@ Transfer-Encoding: chunked ``` ### Config File -The `-c` or `--config-path` global variable allows you to provide a TOML configuration file for Terrascan. This file can be use to configure the webhook notifications. Here's an example configuration: +The `-c` or `--config-path` global variable allows you to provide a TOML configuration file for Terrascan. This file can be used to configure the webhook notifications. Here's an example configuration: ``` TOML [notifications] From 20f32e0035faeae014db9e735b69da550b4dedb9 Mon Sep 17 00:00:00 2001 From: Cesar Rodriguez Date: Wed, 11 Nov 2020 22:16:35 -0500 Subject: [PATCH 8/9] rewording --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 7fb12aee9..ecf86e4c8 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -154,7 +154,7 @@ Helm chart can be scanned by specifying "helm" on the -i flag as follows: $ terrascan scan -t k8s -i helm ``` -This command will recursively look for Chart.yaml files in the current directory and the corresponding /templates directory and will scan any .yaml, .yml, and .tpl files. +This command will recursively look for Chart.yaml files in the current directory and render .yaml, .yml, .tpl template files found under the corresponding /templates directory. A specific directory to scan can be specified using the `-d` flag. The Helm IaC provider does not support scanning of individual files using the `-f` flag. From aea0e0bc6b108ae1e984b972aea9d5fd6ff0ceb0 Mon Sep 17 00:00:00 2001 From: Cesar Rodriguez Date: Wed, 11 Nov 2020 22:18:15 -0500 Subject: [PATCH 9/9] rewording --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index ecf86e4c8..37d8dc50c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -154,7 +154,7 @@ Helm chart can be scanned by specifying "helm" on the -i flag as follows: $ terrascan scan -t k8s -i helm ``` -This command will recursively look for Chart.yaml files in the current directory and render .yaml, .yml, .tpl template files found under the corresponding /templates directory. +This command will recursively look for Chart.yaml files in the current directory and scans rendered .yaml, .yml, .tpl template files found under the corresponding /templates directory. A specific directory to scan can be specified using the `-d` flag. The Helm IaC provider does not support scanning of individual files using the `-f` flag.