From 669d2e4f3ce79948ab8027461f61c14767580b52 Mon Sep 17 00:00:00 2001 From: Nathan Martins Date: Sat, 26 Feb 2022 01:31:19 +0000 Subject: [PATCH] gitleaks:bugfix - updating formatter to gitleaks v8 When the v8 version of gitaleks was released, the way to do an analysis and the output of that analysis changed. This pr updates the formatter to the new format and updates gitleaks to v8.3.0. It also contains some improvements in the formatter code. Signed-off-by: Nathan Martins --- e2e/analysis/test_case.go | 1 + internal/controllers/analyzer/analyzer.go | 3 +- internal/enums/images/images.go | 2 +- internal/helpers/messages/warn.go | 1 + .../formatters/leaks/deployments/Dockerfile | 2 +- .../formatters/leaks/gitleaks/config.go | 16 +-- .../formatters/leaks/gitleaks/formatter.go | 105 +++++++++--------- .../leaks/gitleaks/{entities => }/output.go | 27 +++-- 8 files changed, 80 insertions(+), 77 deletions(-) rename internal/services/formatters/leaks/gitleaks/{entities => }/output.go (59%) diff --git a/e2e/analysis/test_case.go b/e2e/analysis/test_case.go index eda6877ef..38a934caf 100644 --- a/e2e/analysis/test_case.go +++ b/e2e/analysis/test_case.go @@ -661,6 +661,7 @@ func NewTestCase() []*TestCase { fmt.Sprintf("{HORUSEC_CLI} %s - %s is finished in analysisID:", tools.GitLeaks, languages.Leaks), fmt.Sprintf("{HORUSEC_CLI} Running %s - %s", tools.HorusecEngine, languages.Leaks), fmt.Sprintf("{HORUSEC_CLI} %s - %s is finished in analysisID:", tools.HorusecEngine, languages.Leaks), + fmt.Sprintf("{HORUSEC_CLI} The current path it's not a valid git repository"), }, OutputsNotContains: []string{ fmt.Sprintf("{HORUSEC_CLI} Something error went wrong in %s tool", tools.GitLeaks), diff --git a/internal/controllers/analyzer/analyzer.go b/internal/controllers/analyzer/analyzer.go index af0a5acd3..2ef7291c5 100644 --- a/internal/controllers/analyzer/analyzer.go +++ b/internal/controllers/analyzer/analyzer.go @@ -452,5 +452,6 @@ func (a *Analyzer) isWarning(err string) bool { return strings.Contains(err, messages.MsgErrorPacketJSONNotFound) || strings.Contains(err, messages.MsgErrorYarnLockNotFound) || strings.Contains(err, messages.MsgErrorGemLockNotFound) || - strings.Contains(err, messages.MsgErrorNotFoundRequirementsTxt) + strings.Contains(err, messages.MsgErrorNotFoundRequirementsTxt) || + strings.Contains(err, messages.MsgWarnPathIsInvalidGitRepository) } diff --git a/internal/enums/images/images.go b/internal/enums/images/images.go index 928424417..9b6161539 100644 --- a/internal/enums/images/images.go +++ b/internal/enums/images/images.go @@ -25,7 +25,7 @@ const ( Go = "horuszup/horusec-go:v1.2.1" HCL = "horuszup/horusec-hcl:v1.1.0" Javascript = "horuszup/horusec-js:v1.2.0" - Leaks = "horuszup/horusec-leaks:v1.1.0" + Leaks = "horuszup/horusec-leaks:v1.2.0" PHP = "horuszup/horusec-php:v1.0.1" Python = "horuszup/horusec-python:v1.0.0" Ruby = "horuszup/horusec-ruby:v1.1.1" diff --git a/internal/helpers/messages/warn.go b/internal/helpers/messages/warn.go index 7e6f3597b..4d7ed341b 100644 --- a/internal/helpers/messages/warn.go +++ b/internal/helpers/messages/warn.go @@ -55,4 +55,5 @@ const ( // TODO: Remove MsgWarnAnalysisContainsOutdatedHash before release v2.10.0 MsgWarnAnalysisContainsOutdatedHash = "{HORUSEC_CLI} YOUR CONFIGURATION FILE CONTAINS SOME HASHES THAT WILL NO " + "LONGER BE VALID AS OF v2.10.0 IS RELEASED. PLEASE UPDATE YOUR CONFIGURATION FILE WITH THE FOLLOWING HASHES:" + MsgWarnPathIsInvalidGitRepository = "{HORUSEC_CLI} The current path it's not a valid git repository" ) diff --git a/internal/services/formatters/leaks/deployments/Dockerfile b/internal/services/formatters/leaks/deployments/Dockerfile index a151dbc67..af899ca79 100644 --- a/internal/services/formatters/leaks/deployments/Dockerfile +++ b/internal/services/formatters/leaks/deployments/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM zricethezav/gitleaks:v8.2.7 +FROM zricethezav/gitleaks:v8.3.0 COPY ./internal/services/formatters/leaks/deployments/rules.toml /rules/rules.toml diff --git a/internal/services/formatters/leaks/gitleaks/config.go b/internal/services/formatters/leaks/gitleaks/config.go index 9b097f650..41a00a450 100644 --- a/internal/services/formatters/leaks/gitleaks/config.go +++ b/internal/services/formatters/leaks/gitleaks/config.go @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -//nolint package gitleaks +// CMD contains the necessary code to execute Gitleaks inside the container. The 'git config diff.renames 0' command +// it's necessary to avoid the 'inexact rename detection was skipped due to too many files' error in big projects. const CMD = ` - {{WORK_DIR}} - if ! gitleaks --config-path /rules/rules.toml --path . --leaks-exit-code 0 --format json --report /tmp/leaks-result.json &> /tmp/leaks-runner-output.txt; then - echo /tmp/leaks-runner-output.txt - else - cat /tmp/leaks-result.json - fi + {{WORK_DIR}} + git config diff.renames 0 + if ! gitleaks detect -c /rules/rules.toml -f json -r /tmp/leaks.json --exit-code 0 &> /tmp/leaks-output.txt; then + cat /tmp/leaks-output.txt + else + cat /tmp/leaks.json + fi ` diff --git a/internal/services/formatters/leaks/gitleaks/formatter.go b/internal/services/formatters/leaks/gitleaks/formatter.go index 049671129..fe2f75a57 100644 --- a/internal/services/formatters/leaks/gitleaks/formatter.go +++ b/internal/services/formatters/leaks/gitleaks/formatter.go @@ -17,6 +17,7 @@ package gitleaks import ( "encoding/json" "errors" + "strconv" "strings" "github.com/ZupIT/horusec-devkit/pkg/entities/vulnerability" @@ -29,7 +30,6 @@ import ( "github.com/ZupIT/horusec/internal/enums/images" "github.com/ZupIT/horusec/internal/helpers/messages" "github.com/ZupIT/horusec/internal/services/formatters" - "github.com/ZupIT/horusec/internal/services/formatters/leaks/gitleaks/entities" vulnhash "github.com/ZupIT/horusec/internal/utils/vuln_hash" ) @@ -62,78 +62,73 @@ func (f *Formatter) startGitLeaks(projectSubPath string) (string, error) { return output, err } - return output, f.formatOutputGitLeaks(output) + if err := f.checkOutputErrors(output); err != nil { + return output, err + } + + return output, f.parseOutput(output) } -func (f *Formatter) formatOutputGitLeaks(output string) error { - if output == "" || (len(output) >= 4 && output[:4] == "null") { +func (f *Formatter) getDockerConfig(projectSubPath string) *dockerEntities.AnalysisData { + analysisData := &dockerEntities.AnalysisData{ + CMD: f.AddWorkDirInCmd(CMD, projectSubPath, tools.GitLeaks), + Language: languages.Leaks, + } + + return analysisData.SetImage(f.GetCustomImageByLanguage(languages.Leaks), images.Leaks) +} + +func (f *Formatter) parseOutput(output string) error { + issues := make([]*Issue, 0) + + if err := json.Unmarshal([]byte(output), &issues); err != nil { + return err + } + + if len(issues) == 0 { logger.LogDebugWithLevel(messages.MsgDebugOutputEmpty, map[string]interface{}{"tool": tools.GitLeaks.ToString()}) - f.setGitLeaksOutPutInHorusecAnalysis([]entities.Issue{}) return nil } - issues, err := f.parseOutputToIssues(output) - if err != nil { - return err - } + f.forEachIssueCreateNewVuln(issues) - f.setGitLeaksOutPutInHorusecAnalysis(issues) return nil } -func (f *Formatter) parseOutputToIssues(output string) ([]entities.Issue, error) { - var issues []entities.Issue - err := json.Unmarshal([]byte(output), &issues) - if err != nil && strings.Contains(err.Error(), "invalid character") { - err = errors.New(output) +func (f *Formatter) forEachIssueCreateNewVuln(issues []*Issue) { + for _, issue := range issues { + f.AddNewVulnerabilityIntoAnalysis(f.newVulnerability(issue)) } - return issues, err } -func (f *Formatter) setGitLeaksOutPutInHorusecAnalysis(issues []entities.Issue) { - for key := range issues { - vuln := f.setupVulnerabilitiesSeveritiesGitLeaks(&issues[key]) - f.AddNewVulnerabilityIntoAnalysis(vuln) +//nolint:funlen // necessary to be long +func (f *Formatter) newVulnerability(issue *Issue) *vulnerability.Vulnerability { + vuln := &vulnerability.Vulnerability{ + Language: languages.Leaks, + SecurityTool: tools.GitLeaks, + Severity: severities.Critical, + RuleID: vulnhash.HashRuleID(issue.Description), + Details: issue.Description, + Code: f.GetCodeWithMaxCharacters(issue.Secret, 0), + File: issue.File, + Line: strconv.Itoa(issue.StartLine), + Column: strconv.Itoa(issue.StartColumn), + CommitAuthor: issue.Author, + CommitMessage: f.GetCodeWithMaxCharacters(issue.Message, 0), + CommitEmail: issue.Email, + CommitDate: issue.Date, + CommitHash: issue.Commit, } -} - -func (f *Formatter) setupVulnerabilitiesSeveritiesGitLeaks(issue *entities.Issue) ( - vulnerabilitySeverity *vulnerability.Vulnerability, -) { - vulnerabilitySeverity = f.getDefaultSeverity() - vulnerabilitySeverity.Severity = severities.Critical - vulnerabilitySeverity.RuleID = vulnhash.HashRuleID(issue.Rule) - vulnerabilitySeverity.Details = issue.Rule - vulnerabilitySeverity.Code = f.GetCodeWithMaxCharacters(issue.Line, 0) - vulnerabilitySeverity.File = issue.File - vulnerabilitySeverity = vulnhash.Bind(vulnerabilitySeverity) - return f.setCommitAuthor(vulnerabilitySeverity, issue) -} -func (f *Formatter) setCommitAuthor(vuln *vulnerability.Vulnerability, - issue *entities.Issue, -) *vulnerability.Vulnerability { - vuln.CommitAuthor = issue.Author - vuln.CommitMessage = strings.ReplaceAll(issue.CommitMessage, "\n", "") - vuln.CommitEmail = issue.Email - vuln.CommitDate = issue.Date - vuln.CommitHash = issue.Commit - return vuln + return vulnhash.Bind(vuln) } -func (f *Formatter) getDockerConfig(projectSubPath string) *dockerEntities.AnalysisData { - analysisData := &dockerEntities.AnalysisData{ - CMD: f.AddWorkDirInCmd(CMD, projectSubPath, tools.GitLeaks), - Language: languages.Leaks, +func (f *Formatter) checkOutputErrors(output string) error { + if strings.Contains(output, "fatal: not a git repository") || + strings.Contains(output, "fatal: cannot chdir to") { + return errors.New(messages.MsgWarnPathIsInvalidGitRepository) } - return analysisData.SetImage(f.GetCustomImageByLanguage(languages.Leaks), images.Leaks) -} - -func (f *Formatter) getDefaultSeverity() *vulnerability.Vulnerability { - vulnerabilitySeverity := &vulnerability.Vulnerability{} - vulnerabilitySeverity.Language = languages.Leaks - vulnerabilitySeverity.SecurityTool = tools.GitLeaks - return vulnerabilitySeverity + return nil } diff --git a/internal/services/formatters/leaks/gitleaks/entities/output.go b/internal/services/formatters/leaks/gitleaks/output.go similarity index 59% rename from internal/services/formatters/leaks/gitleaks/entities/output.go rename to internal/services/formatters/leaks/gitleaks/output.go index 58826f419..bf6b89c2f 100644 --- a/internal/services/formatters/leaks/gitleaks/entities/output.go +++ b/internal/services/formatters/leaks/gitleaks/output.go @@ -12,18 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package entities +package gitleaks type Issue struct { - Line string `json:"line"` - Offender string `json:"offender"` - Commit string `json:"commit"` - Repo string `json:"repo"` - Rule string `json:"rule"` - CommitMessage string `json:"commitMessage"` - Author string `json:"author"` - Email string `json:"email"` - File string `json:"file"` - Date string `json:"date"` - Tags string `json:"tags"` + Description string + StartLine int + EndLine int + StartColumn int + EndColumn int + Match string + Secret string + File string + Commit string + Entropy int + Author string + Email string + Date string + Message string }