Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Implement concurrency mechanism #52

Merged
merged 6 commits into from
Apr 26, 2023
Merged

Implement concurrency mechanism #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f70e17f
Implement concurrency mechanism && Add confluence window constant to …
joaopedrocsilva Apr 19, 2023
c2a11aa
Change context since we already have a mechanism to listen for Interr…
joaopedrocsilva Apr 19, 2023
f547867
Add rate limiter
joaopedrocsilva Apr 20, 2023
d9f2b70
Remove unnecessary variables
joaopedrocsilva Apr 20, 2023
03ac40b
Fix limit name
joaopedrocsilva Apr 20, 2023
fee97bd
Fix issue
joaopedrocsilva Apr 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 38 additions & 25 deletions cmd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,16 @@ import (
"github.com/checkmarx/2ms/secrets"
"os"
"strings"
"sync"
"time"

"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
)

const timeSleepInterval = 50

var rootCmd = &cobra.Command{
Use: "2ms",
Short: "2ms Secrets Detection",
Expand Down Expand Up @@ -90,9 +94,17 @@ func execute(cmd *cobra.Command, args []string) {

validateTags(tags)

secrets := secrets.Init(tags)
report := reporting.Init()

var itemsChannel = make(chan plugins.Item)
var secretsChannel = make(chan reporting.Secret)
var errorsChannel = make(chan error)

var wg sync.WaitGroup

// -------------------------------------
// Get content from plugins

pluginsInitialized := 0
for _, plugin := range allPlugins {
err := plugin.Initialize(cmd)
Expand All @@ -108,41 +120,42 @@ func execute(cmd *cobra.Command, args []string) {
os.Exit(1)
}

items := make([]plugins.Item, 0)
for _, plugin := range allPlugins {
if !plugin.IsEnabled() {
continue
}

pluginItems, err := plugin.GetItems()
if err != nil {
log.Fatal().Msg(err.Error())
}
items = append(items, *pluginItems...)
wg.Add(1)
go plugin.GetItems(itemsChannel, errorsChannel, &wg)
}

report := reporting.Report{}
report.Results = make(map[string][]reporting.Secret)

// -------------------------------------
// Detect Secrets

secrets := secrets.Init(tags)

for _, item := range items {
secrets := secrets.Detect(item.Content)
if len(secrets) > 0 {
report.TotalSecretsFound = report.TotalSecretsFound + len(secrets)
report.Results[item.ID] = append(report.Results[item.ID], secrets...)
go func() {
for {
select {
case item := <-itemsChannel:
report.TotalItemsScanned++
wg.Add(1)
go secrets.Detect(secretsChannel, item, &wg)
case secret := <-secretsChannel:
report.TotalSecretsFound++
report.Results[secret.ID] = append(report.Results[secret.ID], secret)
case err, ok := <-errorsChannel:
if !ok {
return
}
log.Fatal().Msg(err.Error())
}
}
}
report.TotalItemsScanned = len(items)
}()
wg.Wait()

// Wait for last secret to be added to report
time.Sleep(time.Millisecond * timeSleepInterval)
Comment on lines +152 to +153
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @joaopedrocsilva , very nice work, I added this to my PR #53.

I have one question... why we are sleeping here? Why the waitGroup is not helping us?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey @baruchiro.

Couln't understand why but without the sleep we can't get all elements.

Maybe it would make sense to spend some time investigating if it's realated to issues on goroutines management or it's related to some kind of limitations.


// -------------------------------------
// Show Report

if len(items) > 0 {
reporting.ShowReport(report)
if report.TotalItemsScanned > 0 {
report.ShowReport()
} else {
log.Error().Msg("Scan completed with empty content")
os.Exit(0)
Expand Down
3 changes: 3 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ go 1.20
require (
github.com/rs/zerolog v1.29.0
github.com/spf13/cobra v1.6.1
github.com/stretchr/testify v1.8.1
github.com/zricethezav/gitleaks/v8 v8.16.1
)

require (
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/charmbracelet/lipgloss v0.7.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fatih/semgroup v1.2.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gitleaks/go-gitdiff v0.8.0 // indirect
Expand All @@ -28,6 +30,7 @@ require (
github.com/muesli/termenv v0.15.1 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.0 // indirect
Expand Down
94 changes: 55 additions & 39 deletions plugins/confluence.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"strings"
"sync"

"github.com/checkmarx/2ms/lib"
"github.com/rs/zerolog/log"
Expand All @@ -17,6 +18,8 @@ const argConfluenceSpaces = "confluence-spaces"
const argConfluenceUsername = "confluence-username"
const argConfluenceToken = "confluence-token"
const argConfluenceHistory = "history"
const confluenceDefaultWindow = 25
const confluenceMaxRequests = 500

type ConfluencePlugin struct {
Plugin
Expand Down Expand Up @@ -69,46 +72,60 @@ func (p *ConfluencePlugin) Initialize(cmd *cobra.Command) error {
p.Spaces = confluenceSpaces
p.Enabled = true
p.History = runHistory
p.Limit = make(chan struct{}, confluenceMaxRequests)
return nil
}

func (p *ConfluencePlugin) GetItems() (*[]Item, error) {
items := make([]Item, 0)
spaces, err := p.getTotalSpaces()
func (p *ConfluencePlugin) GetItems(items chan Item, errs chan error, wg *sync.WaitGroup) {
defer wg.Done()

go p.getSpacesItems(items, errs, wg)
wg.Add(1)
}

func (p *ConfluencePlugin) getSpacesItems(items chan Item, errs chan error, wg *sync.WaitGroup) {
defer wg.Done()

spaces, err := p.getSpaces()
if err != nil {
return nil, err
errs <- err
}

for _, space := range spaces {
spacePages, err := p.getTotalPages(space)
if err != nil {
return nil, err
}
go p.getSpaceItems(items, errs, wg, space)
wg.Add(1)
}
}

for _, page := range spacePages.Pages {
pageContents, err := p.getContents(page, space)
if err != nil {
return nil, err
}
func (p *ConfluencePlugin) getSpaceItems(items chan Item, errs chan error, wg *sync.WaitGroup, space ConfluenceSpaceResult) {
defer wg.Done()

items = append(items, *pageContents...)
}
pages, err := p.getPages(space)
if err != nil {
errs <- err
return
}

log.Debug().Msg("Confluence plugin completed successfully")
return &items, nil
for _, page := range pages.Pages {
wg.Add(1)
p.Limit <- struct{}{}
go func(page ConfluencePage) {
p.getPageItems(items, errs, wg, page, space)
<-p.Limit
}(page)
}
}

func (p *ConfluencePlugin) getTotalSpaces() ([]ConfluenceSpaceResult, error) {
totalSpaces, err := p.getSpaces(0)
func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) {
totalSpaces, err := p.getSpacesRequest(0)
if err != nil {
return nil, err
}

actualSize := totalSpaces.Size

for actualSize != 0 {
moreSpaces, err := p.getSpaces(totalSpaces.Size)
for actualSize == confluenceDefaultWindow {
moreSpaces, err := p.getSpacesRequest(totalSpaces.Size)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -138,7 +155,7 @@ func (p *ConfluencePlugin) getTotalSpaces() ([]ConfluenceSpaceResult, error) {
return filteredSpaces, nil
}

func (p *ConfluencePlugin) getSpaces(start int) (*ConfluenceSpaceResponse, error) {
func (p *ConfluencePlugin) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) {
url := fmt.Sprintf("%s/rest/api/space?start=%d", p.URL, start)
body, err := lib.HttpRequest(http.MethodGet, url, p)
if err != nil {
Expand All @@ -154,17 +171,17 @@ func (p *ConfluencePlugin) getSpaces(start int) (*ConfluenceSpaceResponse, error
return response, nil
}

func (p *ConfluencePlugin) getTotalPages(space ConfluenceSpaceResult) (*ConfluencePageResult, error) {
totalPages, err := p.getPages(space, 0)
func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePageResult, error) {
totalPages, err := p.getPagesRequest(space, 0)

if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
}

actualSize := len(totalPages.Pages)

for actualSize != 0 {
morePages, err := p.getPages(space, len(totalPages.Pages))
for actualSize == confluenceDefaultWindow {
morePages, err := p.getPagesRequest(space, len(totalPages.Pages))

if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
Expand All @@ -179,7 +196,7 @@ func (p *ConfluencePlugin) getTotalPages(space ConfluenceSpaceResult) (*Confluen
return totalPages, nil
}

func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) {
func (p *ConfluencePlugin) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) {
url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", p.URL, space.Key, start)
body, err := lib.HttpRequest(http.MethodGet, url, p)

Expand All @@ -196,29 +213,28 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult, start int) (*Co
return &response.Results, nil
}

func (p *ConfluencePlugin) getContents(page ConfluencePage, space ConfluenceSpaceResult) (*[]Item, error) {
items := make([]Item, 0)
func (p *ConfluencePlugin) getPageItems(items chan Item, errs chan error, wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) {
defer wg.Done()

actualPage, previousVersion, err := p.getContent(page, space, 0)
actualPage, previousVersion, err := p.getItem(page, space, 0)
if err != nil {
return nil, err
errs <- err
return
}

items = append(items, *actualPage)
items <- *actualPage

// If older versions exist & run history is true
for previousVersion > 0 && p.History {
actualPage, previousVersion, err = p.getContent(page, space, previousVersion)
actualPage, previousVersion, err = p.getItem(page, space, previousVersion)
if err != nil {
return nil, err
errs <- err
return
}
items = append(items, *actualPage)
items <- *actualPage
}

return &items, nil
}

func (p *ConfluencePlugin) getContent(page ConfluencePage, space ConfluenceSpaceResult, version int) (*Item, int, error) {
func (p *ConfluencePlugin) getItem(page ConfluencePage, space ConfluenceSpaceResult, version int) (*Item, int, error) {
var url string
var originalUrl string

Expand Down
8 changes: 6 additions & 2 deletions plugins/plugins.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package plugins

import "github.com/spf13/cobra"
import (
"github.com/spf13/cobra"
"sync"
)

type Item struct {
Content string
Expand All @@ -11,11 +14,12 @@ type Item struct {
type Plugin struct {
ID string
Enabled bool
Limit chan struct{}
}

type IPlugin interface {
DefineCommandLineArgs(cmd *cobra.Command) error
Initialize(cmd *cobra.Command) error
GetItems() (*[]Item, error)
GetItems(chan Item, chan error, *sync.WaitGroup)
IsEnabled() bool
}
Loading