diff --git a/github.go b/github.go index 1caed81..2319f7c 100644 --- a/github.go +++ b/github.go @@ -8,13 +8,12 @@ import ( "github.com/google/go-github/v57/github" "github.com/gregjones/httpcache" + "github.com/sourcegraph/conc" "golang.org/x/text/cases" "golang.org/x/text/language" ) const ( - // perPage is how many links we get by one shoot - perPage int = 100 // repositoriesCount is const for allocation memory to store repositories repositoriesCount = 1000 // langReposCount is const for allocation memory to store langRepo @@ -50,15 +49,34 @@ func (g *GitHub) GetRepositories(ctx context.Context) (langRepoMap map[string][] repositories = make([]Repository, 0, repositoriesCount) langRepoMap = make(map[string][]Repository, langReposCount) - opt := &github.ActivityListStarredOptions{ - ListOptions: github.ListOptions{PerPage: perPage}, + _, resp, err := g.client.Activity.ListStarred(ctx, username, nil) + if err != nil { + log.Fatalln("Error: cannot fetch starred:", err) } - for { - repos, resp, err := g.client.Activity.ListStarred(ctx, username, opt) - if err != nil { - log.Fatalln("Error: cannot fetch starred:", err) + ch := make(chan []*github.StarredRepository, 1) + go func() { + wg := conc.WaitGroup{} + for i := 1; i <= resp.LastPage; i++ { + i := i + wg.Go(func() { + opt := &github.ActivityListStarredOptions{ + ListOptions: github.ListOptions{ + Page: i, + }, + } + repos, _, err := g.client.Activity.ListStarred(ctx, username, opt) + if err != nil { + log.Fatalln("Error: cannot fetch starred:", err) + } + ch <- repos + }) } + wg.Wait() + close(ch) + }() + + for repos := range ch { for _, r := range repos { repo := Repository{ FullName: r.Repository.GetFullName(), @@ -77,11 +95,6 @@ func (g *GitHub) GetRepositories(ctx context.Context) (langRepoMap map[string][] } langRepoMap[lang] = append(langRepoMap[lang], repo) } - - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage } if len(repositories) == 0 { diff --git a/go.mod b/go.mod index ac4b6b5..020d036 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/google/go-github/v57 v57.0.0 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 + github.com/sourcegraph/conc v0.3.0 github.com/spf13/pflag v1.0.5 golang.org/x/text v0.14.0 ) diff --git a/go.sum b/go.sum index 86014ab..d3d4733 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs= @@ -6,8 +7,13 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=