Use pagination when pulling back list of teams from GitHub.

lib/auth/github.go

@@ -142,7 +142,10 @@ func (s *AuthServer) validateGithubAuthCallback(q url.Values) (*GithubAuthRespon
 		token.TokenType, token.Expires, token.Scope)
 	// Github does not support OIDC so user claims have to be populated
 	// by making requests to Github API using the access token
-	claims, err := populateGithubClaims(&githubAPIClient{token: token.AccessToken})
+	claims, err := populateGithubClaims(&githubAPIClient{
+		token:      token.AccessToken,
+		authServer: s,
+	})
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -274,6 +277,8 @@ func populateGithubClaims(client githubAPIClientI) (*services.GithubClaims, erro
 	if err != nil {
 		return nil, trace.Wrap(err, "failed to query Github user teams")
 	}
+	log.Debugf("Retrieved %v teams for GitHub user %v.", len(teams), user.Login)
+
 	orgToTeams := make(map[string][]string)
 	for _, team := range teams {
 		orgToTeams[team.Org.Login] = append(
@@ -334,6 +339,8 @@ type githubAPIClientI interface {
 type githubAPIClient struct {
 	// token is the access token retrieved during OAuth2 flow
 	token string
+	// authServer points to the Auth Server.
+	authServer *AuthServer
 }
 
 // userResponse represents response from "user" API call
@@ -344,7 +351,8 @@ type userResponse struct {
 
 // getEmails retrieves a list of emails for authenticated user
 func (c *githubAPIClient) getUser() (*userResponse, error) {
-	bytes, err := c.get("/user")
+	// Ignore pagination links, we should never get more than a single user here.
+	bytes, _, err := c.get("/user")
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -372,50 +380,108 @@ type orgResponse struct {
 	Login string `json:"login"`
 }
 
-// getTeams retrieves a list of teams authenticated user belongs to
+// getTeams retrieves a list of teams authenticated user belongs to.
 func (c *githubAPIClient) getTeams() ([]teamResponse, error) {
-	bytes, err := c.get("/user/teams")
+	var result []teamResponse
+
+	bytes, nextPage, err := c.get("/user/teams")
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
+
+	// Extract the first page of results and append them to the full result set.
 	var teams []teamResponse
 	err = json.Unmarshal(bytes, &teams)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
-	return teams, nil
+	result = append(result, teams...)
+
+	// If the response returned a next page link, continue following the next
+	// page links until all teams have been retrieved.
+	var count int
+	for nextPage != "" {
+		// To prevent this from looping forever, don't fetch more than a set number
+		// of pages, print an error when it does happen, and return the results up
+		// to that point.
+		if count > MaxPages {
+			warningMessage := "Truncating list of teams used to populate claims: " +
+				"hit maximum number pages that can be fetched from GitHub."
+
+			// Print warning to Teleport logs as well as the Audit Log.
+			log.Warnf(warningMessage)
+			c.authServer.EmitAuditEvent(events.UserLoginEvent, events.EventFields{
+				events.LoginMethod:        events.LoginMethodGithub,
+				events.AuthAttemptMessage: warningMessage,
+			})
+
+			return result, nil
+		}
+
+		u, err := url.Parse(nextPage)
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+
+		bytes, nextPage, err = c.get(u.RequestURI())
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+
+		err = json.Unmarshal(bytes, &teams)
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+
+		// Append this page of teams to full result set.
+		result = append(result, teams...)
+
+		count = count + 1
+	}
+
+	return result, nil
 }
 
 // get makes a GET request to the provided URL using the client's token for auth
-func (c *githubAPIClient) get(url string) ([]byte, error) {
+func (c *githubAPIClient) get(url string) ([]byte, string, error) {
 	request, err := http.NewRequest("GET", fmt.Sprintf("%v%v", GithubAPIURL, url), nil)
 	if err != nil {
-		return nil, trace.Wrap(err)
+		return nil, "", trace.Wrap(err)
 	}
 	request.Header.Set("Authorization", fmt.Sprintf("token %v", c.token))
 	response, err := http.DefaultClient.Do(request)
 	if err != nil {
-		return nil, trace.Wrap(err)
+		return nil, "", trace.Wrap(err)
 	}
 	defer response.Body.Close()
 	bytes, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		return nil, trace.Wrap(err)
+		return nil, "", trace.Wrap(err)
 	}
 	if response.StatusCode != 200 {
-		return nil, trace.AccessDenied("bad response: %v %v",
+		return nil, "", trace.AccessDenied("bad response: %v %v",
 			response.StatusCode, string(bytes))
 	}
-	return bytes, nil
+
+	// Parse web links header to extract any pagination links. This is used to
+	// return the next link which can be used in a loop to pull back all data.
+	wls := utils.ParseWebLinks(response)
+
+	return bytes, wls.NextPage, nil
 }
 
 const (
 	// GithubAuthURL is the Github authorization endpoint
 	GithubAuthURL = "https://github.com/#/oauth/authorize"
+
 	// GithubTokenURL is the Github token exchange endpoint
 	GithubTokenURL = "https://github.com/#/oauth/access_token"
+
 	// GithubAPIURL is the Github base API URL
 	GithubAPIURL = "https://api.github.com"
+
+	// MaxPages is the maximum number of pagination links that will be followed.
+	MaxPages = 99
 )
 
 var (

lib/events/api.go

@@ -112,6 +112,7 @@ const (
 	AuthAttemptEvent   = "auth"
 	AuthAttemptSuccess = "success"
 	AuthAttemptErr     = "error"
+	AuthAttemptMessage = "message"
 
 	// SCPEvent means data transfer that occurred on the server
 	SCPEvent  = "scp"

lib/utils/linking.go

@@ -0,0 +1,121 @@
+/*
+Copyright (c) 2013 The go-github AUTHORS. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/*
+Copyright 2018 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// WebLinks holds the pagination links parsed out of a request header
+// conforming to RFC 8288.
+type WebLinks struct {
+	// NextPage is the next page of pagination links.
+	NextPage string
+
+	// PrevPage is the previous page of pagination links.
+	PrevPage string
+
+	// FirstPage is the first page of pagination links.
+	FirstPage string
+
+	// LastPage is the last page of pagination links.
+	LastPage string
+}
+
+// ParseWebLinks partially implements RFC 8288 parsing, enough to support
+// GitHub pagination links. See https://tools.ietf.org/html/rfc8288 for more
+// details on Web Linking and https://github.com/google/go-github for the API
+// client that this function was original extracted from.
+//
+// Link headers typically look like:
+//
+//   Link: <https://api.github.com/user/teams?page=2>; rel="next",
+//     <https://api.github.com/user/teams?page=34>; rel="last"
+func ParseWebLinks(response *http.Response) WebLinks {
+	wls := WebLinks{}
+
+	if links, ok := response.Header["Link"]; ok && len(links) > 0 {
+		for _, lnk := range links {
+			for _, link := range strings.Split(lnk, ",") {
+				segments := strings.Split(strings.TrimSpace(link), ";")
+
+				// link must at least have href and rel
+				if len(segments) < 2 {
+					continue
+				}
+
+				// ensure href is properly formatted
+				if !strings.HasPrefix(segments[0], "<") || !strings.HasSuffix(segments[0], ">") {
+					continue
+				}
+
+				// try to pull out page parameter
+				link, err := url.Parse(segments[0][1 : len(segments[0])-1])
+				if err != nil {
+					continue
+				}
+
+				for _, segment := range segments[1:] {
+					switch strings.TrimSpace(segment) {
+					case `rel="next"`:
+						wls.NextPage = link.String()
+					case `rel="prev"`:
+						wls.PrevPage = link.String()
+					case `rel="first"`:
+						wls.FirstPage = link.String()
+					case `rel="last"`:
+						wls.LastPage = link.String()
+					}
+
+				}
+			}
+		}
+	}
+
+	return wls
+}

lib/utils/linking_test.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2018 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"net/http"
+
+	"gopkg.in/check.v1"
+)
+
+var _ = fmt.Printf
+
+type WebLinksSuite struct {
+}
+
+var _ = check.Suite(&WebLinksSuite{})
+
+func (s *WebLinksSuite) SetUpSuite(c *check.C) {
+	InitLoggerForTests()
+}
+
+func (s *WebLinksSuite) TestWebLinks(c *check.C) {
+	var tests = []struct {
+		inResponse *http.Response
+		outNext    string
+		outPrev    string
+		outFirst   string
+		outLast    string
+	}{
+		// 0 - Multiple links in single header. Partial list of relations.
+		{
+			inResponse: &http.Response{
+				Header: http.Header{
+					"Link": []string{`<https://api.github.com/user/teams?page=2>; rel="next",
+		                              <https://api.github.com/user/teams?page=34>; rel="last"`},
+				},
+			},
+			outNext:  "https://api.github.com/user/teams?page=2",
+			outPrev:  "",
+			outFirst: "",
+			outLast:  "https://api.github.com/user/teams?page=34",
+		},
+		// 1 - Multiple links in single header. Full list of relations.
+		{
+			inResponse: &http.Response{
+				Header: http.Header{
+					"Link": []string{`<https://api.github.com/user/teams?page=2>; rel="next",
+		                              <https://api.github.com/user/teams?page=1>; rel="prev",
+                                      <https://api.github.com/user/teams?page=1>; rel="first",
+		                              <https://api.github.com/user/teams?page=34>; rel="last"`},
+				},
+			},
+			outNext:  "https://api.github.com/user/teams?page=2",
+			outPrev:  "https://api.github.com/user/teams?page=1",
+			outFirst: "https://api.github.com/user/teams?page=1",
+			outLast:  "https://api.github.com/user/teams?page=34",
+		},
+		// 2 - Multiple links in multiple headers. Full list of relations.
+		{
+			inResponse: &http.Response{
+				Header: http.Header{
+					"Link": []string{
+						`<https://api.github.com/user/teams?page=1>; rel="next"`,
+						`<https://api.github.com/user/teams?page=2>; rel="prev"`,
+						`<https://api.github.com/user/teams?page=3>; rel="first"`,
+						`<https://api.github.com/user/teams?page=4>; rel="last"`,
+					},
+				},
+			},
+			outNext:  "https://api.github.com/user/teams?page=1",
+			outPrev:  "https://api.github.com/user/teams?page=2",
+			outFirst: "https://api.github.com/user/teams?page=3",
+			outLast:  "https://api.github.com/user/teams?page=4",
+		},
+	}
+
+	for _, tt := range tests {
+		wls := ParseWebLinks(tt.inResponse)
+		c.Assert(wls.NextPage, check.Equals, tt.outNext)
+		c.Assert(wls.PrevPage, check.Equals, tt.outPrev)
+		c.Assert(wls.FirstPage, check.Equals, tt.outFirst)
+		c.Assert(wls.LastPage, check.Equals, tt.outLast)
+	}
+}