Add pagination and streaming

lib/tentacat.ex

@@ -6,11 +6,13 @@ defmodule Tentacat do
 
   @type response :: {integer, any} | :jsx.json_term
 
+  @spec process_response_body(binary) :: term
+  def process_response_body(""), do: nil
+  def process_response_body(body), do: JSX.decode!(body)
+
   @spec process_response(HTTPoison.Response.t) :: response
-  def process_response(%HTTPoison.Response{status_code: 200, body: ""}), do: nil
-  def process_response(%HTTPoison.Response{status_code: 200, body: body}), do: JSX.decode!(body)
-  def process_response(%HTTPoison.Response{status_code: status_code, body: ""}), do: { status_code, nil }
-  def process_response(%HTTPoison.Response{status_code: status_code, body: body }), do: { status_code, JSX.decode!(body) }
+  def process_response(%HTTPoison.Response{status_code: 200, body: body}), do: body
+  def process_response(%HTTPoison.Response{status_code: status_code, body: body }), do: { status_code, body }
 
   def delete(path, client, body \\ "") do
     _request(:delete, url(client, path), client.auth, body)
@@ -28,10 +30,34 @@ defmodule Tentacat do
     _request(:put, url(client, path), client.auth, body)
   end
 
-  def get(path, client, params \\ []) do
-    initial_url = url(client, path)
-    url = add_params_to_url(initial_url, params)
-    _request(:get, url, client.auth)
+  @doc """
+  Underlying utility retrieval function. The options passed affect both the
+  return value and, ultimately, the number of requests made to GitHub.
+
+  Options:
+    * `:pagination` - Can be `:none`, `:manual`, `:stream`, or `:auto`. Defaults to :auto
+      `:none` will only return the first page. You won't have access to the headers to manually
+      paginate.
+      `:auto` will block until all the pages have been retrieved and concatenated together. Most
+      of the time, this is what you want. For example, `Tentacat.Repositories.list_users("chrismccord")`
+      and `Tentacat.Repositories.list_users("octocat")` have the same interface though one call
+      will page many times and the other not at all.
+      `:stream` will return a `Stream`, prepopulated with the first page.
+      `:manual` will return a 3 element tuple of `{page_body, url_for_next_page, auth_credentials}`,
+      which will allow you to control the paging yourself.
+  """
+  def get(path, client, params \\ [], options \\ []) do
+    url =
+      client
+      |> url(path)
+      |> add_params_to_url(params)
+    case Keyword.get(options, :pagination, nil) do
+      nil     -> request_stream(:get, url, client.auth) |> realize_if_needed
+      :none   -> request_stream(:get, url, client.auth, "", :one_page)
+      :auto   -> request_stream(:get, url, client.auth) |> realize_if_needed
+      :stream -> request_stream(:get, url, client.auth)
+      :manual -> request_with_pagination(:get, url, client.auth)
+    end
   end
 
   def _request(method, url, auth, body \\ "") do
@@ -42,23 +68,130 @@ defmodule Tentacat do
     raw_request(method, url, JSX.encode!(body), headers, options)
   end
 
+  defp extra_options do
+    Application.get_env(:tentacat, :request_options, [])
+  end
+
   def raw_request(method, url, body \\ "", headers \\ [], options \\ []) do
-    extra_options = Application.get_env(:tentacat, :request_options, [])
     request!(method, url, body, headers, extra_options ++ options) |> process_response
   end
 
+  def request_stream(method, url, auth, body \\ "", override \\ nil) do
+    request_with_pagination(method, url, auth, JSX.encode!(body))
+    |> stream_if_needed(override)
+  end
+  defp stream_if_needed(result = {status_code, _}, _) when is_number(status_code), do: result
+  defp stream_if_needed({body, nil, _}, _), do: body
+  defp stream_if_needed({body, _, _}, :one_page), do: body
+  defp stream_if_needed(initial_results, _) do
+    Stream.resource(
+      fn -> initial_results end,
+      &process_stream/1,
+      fn _ -> nil end)
+  end
+
+  defp realize_if_needed(x) when is_tuple(x) or is_binary(x) or is_list(x) or is_map(x), do: x
+  defp realize_if_needed(stream), do: Enum.to_list(stream)
+
+  defp process_stream({[], nil, _}), do: {:halt, nil}
+  defp process_stream({[], next, auth}) do
+    request_with_pagination(:get, next, auth, "")
+    |> process_stream
+  end
+  defp process_stream({items, next, auth}) when is_list(items) do
+    {items, {[], next, auth}}
+  end
+  defp process_stream({item, next, auth}) do
+    {[item], {[], next, auth}}
+  end
+
+  @spec request_with_pagination(atom, binary, Client.auth, binary) :: {binary, binary, Client.auth}
+  def request_with_pagination(method, url, auth, body \\ "") do
+    resp = request!(method, url, JSX.encode!(body), authorization_header(auth, @user_agent), extra_options)
+    case process_response(resp) do
+      x when is_tuple(x) -> x
+      _ -> pagination_tuple(resp, auth)
+    end
+  end
+
+  @spec pagination_tuple(HTTPoison.Response.t, Client.auth) :: {binary, binary, Client.auth}
+  defp pagination_tuple(%HTTPoison.Response{headers: headers} = resp, auth) do
+    {process_response(resp), next_link(headers), auth}
+  end
+
+  defp next_link(headers) do
+    for {"Link", link_header} <- headers, links <- String.split(link_header, ",") do
+      Regex.named_captures(~r/<(?<link>.*)>;\s*rel=\"(?<rel>.*)\"/, links)
+      |> case do
+        %{"link" => link, "rel" => "next"} -> link
+        _ -> nil
+      end
+    end
+    |> Enum.filter(&(not is_nil(&1)))
+    |> List.first
+  end
+
   @spec url(client :: Client.t, path :: binary) :: binary
   defp url(_client = %Client{endpoint: endpoint}, path) do
     endpoint <> path
   end
 
-  defp add_params_to_url(url, params) do
-    <<url :: binary, build_qs(params) :: binary>>
+  @doc """
+  Take an existing URI and add addition params, appending and replacing as necessary
+
+  ## Examples
+      iex> add_params_to_url("http://example.com/wat", [])
+      "http://example.com/wat"
+
+      iex> add_params_to_url("http://example.com/wat", [q: 1])
+      "http://example.com/wat?q=1"
+
+      iex> add_params_to_url("http://example.com/wat", [q: 1, t: 2])
+      "http://example.com/wat?q=1&t=2"
+
+      iex> add_params_to_url("http://example.com/wat", %{q: 1, t: 2})
+      "http://example.com/wat?q=1&t=2"
+
+      iex> add_params_to_url("http://example.com/wat?q=1&t=2", [])
+      "http://example.com/wat?q=1&t=2"
+
+      iex> add_params_to_url("http://example.com/wat?q=1", [t: 2])
+      "http://example.com/wat?q=1&t=2"
+
+      iex> add_params_to_url("http://example.com/wat?q=1", [q: 3, t: 2])
+      "http://example.com/wat?q=3&t=2"
+
+      iex> add_params_to_url("http://example.com/wat?q=1&s=4", [q: 3, t: 2])
+      "http://example.com/wat?q=3&s=4&t=2"
+
+      iex> add_params_to_url("http://example.com/wat?q=1&s=4", %{q: 3, t: 2})
+      "http://example.com/wat?q=3&s=4&t=2"
+  """
+  @spec add_params_to_url(binary, list) :: binary
+  def add_params_to_url(url, params) do
+    url
+    |> URI.parse
+    |> merge_uri_params(params)
+    |> String.Chars.to_string
   end
 
-  @spec build_qs([{atom, binary}]) :: binary
-  defp build_qs([]), do: ""
-  defp build_qs(kvs), do: to_string('?' ++ URI.encode_query(kvs))
+  @spec merge_uri_params(URI.t, list) :: URI.t
+  defp merge_uri_params(uri, []), do: uri
+  defp merge_uri_params(%URI{query: nil} = uri, params) when is_list(params) or is_map(params) do
+    uri
+    |> Map.put(:query, URI.encode_query(params))
+  end
+  defp merge_uri_params(%URI{} = uri, params) when is_list(params) or is_map(params) do
+    uri
+    |> Map.update!(:query, fn q -> q |> URI.decode_query |> Map.merge(param_list_to_map_with_string_keys(params)) |> URI.encode_query end)
+  end
+
+  @spec param_list_to_map_with_string_keys(list) :: map
+  defp param_list_to_map_with_string_keys(list) when is_list(list) or is_map(list) do
+    for {key, value} <- list, into: Map.new do
+      {"#{key}", value}
+    end
+  end
 
   @doc """
   There are two ways to authenticate through GitHub API v3:

lib/tentacat/followers.ex

@@ -12,8 +12,11 @@ defmodule Tentacat.Users.Followers do
   More info at: http://developer.github.com/v3/users/followers/#list-users-followed-by-another-user
   """
   @spec following(Client.t) :: Tentacat.response
-  def following(client) do
-    get "user/following", client
+  def following(client) when is_map(client) do
+    following(client, [])
+  end
+  def following(client, options) when is_map(client) and is_list(options) do
+    get "user/following", client, [], options
   end
 
   @doc """
@@ -25,8 +28,11 @@ defmodule Tentacat.Users.Followers do
 
   More info at: http://developer.github.com/v3/users/followers/#list-users-followed-by-another-user
   """
-  def following(user_name, client) do
-    get "users/#{user_name}/following", client
+  def following(user_name, client) when is_binary(user_name) and is_map(client) do
+    following(user_name, client, [])
+  end
+  def following(user_name, client, options) when is_binary(user_name) and is_map(client) and is_list(options) do
+    get "users/#{user_name}/following", client, [], options
   end
 
   @doc """

lib/tentacat/repositories.ex

@@ -18,8 +18,8 @@ defmodule Tentacat.Repositories do
   More info at: https://developer.github.com/v3/repos/#list-your-repositories
   """
   @spec list_mine(Client.t, Keyword.t) :: Tentacat.response
-  def list_mine(client, opts \\ []) do
-    get "user/repos", client, opts
+  def list_mine(client, params \\ [], options \\ []) do
+    get "user/repos", client, params, options
   end
 
   @doc """
@@ -32,8 +32,8 @@ defmodule Tentacat.Repositories do
   More info at: https://developer.github.com/v3/repos/#list-user-repositories
   """
   @spec list_users(binary, Client.t) :: Tentacat.response
-  def list_users(owner, client \\ %Client{}) do
-    get "users/#{owner}/repos", client
+  def list_users(owner, client \\ %Client{}, params \\ [], options \\ []) do
+    get "users/#{owner}/repos", client, params, options
   end
 
   @doc """
@@ -46,8 +46,8 @@ defmodule Tentacat.Repositories do
   More info at: https://developer.github.com/v3/repos/#list-organization-repositories
   """
   @spec list_orgs(binary, Client.t) :: Tentacat.response
-  def list_orgs(org, client \\ %Client{}) do
-    get "orgs/#{org}/repos", client
+  def list_orgs(org, client \\ %Client{}, params \\ [], options \\ []) do
+    get "orgs/#{org}/repos", client, params, options
   end
 
   @doc """
@@ -61,8 +61,8 @@ defmodule Tentacat.Repositories do
   More info at: https://developer.github.com/v3/repos/#list-all-public-repositories
   """
   @spec list_public(Client.t) :: Tentacat.response
-  def list_public(client \\ %Client{}) do
-    get "repositories", client
+  def list_public(client \\ %Client{}, params \\ [], options \\ []) do
+    get "repositories", client, params, Keyword.merge([pagination: :none], options)
   end
 
   @doc """
@@ -76,8 +76,8 @@ defmodule Tentacat.Repositories do
   More info at: https://developer.github.com/v3/repos/#get
   """
   @spec repo_get(binary, binary, Client.t) :: Tentacat.response
-  def repo_get(owner, repo, client \\ %Client{}) do
-    get "repos/#{owner}/#{repo}", client
+  def repo_get(owner, repo, client \\ %Client{}, params \\ []) do
+    get "repos/#{owner}/#{repo}", client, params
   end
 
   @doc """

lib/tentacat/users.ex

@@ -42,8 +42,8 @@ defmodule Tentacat.Users do
   More info at: http://developer.github.com/v3/users/#get-all-users
   """
   @spec list(Client.t) :: Tentacat.response
-  def list(client \\ %Client{}) do
-    get "users", client
+  def list(client \\ %Client{}, options \\ []) do
+    get "users", client, [], Keyword.merge([pagination: :none], options)
   end
 
   @doc """
@@ -57,8 +57,8 @@ defmodule Tentacat.Users do
   More info at: http://developer.github.com/v3/users/#get-all-users
   """
   @spec list_since(integer, Client.t) :: Tentacat.response
-  def list_since(since, client \\ %Client{}) do
-    get "users", client, [since: since]
+  def list_since(since, client \\ %Client{}, options \\ []) do
+    get "users", client, [since: since], Keyword.merge([pagination: :none], options)
   end
 
   @doc """

test/fixture/vcr_cassettes/followers#following/2.json

@@ -28,7 +28,6 @@
         "X-Accepted-OAuth-Scopes": "repo",
         "Vary": "Accept, Authorization, Cookie, X-GitHub-OTP",
         "X-GitHub-Media-Type": "github.v3; format=json",
-        "Link": "<https://api.github.com/user/222101/following?page=2>; rel=\"next\", <https://api.github.com/user/222101/following?page=3>; rel=\"last\"",
         "X-XSS-Protection": "1; mode=block",
         "X-Frame-Options": "deny",
         "Content-Security-Policy": "default-src 'none'",

test/fixture/vcr_cassettes/hooks#find.json

@@ -9,7 +9,7 @@
       "method": "get",
       "options": [],
       "request_body": "",
-      "url": "https://api.github.com/repos/tentatest/tentacat/hooks/?"
+      "url": "https://api.github.com/repos/tentatest/tentacat/hooks/1234"
     },
     "response": {
       "body": "{\"message\":\"Not Found\",\"documentation_url\":\"https://developer.github.com/v3\"}",