Use search after instead of page offsets (#68)

* Use search-after header for scrolling instead of pagination.

* Fix dynamic test.

src/search.jl

@@ -208,12 +208,11 @@ function earthdata_search(;
     after::Union{Nothing,DateTime} = nothing,
     all_pages::Bool = true,
     page_size = 2000,
-    page_num = 1,
+    page_num = 1,  # unused
     umm = false,
     verbose = 0,
 )
     q = Dict(
-        "page_num" => page_num,
         "page_size" => page_size,
         "short_name" => short_name,
     )
@@ -234,10 +233,10 @@ function earthdata_search(;
     cgranules = parsef(r)
     granules = Vector{NamedTuple}()
     append!(granules, cgranules)
-    while (length(cgranules) == page_size) && all_pages
+    while !(length(cgranules) < page_size) && haskey(Dict(r.headers), "CMR-Search-After") && all_pages
         @warn "Found more than $page_size granules, requesting another $page_size..."
-        q["page_num"] += 1
-        r = HTTP.get(qurl, query = q, verbose = verbose, status_exception = false)
+        v = get(Dict(r.headers), SubString("CMR-Search-After"), "")
+        r = HTTP.get(qurl, query = q, ["CMR-Search-After" => v], verbose = verbose, status_exception = false)
         HTTP.iserror(r) && error(parse_cmr_error(r))
         cgranules = parsef(r)
         append!(granules, cgranules)

test/runtests.jl

@@ -82,7 +82,7 @@ empty_bbox = (min_x = 4.0, min_y = 40.0, max_x = 5.0, max_y = 50.0)
 
         # Time
         @test length(SpaceLiDAR.search(:ICESat2, :ATL08, after = DateTime(2019, 12, 12), before = DateTime(2019, 12, 13))) == 161
-        @test length(SpaceLiDAR.search(:ICESat2, :ATL08, before = now() - Year(5))) == 0
+        @test length(SpaceLiDAR.search(:ICESat2, :ATL08, before = DateTime(2017, 12, 12))) == 0
         @test length(SpaceLiDAR.search(:ICESat2, :ATL08, after = now())) == 0
         @test_throws ErrorException SpaceLiDAR.search(:ICESat2, :ATL08, after = now() - Month(47), before = now() - Month(48))
     end