fix(pagination): [nan-1958] per page offset pagination

docs-v2/reference/api-configuration.mdx

@@ -323,6 +323,9 @@ slack:
                 <ResponseField name="offset_name_in_request" type="string">
                     The name of the offset parameter in the request.
                 </ResponseField>
+                <ResponseField name="offset_calculation_method" type="string">
+                    The method to calculate the offset in the request. Must be one of: "per-page" or "by-response-size". Optional parameter that defaults to "by-response-size".
+                </ResponseField>
                 <ResponseField name="offset_start_value" type="number">
                     The starting value for the offset.
                 </ResponseField>

packages/cli/lib/services/__snapshots__/model.service.unit.test.ts.snap

@@ -102,6 +102,7 @@ interface LinkPagination extends Pagination {
 interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: 'per-page' | 'by-response-size';
 }
 interface RetryHeaderConfig {
     at?: string;

packages/shared/lib/models/Proxy.ts

@@ -83,7 +83,10 @@ export interface LinkPagination extends Pagination {
     link_path_in_response_body?: string;
 }
 
+export type OffsetCalculationMethod = 'per-page' | 'by-response-size';
+
 export interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: OffsetCalculationMethod;
 }

packages/shared/lib/sdk/sync.ts

@@ -130,6 +130,7 @@ interface LinkPagination extends Pagination {
 interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: 'per-page' | 'by-response-size';
 }
 
 interface RetryHeaderConfig {

packages/shared/lib/services/paginate.service.ts

@@ -1,7 +1,14 @@
 import type { AxiosResponse } from 'axios';
 import parseLinksHeader from 'parse-link-header';
 import get from 'lodash-es/get.js';
-import type { Pagination, UserProvidedProxyConfiguration, CursorPagination, OffsetPagination, LinkPagination } from '../models/Proxy.js';
+import type {
+    Pagination,
+    UserProvidedProxyConfiguration,
+    CursorPagination,
+    OffsetPagination,
+    LinkPagination,
+    OffsetCalculationMethod
+} from '../models/Proxy.js';
 import { PaginationType } from '../models/Proxy.js';
 import { isValidHttpUrl } from '../utils/utils.js';
 
@@ -124,6 +131,7 @@ class PaginationService {
     ): AsyncGenerator<T[], undefined, void> {
         const offsetPagination: OffsetPagination = paginationConfig;
         const offsetParameterName: string = offsetPagination.offset_name_in_request;
+        const offsetCalculatioMethod: OffsetCalculationMethod = offsetPagination.offset_calculation_method || 'by-response-size';
         let offset = offsetPagination.offset_start_value || 0;
 
         while (true) {
@@ -149,7 +157,11 @@ class PaginationService {
                 return;
             }
 
-            offset += responseData.length;
+            if (offsetCalculatioMethod === 'per-page') {
+                offset++;
+            } else {
+                offset += responseData.length;
+            }
         }
     }
 

packages/types/lib/proxy/api.ts

@@ -70,7 +70,10 @@ export interface LinkPagination extends Pagination {
     link_path_in_response_body?: string;
 }
 
+export type OffsetCalculationMethod = 'per-page' | 'by-response-size';
+
 export interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: OffsetCalculationMethod;
 }

scripts/validation/providers/schema.json

@@ -159,6 +159,10 @@
                                 "offset_start_value": {
                                     "type": "number"
                                 },
+                                "offset_calculation_method": {
+                                    "type": "string",
+                                    "enum": ["per-page", "by-response-size"]
+                                },
                                 "response_path": {
                                     "type": "string"
                                 },