From feaa5e9c995f0d9ab5d39ba66c6a4ee503f76c6a Mon Sep 17 00:00:00 2001 From: omotnyk Date: Mon, 9 Oct 2023 14:10:11 +0300 Subject: [PATCH 01/50] Add hardcoded offset-limit pagination helper --- packages/node-client/lib/index.ts | 29 +++++++++++++++++++++++++++++ packages/node-client/lib/types.ts | 20 ++++++++++++++++++++ packages/shared/lib/sdk/sync.ts | 4 ++++ 3 files changed, 53 insertions(+) diff --git a/packages/node-client/lib/index.ts b/packages/node-client/lib/index.ts index 5cf98d944ba..dafe730e623 100644 --- a/packages/node-client/lib/index.ts +++ b/packages/node-client/lib/index.ts @@ -561,6 +561,35 @@ export class Nango { }); } + public async *paginate(config: ProxyConfiguration, nangoProxyFunction: (config: ProxyConfiguration) => Promise>): AsyncGenerator { + if (!config.pagination) { + throw Error(`Pagination config is not specififed for '${config.providerConfigKey}' provider nor it's passed as an override to 'paginate' method`); + } + + let page = 1; + const defaultMaxValuePerPage:number = 100; + const limit: number = config.pagination?.limit ?? defaultMaxValuePerPage; + const endpoint:string = config.endpoint; + + while (true) { + const resp: AxiosResponse = await nangoProxyFunction({ + endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` + }); + + if (!resp.data.length) { + return; + } + + yield resp.data; + + if (resp.data.length < limit) { + return; + } + + page += 1; + } + } + private async listConnectionDetails(connectionId?: string): Promise> { let url = `${this.serverUrl}/connection?`; if (connectionId) { diff --git a/packages/node-client/lib/types.ts b/packages/node-client/lib/types.ts index 4a3699e1323..53026151680 100644 --- a/packages/node-client/lib/types.ts +++ b/packages/node-client/lib/types.ts @@ -26,6 +26,25 @@ export interface OAuth2Credentials extends CredentialsCommon { expires_at?: Date | undefined; } +export enum PaginationType { + CURSOR = 'cursor', + OFFSET = 'offset', +} + +interface Pagination { + type: PaginationType; + limit?: number; +} + +export interface CursorPagination extends Pagination { + nextCursorParameterPath: string; + cursorParameterName: string; +} + +export interface OffsetPagination extends Pagination { + offset: number; +} + export interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -38,6 +57,7 @@ export interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; + pagination?: OffsetPagination; } export interface GetRecordsRequestConfig { diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index fd2a8be6c32..e55b38ffebf 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -243,6 +243,10 @@ export class NangoAction { }); } + public async paginate(config: ProxyConfiguration, nangoProxyFunction: (config: ProxyConfiguration) => Promise>): Promise> { + return this.nango.paginate(config, nangoProxyFunction); + } + public async getConnection(): Promise { return this.nango.getConnection(this.providerConfigKey as string, this.connectionId as string); } From 99f1792ee6e970d5f8ada9228e478fa6823ce27f Mon Sep 17 00:00:00 2001 From: omotnyk Date: Mon, 9 Oct 2023 16:32:16 +0300 Subject: [PATCH 02/50] Use generics in `paginate` helper --- packages/node-client/lib/index.ts | 15 +++++++++------ packages/node-client/lib/types.ts | 4 ++-- packages/shared/lib/sdk/sync.ts | 5 ++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/node-client/lib/index.ts b/packages/node-client/lib/index.ts index dafe730e623..6122ff8faa0 100644 --- a/packages/node-client/lib/index.ts +++ b/packages/node-client/lib/index.ts @@ -561,28 +561,31 @@ export class Nango { }); } - public async *paginate(config: ProxyConfiguration, nangoProxyFunction: (config: ProxyConfiguration) => Promise>): AsyncGenerator { + public async *paginate( + config: ProxyConfiguration, + nangoProxyFunction: (config: ProxyConfiguration) => Promise> + ): AsyncGenerator { if (!config.pagination) { throw Error(`Pagination config is not specififed for '${config.providerConfigKey}' provider nor it's passed as an override to 'paginate' method`); } let page = 1; - const defaultMaxValuePerPage:number = 100; + const defaultMaxValuePerPage: number = 100; const limit: number = config.pagination?.limit ?? defaultMaxValuePerPage; - const endpoint:string = config.endpoint; + const endpoint: string = config.endpoint; while (true) { - const resp: AxiosResponse = await nangoProxyFunction({ + const resp: AxiosResponse = await nangoProxyFunction.call(this, { endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` }); - if (!resp.data.length) { + if (!(resp.data as any).length) { return; } yield resp.data; - if (resp.data.length < limit) { + if ((resp.data as any).length < limit) { return; } diff --git a/packages/node-client/lib/types.ts b/packages/node-client/lib/types.ts index 53026151680..0083e1de7b3 100644 --- a/packages/node-client/lib/types.ts +++ b/packages/node-client/lib/types.ts @@ -28,7 +28,7 @@ export interface OAuth2Credentials extends CredentialsCommon { export enum PaginationType { CURSOR = 'cursor', - OFFSET = 'offset', + OFFSET = 'offset' } interface Pagination { @@ -42,7 +42,7 @@ export interface CursorPagination extends Pagination { } export interface OffsetPagination extends Pagination { - offset: number; + offset: string; } export interface ProxyConfiguration { diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index e55b38ffebf..e568af31ada 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -243,7 +243,10 @@ export class NangoAction { }); } - public async paginate(config: ProxyConfiguration, nangoProxyFunction: (config: ProxyConfiguration) => Promise>): Promise> { + public paginate( + config: ProxyConfiguration, + nangoProxyFunction: (config: ProxyConfiguration) => Promise> + ): AsyncGenerator { return this.nango.paginate(config, nangoProxyFunction); } From 800c78c0ce766c81db4f618375a83aa2602974c7 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Mon, 9 Oct 2023 19:01:38 +0300 Subject: [PATCH 03/50] Move pagination from proxy to sync --- packages/node-client/lib/index.ts | 32 ---------- packages/node-client/lib/types.ts | 20 ------- packages/shared/lib/models/Provider.ts | 2 + packages/shared/lib/sdk/sync.ts | 81 +++++++++++++++++++++++++- 4 files changed, 81 insertions(+), 54 deletions(-) diff --git a/packages/node-client/lib/index.ts b/packages/node-client/lib/index.ts index 6122ff8faa0..5cf98d944ba 100644 --- a/packages/node-client/lib/index.ts +++ b/packages/node-client/lib/index.ts @@ -561,38 +561,6 @@ export class Nango { }); } - public async *paginate( - config: ProxyConfiguration, - nangoProxyFunction: (config: ProxyConfiguration) => Promise> - ): AsyncGenerator { - if (!config.pagination) { - throw Error(`Pagination config is not specififed for '${config.providerConfigKey}' provider nor it's passed as an override to 'paginate' method`); - } - - let page = 1; - const defaultMaxValuePerPage: number = 100; - const limit: number = config.pagination?.limit ?? defaultMaxValuePerPage; - const endpoint: string = config.endpoint; - - while (true) { - const resp: AxiosResponse = await nangoProxyFunction.call(this, { - endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` - }); - - if (!(resp.data as any).length) { - return; - } - - yield resp.data; - - if ((resp.data as any).length < limit) { - return; - } - - page += 1; - } - } - private async listConnectionDetails(connectionId?: string): Promise> { let url = `${this.serverUrl}/connection?`; if (connectionId) { diff --git a/packages/node-client/lib/types.ts b/packages/node-client/lib/types.ts index 0083e1de7b3..4a3699e1323 100644 --- a/packages/node-client/lib/types.ts +++ b/packages/node-client/lib/types.ts @@ -26,25 +26,6 @@ export interface OAuth2Credentials extends CredentialsCommon { expires_at?: Date | undefined; } -export enum PaginationType { - CURSOR = 'cursor', - OFFSET = 'offset' -} - -interface Pagination { - type: PaginationType; - limit?: number; -} - -export interface CursorPagination extends Pagination { - nextCursorParameterPath: string; - cursorParameterName: string; -} - -export interface OffsetPagination extends Pagination { - offset: string; -} - export interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -57,7 +38,6 @@ export interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; - pagination?: OffsetPagination; } export interface GetRecordsRequestConfig { diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index 100d283ecde..a8c6b9e0ca2 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,3 +1,4 @@ +import type { CursorPagination, OffsetPagination } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -25,6 +26,7 @@ export interface Template { at?: string; after?: string; }; + paginate?: OffsetPagination | CursorPagination; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index e568af31ada..10c45e180a4 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -8,6 +8,8 @@ import errorManager, { ErrorSourceEnum } from '../utils/error.manager.js'; import { LogActionEnum } from '../models/Activity.js'; import { Nango } from '@nangohq/node'; +import configService from '../services/config.service.js'; +import type { Template } from '../models/index.js'; type LogLevel = 'info' | 'debug' | 'error' | 'warn' | 'http' | 'verbose' | 'silly'; @@ -59,6 +61,24 @@ interface DataResponse { [index: string]: unknown | undefined | string | number | boolean | Record; } +export enum PaginationType { + CURSOR = 'cursor', + OFFSET = 'offset' +} + +interface Pagination { + type: PaginationType; + limit?: number; +} + +export interface CursorPagination extends Pagination { + nextCursorParameterPath: string; + cursorParameterName: string; +} + +export interface OffsetPagination extends Pagination { +} + interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -71,6 +91,7 @@ interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; + pagination?: Record; // Supported only by Syncs and Actions ATM } enum AuthModes { @@ -243,11 +264,67 @@ export class NangoAction { }); } - public paginate( + public async *paginate( config: ProxyConfiguration, nangoProxyFunction: (config: ProxyConfiguration) => Promise> ): AsyncGenerator { - return this.nango.paginate(config, nangoProxyFunction); + if (!this.providerConfigKey) { + throw Error(`Please, specify provider config key`); + } + + const providerConfigKey: string = this.providerConfigKey; + const template: Template = configService.getTemplate(providerConfigKey); + let paginationConfig: OffsetPagination | CursorPagination | undefined = template.proxy?.paginate ?? { + type: PaginationType.OFFSET // Add UNKNOWN type + }; + const paginationConfigOverride: Record | boolean = config.pagination ?? false; + + if (paginationConfigOverride) { + paginationConfig = { ...paginationConfig, ...paginationConfigOverride}; + } + + console.log(`Pagination config: ${JSON.stringify(paginationConfig)}`); + console.log(`Template: ${JSON.stringify(template)}`); + + if (!paginationConfig) { + const template: Template = configService.getTemplate(providerConfigKey); + paginationConfig = template.proxy?.paginate; + } + + if (!paginationConfig) { + // TODO: We should check that the valid Pagination object is passed as an override and do not throw error if so + throw Error(`Pagination config is not specififed for '${config.providerConfigKey}' nor it's passed as an override to 'paginate' method`); + } + + switch (paginationConfig.type) { + case PaginationType.OFFSET: + const offsetPaginationConfig: OffsetPagination = paginationConfig as OffsetPagination; + let page = 1; + const defaultMaxValuePerPage: number = 100; + const limit: number = offsetPaginationConfig.limit ?? defaultMaxValuePerPage; + const endpoint: string = config.endpoint; + + while (true) { + console.log('llllllllllll') + const resp: AxiosResponse = await nangoProxyFunction.call(this, { + endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` + }); + + if (!(resp.data as any).length) { + return; + } + + yield resp.data; + + if ((resp.data as any).length < limit) { + return; + } + + page += 1; + } + default: + throw Error(`${paginationConfig.type} pagination is not supported`); + } } public async getConnection(): Promise { From f7598c117f16b6d042775f577a5c8ae8a0ba1b6d Mon Sep 17 00:00:00 2001 From: omotnyk Date: Mon, 9 Oct 2023 20:59:45 +0300 Subject: [PATCH 04/50] Add pagination based on `paginate` param --- packages/shared/lib/sdk/sync.ts | 146 +++++++++++++++++--------------- 1 file changed, 77 insertions(+), 69 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 10c45e180a4..e861e938508 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -76,8 +76,7 @@ export interface CursorPagination extends Pagination { cursorParameterName: string; } -export interface OffsetPagination extends Pagination { -} +export interface OffsetPagination extends Pagination { } interface ProxyConfiguration { endpoint: string; @@ -91,7 +90,7 @@ interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; - pagination?: Record; // Supported only by Syncs and Actions ATM + paginate?: Record | boolean; // Supported only by Syncs and Actions ATM } enum AuthModes { @@ -232,100 +231,68 @@ export class NangoAction { } } - public async proxy(config: ProxyConfiguration): Promise> { + public async proxy(config: ProxyConfiguration): Promise | AsyncGenerator> { + if (config.paginate) { + if (!this.providerConfigKey) { + throw Error(`Please, specify provider config key`); + } + + const providerConfigKey: string = this.providerConfigKey; + const template: Template = configService.getTemplate(providerConfigKey); + const templatePaginationConfig: OffsetPagination | CursorPagination | undefined = template.proxy?.paginate; + + if(!templatePaginationConfig) { + throw Error(`Please, add pagination config to 'providers.yaml' file`); + } + + let paginationConfig: OffsetPagination | CursorPagination = templatePaginationConfig; + if (typeof config.paginate === 'boolean') { + if (!templatePaginationConfig) { + throw Error(`Pagination is not supported for ${this.providerConfigKey} provider. Please, specify pagination config in 'providers.yaml' file or in proxy configuration while calling this API.`); + } + } else if (typeof config.paginate === 'object') { + const paginationConfigOverride: Record = config.paginate as Record; + + if (paginationConfigOverride) { + paginationConfig = { ...paginationConfig, ...paginationConfigOverride }; + } + } + + return this.paginate(config, paginationConfig, this.nango.proxy); + } + return this.nango.proxy(config); } - public async get(config: ProxyConfiguration): Promise> { + public async get(config: ProxyConfiguration): Promise | AsyncGenerator> { return this.proxy({ ...config, method: 'GET' }); } - public async post(config: ProxyConfiguration): Promise> { + public async post < T = any > (config: ProxyConfiguration): Promise | AsyncGenerator> { return this.proxy({ ...config, method: 'POST' }); } - public async patch(config: ProxyConfiguration): Promise> { + public async patch(config: ProxyConfiguration): Promise | AsyncGenerator> { return this.proxy({ ...config, method: 'PATCH' }); } - public async delete(config: ProxyConfiguration): Promise> { + public async delete(config: ProxyConfiguration): Promise | AsyncGenerator> { return this.proxy({ ...config, method: 'DELETE' }); } - public async *paginate( - config: ProxyConfiguration, - nangoProxyFunction: (config: ProxyConfiguration) => Promise> - ): AsyncGenerator { - if (!this.providerConfigKey) { - throw Error(`Please, specify provider config key`); - } - - const providerConfigKey: string = this.providerConfigKey; - const template: Template = configService.getTemplate(providerConfigKey); - let paginationConfig: OffsetPagination | CursorPagination | undefined = template.proxy?.paginate ?? { - type: PaginationType.OFFSET // Add UNKNOWN type - }; - const paginationConfigOverride: Record | boolean = config.pagination ?? false; - if (paginationConfigOverride) { - paginationConfig = { ...paginationConfig, ...paginationConfigOverride}; - } - - console.log(`Pagination config: ${JSON.stringify(paginationConfig)}`); - console.log(`Template: ${JSON.stringify(template)}`); - - if (!paginationConfig) { - const template: Template = configService.getTemplate(providerConfigKey); - paginationConfig = template.proxy?.paginate; - } - - if (!paginationConfig) { - // TODO: We should check that the valid Pagination object is passed as an override and do not throw error if so - throw Error(`Pagination config is not specififed for '${config.providerConfigKey}' nor it's passed as an override to 'paginate' method`); - } - - switch (paginationConfig.type) { - case PaginationType.OFFSET: - const offsetPaginationConfig: OffsetPagination = paginationConfig as OffsetPagination; - let page = 1; - const defaultMaxValuePerPage: number = 100; - const limit: number = offsetPaginationConfig.limit ?? defaultMaxValuePerPage; - const endpoint: string = config.endpoint; - - while (true) { - console.log('llllllllllll') - const resp: AxiosResponse = await nangoProxyFunction.call(this, { - endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` - }); - - if (!(resp.data as any).length) { - return; - } - - yield resp.data; - - if ((resp.data as any).length < limit) { - return; - } - - page += 1; - } - default: - throw Error(`${paginationConfig.type} pagination is not supported`); - } - } public async getConnection(): Promise { return this.nango.getConnection(this.providerConfigKey as string, this.connectionId as string); @@ -383,6 +350,47 @@ export class NangoAction { return this.attributes as A; } + + private async *paginate( + config: ProxyConfiguration, + paginationConfig: Pagination, + nangoProxyFunction: (config: ProxyConfiguration) => Promise> + ): AsyncGenerator { + if (!this.providerConfigKey) { + throw Error(`Failed to find provider config key`); + } + + switch (paginationConfig.type) { + case PaginationType.OFFSET: + const offsetPaginationConfig: OffsetPagination = paginationConfig as OffsetPagination; + let page = 1; + const defaultMaxValuePerPage: number = 100; + const limit: number = offsetPaginationConfig.limit ?? defaultMaxValuePerPage; + const endpoint: string = config.endpoint; + + while (true) { + const resp: AxiosResponse = await nangoProxyFunction.call(this.nango, { + ...config, ...{ + endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` + } + }); + + if (!(resp.data as any).length) { + return; + } + + yield resp.data; + + if ((resp.data as any).length < limit) { + return; + } + + page += 1; + } + default: + throw Error(`${paginationConfig.type} pagination is not supported`); + } + } } export class NangoSync extends NangoAction { From 2500967f86d0073e5531fb251ded32ddd3290a38 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Mon, 9 Oct 2023 22:05:30 +0300 Subject: [PATCH 05/50] Add GitHub pagination config to `providers.yaml` --- packages/shared/providers.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 61cfbe7cfdb..98ef806eb0a 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -361,6 +361,9 @@ github: base_url: https://api.github.com retry: at: 'x-ratelimit-reset' + paginate: + type: offset + limit: 100 docs: https://docs.github.com/en/rest gitlab: auth_mode: OAUTH2 From 4445f2a8c32dbfb06402576f042954daae02ae46 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Mon, 9 Oct 2023 23:07:52 +0300 Subject: [PATCH 06/50] Simplify ifs --- packages/shared/lib/sdk/sync.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index e861e938508..ece9e9212fc 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -76,7 +76,7 @@ export interface CursorPagination extends Pagination { cursorParameterName: string; } -export interface OffsetPagination extends Pagination { } +export interface OffsetPagination extends Pagination {} interface ProxyConfiguration { endpoint: string; @@ -247,9 +247,7 @@ export class NangoAction { let paginationConfig: OffsetPagination | CursorPagination = templatePaginationConfig; if (typeof config.paginate === 'boolean') { - if (!templatePaginationConfig) { - throw Error(`Pagination is not supported for ${this.providerConfigKey} provider. Please, specify pagination config in 'providers.yaml' file or in proxy configuration while calling this API.`); - } + console.debug(`Paginating using the default config from providers.yaml`); } else if (typeof config.paginate === 'object') { const paginationConfigOverride: Record = config.paginate as Record; @@ -292,8 +290,6 @@ export class NangoAction { }); } - - public async getConnection(): Promise { return this.nango.getConnection(this.providerConfigKey as string, this.connectionId as string); } @@ -370,7 +366,8 @@ export class NangoAction { while (true) { const resp: AxiosResponse = await nangoProxyFunction.call(this.nango, { - ...config, ...{ + ...config, + ...{ endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` } }); From c0b9eaf8b08e41ece14321924d546c84c3cc7448 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Mon, 9 Oct 2023 23:26:36 +0300 Subject: [PATCH 07/50] Implement separate method for pagination helper --- packages/shared/lib/sdk/sync.ts | 59 ++++++++++++++------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index ece9e9212fc..471d6aac698 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -231,45 +231,18 @@ export class NangoAction { } } - public async proxy(config: ProxyConfiguration): Promise | AsyncGenerator> { - if (config.paginate) { - if (!this.providerConfigKey) { - throw Error(`Please, specify provider config key`); - } - - const providerConfigKey: string = this.providerConfigKey; - const template: Template = configService.getTemplate(providerConfigKey); - const templatePaginationConfig: OffsetPagination | CursorPagination | undefined = template.proxy?.paginate; - - if(!templatePaginationConfig) { - throw Error(`Please, add pagination config to 'providers.yaml' file`); - } - - let paginationConfig: OffsetPagination | CursorPagination = templatePaginationConfig; - if (typeof config.paginate === 'boolean') { - console.debug(`Paginating using the default config from providers.yaml`); - } else if (typeof config.paginate === 'object') { - const paginationConfigOverride: Record = config.paginate as Record; - - if (paginationConfigOverride) { - paginationConfig = { ...paginationConfig, ...paginationConfigOverride }; - } - } - - return this.paginate(config, paginationConfig, this.nango.proxy); - } - + public async proxy(config: ProxyConfiguration): Promise> { return this.nango.proxy(config); } - public async get(config: ProxyConfiguration): Promise | AsyncGenerator> { + public async get(config: ProxyConfiguration): Promise> { return this.proxy({ ...config, method: 'GET' }); } - public async post < T = any > (config: ProxyConfiguration): Promise | AsyncGenerator> { + public async post(config: ProxyConfiguration): Promise | AsyncGenerator> { return this.proxy({ ...config, method: 'POST' @@ -347,13 +320,31 @@ export class NangoAction { return this.attributes as A; } - private async *paginate( + public async *paginate( config: ProxyConfiguration, - paginationConfig: Pagination, nangoProxyFunction: (config: ProxyConfiguration) => Promise> ): AsyncGenerator { if (!this.providerConfigKey) { - throw Error(`Failed to find provider config key`); + throw Error(`Please, specify provider config key`); + } + + const providerConfigKey: string = this.providerConfigKey; + const template: Template = configService.getTemplate(providerConfigKey); + const templatePaginationConfig: OffsetPagination | CursorPagination | undefined = template.proxy?.paginate; + + if (!templatePaginationConfig) { + throw Error(`Please, add pagination config to 'providers.yaml' file`); + } + + let paginationConfig: OffsetPagination | CursorPagination = templatePaginationConfig; + if (typeof config.paginate === 'boolean') { + console.debug(`Paginating using the default config from providers.yaml`); + } else if (typeof config.paginate === 'object') { + const paginationConfigOverride: Record = config.paginate as Record; + + if (paginationConfigOverride) { + paginationConfig = { ...paginationConfig, ...paginationConfigOverride }; + } } switch (paginationConfig.type) { @@ -365,7 +356,7 @@ export class NangoAction { const endpoint: string = config.endpoint; while (true) { - const resp: AxiosResponse = await nangoProxyFunction.call(this.nango, { + const resp: AxiosResponse = await nangoProxyFunction.call(this, { ...config, ...{ endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` From 63dc50dccbee265584c307464afba67bdcb3dc56 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 12:24:52 +0300 Subject: [PATCH 08/50] Support cursor based pagination --- packages/shared/lib/models/Provider.ts | 4 +- packages/shared/lib/sdk/sync.ts | 84 ++++++++++++++++++++------ packages/shared/providers.yaml | 11 +++- 3 files changed, 76 insertions(+), 23 deletions(-) diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index a8c6b9e0ca2..937bb12cfdd 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,4 +1,4 @@ -import type { CursorPagination, OffsetPagination } from '../sdk/sync.js'; +import type { CursorPagination as CursorBased, OffsetIncrement as OffsetIncrement, PageIncrement } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -26,7 +26,7 @@ export interface Template { at?: string; after?: string; }; - paginate?: OffsetPagination | CursorPagination; + paginate?: OffsetIncrement | PageIncrement | CursorBased; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 471d6aac698..da82068d69d 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -7,6 +7,7 @@ import { updateSyncJobResult } from '../services/sync/job.service.js'; import errorManager, { ErrorSourceEnum } from '../utils/error.manager.js'; import { LogActionEnum } from '../models/Activity.js'; + import { Nango } from '@nangohq/node'; import configService from '../services/config.service.js'; import type { Template } from '../models/index.js'; @@ -62,13 +63,15 @@ interface DataResponse { } export enum PaginationType { - CURSOR = 'cursor', - OFFSET = 'offset' + CURSOR_BASED = 'CursorBased', + OFFSET_INCREMENT = 'OffsetIncrement', + PAGE_INCREMENT = 'PageIncrement' } interface Pagination { type: PaginationType; limit?: number; + responsePath?: string; } export interface CursorPagination extends Pagination { @@ -76,7 +79,9 @@ export interface CursorPagination extends Pagination { cursorParameterName: string; } -export interface OffsetPagination extends Pagination {} +export interface PageIncrement extends Pagination { } + +export interface OffsetIncrement extends Pagination { } interface ProxyConfiguration { endpoint: string; @@ -231,7 +236,7 @@ export class NangoAction { } } - public async proxy(config: ProxyConfiguration): Promise> { + public async proxy(config: ProxyConfiguration): Promise> { return this.nango.proxy(config); } @@ -330,16 +335,14 @@ export class NangoAction { const providerConfigKey: string = this.providerConfigKey; const template: Template = configService.getTemplate(providerConfigKey); - const templatePaginationConfig: OffsetPagination | CursorPagination | undefined = template.proxy?.paginate; + const templatePaginationConfig: Pagination | undefined = template.proxy?.paginate; if (!templatePaginationConfig) { throw Error(`Please, add pagination config to 'providers.yaml' file`); } - let paginationConfig: OffsetPagination | CursorPagination = templatePaginationConfig; - if (typeof config.paginate === 'boolean') { - console.debug(`Paginating using the default config from providers.yaml`); - } else if (typeof config.paginate === 'object') { + let paginationConfig: Pagination = templatePaginationConfig; + if (config.paginate) { const paginationConfigOverride: Record = config.paginate as Record; if (paginationConfigOverride) { @@ -347,21 +350,19 @@ export class NangoAction { } } + const updatedConfigParams: Record = config.params as Record ?? {}; + const defaultMaxValuePerPage: string = '100'; + const limit: string = updatedConfigParams.limit || paginationConfig.limit as unknown as string || defaultMaxValuePerPage; + updatedConfigParams.limit = limit; switch (paginationConfig.type) { - case PaginationType.OFFSET: - const offsetPaginationConfig: OffsetPagination = paginationConfig as OffsetPagination; + case PaginationType.PAGE_INCREMENT: { let page = 1; - const defaultMaxValuePerPage: number = 100; - const limit: number = offsetPaginationConfig.limit ?? defaultMaxValuePerPage; - const endpoint: string = config.endpoint; while (true) { - const resp: AxiosResponse = await nangoProxyFunction.call(this, { - ...config, - ...{ - endpoint: endpoint + (endpoint.includes('?') ? '&' : '?') + `limit=${limit}&page=${page}` - } - }); + updatedConfigParams.page = `${page}`; + updatedConfigParams.limit = `${limit}`; + + const resp: AxiosResponse = await nangoProxyFunction.call(this, config); if (!(resp.data as any).length) { return; @@ -375,10 +376,53 @@ export class NangoAction { page += 1; } + } + case PaginationType.CURSOR_BASED: { + const cursorBasedPagination: CursorPagination = paginationConfig as CursorPagination; + + let nextCursor: string | undefined; + while (true) { + if (nextCursor) { + updatedConfigParams[cursorBasedPagination.cursorParameterName] = `${nextCursor}`; + } + + config.params = updatedConfigParams; + + const resp: AxiosResponse = await nangoProxyFunction.call(this, config); + + const responseData = cursorBasedPagination.responsePath ? this.getNestedField(resp.data, cursorBasedPagination.responsePath) : resp.data; + if (!responseData.length) { + return; + } + + yield responseData; + + nextCursor = this.getNestedField(resp.data, cursorBasedPagination.nextCursorParameterPath); + + if (!nextCursor || nextCursor.trim().length === 0) { + return; + } + } + } default: throw Error(`${paginationConfig.type} pagination is not supported`); } } + + private getNestedField(object: any, path: string, defaultValue?: any): any { // TODO: extract to util or figure out how to use lodash + const keys = path.split('.'); + let result = object; + + for (const key of keys) { + if (result && typeof result === 'object' && key in result) { + result = result[key]; + } else { + return defaultValue; + } + } + + return result !== undefined ? result : defaultValue; + } } export class NangoSync extends NangoAction { diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 98ef806eb0a..44698e70428 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -362,7 +362,7 @@ github: retry: at: 'x-ratelimit-reset' paginate: - type: offset + type: PageIncrement limit: 100 docs: https://docs.github.com/en/rest gitlab: @@ -468,6 +468,11 @@ hubspot: token_url: https://api.hubapi.com/oauth/v1/token proxy: base_url: https://api.hubapi.com + paginate: + type: CursorBased + nextCursorParameterPath: 'paging.next.after' + cursorParameterName: 'after' + responsePath: 'results' docs: https://developers.hubspot.com/docs/api/overview instagram: auth_mode: OAUTH2 @@ -843,6 +848,10 @@ slack: - incoming_webhook.url proxy: base_url: https://slack.com/api + paginate: + type: CursorBased + nextCursorParameterPath: 'response_metadata.next_cursor' + cursorParameterName: 'cursor' docs: https://api.slack.com/apis smugmug: auth_mode: OAUTH1 From e22a6a6e27703339f1daa378a52fcea6caefbd09 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 12:31:14 +0300 Subject: [PATCH 09/50] Refactor pagination interface --- packages/shared/lib/sdk/sync.ts | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index da82068d69d..1a5acecdc1d 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -7,7 +7,6 @@ import { updateSyncJobResult } from '../services/sync/job.service.js'; import errorManager, { ErrorSourceEnum } from '../utils/error.manager.js'; import { LogActionEnum } from '../models/Activity.js'; - import { Nango } from '@nangohq/node'; import configService from '../services/config.service.js'; import type { Template } from '../models/index.js'; @@ -326,8 +325,7 @@ export class NangoAction { } public async *paginate( - config: ProxyConfiguration, - nangoProxyFunction: (config: ProxyConfiguration) => Promise> + config: ProxyConfiguration ): AsyncGenerator { if (!this.providerConfigKey) { throw Error(`Please, specify provider config key`); @@ -350,19 +348,23 @@ export class NangoAction { } } - const updatedConfigParams: Record = config.params as Record ?? {}; + if (!config.method) { // default to get if user doesn't specify a different method themselves + config.method = 'GET'; + } + + const updatedConfigParams: Record = (config.params as Record) ?? {}; const defaultMaxValuePerPage: string = '100'; - const limit: string = updatedConfigParams.limit || paginationConfig.limit as unknown as string || defaultMaxValuePerPage; - updatedConfigParams.limit = limit; + const limit: string = updatedConfigParams['limit'] || (paginationConfig['limit'] as unknown as string) || defaultMaxValuePerPage; + updatedConfigParams['limit'] = limit; switch (paginationConfig.type) { case PaginationType.PAGE_INCREMENT: { let page = 1; while (true) { - updatedConfigParams.page = `${page}`; - updatedConfigParams.limit = `${limit}`; + updatedConfigParams['page'] = `${page}`; - const resp: AxiosResponse = await nangoProxyFunction.call(this, config); + config.params = updatedConfigParams; + const resp: AxiosResponse = await this.proxy(config); if (!(resp.data as any).length) { return; @@ -388,7 +390,7 @@ export class NangoAction { config.params = updatedConfigParams; - const resp: AxiosResponse = await nangoProxyFunction.call(this, config); + const resp: AxiosResponse = await this.proxy(config); const responseData = cursorBasedPagination.responsePath ? this.getNestedField(resp.data, cursorBasedPagination.responsePath) : resp.data; if (!responseData.length) { @@ -409,7 +411,8 @@ export class NangoAction { } } - private getNestedField(object: any, path: string, defaultValue?: any): any { // TODO: extract to util or figure out how to use lodash + private getNestedField(object: any, path: string, defaultValue?: any): any { + // TODO: extract to util or figure out how to use lodash const keys = path.split('.'); let result = object; From 23cc70437afc4bcdae93f6bbf8abb61ea2a56f28 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 12:47:52 +0300 Subject: [PATCH 10/50] Revert generator return types --- packages/shared/lib/sdk/sync.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 1a5acecdc1d..d143047d58e 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -78,9 +78,9 @@ export interface CursorPagination extends Pagination { cursorParameterName: string; } -export interface PageIncrement extends Pagination { } +export interface PageIncrement extends Pagination {} -export interface OffsetIncrement extends Pagination { } +export interface OffsetIncrement extends Pagination {} interface ProxyConfiguration { endpoint: string; @@ -246,21 +246,21 @@ export class NangoAction { }); } - public async post(config: ProxyConfiguration): Promise | AsyncGenerator> { + public async post(config: ProxyConfiguration): Promise> { return this.proxy({ ...config, method: 'POST' }); } - public async patch(config: ProxyConfiguration): Promise | AsyncGenerator> { + public async patch(config: ProxyConfiguration): Promise> { return this.proxy({ ...config, method: 'PATCH' }); } - public async delete(config: ProxyConfiguration): Promise | AsyncGenerator> { + public async delete(config: ProxyConfiguration): Promise> { return this.proxy({ ...config, method: 'DELETE' @@ -324,9 +324,7 @@ export class NangoAction { return this.attributes as A; } - public async *paginate( - config: ProxyConfiguration - ): AsyncGenerator { + public async *paginate(config: ProxyConfiguration): AsyncGenerator { if (!this.providerConfigKey) { throw Error(`Please, specify provider config key`); } @@ -348,14 +346,16 @@ export class NangoAction { } } - if (!config.method) { // default to get if user doesn't specify a different method themselves + if (!config.method) { + // default to get if user doesn't specify a different method themselves config.method = 'GET'; } const updatedConfigParams: Record = (config.params as Record) ?? {}; const defaultMaxValuePerPage: string = '100'; - const limit: string = updatedConfigParams['limit'] || (paginationConfig['limit'] as unknown as string) || defaultMaxValuePerPage; + const limit: string = (paginationConfig['limit'] as unknown as string) || updatedConfigParams['limit'] || defaultMaxValuePerPage; updatedConfigParams['limit'] = limit; + switch (paginationConfig.type) { case PaginationType.PAGE_INCREMENT: { let page = 1; From 3733549a11854f472f3360a4775855441a590f7a Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 12:48:40 +0300 Subject: [PATCH 11/50] Prohibit passing boolean param for `paginate` --- packages/shared/lib/sdk/sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index d143047d58e..e76ff538fcc 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -94,7 +94,7 @@ interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; - paginate?: Record | boolean; // Supported only by Syncs and Actions ATM + paginate?: Record; // Supported only by Syncs and Actions ATM } enum AuthModes { From daae1fe9bf4bc7163006eeb2ceb2e049faedd4cf Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 12:50:03 +0300 Subject: [PATCH 12/50] Specify default limit for Slack and HubSpot --- packages/shared/providers.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 44698e70428..0c637792e45 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -473,6 +473,7 @@ hubspot: nextCursorParameterPath: 'paging.next.after' cursorParameterName: 'after' responsePath: 'results' + limit: 100 docs: https://developers.hubspot.com/docs/api/overview instagram: auth_mode: OAUTH2 @@ -852,6 +853,7 @@ slack: type: CursorBased nextCursorParameterPath: 'response_metadata.next_cursor' cursorParameterName: 'cursor' + limit: 100 docs: https://api.slack.com/apis smugmug: auth_mode: OAUTH1 From a06baf6d042e38250e6b7ac7b4de27ff6baf93a5 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 13:53:25 +0300 Subject: [PATCH 13/50] Return batch from generator --- packages/shared/lib/sdk/sync.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index e76ff538fcc..bdbb44d6cdb 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -78,9 +78,9 @@ export interface CursorPagination extends Pagination { cursorParameterName: string; } -export interface PageIncrement extends Pagination {} +export interface PageIncrement extends Pagination { } -export interface OffsetIncrement extends Pagination {} +export interface OffsetIncrement extends Pagination { } interface ProxyConfiguration { endpoint: string; @@ -324,7 +324,7 @@ export class NangoAction { return this.attributes as A; } - public async *paginate(config: ProxyConfiguration): AsyncGenerator { + public async *paginate(config: ProxyConfiguration): AsyncGenerator { if (!this.providerConfigKey) { throw Error(`Please, specify provider config key`); } @@ -364,13 +364,14 @@ export class NangoAction { updatedConfigParams['page'] = `${page}`; config.params = updatedConfigParams; - const resp: AxiosResponse = await this.proxy(config); + const resp: AxiosResponse = await this.proxy(config); - if (!(resp.data as any).length) { + const responseData: T[] = paginationConfig.responsePath ? this.getNestedField(resp.data, paginationConfig.responsePath) : resp.data; + if (!responseData.length) { return; } - yield resp.data; + yield responseData; if ((resp.data as any).length < limit) { return; @@ -390,9 +391,9 @@ export class NangoAction { config.params = updatedConfigParams; - const resp: AxiosResponse = await this.proxy(config); + const resp: AxiosResponse = await this.proxy(config); - const responseData = cursorBasedPagination.responsePath ? this.getNestedField(resp.data, cursorBasedPagination.responsePath) : resp.data; + const responseData: T[] = cursorBasedPagination.responsePath ? this.getNestedField(resp.data, cursorBasedPagination.responsePath) : resp.data; if (!responseData.length) { return; } From adb682aaf87e993d029151e5882c9cc2fb094304 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 15:22:11 +0300 Subject: [PATCH 14/50] Support passing pagination params in body --- packages/shared/lib/sdk/sync.ts | 37 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index bdbb44d6cdb..301b00172a1 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -78,9 +78,9 @@ export interface CursorPagination extends Pagination { cursorParameterName: string; } -export interface PageIncrement extends Pagination { } +export interface PageIncrement extends Pagination {} -export interface OffsetIncrement extends Pagination { } +export interface OffsetIncrement extends Pagination {} interface ProxyConfiguration { endpoint: string; @@ -351,19 +351,30 @@ export class NangoAction { config.method = 'GET'; } - const updatedConfigParams: Record = (config.params as Record) ?? {}; + const configMethod: string = config.method.toLocaleLowerCase(); + let passPaginationParamsInBody: boolean = false; + if (['post', 'put', 'patch'].includes(configMethod)) { + passPaginationParamsInBody = true; + } + + const updatedBodyOrParams: Record = (passPaginationParamsInBody ? config.data : config.params) as Record ?? {}; const defaultMaxValuePerPage: string = '100'; - const limit: string = (paginationConfig['limit'] as unknown as string) || updatedConfigParams['limit'] || defaultMaxValuePerPage; - updatedConfigParams['limit'] = limit; + const limit: string = (paginationConfig['limit'] as unknown as string) || updatedBodyOrParams['limit'] || defaultMaxValuePerPage; + updatedBodyOrParams['limit'] = limit; switch (paginationConfig.type) { case PaginationType.PAGE_INCREMENT: { let page = 1; while (true) { - updatedConfigParams['page'] = `${page}`; + updatedBodyOrParams['page'] = `${page}`; + + if (passPaginationParamsInBody) { + config.data = updatedBodyOrParams; + } else { + config.params = updatedBodyOrParams; + } - config.params = updatedConfigParams; const resp: AxiosResponse = await this.proxy(config); const responseData: T[] = paginationConfig.responsePath ? this.getNestedField(resp.data, paginationConfig.responsePath) : resp.data; @@ -386,14 +397,20 @@ export class NangoAction { let nextCursor: string | undefined; while (true) { if (nextCursor) { - updatedConfigParams[cursorBasedPagination.cursorParameterName] = `${nextCursor}`; + updatedBodyOrParams[cursorBasedPagination.cursorParameterName] = `${nextCursor}`; } - config.params = updatedConfigParams; + if (passPaginationParamsInBody) { + config.data = updatedBodyOrParams; + } else { + config.params = updatedBodyOrParams; + } const resp: AxiosResponse = await this.proxy(config); - const responseData: T[] = cursorBasedPagination.responsePath ? this.getNestedField(resp.data, cursorBasedPagination.responsePath) : resp.data; + const responseData: T[] = cursorBasedPagination.responsePath + ? this.getNestedField(resp.data, cursorBasedPagination.responsePath) + : resp.data; if (!responseData.length) { return; } From 8d1ef8cb91647ab89a0500919ee81df73f369785 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 15:49:12 +0300 Subject: [PATCH 15/50] Extract body/params update into helper method --- packages/shared/lib/sdk/sync.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 301b00172a1..13e336ec635 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -357,7 +357,7 @@ export class NangoAction { passPaginationParamsInBody = true; } - const updatedBodyOrParams: Record = (passPaginationParamsInBody ? config.data : config.params) as Record ?? {}; + const updatedBodyOrParams: Record = ((passPaginationParamsInBody ? config.data : config.params) as Record) ?? {}; const defaultMaxValuePerPage: string = '100'; const limit: string = (paginationConfig['limit'] as unknown as string) || updatedBodyOrParams['limit'] || defaultMaxValuePerPage; updatedBodyOrParams['limit'] = limit; @@ -369,11 +369,7 @@ export class NangoAction { while (true) { updatedBodyOrParams['page'] = `${page}`; - if (passPaginationParamsInBody) { - config.data = updatedBodyOrParams; - } else { - config.params = updatedBodyOrParams; - } + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); const resp: AxiosResponse = await this.proxy(config); @@ -400,11 +396,7 @@ export class NangoAction { updatedBodyOrParams[cursorBasedPagination.cursorParameterName] = `${nextCursor}`; } - if (passPaginationParamsInBody) { - config.data = updatedBodyOrParams; - } else { - config.params = updatedBodyOrParams; - } + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); const resp: AxiosResponse = await this.proxy(config); @@ -429,6 +421,14 @@ export class NangoAction { } } + private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { + if (passPaginationParamsInBody) { + config.data = updatedBodyOrParams; + } else { + config.params = updatedBodyOrParams; + } + } + private getNestedField(object: any, path: string, defaultValue?: any): any { // TODO: extract to util or figure out how to use lodash const keys = path.split('.'); From 9cf10aab5b9ab44d391a644094efdd36c928eac5 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 15:54:11 +0300 Subject: [PATCH 16/50] Fix limit --- packages/shared/lib/sdk/sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 13e336ec635..cd7eb5f7a2b 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -380,7 +380,7 @@ export class NangoAction { yield responseData; - if ((resp.data as any).length < limit) { + if (responseData.length < +limit) { return; } From 198653c9c915d608e014eead17638d715f990cdd Mon Sep 17 00:00:00 2001 From: omotnyk Date: Tue, 10 Oct 2023 16:01:40 +0300 Subject: [PATCH 17/50] Allow specifying page parameter name --- packages/shared/lib/sdk/sync.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index cd7eb5f7a2b..25f55fb8f20 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -78,7 +78,9 @@ export interface CursorPagination extends Pagination { cursorParameterName: string; } -export interface PageIncrement extends Pagination {} +export interface PageIncrement extends Pagination { + pageParameterName?: string; +} export interface OffsetIncrement extends Pagination {} @@ -364,10 +366,12 @@ export class NangoAction { switch (paginationConfig.type) { case PaginationType.PAGE_INCREMENT: { + const pageIncderementPaginationConfig: PageIncrement = paginationConfig as PageIncrement; let page = 1; + const pageParameterName: string = pageIncderementPaginationConfig.pageParameterName ?? 'page'; while (true) { - updatedBodyOrParams['page'] = `${page}`; + updatedBodyOrParams[pageParameterName] = `${page}`; this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); From 4d93b69568523c52d22856405ec8c6121d5bd403 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 12:02:47 +0300 Subject: [PATCH 18/50] Simplify cursor type names --- packages/shared/lib/models/Provider.ts | 4 +-- packages/shared/lib/sdk/sync.ts | 38 ++++++++++++-------------- packages/shared/providers.yaml | 12 ++++++-- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index 937bb12cfdd..7a861c92a8d 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,4 +1,4 @@ -import type { CursorPagination as CursorBased, OffsetIncrement as OffsetIncrement, PageIncrement } from '../sdk/sync.js'; +import type { CursorPagination, PagePagination } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -26,7 +26,7 @@ export interface Template { at?: string; after?: string; }; - paginate?: OffsetIncrement | PageIncrement | CursorBased; + paginate?: PagePagination | CursorPagination; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 25f55fb8f20..d4972de33f1 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -62,9 +62,8 @@ interface DataResponse { } export enum PaginationType { - CURSOR_BASED = 'CursorBased', - OFFSET_INCREMENT = 'OffsetIncrement', - PAGE_INCREMENT = 'PageIncrement' + CURSOR = 'cursor', + PAGE = 'page' } interface Pagination { @@ -74,16 +73,16 @@ interface Pagination { } export interface CursorPagination extends Pagination { + type: PaginationType.CURSOR; nextCursorParameterPath: string; cursorParameterName: string; } -export interface PageIncrement extends Pagination { +export interface PagePagination extends Pagination { + type: PaginationType.PAGE; pageParameterName?: string; } -export interface OffsetIncrement extends Pagination {} - interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -336,7 +335,7 @@ export class NangoAction { const templatePaginationConfig: Pagination | undefined = template.proxy?.paginate; if (!templatePaginationConfig) { - throw Error(`Please, add pagination config to 'providers.yaml' file`); + throw Error(`Pagination is not supported for ${providerConfigKey}. Please, add pagination config to 'providers.yaml' file`); } let paginationConfig: Pagination = templatePaginationConfig; @@ -354,10 +353,7 @@ export class NangoAction { } const configMethod: string = config.method.toLocaleLowerCase(); - let passPaginationParamsInBody: boolean = false; - if (['post', 'put', 'patch'].includes(configMethod)) { - passPaginationParamsInBody = true; - } + let passPaginationParamsInBody: boolean = ['post', 'put', 'patch'].includes(configMethod); const updatedBodyOrParams: Record = ((passPaginationParamsInBody ? config.data : config.params) as Record) ?? {}; const defaultMaxValuePerPage: string = '100'; @@ -365,8 +361,8 @@ export class NangoAction { updatedBodyOrParams['limit'] = limit; switch (paginationConfig.type) { - case PaginationType.PAGE_INCREMENT: { - const pageIncderementPaginationConfig: PageIncrement = paginationConfig as PageIncrement; + case PaginationType.PAGE: { + const pageIncderementPaginationConfig: PagePagination = paginationConfig as PagePagination; let page = 1; const pageParameterName: string = pageIncderementPaginationConfig.pageParameterName ?? 'page'; @@ -375,9 +371,9 @@ export class NangoAction { this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - const resp: AxiosResponse = await this.proxy(config); + const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.responsePath ? this.getNestedField(resp.data, paginationConfig.responsePath) : resp.data; + const responseData: T[] = paginationConfig.responsePath ? this.getNestedField(response.data, paginationConfig.responsePath) : response.data; if (!responseData.length) { return; } @@ -391,7 +387,7 @@ export class NangoAction { page += 1; } } - case PaginationType.CURSOR_BASED: { + case PaginationType.CURSOR: { const cursorBasedPagination: CursorPagination = paginationConfig as CursorPagination; let nextCursor: string | undefined; @@ -402,18 +398,18 @@ export class NangoAction { this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - const resp: AxiosResponse = await this.proxy(config); + const response: AxiosResponse = await this.proxy(config); const responseData: T[] = cursorBasedPagination.responsePath - ? this.getNestedField(resp.data, cursorBasedPagination.responsePath) - : resp.data; + ? this.getNestedField(response.data, cursorBasedPagination.responsePath) + : response.data; if (!responseData.length) { return; } yield responseData; - nextCursor = this.getNestedField(resp.data, cursorBasedPagination.nextCursorParameterPath); + nextCursor = this.getNestedField(response.data, cursorBasedPagination.nextCursorParameterPath); if (!nextCursor || nextCursor.trim().length === 0) { return; @@ -421,7 +417,7 @@ export class NangoAction { } } default: - throw Error(`${paginationConfig.type} pagination is not supported`); + throw Error(`'${paginationConfig.type}' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)}`); } } diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 0c637792e45..0585898c683 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -70,6 +70,12 @@ asana: grant_type: refresh_token proxy: base_url: https://app.asana.com + paginate: + type: cursor + nextCursorParameterPath: 'next_page.offset' + cursorParameterName: 'offset' + responsePath: 'data' + limit: 100 docs: https://developers.asana.com/reference ashby: auth_mode: BASIC @@ -362,7 +368,7 @@ github: retry: at: 'x-ratelimit-reset' paginate: - type: PageIncrement + type: page limit: 100 docs: https://docs.github.com/en/rest gitlab: @@ -469,7 +475,7 @@ hubspot: proxy: base_url: https://api.hubapi.com paginate: - type: CursorBased + type: cursor nextCursorParameterPath: 'paging.next.after' cursorParameterName: 'after' responsePath: 'results' @@ -850,7 +856,7 @@ slack: proxy: base_url: https://slack.com/api paginate: - type: CursorBased + type: cursor nextCursorParameterPath: 'response_metadata.next_cursor' cursorParameterName: 'cursor' limit: 100 From 9a3cbd191fe9912cab1b9cc738faa93bf1b09ca2 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 16:58:39 +0300 Subject: [PATCH 19/50] Fix config names & allow custom limit param name --- packages/shared/lib/sdk/sync.ts | 30 +++++++++++++++++------------- packages/shared/providers.yaml | 21 +++++++++------------ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index d4972de33f1..bd8bb852f2f 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -69,18 +69,19 @@ export enum PaginationType { interface Pagination { type: PaginationType; limit?: number; - responsePath?: string; + response_path?: string; + limit_parameter_name: string; } export interface CursorPagination extends Pagination { type: PaginationType.CURSOR; - nextCursorParameterPath: string; - cursorParameterName: string; + next_cursor_parameter_path: string; + cursor_parameter_name: string; } export interface PagePagination extends Pagination { type: PaginationType.PAGE; - pageParameterName?: string; + page_parameter_name?: string; } interface ProxyConfiguration { @@ -356,15 +357,18 @@ export class NangoAction { let passPaginationParamsInBody: boolean = ['post', 'put', 'patch'].includes(configMethod); const updatedBodyOrParams: Record = ((passPaginationParamsInBody ? config.data : config.params) as Record) ?? {}; - const defaultMaxValuePerPage: string = '100'; - const limit: string = (paginationConfig['limit'] as unknown as string) || updatedBodyOrParams['limit'] || defaultMaxValuePerPage; - updatedBodyOrParams['limit'] = limit; + const defaultMaxValuePerPage: string = '10'; + const limitParameterName: string = paginationConfig.limit_parameter_name; + + const limit: string = (paginationConfig['limit'] as unknown as string) || updatedBodyOrParams[limitParameterName] || defaultMaxValuePerPage; + + updatedBodyOrParams[limitParameterName] = limit; switch (paginationConfig.type) { case PaginationType.PAGE: { const pageIncderementPaginationConfig: PagePagination = paginationConfig as PagePagination; let page = 1; - const pageParameterName: string = pageIncderementPaginationConfig.pageParameterName ?? 'page'; + const pageParameterName: string = pageIncderementPaginationConfig.page_parameter_name ?? 'page'; while (true) { updatedBodyOrParams[pageParameterName] = `${page}`; @@ -373,7 +377,7 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.responsePath ? this.getNestedField(response.data, paginationConfig.responsePath) : response.data; + const responseData: T[] = paginationConfig.response_path ? this.getNestedField(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; } @@ -393,15 +397,15 @@ export class NangoAction { let nextCursor: string | undefined; while (true) { if (nextCursor) { - updatedBodyOrParams[cursorBasedPagination.cursorParameterName] = `${nextCursor}`; + updatedBodyOrParams[cursorBasedPagination.cursor_parameter_name] = `${nextCursor}`; } this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = cursorBasedPagination.responsePath - ? this.getNestedField(response.data, cursorBasedPagination.responsePath) + const responseData: T[] = cursorBasedPagination.response_path + ? this.getNestedField(response.data, cursorBasedPagination.response_path) : response.data; if (!responseData.length) { return; @@ -409,7 +413,7 @@ export class NangoAction { yield responseData; - nextCursor = this.getNestedField(response.data, cursorBasedPagination.nextCursorParameterPath); + nextCursor = this.getNestedField(response.data, cursorBasedPagination.next_cursor_parameter_path); if (!nextCursor || nextCursor.trim().length === 0) { return; diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 0585898c683..b5dbf148502 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -72,10 +72,9 @@ asana: base_url: https://app.asana.com paginate: type: cursor - nextCursorParameterPath: 'next_page.offset' - cursorParameterName: 'offset' - responsePath: 'data' - limit: 100 + next_cursor_parameter_path: 'next_page.offset' + cursor_parameter_name: 'offset' + response_path: 'data' docs: https://developers.asana.com/reference ashby: auth_mode: BASIC @@ -369,7 +368,7 @@ github: at: 'x-ratelimit-reset' paginate: type: page - limit: 100 + limit_parameter_name: 'per_page' docs: https://docs.github.com/en/rest gitlab: auth_mode: OAUTH2 @@ -476,10 +475,9 @@ hubspot: base_url: https://api.hubapi.com paginate: type: cursor - nextCursorParameterPath: 'paging.next.after' - cursorParameterName: 'after' - responsePath: 'results' - limit: 100 + next_cursor_parameter_path: 'paging.next.after' + cursor_parameter_name: 'after' + response_path: 'results' docs: https://developers.hubspot.com/docs/api/overview instagram: auth_mode: OAUTH2 @@ -857,9 +855,8 @@ slack: base_url: https://slack.com/api paginate: type: cursor - nextCursorParameterPath: 'response_metadata.next_cursor' - cursorParameterName: 'cursor' - limit: 100 + next_cursor_parameter_path: 'response_metadata.next_cursor' + cursor_parameter_name: 'cursor' docs: https://api.slack.com/apis smugmug: auth_mode: OAUTH1 From 6c8656a1719faf734ec638ab1e04de676830e60d Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 17:50:43 +0300 Subject: [PATCH 20/50] Add support for header link rel pagination --- packages/shared/lib/sdk/sync.ts | 62 ++++++++++++++++++++++++++++----- packages/shared/providers.yaml | 3 +- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index bd8bb852f2f..06dc9bc72db 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -10,6 +10,8 @@ import { LogActionEnum } from '../models/Activity.js'; import { Nango } from '@nangohq/node'; import configService from '../services/config.service.js'; import type { Template } from '../models/index.js'; +import { isValidHttpUrl } from '../utils/utils.js'; +import parseLinksHeader from 'parse-link-header'; type LogLevel = 'info' | 'debug' | 'error' | 'warn' | 'http' | 'verbose' | 'silly'; @@ -63,7 +65,8 @@ interface DataResponse { export enum PaginationType { CURSOR = 'cursor', - PAGE = 'page' + PAGE = 'page', + LINK_REL = 'link_rel' } interface Pagination { @@ -84,6 +87,11 @@ export interface PagePagination extends Pagination { page_parameter_name?: string; } +export interface LinkRelPagination extends Pagination { + type: PaginationType.LINK_REL; + link_rel: string; +} + interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -377,7 +385,9 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_path ? this.getNestedField(response.data, paginationConfig.response_path) : response.data; + const responseData: T[] = paginationConfig.response_path + ? this.getNestedField(response.data, paginationConfig.response_path) + : response.data; if (!responseData.length) { return; } @@ -420,8 +430,42 @@ export class NangoAction { } } } + case PaginationType.LINK_REL: { + const linkRelPagination: LinkRelPagination = paginationConfig as LinkRelPagination; + + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); + while (true) { + const response: AxiosResponse = await this.proxy(config); + + const responseData: T[] = paginationConfig.response_path + ? this.getNestedField(response.data, paginationConfig.response_path) + : response.data; + if (!responseData.length) { + return; + } + + yield responseData; + + const linkHeader = parseLinksHeader(response.headers['link']); + let nextPageUrl: string | undefined = linkHeader?.[linkRelPagination.link_rel]?.url; + + if (nextPageUrl && isValidHttpUrl(nextPageUrl)) { + const url = new URL(nextPageUrl); + const searchParams: URLSearchParams = url.searchParams; + const path = url.pathname; + config.endpoint = path; + config.params = { + ...config.params as Record, + ...Object.fromEntries(searchParams.entries()) + }; + continue; + } else { + return; + } + } + } default: - throw Error(`'${paginationConfig.type}' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)}`); + throw Error(`'${paginationConfig.type} ' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)} `); } } @@ -511,7 +555,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'error', activity_log_id: this.activityLogId as number, - content: `There was an issue with the batch save. ${error?.message}`, + content: `There was an issue with the batch save.${error?.message} `, timestamp: Date.now() }); } @@ -554,7 +598,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'info', activity_log_id: this.activityLogId as number, - content: `Batch save was a success and resulted in ${JSON.stringify(updatedResults, null, 2)}`, + content: `Batch save was a success and resulted in ${JSON.stringify(updatedResults, null, 2)} `, timestamp: Date.now() }); @@ -562,7 +606,7 @@ export class NangoSync extends NangoAction { return true; } else { - const content = `There was an issue with the batch save. ${responseResults?.error}`; + const content = `There was an issue with the batch save.${responseResults?.error} `; if (!this.dryRun) { await createActivityLogMessage({ @@ -613,7 +657,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'error', activity_log_id: this.activityLogId as number, - content: `There was an issue with the batch delete. ${error?.message}`, + content: `There was an issue with the batch delete.${error?.message} `, timestamp: Date.now() }); } @@ -657,7 +701,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'info', activity_log_id: this.activityLogId as number, - content: `Batch delete was a success and resulted in ${JSON.stringify(updatedResults, null, 2)}`, + content: `Batch delete was a success and resulted in ${JSON.stringify(updatedResults, null, 2)} `, timestamp: Date.now() }); @@ -665,7 +709,7 @@ export class NangoSync extends NangoAction { return true; } else { - const content = `There was an issue with the batch delete. ${responseResults?.error}`; + const content = `There was an issue with the batch delete.${responseResults?.error} `; if (!this.dryRun) { await createActivityLogMessage({ diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index b5dbf148502..66e9055096f 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -367,8 +367,9 @@ github: retry: at: 'x-ratelimit-reset' paginate: - type: page limit_parameter_name: 'per_page' + type: link_rel + link_rel: next docs: https://docs.github.com/en/rest gitlab: auth_mode: OAUTH2 From 7787ed59633d97a29ef94d709ff9290eb69445f9 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 17:52:05 +0300 Subject: [PATCH 21/50] Remove 'page' pagination support --- packages/shared/lib/models/Provider.ts | 4 +-- packages/shared/lib/sdk/sync.ts | 36 +------------------------- 2 files changed, 3 insertions(+), 37 deletions(-) diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index 7a861c92a8d..2640fc0791b 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,4 +1,4 @@ -import type { CursorPagination, PagePagination } from '../sdk/sync.js'; +import type { CursorPagination, LinkRelPagination } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -26,7 +26,7 @@ export interface Template { at?: string; after?: string; }; - paginate?: PagePagination | CursorPagination; + paginate?: LinkRelPagination | CursorPagination; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 06dc9bc72db..46583eec76f 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -65,7 +65,6 @@ interface DataResponse { export enum PaginationType { CURSOR = 'cursor', - PAGE = 'page', LINK_REL = 'link_rel' } @@ -82,11 +81,6 @@ export interface CursorPagination extends Pagination { cursor_parameter_name: string; } -export interface PagePagination extends Pagination { - type: PaginationType.PAGE; - page_parameter_name?: string; -} - export interface LinkRelPagination extends Pagination { type: PaginationType.LINK_REL; link_rel: string; @@ -373,34 +367,6 @@ export class NangoAction { updatedBodyOrParams[limitParameterName] = limit; switch (paginationConfig.type) { - case PaginationType.PAGE: { - const pageIncderementPaginationConfig: PagePagination = paginationConfig as PagePagination; - let page = 1; - const pageParameterName: string = pageIncderementPaginationConfig.page_parameter_name ?? 'page'; - - while (true) { - updatedBodyOrParams[pageParameterName] = `${page}`; - - this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - - const response: AxiosResponse = await this.proxy(config); - - const responseData: T[] = paginationConfig.response_path - ? this.getNestedField(response.data, paginationConfig.response_path) - : response.data; - if (!responseData.length) { - return; - } - - yield responseData; - - if (responseData.length < +limit) { - return; - } - - page += 1; - } - } case PaginationType.CURSOR: { const cursorBasedPagination: CursorPagination = paginationConfig as CursorPagination; @@ -455,7 +421,7 @@ export class NangoAction { const path = url.pathname; config.endpoint = path; config.params = { - ...config.params as Record, + ...(config.params as Record), ...Object.fromEntries(searchParams.entries()) }; continue; From cbf9b9a54cfa6c8ae3436069bb4ef01b5324e7d2 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 17:56:04 +0300 Subject: [PATCH 22/50] Do not support `limit` parameter We are going to rely on API defaults and only allow passing limit as an override --- packages/shared/lib/sdk/sync.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 46583eec76f..8f0e7389305 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -359,12 +359,12 @@ export class NangoAction { let passPaginationParamsInBody: boolean = ['post', 'put', 'patch'].includes(configMethod); const updatedBodyOrParams: Record = ((passPaginationParamsInBody ? config.data : config.params) as Record) ?? {}; - const defaultMaxValuePerPage: string = '10'; const limitParameterName: string = paginationConfig.limit_parameter_name; - const limit: string = (paginationConfig['limit'] as unknown as string) || updatedBodyOrParams[limitParameterName] || defaultMaxValuePerPage; - - updatedBodyOrParams[limitParameterName] = limit; + if (paginationConfig['limit']) { + const limit: string = paginationConfig['limit'] as unknown as string; + updatedBodyOrParams[limitParameterName] = limit; + } switch (paginationConfig.type) { case PaginationType.CURSOR: { From a4593ab6cfe810eae6252e8059a75b9930f6b9bf Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 18:03:40 +0300 Subject: [PATCH 23/50] Support gettign next URL from reponse body --- packages/shared/lib/sdk/sync.ts | 55 ++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 8f0e7389305..09d47420dec 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -65,7 +65,8 @@ interface DataResponse { export enum PaginationType { CURSOR = 'cursor', - LINK_REL = 'link_rel' + LINK_REL = 'link_rel', + URL = 'url' } interface Pagination { @@ -86,6 +87,11 @@ export interface LinkRelPagination extends Pagination { link_rel: string; } +export interface UrlPagination extends Pagination { + type: PaginationType.URL; + next_url_parameter_path: string; +} + interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -413,18 +419,34 @@ export class NangoAction { yield responseData; const linkHeader = parseLinksHeader(response.headers['link']); - let nextPageUrl: string | undefined = linkHeader?.[linkRelPagination.link_rel]?.url; + const nextPageUrl: string | undefined = linkHeader?.[linkRelPagination.link_rel]?.url; + + if (nextPageUrl && isValidHttpUrl(nextPageUrl)) { + this.updateProxyConfigEndpointAndParams(nextPageUrl, config); + } else { + return; + } + } + } + case PaginationType.URL:{ + const urlPagination: UrlPagination = paginationConfig as UrlPagination; + + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); + while (true) { + const response: AxiosResponse = await this.proxy(config); + + const responseData: T[] = paginationConfig.response_path + ? this.getNestedField(response.data, paginationConfig.response_path) + : response.data; + if (!responseData.length) { + return; + } + + yield responseData; + const nextPageUrl: string | undefined = this.getNestedField(response.data, urlPagination.next_url_parameter_path); if (nextPageUrl && isValidHttpUrl(nextPageUrl)) { - const url = new URL(nextPageUrl); - const searchParams: URLSearchParams = url.searchParams; - const path = url.pathname; - config.endpoint = path; - config.params = { - ...(config.params as Record), - ...Object.fromEntries(searchParams.entries()) - }; - continue; + this.updateProxyConfigEndpointAndParams(nextPageUrl, config); } else { return; } @@ -435,6 +457,17 @@ export class NangoAction { } } + private updateProxyConfigEndpointAndParams(nextPageUrl: string, config: ProxyConfiguration) { + const url = new URL(nextPageUrl); + const searchParams: URLSearchParams = url.searchParams; + const path = url.pathname; + config.endpoint = path; + config.params = { + ...(config.params as Record), + ...Object.fromEntries(searchParams.entries()) + }; + } + private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { if (passPaginationParamsInBody) { config.data = updatedBodyOrParams; From d534f90fd7cde3194761702ab486e9c25d5072e8 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 18:04:22 +0300 Subject: [PATCH 24/50] Add missing 'parse-link-header' dependency --- packages/shared/package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/shared/package.json b/packages/shared/package.json index 57c5cedae8b..97a01177317 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -39,6 +39,7 @@ "lodash": "^4.17.21", "md5": "^2.3.0", "ms": "^2.1.3", + "parse-link-header": "^2.0.0", "pg": "^8.8.0", "posthog-node": "^2.2.3", "rimraf": "^5.0.1", @@ -61,8 +62,9 @@ "@types/debug": "^4.1.7", "@types/human-to-cron": "^0.3.0", "@types/js-yaml": "^4.0.5", - "@types/lodash": "^4.14.195", + "@types/lodash": "^4.14.199", "@types/node": "^18.7.6", + "@types/parse-link-header": "^2.0.1", "@types/uuid": "^9.0.0", "typescript": "^4.7.4" } From 6f993f665ea5b039114535d8ed82c0896407ff35 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 18:21:46 +0300 Subject: [PATCH 25/50] Rename params for clarity --- packages/shared/lib/models/Provider.ts | 4 ++-- packages/shared/lib/sdk/sync.ts | 16 ++++++++-------- packages/shared/providers.yaml | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index 2640fc0791b..c7e6074ed86 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,4 +1,4 @@ -import type { CursorPagination, LinkRelPagination } from '../sdk/sync.js'; +import type { CursorPagination, LinkRelPagination, UrlPagination } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -26,7 +26,7 @@ export interface Template { at?: string; after?: string; }; - paginate?: LinkRelPagination | CursorPagination; + paginate?: LinkRelPagination | CursorPagination | UrlPagination; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 09d47420dec..cb6228a2f40 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -72,7 +72,7 @@ export enum PaginationType { interface Pagination { type: PaginationType; limit?: number; - response_path?: string; + response_data_path?: string; limit_parameter_name: string; } @@ -386,8 +386,8 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = cursorBasedPagination.response_path - ? this.getNestedField(response.data, cursorBasedPagination.response_path) + const responseData: T[] = cursorBasedPagination.response_data_path + ? this.getNestedField(response.data, cursorBasedPagination.response_data_path) : response.data; if (!responseData.length) { return; @@ -409,8 +409,8 @@ export class NangoAction { while (true) { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_path - ? this.getNestedField(response.data, paginationConfig.response_path) + const responseData: T[] = paginationConfig.response_data_path + ? this.getNestedField(response.data, paginationConfig.response_data_path) : response.data; if (!responseData.length) { return; @@ -428,15 +428,15 @@ export class NangoAction { } } } - case PaginationType.URL:{ + case PaginationType.URL: { const urlPagination: UrlPagination = paginationConfig as UrlPagination; this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); while (true) { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_path - ? this.getNestedField(response.data, paginationConfig.response_path) + const responseData: T[] = paginationConfig.response_data_path + ? this.getNestedField(response.data, paginationConfig.response_data_path) : response.data; if (!responseData.length) { return; diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 66e9055096f..f74b981460c 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -74,7 +74,7 @@ asana: type: cursor next_cursor_parameter_path: 'next_page.offset' cursor_parameter_name: 'offset' - response_path: 'data' + response_data_path: 'data' docs: https://developers.asana.com/reference ashby: auth_mode: BASIC @@ -478,7 +478,7 @@ hubspot: type: cursor next_cursor_parameter_path: 'paging.next.after' cursor_parameter_name: 'after' - response_path: 'results' + response_data_path: 'results' docs: https://developers.hubspot.com/docs/api/overview instagram: auth_mode: OAUTH2 From 969e4879c3a2674ff404eace35ef29c3a054a5a7 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 18:36:26 +0300 Subject: [PATCH 26/50] Support offset(by count) pagination --- packages/shared/lib/sdk/sync.ts | 47 ++++++++++++++++++++++++++++----- packages/shared/providers.yaml | 3 +++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index cb6228a2f40..9d739f0a0bd 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -66,7 +66,8 @@ interface DataResponse { export enum PaginationType { CURSOR = 'cursor', LINK_REL = 'link_rel', - URL = 'url' + URL = 'url', + OFFSET = 'offset' } interface Pagination { @@ -92,6 +93,11 @@ export interface UrlPagination extends Pagination { next_url_parameter_path: string; } +export interface OffsetPagination extends Pagination { + type: PaginationType.OFFSET; + offset_parameter_name: string; +} + interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -374,20 +380,20 @@ export class NangoAction { switch (paginationConfig.type) { case PaginationType.CURSOR: { - const cursorBasedPagination: CursorPagination = paginationConfig as CursorPagination; + const cursorPagination: CursorPagination = paginationConfig as CursorPagination; let nextCursor: string | undefined; while (true) { if (nextCursor) { - updatedBodyOrParams[cursorBasedPagination.cursor_parameter_name] = `${nextCursor}`; + updatedBodyOrParams[cursorPagination.cursor_parameter_name] = `${nextCursor}`; } this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = cursorBasedPagination.response_data_path - ? this.getNestedField(response.data, cursorBasedPagination.response_data_path) + const responseData: T[] = cursorPagination.response_data_path + ? this.getNestedField(response.data, cursorPagination.response_data_path) : response.data; if (!responseData.length) { return; @@ -395,7 +401,7 @@ export class NangoAction { yield responseData; - nextCursor = this.getNestedField(response.data, cursorBasedPagination.next_cursor_parameter_path); + nextCursor = this.getNestedField(response.data, cursorPagination.next_cursor_parameter_path); if (!nextCursor || nextCursor.trim().length === 0) { return; @@ -452,6 +458,35 @@ export class NangoAction { } } } + case PaginationType.OFFSET: { + const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; + const offsetParameterName: string = offsetPagination.offset_parameter_name; + let offset: number = 0; + + while (true) { + updatedBodyOrParams[offsetParameterName] = `${offset}`; + + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); + + const response: AxiosResponse = await this.proxy(config); + + const responseData: T[] = paginationConfig.response_data_path + ? this.getNestedField(response.data, paginationConfig.response_data_path) + : response.data; + if (!responseData.length) { + return; + } + + yield responseData; + + if (responseData.length < 1) { + // Last page was empty so no need to fetch further + return; + } + + offset += responseData.length; + } + } default: throw Error(`'${paginationConfig.type} ' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)} `); } diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index f74b981460c..fcf6eab0c79 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -507,6 +507,9 @@ jira: prompt: consent proxy: base_url: https://api.atlassian.com + paginate: + type: offset + offset_parameter_name: start keap: auth_mode: OAUTH2 authorization_url: https://accounts.infusionsoft.com/app/oauth/authorize From 91ee0003109f4f674f2886e5dc25582ecce0ead4 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 18:45:46 +0300 Subject: [PATCH 27/50] Backfill pagination params in `providers.yaml` --- packages/shared/providers.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index fcf6eab0c79..1aee7098577 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -72,9 +72,10 @@ asana: base_url: https://app.asana.com paginate: type: cursor - next_cursor_parameter_path: 'next_page.offset' - cursor_parameter_name: 'offset' - response_data_path: 'data' + next_cursor_parameter_path: next_page.offset + cursor_parameter_name: offset + response_data_path: data + limit_parameter_name: limit docs: https://developers.asana.com/reference ashby: auth_mode: BASIC @@ -367,7 +368,7 @@ github: retry: at: 'x-ratelimit-reset' paginate: - limit_parameter_name: 'per_page' + limit_parameter_name: per_page type: link_rel link_rel: next docs: https://docs.github.com/en/rest @@ -476,9 +477,10 @@ hubspot: base_url: https://api.hubapi.com paginate: type: cursor - next_cursor_parameter_path: 'paging.next.after' - cursor_parameter_name: 'after' - response_data_path: 'results' + next_cursor_parameter_path: paging.next.after + limit_parameter_name: limit + cursor_parameter_name: after + response_data_path: results docs: https://developers.hubspot.com/docs/api/overview instagram: auth_mode: OAUTH2 @@ -510,6 +512,7 @@ jira: paginate: type: offset offset_parameter_name: start + limit_parameter_name: limit keap: auth_mode: OAUTH2 authorization_url: https://accounts.infusionsoft.com/app/oauth/authorize @@ -859,8 +862,9 @@ slack: base_url: https://slack.com/api paginate: type: cursor - next_cursor_parameter_path: 'response_metadata.next_cursor' - cursor_parameter_name: 'cursor' + next_cursor_parameter_path: response_metadata.next_cursor + cursor_parameter_name: cursor + limit_parameter_name: limit docs: https://api.slack.com/apis smugmug: auth_mode: OAUTH1 From 5e8ef30df1ebf6ace522b68471273ad68ad49e2a Mon Sep 17 00:00:00 2001 From: omotnyk Date: Wed, 11 Oct 2023 18:49:13 +0300 Subject: [PATCH 28/50] Add error logging in case next URL is invalid --- packages/shared/lib/sdk/sync.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 9d739f0a0bd..85005889a83 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -427,8 +427,11 @@ export class NangoAction { const linkHeader = parseLinksHeader(response.headers['link']); const nextPageUrl: string | undefined = linkHeader?.[linkRelPagination.link_rel]?.url; - if (nextPageUrl && isValidHttpUrl(nextPageUrl)) { - this.updateProxyConfigEndpointAndParams(nextPageUrl, config); + if (nextPageUrl) { + if (!isValidHttpUrl(nextPageUrl)) { + throw Error(`Next page URL ${nextPageUrl} returned from ${this.providerConfigKey} is invalid`); + } + this.updateEndpointAndParams(nextPageUrl, config); } else { return; } @@ -451,8 +454,11 @@ export class NangoAction { yield responseData; const nextPageUrl: string | undefined = this.getNestedField(response.data, urlPagination.next_url_parameter_path); - if (nextPageUrl && isValidHttpUrl(nextPageUrl)) { - this.updateProxyConfigEndpointAndParams(nextPageUrl, config); + if (nextPageUrl) { + if (!isValidHttpUrl(nextPageUrl)) { + throw Error(`Next page URL ${nextPageUrl} returned from ${this.providerConfigKey} is invalid`); + } + this.updateEndpointAndParams(nextPageUrl, config); } else { return; } @@ -492,7 +498,7 @@ export class NangoAction { } } - private updateProxyConfigEndpointAndParams(nextPageUrl: string, config: ProxyConfiguration) { + private updateEndpointAndParams(nextPageUrl: string, config: ProxyConfiguration) { const url = new URL(nextPageUrl); const searchParams: URLSearchParams = url.searchParams; const path = url.pathname; From cc01c72a4be149f55f3df89394484989cc9990e2 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 12:44:32 +0300 Subject: [PATCH 29/50] Unify pagination interfaces --- packages/shared/lib/models/Provider.ts | 4 +- packages/shared/lib/sdk/sync.ts | 127 ++++++------------------- packages/shared/providers.yaml | 8 +- 3 files changed, 35 insertions(+), 104 deletions(-) diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index c7e6074ed86..c9a72c16f9c 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,4 +1,4 @@ -import type { CursorPagination, LinkRelPagination, UrlPagination } from '../sdk/sync.js'; +import type { CursorPagination, NextUrlPagination } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -26,7 +26,7 @@ export interface Template { at?: string; after?: string; }; - paginate?: LinkRelPagination | CursorPagination | UrlPagination; + paginate?: NextUrlPagination | CursorPagination; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 85005889a83..30c805387c4 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -65,9 +65,7 @@ interface DataResponse { export enum PaginationType { CURSOR = 'cursor', - LINK_REL = 'link_rel', - URL = 'url', - OFFSET = 'offset' + NEXT_URL = 'next_url' } interface Pagination { @@ -83,19 +81,10 @@ export interface CursorPagination extends Pagination { cursor_parameter_name: string; } -export interface LinkRelPagination extends Pagination { - type: PaginationType.LINK_REL; - link_rel: string; -} - -export interface UrlPagination extends Pagination { - type: PaginationType.URL; - next_url_parameter_path: string; -} - -export interface OffsetPagination extends Pagination { - type: PaginationType.OFFSET; - offset_parameter_name: string; +export interface NextUrlPagination extends Pagination { + type: PaginationType.NEXT_URL; + link_rel?: string; + next_url_body_parameter_path?: string; } interface ProxyConfiguration { @@ -341,11 +330,7 @@ export class NangoAction { } public async *paginate(config: ProxyConfiguration): AsyncGenerator { - if (!this.providerConfigKey) { - throw Error(`Please, specify provider config key`); - } - - const providerConfigKey: string = this.providerConfigKey; + const providerConfigKey: string = this.providerConfigKey as string; const template: Template = configService.getTemplate(providerConfigKey); const templatePaginationConfig: Pagination | undefined = template.proxy?.paginate; @@ -363,19 +348,18 @@ export class NangoAction { } if (!config.method) { - // default to get if user doesn't specify a different method themselves + // default to GET if user doesn't specify a different method themselves config.method = 'GET'; } const configMethod: string = config.method.toLocaleLowerCase(); - let passPaginationParamsInBody: boolean = ['post', 'put', 'patch'].includes(configMethod); + const passPaginationParamsInBody: boolean = ['post', 'put', 'patch'].includes(configMethod); - const updatedBodyOrParams: Record = ((passPaginationParamsInBody ? config.data : config.params) as Record) ?? {}; + const updatedBodyOrParams: Record = ((passPaginationParamsInBody ? config.data : config.params) as Record) ?? {}; const limitParameterName: string = paginationConfig.limit_parameter_name; if (paginationConfig['limit']) { - const limit: string = paginationConfig['limit'] as unknown as string; - updatedBodyOrParams[limitParameterName] = limit; + updatedBodyOrParams[limitParameterName] = paginationConfig['limit']; } switch (paginationConfig.type) { @@ -385,7 +369,7 @@ export class NangoAction { let nextCursor: string | undefined; while (true) { if (nextCursor) { - updatedBodyOrParams[cursorPagination.cursor_parameter_name] = `${nextCursor}`; + updatedBodyOrParams[cursorPagination.cursor_parameter_name] = nextCursor; } this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); @@ -408,8 +392,8 @@ export class NangoAction { } } } - case PaginationType.LINK_REL: { - const linkRelPagination: LinkRelPagination = paginationConfig as LinkRelPagination; + case PaginationType.NEXT_URL: { + const nextUrlPagination: NextUrlPagination = paginationConfig as NextUrlPagination; this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); while (true) { @@ -424,73 +408,29 @@ export class NangoAction { yield responseData; - const linkHeader = parseLinksHeader(response.headers['link']); - const nextPageUrl: string | undefined = linkHeader?.[linkRelPagination.link_rel]?.url; + let nextPageUrl: string | undefined; - if (nextPageUrl) { - if (!isValidHttpUrl(nextPageUrl)) { - throw Error(`Next page URL ${nextPageUrl} returned from ${this.providerConfigKey} is invalid`); - } - this.updateEndpointAndParams(nextPageUrl, config); + if (nextUrlPagination.link_rel) { + const linkHeader = parseLinksHeader(response.headers['link']); + nextPageUrl = linkHeader?.[nextUrlPagination.link_rel]?.url; + } else if (nextUrlPagination.next_url_body_parameter_path) { + nextPageUrl = this.getNestedField(response.data, nextUrlPagination.next_url_body_parameter_path); } else { - return; + throw Error(`Either 'link_rel' or 'next_url_body_parameter_path' should be specified for '${paginationConfig.type}' pagination`); } - } - } - case PaginationType.URL: { - const urlPagination: UrlPagination = paginationConfig as UrlPagination; - - this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - while (true) { - const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_data_path - ? this.getNestedField(response.data, paginationConfig.response_data_path) - : response.data; - if (!responseData.length) { - return; + if (!nextPageUrl) { + return } - yield responseData; - - const nextPageUrl: string | undefined = this.getNestedField(response.data, urlPagination.next_url_parameter_path); - if (nextPageUrl) { - if (!isValidHttpUrl(nextPageUrl)) { - throw Error(`Next page URL ${nextPageUrl} returned from ${this.providerConfigKey} is invalid`); - } - this.updateEndpointAndParams(nextPageUrl, config); + if (!isValidHttpUrl(nextPageUrl)) { + // some providers only send path+query params in the link so we can immediately assign those to the endpoint + config.endpoint = nextPageUrl; } else { - return; - } - } - } - case PaginationType.OFFSET: { - const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; - const offsetParameterName: string = offsetPagination.offset_parameter_name; - let offset: number = 0; - - while (true) { - updatedBodyOrParams[offsetParameterName] = `${offset}`; - - this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - - const response: AxiosResponse = await this.proxy(config); - - const responseData: T[] = paginationConfig.response_data_path - ? this.getNestedField(response.data, paginationConfig.response_data_path) - : response.data; - if (!responseData.length) { - return; + const url: URL = new URL(nextPageUrl); + config.endpoint = url.pathname + url.search; } - - yield responseData; - - if (responseData.length < 1) { - // Last page was empty so no need to fetch further - return; - } - - offset += responseData.length; + delete config.params; } } default: @@ -498,17 +438,6 @@ export class NangoAction { } } - private updateEndpointAndParams(nextPageUrl: string, config: ProxyConfiguration) { - const url = new URL(nextPageUrl); - const searchParams: URLSearchParams = url.searchParams; - const path = url.pathname; - config.endpoint = path; - config.params = { - ...(config.params as Record), - ...Object.fromEntries(searchParams.entries()) - }; - } - private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { if (passPaginationParamsInBody) { config.data = updatedBodyOrParams; diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 1aee7098577..b534df5a592 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -368,8 +368,8 @@ github: retry: at: 'x-ratelimit-reset' paginate: + type: next_url limit_parameter_name: per_page - type: link_rel link_rel: next docs: https://docs.github.com/en/rest gitlab: @@ -510,9 +510,11 @@ jira: proxy: base_url: https://api.atlassian.com paginate: - type: offset - offset_parameter_name: start + type: next_url + link_rel: next limit_parameter_name: limit + response_data_path: results + next_url_body_parameter_path: _links.next keap: auth_mode: OAUTH2 authorization_url: https://accounts.infusionsoft.com/app/oauth/authorize From 0bbf118c0f04f27889524aa9a476c5506de2f3d8 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 12:52:13 +0300 Subject: [PATCH 30/50] Extact next URL extration into separate method --- packages/shared/lib/sdk/sync.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 30c805387c4..21f9298c940 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -408,19 +408,10 @@ export class NangoAction { yield responseData; - let nextPageUrl: string | undefined; - - if (nextUrlPagination.link_rel) { - const linkHeader = parseLinksHeader(response.headers['link']); - nextPageUrl = linkHeader?.[nextUrlPagination.link_rel]?.url; - } else if (nextUrlPagination.next_url_body_parameter_path) { - nextPageUrl = this.getNestedField(response.data, nextUrlPagination.next_url_body_parameter_path); - } else { - throw Error(`Either 'link_rel' or 'next_url_body_parameter_path' should be specified for '${paginationConfig.type}' pagination`); - } + let nextPageUrl: string | undefined = this.getNextPageUrlFromBodyOrHeaders(nextUrlPagination, response, paginationConfig); if (!nextPageUrl) { - return + return; } if (!isValidHttpUrl(nextPageUrl)) { @@ -438,6 +429,17 @@ export class NangoAction { } } + private getNextPageUrlFromBodyOrHeaders(nextUrlPagination: NextUrlPagination, response: AxiosResponse, paginationConfig: Pagination) { + if (nextUrlPagination.link_rel) { + const linkHeader = parseLinksHeader(response.headers['link']); + return linkHeader?.[nextUrlPagination.link_rel]?.url; + } else if (nextUrlPagination.next_url_body_parameter_path) { + return this.getNestedField(response.data, nextUrlPagination.next_url_body_parameter_path); + } + + throw Error(`Either 'link_rel' or 'next_url_body_parameter_path' should be specified for '${paginationConfig.type}' pagination`); + } + private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { if (passPaginationParamsInBody) { config.data = updatedBodyOrParams; From 2722926dcb0cda8143e618acf28d0a1033dc4b44 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 13:02:45 +0300 Subject: [PATCH 31/50] Ensure override type safety on compile stage --- packages/shared/lib/sdk/sync.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 21f9298c940..bdb7023f99a 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -99,7 +99,7 @@ interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; - paginate?: Record; // Supported only by Syncs and Actions ATM + paginate?: Partial | Partial; // Supported only by Syncs and Actions ATM } enum AuthModes { @@ -339,6 +339,8 @@ export class NangoAction { } let paginationConfig: Pagination = templatePaginationConfig; + delete paginationConfig.limit; + if (config.paginate) { const paginationConfigOverride: Record = config.paginate as Record; From 45920e3cafc3910cbf97fb3c4cdb864878a581f0 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 14:05:01 +0300 Subject: [PATCH 32/50] Migrate GitHub integration templates --- integration-templates/github/github-issues.ts | 71 ++++------ .../github/github-list-files-sync.ts | 123 +++++++++--------- .../github/github-list-repos-action.ts | 33 ++--- 3 files changed, 97 insertions(+), 130 deletions(-) diff --git a/integration-templates/github/github-issues.ts b/integration-templates/github/github-issues.ts index f9d467ee7ba..da7f784e523 100644 --- a/integration-templates/github/github-issues.ts +++ b/integration-templates/github/github-issues.ts @@ -1,57 +1,40 @@ import type { NangoSync, GithubIssue } from './models'; export default async function fetchData(nango: NangoSync) { - const repos = await paginate(nango, '/user/repos'); + const repos: any[] = await getAll(nango, '/user/repos'); for (let repo of repos) { - let issues = await paginate(nango, `/repos/${repo.owner.login}/${repo.name}/issues`); - - // Filter out pull requests - issues = issues.filter((issue) => !('pull_request' in issue)); - - const mappedIssues: GithubIssue[] = issues.map((issue) => ({ - id: issue.id, - owner: repo.owner.login, - repo: repo.name, - issue_number: issue.number, - title: issue.title, - state: issue.state, - author: issue.user.login, - author_id: issue.user.id, - body: issue.body, - date_created: issue.created_at, - date_last_modified: issue.updated_at - })); - - if (mappedIssues.length > 0) { - await nango.batchSave(mappedIssues, 'GithubIssue'); - await nango.log(`Sent ${mappedIssues.length} issues from ${repo.owner.login}/${repo.name}`); + for await (const issueBatch of nango.paginate({ endpoint: `/repos/${repo.owner.login}/${repo.name}/issues` })) { + let issues: any[] = issueBatch.filter((issue) => !('pull_request' in issue)); + + const mappedIssues: GithubIssue[] = issues.map((issue) => ({ + id: issue.id, + owner: repo.owner.login, + repo: repo.name, + issue_number: issue.number, + title: issue.title, + state: issue.state, + author: issue.user.login, + author_id: issue.user.id, + body: issue.body, + date_created: issue.created_at, + date_last_modified: issue.updated_at + })); + + if (mappedIssues.length > 0) { + await nango.batchSave(mappedIssues, 'GithubIssue'); + await nango.log(`Sent ${mappedIssues.length} issues from ${repo.owner.login}/${repo.name}`); + } } } } -async function paginate(nango: NangoSync, endpoint: string) { - const MAX_PAGE = 100; +async function getAll(nango: NangoSync, endpoint: string) { + const records: any[] = []; - let results: any[] = []; - let page = 1; - while (true) { - const resp = await nango.get({ - endpoint: endpoint, - params: { - limit: `${MAX_PAGE}`, - page: `${page}` - } - }); - - results = results.concat(resp.data); - - if (resp.data.length == MAX_PAGE) { - page += 1; - } else { - break; - } + for await (const recordBatch of nango.paginate({ endpoint })) { + records.push(...recordBatch); } - return results; + return records; } diff --git a/integration-templates/github/github-list-files-sync.ts b/integration-templates/github/github-list-files-sync.ts index 45223e6c569..861eba008ed 100644 --- a/integration-templates/github/github-list-files-sync.ts +++ b/integration-templates/github/github-list-files-sync.ts @@ -1,11 +1,5 @@ import type { NangoSync, GithubRepoFile } from './models'; -enum PaginationType { - RepoFile, - CommitFile, - Commit -} - enum Models { GithubRepoFile = 'GithubRepoFile' } @@ -17,81 +11,88 @@ interface Metadata { } export default async function fetchData(nango: NangoSync) { - const { owner, repo, branch } = await nango.getMetadata(); - + let { owner, repo, branch } = await nango.getMetadata(); // On the first run, fetch all files. On subsequent runs, fetch only updated files. if (!nango.lastSyncDate) { - await getAllFilesFromRepo(nango, owner, repo, branch); + await saveAllRepositoryFiles(nango, owner, repo, branch); } else { - await getUpdatedFiles(nango, owner, repo, nango.lastSyncDate); + await saveFileUpdates(nango, owner, repo, nango.lastSyncDate); } } -async function getAllFilesFromRepo(nango: NangoSync, owner: string, repo: string, branch: string) { - await paginate(nango, `/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`, PaginationType.RepoFile); +async function saveAllRepositoryFiles(nango: NangoSync, owner: string, repo: string, branch: string) { + let count = 0; + + const endpoint: string = `/repos/${owner}/${repo}/git/trees/${branch}`; + const proxyConfig = { + endpoint, + params: { recursive: '1' }, + paginate: { response_data_path: 'tree' } + }; + + await nango.log(`Fetching files from endpoint ${endpoint}.`); + + for await (const fileBatch of nango.paginate(proxyConfig)) { + const blobFiles = fileBatch.filter((item: any) => item.type === 'blob'); + count += blobFiles.length; + await nango.batchSave(blobFiles.map(mapToFile), Models.GithubRepoFile); + } + await nango.log(`Got ${count} file(s).`); } -async function getUpdatedFiles(nango: NangoSync, owner: string, repo: string, since: Date) { - const commitsSinceLastSync = await paginate(nango, `/repos/${owner}/${repo}/commits`, PaginationType.Commit, { since: since.toISOString() }); +async function saveFileUpdates(nango: NangoSync, owner: string, repo: string, since: Date) { + const commitsSinceLastSync: any[] = await getCommitsSinceLastSync(owner, repo, since, nango); for (const commitSummary of commitsSinceLastSync) { - await paginate(nango, `/repos/${owner}/${repo}/commits/${commitSummary.sha}`, PaginationType.CommitFile); + await saveFilesUpdatedByCommit(owner, repo, commitSummary, nango); } } -function mapToFile(file: any): GithubRepoFile { - return { - id: file.sha, - name: file.path || file.filename, - url: file.url || file.blob_url, - last_modified_date: file.committer?.date ? new Date(file.committer?.date) : new Date() // Use commit date or current date +async function getCommitsSinceLastSync(owner: string, repo: string, since: Date, nango: NangoSync) { + let count = 0; + const endpoint: string = `/repos/${owner}/${repo}/commits`; + + const proxyConfig = { + endpoint, + params: { since: since.toISOString() } }; + + await nango.log(`Fetching commits from endpoint ${endpoint}.`); + + const commitsSinceLastSync: any[] = []; + for await (const commitBatch of nango.paginate(proxyConfig)) { + count += commitBatch.length; + commitsSinceLastSync.push(...commitBatch); + } + await nango.log(`Got ${count} commits(s).`); + return commitsSinceLastSync; } -async function paginate(nango: NangoSync, endpoint: string, type: PaginationType, params?: any): Promise { - let page = 1; - const PER_PAGE = 100; - const results: any[] = []; +async function saveFilesUpdatedByCommit(owner: string, repo: string, commitSummary: any, nango: NangoSync) { let count = 0; - const objectType = type === PaginationType.Commit ? 'commit' : 'file'; - - await nango.log(`Fetching ${objectType}(s) from endpoint ${endpoint}.`); - - while (true) { - const response = await nango.get({ - endpoint: endpoint, - params: { - ...params, - page: page, - per_page: PER_PAGE - } - }); - - switch (type) { - case PaginationType.RepoFile: - const files = response.data.tree.filter((item: any) => item.type === 'blob'); - count += files.length; - await nango.batchSave(files.map(mapToFile), Models.GithubRepoFile); - break; - case PaginationType.CommitFile: - count += response.data.files.length; - await nango.batchSave(response.data.files.filter((file: any) => file.status !== 'removed').map(mapToFile), Models.GithubRepoFile); - await nango.batchDelete(response.data.files.filter((file: any) => file.status === 'removed').map(mapToFile), Models.GithubRepoFile); - break; - case PaginationType.Commit: - count += response.data.length; - results.push(...response.data); - break; + const endpoint: string = `/repos/${owner}/${repo}/commits/${commitSummary.sha}`; + const proxyConfig = { + endpoint, + paginate: { + response_data_path: 'files' } + }; - if (!response.headers.link || !response.headers.link.includes('rel="next"')) { - break; - } + await nango.log(`Fetching files from endpoint ${endpoint}.`); - page++; + for await (const fileBatch of nango.paginate(proxyConfig)) { + count += fileBatch.length; + await nango.batchSave(fileBatch.filter((file: any) => file.status !== 'removed').map(mapToFile), Models.GithubRepoFile); + await nango.batchDelete(fileBatch.filter((file: any) => file.status === 'removed').map(mapToFile), Models.GithubRepoFile); } + await nango.log(`Got ${count} file(s).`); +} - await nango.log(`Got ${count} ${objectType}(s).`); - - return results; +function mapToFile(file: any): GithubRepoFile { + return { + id: file.sha, + name: file.path || file.filename, + url: file.url || file.blob_url, + last_modified_date: file.committer?.date ? new Date(file.committer?.date) : new Date() // Use commit date or current date + }; } diff --git a/integration-templates/github/github-list-repos-action.ts b/integration-templates/github/github-list-repos-action.ts index aa3bcdda52a..df8b1cf6ba2 100644 --- a/integration-templates/github/github-list-repos-action.ts +++ b/integration-templates/github/github-list-repos-action.ts @@ -4,15 +4,15 @@ export default async function runAction(nango: NangoSync): Promise<{ repos: Gith let allRepos: any[] = []; // Fetch user's personal repositories. - const personalRepos = await paginate(nango, '/user/repos'); + const personalRepos = await getAll(nango, '/user/repos'); allRepos = allRepos.concat(personalRepos); // Fetch organizations the user is a part of. - const organizations = await paginate(nango, '/user/orgs'); + const organizations = await getAll(nango, '/user/orgs'); // For each organization, fetch its repositories. for (const org of organizations) { - const orgRepos = await paginate(nango, `/orgs/${org.login}/repos`); + const orgRepos = await getAll(nango, `/orgs/${org.login}/repos`); allRepos = allRepos.concat(orgRepos); } @@ -30,29 +30,12 @@ export default async function runAction(nango: NangoSync): Promise<{ repos: Gith return { repos: mappedRepos }; } -async function paginate(nango: NangoSync, endpoint: string) { - const MAX_PAGE = 100; +async function getAll(nango: NangoSync, endpoint: string) { + const records: any[] = []; - let results: any[] = []; - let page = 1; - - while (true) { - const resp = await nango.get({ - endpoint: endpoint, - params: { - limit: `${MAX_PAGE}`, - page: `${page}` - } - }); - - results = results.concat(resp.data); - - if (resp.data.length == MAX_PAGE) { - page += 1; - } else { - break; - } + for await (const recordBatch of nango.paginate({ endpoint })) { + records.push(...recordBatch); } - return results; + return records; } From 71434e388146af61b9f957d37a0560703b695aee Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 14:40:06 +0300 Subject: [PATCH 33/50] Migrate Slack integration templates --- integration-templates/slack/slack-channels.ts | 34 ++++++--------- integration-templates/slack/slack-messages.ts | 42 +++++++----------- integration-templates/slack/slack-users.ts | 43 +++++++++---------- 3 files changed, 49 insertions(+), 70 deletions(-) diff --git a/integration-templates/slack/slack-channels.ts b/integration-templates/slack/slack-channels.ts index 7b6ed8df322..56f5802b755 100644 --- a/integration-templates/slack/slack-channels.ts +++ b/integration-templates/slack/slack-channels.ts @@ -1,7 +1,7 @@ import type { SlackChannel, NangoSync } from './models'; export default async function fetchData(nango: NangoSync) { - const responses = await getAllPages(nango, 'conversations.list'); + const responses = await getAllChannels(nango, 'conversations.list'); const mappedChannels: SlackChannel[] = responses.map((record: any) => { return { @@ -27,6 +27,7 @@ export default async function fetchData(nango: NangoSync) { // Now let's also join all public channels where we are not yet a member await joinPublicChannels(nango, mappedChannels); + // console.log(mappedChannels) // Save channels await nango.batchSave(mappedChannels, 'SlackChannel'); } @@ -34,7 +35,7 @@ export default async function fetchData(nango: NangoSync) { // Checks for public channels where the bot is not a member yet and joins them async function joinPublicChannels(nango: NangoSync, channels: SlackChannel[]) { // Get ID of all channels where we are already a member - const joinedChannelsResponse = await getAllPages(nango, 'users.conversations'); + const joinedChannelsResponse = await getAllChannels(nango, 'users.conversations'); const channelIds = joinedChannelsResponse.map((record: any) => { return record.id; }); @@ -52,27 +53,18 @@ async function joinPublicChannels(nango: NangoSync, channels: SlackChannel[]) { } } -async function getAllPages(nango: NangoSync, endpoint: string) { - var nextCursor = 'x'; - var responses: any[] = []; +async function getAllChannels(nango: NangoSync, endpoint: string) { + const channels: any[] = []; - while (nextCursor !== '') { - const response = await nango.get({ - endpoint: endpoint, - params: { - limit: '200', - cursor: nextCursor !== 'x' ? nextCursor : '' - } - }); - - if (!response.data.ok) { - await nango.log(`Received a Slack API error (for ${endpoint}): ${JSON.stringify(response.data, null, 2)}`); + const proxyConfig = { + endpoint, + paginate: { + response_data_path: 'channels' } - - const { channels, response_metadata } = response.data; - responses = responses.concat(channels); - nextCursor = response_metadata.next_cursor; + }; + for await (const channelBatch of nango.paginate(proxyConfig)) { + channels.push(...channelBatch); } - return responses; + return channels; } diff --git a/integration-templates/slack/slack-messages.ts b/integration-templates/slack/slack-messages.ts index 65b059ff625..997461f585a 100644 --- a/integration-templates/slack/slack-messages.ts +++ b/integration-templates/slack/slack-messages.ts @@ -3,7 +3,7 @@ import { createHash } from 'crypto'; export default async function fetchData(nango: NangoSync) { // Get all channels we are part of - let channels = await getAllPages(nango, 'users.conversations', {}, 'channels'); + let channels = await getAllRecords(nango, 'users.conversations', {}, 'channels'); await nango.log(`Bot is part of ${channels.length} channels`); @@ -16,7 +16,7 @@ export default async function fetchData(nango: NangoSync) { // For every channel read messages, replies & reactions for (let channel of channels) { - let allMessages = await getAllPages(nango, 'conversations.history', { channel: channel.id, oldest: oldestTimestamp.toString() }, 'messages'); + let allMessages = await getAllRecords(nango, 'conversations.history', { channel: channel.id, oldest: oldestTimestamp.toString() }, 'messages'); for (let message of allMessages) { const mappedMessage: SlackMessage = { @@ -71,7 +71,7 @@ export default async function fetchData(nango: NangoSync) { // Replies to fetch? if (message.reply_count > 0) { - const allReplies = await getAllPages(nango, 'conversations.replies', { channel: channel.id, ts: message.thread_ts }, 'messages'); + const allReplies = await getAllRecords(nango, 'conversations.replies', { channel: channel.id, ts: message.thread_ts }, 'messages'); for (let reply of allReplies) { if (reply.ts === message.ts) { @@ -141,31 +141,21 @@ export default async function fetchData(nango: NangoSync) { await nango.batchSave(batchReactions, 'SlackMessageReaction'); } -async function getAllPages(nango: NangoSync, endpoint: string, params: Record, resultsKey: string) { - let nextCursor = 'x'; - let responses: any[] = []; - - while (nextCursor !== '') { - const response = await nango.get({ - endpoint: endpoint, - params: { - limit: '200', - cursor: nextCursor !== 'x' ? nextCursor : '', - ...params - }, - retries: 10 - }); - - if (!response.data.ok) { - await nango.log(`Received a Slack API error (for ${endpoint}): ${JSON.stringify(response.data, null, 2)}`); - } +async function getAllRecords(nango: NangoSync, endpoint: string, params: Record, resultsKey: string) { + let records: any[] = []; - const results = response.data[resultsKey]; - const response_metadata = response.data.response_metadata; + const proxyConfig = { + endpoint: endpoint, + params, + retries: 10, + paginate: { + response_data_path: resultsKey + } + }; - responses = responses.concat(results); - nextCursor = response_metadata && response_metadata.next_cursor ? response_metadata.next_cursor : ''; + for await (const recordBatch of nango.paginate(proxyConfig)) { + records.push(...recordBatch); } - return responses; + return records; } diff --git a/integration-templates/slack/slack-users.ts b/integration-templates/slack/slack-users.ts index 9cc49d6be1d..175f1b4d233 100644 --- a/integration-templates/slack/slack-users.ts +++ b/integration-templates/slack/slack-users.ts @@ -2,30 +2,10 @@ import type { SlackUser, NangoSync } from './models'; export default async function fetchData(nango: NangoSync) { // Fetch all users (paginated) - let nextCursor = 'x'; - let responses: any[] = []; - - while (nextCursor !== '') { - const response = await nango.get({ - endpoint: 'users.list', - retries: 10, - params: { - limit: '200', - cursor: nextCursor !== 'x' ? nextCursor : '' - } - }); - - if (!response.data.ok) { - await nango.log(`Received a Slack API error: ${JSON.stringify(response.data, null, 2)}`); - } - - const { members, response_metadata } = response.data; - responses = responses.concat(members); - nextCursor = response_metadata.next_cursor; - } + const users: any[] = await getAllUsers(nango); // Transform users into our data model - const users: SlackUser[] = responses.map((record: any) => { + const mappedUsers: SlackUser[] = users.map((record: any) => { return { id: record.id, team_id: record.team_id, @@ -55,5 +35,22 @@ export default async function fetchData(nango: NangoSync) { }; }); - await nango.batchSave(users, 'SlackUser'); + await nango.batchSave(mappedUsers, 'SlackUser'); +} +async function getAllUsers(nango: NangoSync) { + const users: any[] = []; + + const proxyConfig = { + endpoint: 'users.list', + retries: 10, + paginate: { + response_data_path: 'members' + } + }; + + for await (const userBatch of nango.paginate(proxyConfig)) { + users.push(...userBatch); + } + + return users; } From 394eedb2d5b7c59252f3fec6d4e72a84e23ac2bc Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 15:15:01 +0300 Subject: [PATCH 34/50] Migrate Confluence integration templates --- .../confluence/confluence-pages.ts | 68 ++++--------------- .../confluence/confluence-spaces.ts | 49 +++++-------- 2 files changed, 31 insertions(+), 86 deletions(-) diff --git a/integration-templates/confluence/confluence-pages.ts b/integration-templates/confluence/confluence-pages.ts index aca6f514a24..3f3f28e1a79 100644 --- a/integration-templates/confluence/confluence-pages.ts +++ b/integration-templates/confluence/confluence-pages.ts @@ -6,71 +6,29 @@ async function getCloudId(nango: NangoSync): Promise { endpoint: `oauth/token/accessible-resources`, retries: 10 // Exponential backoff + long-running job = handles rate limits well. }); - return response.data[0].id; -} -interface ResultPage { - pageNumber: number; - results: any[]; - nextPageEndpoint: string; - totalResultCount: number; + return response.data[0].id; } export default async function fetchData(nango: NangoSync) { - let cloudId = await getCloudId(nango); - - let resultPage: ResultPage | null = null; - while (true) { - resultPage = await getNextPage(nango, 'get', 'wiki/api/v2/pages', resultPage, 2, cloudId); - - if (!resultPage) { - break; - } + let totalRecords: number = 0; - let confluencePages = mapConfluencePages(resultPage.results); + let cloudId: string = await getCloudId(nango); + const proxyConfig = { + baseUrlOverride: `https://api.atlassian.com/ex/confluence/${cloudId}`, // The base URL is specific for user because of the cloud ID path param + endpoint: `/wiki/api/v2/pages`, + retries: 10 + }; + for await (const pageBatch of nango.paginate(proxyConfig)) { + const confluencePages = mapConfluencePages(pageBatch); + const batchSize: number = confluencePages.length; + totalRecords += batchSize; + await nango.log(`Saving batch of ${batchSize} pages (total records: ${totalRecords})`); await nango.batchSave(confluencePages, 'ConfluencePage'); } } -async function getNextPage( - nango: NangoSync, - method: 'get' | 'post', - endpoint: string, - prevResultPage: ResultPage | null, - pageSize = 250, - cloudId: string -): Promise { - if (prevResultPage && !prevResultPage.nextPageEndpoint) { - return null; - } - - await nango.log(`Fetching Confluence Pages - with pageCounter = ${prevResultPage ? prevResultPage.pageNumber : 0} & pageSize = ${pageSize}`); - - const res = await nango.get({ - baseUrlOverride: `https://api.atlassian.com`, // Optional - endpoint: `ex/confluence/${cloudId}/${prevResultPage ? prevResultPage.nextPageEndpoint : endpoint}`, - method: method, - params: { limit: `${pageSize}`, 'body-format': 'storage' }, // Page format storage or atlas_doc_format - retries: 10 // Exponential backoff + long-running job = handles rate limits well. - }); - - if (!res.data) { - return null; - } - - const resultPage = { - pageNumber: prevResultPage ? prevResultPage.pageNumber + 1 : 1, - results: res.data.results, - nextPageEndpoint: res.data['_links'].next ? res.data['_links'].next : '', - totalResultCount: prevResultPage ? prevResultPage.totalResultCount + res.data.results.length : res.data.results.length - }; - - await nango.log(`Saving page with ${resultPage.results.length} records (total records: ${resultPage.totalResultCount})`); - - return resultPage; -} - function mapConfluencePages(results: any[]): ConfluencePage[] { return results.map((page: any) => { return { diff --git a/integration-templates/confluence/confluence-spaces.ts b/integration-templates/confluence/confluence-spaces.ts index 2bb05d314f0..5a59c4a5a99 100644 --- a/integration-templates/confluence/confluence-spaces.ts +++ b/integration-templates/confluence/confluence-spaces.ts @@ -11,9 +11,25 @@ async function getCloudId(nango: NangoSync): Promise { export default async function fetchData(nango: NangoSync) { let cloudId = await getCloudId(nango); - const results = await paginate(nango, 'get', 'wiki/api/v2/spaces', 'Confluence spaces', 250, cloudId); + let totalRecords: number = 0; - let spaces: ConfluenceSpace[] = results?.map((space: any) => { + const proxyConfig = { + baseUrlOverride: `https://api.atlassian.com/ex/confluence/${cloudId}`, // The base URL is specific for user because of the cloud ID path param + endpoint: `/wiki/api/v2/spaces`, + retries: 10 + }; + for await (const spaceBatch of nango.paginate(proxyConfig)) { + const confluenceSpaces = mapConfluenceSpaces(spaceBatch); + const batchSize: number = confluenceSpaces.length; + totalRecords += batchSize; + + await nango.log(`Saving batch of ${batchSize} spaces (total records: ${totalRecords})`); + await nango.batchSave(confluenceSpaces, 'ConfluenceSpace'); + } +} + +function mapConfluenceSpaces(spaces: any[]): ConfluenceSpace[] { + return spaces.map((space: any) => { return { id: space.id, key: space.key, @@ -26,33 +42,4 @@ export default async function fetchData(nango: NangoSync) { description: space.description || '' }; }); - - await nango.log(`Fetching ${spaces.length}`); - await nango.batchSave(spaces, 'ConfluenceSpace'); -} - -async function paginate(nango: NangoSync, method: 'get' | 'post', endpoint: string, desc: string, pageSize = 250, cloudId: string) { - let pageCounter = 0; - let results: any[] = []; - - while (true) { - await nango.log(`Fetching ${desc} - with pageCounter = ${pageCounter} & pageSize = ${pageSize}`); - const res = await nango.get({ - baseUrlOverride: `https://api.atlassian.com`, // Optional - endpoint: `ex/confluence/${cloudId}/${endpoint}`, - method: method, - params: { limit: `${pageSize}` }, - retries: 10 // Exponential backoff + long-running job = handles rate limits well. - }); - await nango.log(`Appending records of count ${res.data.results.length} to results of count ${results.length}`); - if (res.data) { - results = [...results, ...res.data.results]; - } - if (res.data['_links'].next) { - endpoint = res.data['_links'].next; - pageCounter += 1; - } else { - return results; - } - } } From 6fc0a9eb20efab41983e50bfcba70c978118a8c0 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 16:33:15 +0300 Subject: [PATCH 35/50] Support offset pagination --- packages/shared/lib/sdk/sync.ts | 41 +++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index bdb7023f99a..3fcbe312a52 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -65,7 +65,8 @@ interface DataResponse { export enum PaginationType { CURSOR = 'cursor', - NEXT_URL = 'next_url' + NEXT_URL = 'next_url', + OFFSET = 'offset' } interface Pagination { @@ -87,6 +88,11 @@ export interface NextUrlPagination extends Pagination { next_url_body_parameter_path?: string; } +export interface OffsetPagination extends Pagination { + type: PaginationType.OFFSET; + offset_parameter_name: string; +} + interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; @@ -99,7 +105,7 @@ interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; - paginate?: Partial | Partial; // Supported only by Syncs and Actions ATM + paginate?: Partial | Partial | Partial; // Supported only by Syncs and Actions ATM } enum AuthModes { @@ -364,6 +370,7 @@ export class NangoAction { updatedBodyOrParams[limitParameterName] = paginationConfig['limit']; } + // TODO: Consider creating 'Paginator' interface and moving the case block to specific implementations of 'Paginator' switch (paginationConfig.type) { case PaginationType.CURSOR: { const cursorPagination: CursorPagination = paginationConfig as CursorPagination; @@ -381,6 +388,7 @@ export class NangoAction { const responseData: T[] = cursorPagination.response_data_path ? this.getNestedField(response.data, cursorPagination.response_data_path) : response.data; + if (!responseData.length) { return; } @@ -426,6 +434,35 @@ export class NangoAction { delete config.params; } } + case PaginationType.OFFSET: { + const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; + const offsetParameterName: string = offsetPagination.offset_parameter_name; + let offset: number = 0; + + while (true) { + updatedBodyOrParams[offsetParameterName] = `${offset}`; + + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); + + const response: AxiosResponse = await this.proxy(config); + + const responseData: T[] = paginationConfig.response_data_path + ? this.getNestedField(response.data, paginationConfig.response_data_path) + : response.data; + if (!responseData.length) { + return; + } + + yield responseData; + + if (responseData.length < 1) { + // Last page was empty so no need to fetch further + return; + } + + offset += responseData.length; + } + } default: throw Error(`'${paginationConfig.type} ' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)} `); } From d42d8cbcd33944b211259c6de0c3d0d5cfded4b5 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 17:25:19 +0300 Subject: [PATCH 36/50] Convert pagination type to string --- packages/shared/lib/sdk/sync.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 3fcbe312a52..1dfa0c5817d 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -70,26 +70,23 @@ export enum PaginationType { } interface Pagination { - type: PaginationType; + type: string; limit?: number; response_data_path?: string; limit_parameter_name: string; } export interface CursorPagination extends Pagination { - type: PaginationType.CURSOR; next_cursor_parameter_path: string; cursor_parameter_name: string; } export interface NextUrlPagination extends Pagination { - type: PaginationType.NEXT_URL; link_rel?: string; next_url_body_parameter_path?: string; } export interface OffsetPagination extends Pagination { - type: PaginationType.OFFSET; offset_parameter_name: string; } From 299bb04036c5c2d4dca7e8f887f86428b9eaeed2 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 17:26:58 +0300 Subject: [PATCH 37/50] Migrate Jira integration template --- integration-templates/jira/jira-issues.ts | 45 ++++++++++------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/integration-templates/jira/jira-issues.ts b/integration-templates/jira/jira-issues.ts index 869b04a19de..e294998c510 100644 --- a/integration-templates/jira/jira-issues.ts +++ b/integration-templates/jira/jira-issues.ts @@ -2,35 +2,30 @@ import type { NangoSync, JiraIssue } from './models'; export default async function fetchData(nango: NangoSync) { const jql = nango.lastSyncDate ? `updated >= "${nango.lastSyncDate?.toISOString().slice(0, -8).replace('T', ' ')}"` : ''; - let startAt: number = 0; - const maxResults: number = 50; const fields = 'id,key,summary,description,issuetype,status,assignee,reporter,project,created,updated'; const cloudId = await getCloudId(nango); - while (true) { - const response = await nango.get({ - baseUrlOverride: 'https://api.atlassian.com', - endpoint: `ex/jira/${cloudId}/rest/api/3/search`, - params: { - jql: jql, - startAt: `${startAt}`, - maxResults: `${maxResults}`, - fields: fields - }, - headers: { - 'X-Atlassian-Token': 'no-check' - }, - retries: 10 // Exponential backoff + long-running job = handles rate limits well. - }); - - const issues = response.data.issues; - await nango.batchSave(mapIssues(issues), 'JiraIssue'); + const proxyConfig = { + baseUrlOverride: `https://api.atlassian.com/ex/jira/${cloudId}`, + endpoint: `/rest/api/3/search`, + paginate: { + type: 'offset', + offset_parameter_name: 'startAt', + limit_parameter_name: 'maxResults', + response_data_path: 'issues' + }, + params: { + jql: jql, + fields: fields + }, + headers: { + 'X-Atlassian-Token': 'no-check' + }, + retries: 10 // Exponential backoff + long-running job = handles rate limits well. + }; - if (issues.length < maxResults) { - break; - } else { - startAt += maxResults; - } + for await (const issueBatch of nango.paginate(proxyConfig)) { + await nango.batchSave(mapIssues(issueBatch), 'JiraIssue'); } } From 3e9fecc0c0ee080852c0af113fab53694633e0e2 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 18:32:16 +0300 Subject: [PATCH 38/50] Revert template updates --- .../confluence/confluence-pages.ts | 68 ++++++++-- .../confluence/confluence-spaces.ts | 49 ++++--- integration-templates/github/github-issues.ts | 71 ++++++---- .../github/github-list-files-sync.ts | 123 +++++++++--------- .../github/github-list-repos-action.ts | 33 +++-- integration-templates/jira/jira-issues.ts | 45 ++++--- integration-templates/slack/slack-channels.ts | 34 +++-- integration-templates/slack/slack-messages.ts | 42 +++--- integration-templates/slack/slack-users.ts | 43 +++--- 9 files changed, 311 insertions(+), 197 deletions(-) diff --git a/integration-templates/confluence/confluence-pages.ts b/integration-templates/confluence/confluence-pages.ts index 3f3f28e1a79..aca6f514a24 100644 --- a/integration-templates/confluence/confluence-pages.ts +++ b/integration-templates/confluence/confluence-pages.ts @@ -6,29 +6,71 @@ async function getCloudId(nango: NangoSync): Promise { endpoint: `oauth/token/accessible-resources`, retries: 10 // Exponential backoff + long-running job = handles rate limits well. }); - return response.data[0].id; } +interface ResultPage { + pageNumber: number; + results: any[]; + nextPageEndpoint: string; + totalResultCount: number; +} + export default async function fetchData(nango: NangoSync) { - let totalRecords: number = 0; + let cloudId = await getCloudId(nango); - let cloudId: string = await getCloudId(nango); - const proxyConfig = { - baseUrlOverride: `https://api.atlassian.com/ex/confluence/${cloudId}`, // The base URL is specific for user because of the cloud ID path param - endpoint: `/wiki/api/v2/pages`, - retries: 10 - }; - for await (const pageBatch of nango.paginate(proxyConfig)) { - const confluencePages = mapConfluencePages(pageBatch); - const batchSize: number = confluencePages.length; - totalRecords += batchSize; + let resultPage: ResultPage | null = null; + while (true) { + resultPage = await getNextPage(nango, 'get', 'wiki/api/v2/pages', resultPage, 2, cloudId); + + if (!resultPage) { + break; + } + + let confluencePages = mapConfluencePages(resultPage.results); - await nango.log(`Saving batch of ${batchSize} pages (total records: ${totalRecords})`); await nango.batchSave(confluencePages, 'ConfluencePage'); } } +async function getNextPage( + nango: NangoSync, + method: 'get' | 'post', + endpoint: string, + prevResultPage: ResultPage | null, + pageSize = 250, + cloudId: string +): Promise { + if (prevResultPage && !prevResultPage.nextPageEndpoint) { + return null; + } + + await nango.log(`Fetching Confluence Pages - with pageCounter = ${prevResultPage ? prevResultPage.pageNumber : 0} & pageSize = ${pageSize}`); + + const res = await nango.get({ + baseUrlOverride: `https://api.atlassian.com`, // Optional + endpoint: `ex/confluence/${cloudId}/${prevResultPage ? prevResultPage.nextPageEndpoint : endpoint}`, + method: method, + params: { limit: `${pageSize}`, 'body-format': 'storage' }, // Page format storage or atlas_doc_format + retries: 10 // Exponential backoff + long-running job = handles rate limits well. + }); + + if (!res.data) { + return null; + } + + const resultPage = { + pageNumber: prevResultPage ? prevResultPage.pageNumber + 1 : 1, + results: res.data.results, + nextPageEndpoint: res.data['_links'].next ? res.data['_links'].next : '', + totalResultCount: prevResultPage ? prevResultPage.totalResultCount + res.data.results.length : res.data.results.length + }; + + await nango.log(`Saving page with ${resultPage.results.length} records (total records: ${resultPage.totalResultCount})`); + + return resultPage; +} + function mapConfluencePages(results: any[]): ConfluencePage[] { return results.map((page: any) => { return { diff --git a/integration-templates/confluence/confluence-spaces.ts b/integration-templates/confluence/confluence-spaces.ts index 5a59c4a5a99..2bb05d314f0 100644 --- a/integration-templates/confluence/confluence-spaces.ts +++ b/integration-templates/confluence/confluence-spaces.ts @@ -11,25 +11,9 @@ async function getCloudId(nango: NangoSync): Promise { export default async function fetchData(nango: NangoSync) { let cloudId = await getCloudId(nango); - let totalRecords: number = 0; + const results = await paginate(nango, 'get', 'wiki/api/v2/spaces', 'Confluence spaces', 250, cloudId); - const proxyConfig = { - baseUrlOverride: `https://api.atlassian.com/ex/confluence/${cloudId}`, // The base URL is specific for user because of the cloud ID path param - endpoint: `/wiki/api/v2/spaces`, - retries: 10 - }; - for await (const spaceBatch of nango.paginate(proxyConfig)) { - const confluenceSpaces = mapConfluenceSpaces(spaceBatch); - const batchSize: number = confluenceSpaces.length; - totalRecords += batchSize; - - await nango.log(`Saving batch of ${batchSize} spaces (total records: ${totalRecords})`); - await nango.batchSave(confluenceSpaces, 'ConfluenceSpace'); - } -} - -function mapConfluenceSpaces(spaces: any[]): ConfluenceSpace[] { - return spaces.map((space: any) => { + let spaces: ConfluenceSpace[] = results?.map((space: any) => { return { id: space.id, key: space.key, @@ -42,4 +26,33 @@ function mapConfluenceSpaces(spaces: any[]): ConfluenceSpace[] { description: space.description || '' }; }); + + await nango.log(`Fetching ${spaces.length}`); + await nango.batchSave(spaces, 'ConfluenceSpace'); +} + +async function paginate(nango: NangoSync, method: 'get' | 'post', endpoint: string, desc: string, pageSize = 250, cloudId: string) { + let pageCounter = 0; + let results: any[] = []; + + while (true) { + await nango.log(`Fetching ${desc} - with pageCounter = ${pageCounter} & pageSize = ${pageSize}`); + const res = await nango.get({ + baseUrlOverride: `https://api.atlassian.com`, // Optional + endpoint: `ex/confluence/${cloudId}/${endpoint}`, + method: method, + params: { limit: `${pageSize}` }, + retries: 10 // Exponential backoff + long-running job = handles rate limits well. + }); + await nango.log(`Appending records of count ${res.data.results.length} to results of count ${results.length}`); + if (res.data) { + results = [...results, ...res.data.results]; + } + if (res.data['_links'].next) { + endpoint = res.data['_links'].next; + pageCounter += 1; + } else { + return results; + } + } } diff --git a/integration-templates/github/github-issues.ts b/integration-templates/github/github-issues.ts index da7f784e523..f9d467ee7ba 100644 --- a/integration-templates/github/github-issues.ts +++ b/integration-templates/github/github-issues.ts @@ -1,40 +1,57 @@ import type { NangoSync, GithubIssue } from './models'; export default async function fetchData(nango: NangoSync) { - const repos: any[] = await getAll(nango, '/user/repos'); + const repos = await paginate(nango, '/user/repos'); for (let repo of repos) { - for await (const issueBatch of nango.paginate({ endpoint: `/repos/${repo.owner.login}/${repo.name}/issues` })) { - let issues: any[] = issueBatch.filter((issue) => !('pull_request' in issue)); - - const mappedIssues: GithubIssue[] = issues.map((issue) => ({ - id: issue.id, - owner: repo.owner.login, - repo: repo.name, - issue_number: issue.number, - title: issue.title, - state: issue.state, - author: issue.user.login, - author_id: issue.user.id, - body: issue.body, - date_created: issue.created_at, - date_last_modified: issue.updated_at - })); - - if (mappedIssues.length > 0) { - await nango.batchSave(mappedIssues, 'GithubIssue'); - await nango.log(`Sent ${mappedIssues.length} issues from ${repo.owner.login}/${repo.name}`); - } + let issues = await paginate(nango, `/repos/${repo.owner.login}/${repo.name}/issues`); + + // Filter out pull requests + issues = issues.filter((issue) => !('pull_request' in issue)); + + const mappedIssues: GithubIssue[] = issues.map((issue) => ({ + id: issue.id, + owner: repo.owner.login, + repo: repo.name, + issue_number: issue.number, + title: issue.title, + state: issue.state, + author: issue.user.login, + author_id: issue.user.id, + body: issue.body, + date_created: issue.created_at, + date_last_modified: issue.updated_at + })); + + if (mappedIssues.length > 0) { + await nango.batchSave(mappedIssues, 'GithubIssue'); + await nango.log(`Sent ${mappedIssues.length} issues from ${repo.owner.login}/${repo.name}`); } } } -async function getAll(nango: NangoSync, endpoint: string) { - const records: any[] = []; +async function paginate(nango: NangoSync, endpoint: string) { + const MAX_PAGE = 100; - for await (const recordBatch of nango.paginate({ endpoint })) { - records.push(...recordBatch); + let results: any[] = []; + let page = 1; + while (true) { + const resp = await nango.get({ + endpoint: endpoint, + params: { + limit: `${MAX_PAGE}`, + page: `${page}` + } + }); + + results = results.concat(resp.data); + + if (resp.data.length == MAX_PAGE) { + page += 1; + } else { + break; + } } - return records; + return results; } diff --git a/integration-templates/github/github-list-files-sync.ts b/integration-templates/github/github-list-files-sync.ts index 861eba008ed..45223e6c569 100644 --- a/integration-templates/github/github-list-files-sync.ts +++ b/integration-templates/github/github-list-files-sync.ts @@ -1,5 +1,11 @@ import type { NangoSync, GithubRepoFile } from './models'; +enum PaginationType { + RepoFile, + CommitFile, + Commit +} + enum Models { GithubRepoFile = 'GithubRepoFile' } @@ -11,88 +17,81 @@ interface Metadata { } export default async function fetchData(nango: NangoSync) { - let { owner, repo, branch } = await nango.getMetadata(); + const { owner, repo, branch } = await nango.getMetadata(); + // On the first run, fetch all files. On subsequent runs, fetch only updated files. if (!nango.lastSyncDate) { - await saveAllRepositoryFiles(nango, owner, repo, branch); + await getAllFilesFromRepo(nango, owner, repo, branch); } else { - await saveFileUpdates(nango, owner, repo, nango.lastSyncDate); + await getUpdatedFiles(nango, owner, repo, nango.lastSyncDate); } } -async function saveAllRepositoryFiles(nango: NangoSync, owner: string, repo: string, branch: string) { - let count = 0; - - const endpoint: string = `/repos/${owner}/${repo}/git/trees/${branch}`; - const proxyConfig = { - endpoint, - params: { recursive: '1' }, - paginate: { response_data_path: 'tree' } - }; - - await nango.log(`Fetching files from endpoint ${endpoint}.`); - - for await (const fileBatch of nango.paginate(proxyConfig)) { - const blobFiles = fileBatch.filter((item: any) => item.type === 'blob'); - count += blobFiles.length; - await nango.batchSave(blobFiles.map(mapToFile), Models.GithubRepoFile); - } - await nango.log(`Got ${count} file(s).`); +async function getAllFilesFromRepo(nango: NangoSync, owner: string, repo: string, branch: string) { + await paginate(nango, `/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`, PaginationType.RepoFile); } -async function saveFileUpdates(nango: NangoSync, owner: string, repo: string, since: Date) { - const commitsSinceLastSync: any[] = await getCommitsSinceLastSync(owner, repo, since, nango); +async function getUpdatedFiles(nango: NangoSync, owner: string, repo: string, since: Date) { + const commitsSinceLastSync = await paginate(nango, `/repos/${owner}/${repo}/commits`, PaginationType.Commit, { since: since.toISOString() }); for (const commitSummary of commitsSinceLastSync) { - await saveFilesUpdatedByCommit(owner, repo, commitSummary, nango); + await paginate(nango, `/repos/${owner}/${repo}/commits/${commitSummary.sha}`, PaginationType.CommitFile); } } -async function getCommitsSinceLastSync(owner: string, repo: string, since: Date, nango: NangoSync) { - let count = 0; - const endpoint: string = `/repos/${owner}/${repo}/commits`; - - const proxyConfig = { - endpoint, - params: { since: since.toISOString() } +function mapToFile(file: any): GithubRepoFile { + return { + id: file.sha, + name: file.path || file.filename, + url: file.url || file.blob_url, + last_modified_date: file.committer?.date ? new Date(file.committer?.date) : new Date() // Use commit date or current date }; - - await nango.log(`Fetching commits from endpoint ${endpoint}.`); - - const commitsSinceLastSync: any[] = []; - for await (const commitBatch of nango.paginate(proxyConfig)) { - count += commitBatch.length; - commitsSinceLastSync.push(...commitBatch); - } - await nango.log(`Got ${count} commits(s).`); - return commitsSinceLastSync; } -async function saveFilesUpdatedByCommit(owner: string, repo: string, commitSummary: any, nango: NangoSync) { +async function paginate(nango: NangoSync, endpoint: string, type: PaginationType, params?: any): Promise { + let page = 1; + const PER_PAGE = 100; + const results: any[] = []; let count = 0; - const endpoint: string = `/repos/${owner}/${repo}/commits/${commitSummary.sha}`; - const proxyConfig = { - endpoint, - paginate: { - response_data_path: 'files' + const objectType = type === PaginationType.Commit ? 'commit' : 'file'; + + await nango.log(`Fetching ${objectType}(s) from endpoint ${endpoint}.`); + + while (true) { + const response = await nango.get({ + endpoint: endpoint, + params: { + ...params, + page: page, + per_page: PER_PAGE + } + }); + + switch (type) { + case PaginationType.RepoFile: + const files = response.data.tree.filter((item: any) => item.type === 'blob'); + count += files.length; + await nango.batchSave(files.map(mapToFile), Models.GithubRepoFile); + break; + case PaginationType.CommitFile: + count += response.data.files.length; + await nango.batchSave(response.data.files.filter((file: any) => file.status !== 'removed').map(mapToFile), Models.GithubRepoFile); + await nango.batchDelete(response.data.files.filter((file: any) => file.status === 'removed').map(mapToFile), Models.GithubRepoFile); + break; + case PaginationType.Commit: + count += response.data.length; + results.push(...response.data); + break; } - }; - await nango.log(`Fetching files from endpoint ${endpoint}.`); + if (!response.headers.link || !response.headers.link.includes('rel="next"')) { + break; + } - for await (const fileBatch of nango.paginate(proxyConfig)) { - count += fileBatch.length; - await nango.batchSave(fileBatch.filter((file: any) => file.status !== 'removed').map(mapToFile), Models.GithubRepoFile); - await nango.batchDelete(fileBatch.filter((file: any) => file.status === 'removed').map(mapToFile), Models.GithubRepoFile); + page++; } - await nango.log(`Got ${count} file(s).`); -} -function mapToFile(file: any): GithubRepoFile { - return { - id: file.sha, - name: file.path || file.filename, - url: file.url || file.blob_url, - last_modified_date: file.committer?.date ? new Date(file.committer?.date) : new Date() // Use commit date or current date - }; + await nango.log(`Got ${count} ${objectType}(s).`); + + return results; } diff --git a/integration-templates/github/github-list-repos-action.ts b/integration-templates/github/github-list-repos-action.ts index df8b1cf6ba2..aa3bcdda52a 100644 --- a/integration-templates/github/github-list-repos-action.ts +++ b/integration-templates/github/github-list-repos-action.ts @@ -4,15 +4,15 @@ export default async function runAction(nango: NangoSync): Promise<{ repos: Gith let allRepos: any[] = []; // Fetch user's personal repositories. - const personalRepos = await getAll(nango, '/user/repos'); + const personalRepos = await paginate(nango, '/user/repos'); allRepos = allRepos.concat(personalRepos); // Fetch organizations the user is a part of. - const organizations = await getAll(nango, '/user/orgs'); + const organizations = await paginate(nango, '/user/orgs'); // For each organization, fetch its repositories. for (const org of organizations) { - const orgRepos = await getAll(nango, `/orgs/${org.login}/repos`); + const orgRepos = await paginate(nango, `/orgs/${org.login}/repos`); allRepos = allRepos.concat(orgRepos); } @@ -30,12 +30,29 @@ export default async function runAction(nango: NangoSync): Promise<{ repos: Gith return { repos: mappedRepos }; } -async function getAll(nango: NangoSync, endpoint: string) { - const records: any[] = []; +async function paginate(nango: NangoSync, endpoint: string) { + const MAX_PAGE = 100; - for await (const recordBatch of nango.paginate({ endpoint })) { - records.push(...recordBatch); + let results: any[] = []; + let page = 1; + + while (true) { + const resp = await nango.get({ + endpoint: endpoint, + params: { + limit: `${MAX_PAGE}`, + page: `${page}` + } + }); + + results = results.concat(resp.data); + + if (resp.data.length == MAX_PAGE) { + page += 1; + } else { + break; + } } - return records; + return results; } diff --git a/integration-templates/jira/jira-issues.ts b/integration-templates/jira/jira-issues.ts index e294998c510..869b04a19de 100644 --- a/integration-templates/jira/jira-issues.ts +++ b/integration-templates/jira/jira-issues.ts @@ -2,30 +2,35 @@ import type { NangoSync, JiraIssue } from './models'; export default async function fetchData(nango: NangoSync) { const jql = nango.lastSyncDate ? `updated >= "${nango.lastSyncDate?.toISOString().slice(0, -8).replace('T', ' ')}"` : ''; + let startAt: number = 0; + const maxResults: number = 50; const fields = 'id,key,summary,description,issuetype,status,assignee,reporter,project,created,updated'; const cloudId = await getCloudId(nango); - const proxyConfig = { - baseUrlOverride: `https://api.atlassian.com/ex/jira/${cloudId}`, - endpoint: `/rest/api/3/search`, - paginate: { - type: 'offset', - offset_parameter_name: 'startAt', - limit_parameter_name: 'maxResults', - response_data_path: 'issues' - }, - params: { - jql: jql, - fields: fields - }, - headers: { - 'X-Atlassian-Token': 'no-check' - }, - retries: 10 // Exponential backoff + long-running job = handles rate limits well. - }; + while (true) { + const response = await nango.get({ + baseUrlOverride: 'https://api.atlassian.com', + endpoint: `ex/jira/${cloudId}/rest/api/3/search`, + params: { + jql: jql, + startAt: `${startAt}`, + maxResults: `${maxResults}`, + fields: fields + }, + headers: { + 'X-Atlassian-Token': 'no-check' + }, + retries: 10 // Exponential backoff + long-running job = handles rate limits well. + }); + + const issues = response.data.issues; + await nango.batchSave(mapIssues(issues), 'JiraIssue'); - for await (const issueBatch of nango.paginate(proxyConfig)) { - await nango.batchSave(mapIssues(issueBatch), 'JiraIssue'); + if (issues.length < maxResults) { + break; + } else { + startAt += maxResults; + } } } diff --git a/integration-templates/slack/slack-channels.ts b/integration-templates/slack/slack-channels.ts index 56f5802b755..7b6ed8df322 100644 --- a/integration-templates/slack/slack-channels.ts +++ b/integration-templates/slack/slack-channels.ts @@ -1,7 +1,7 @@ import type { SlackChannel, NangoSync } from './models'; export default async function fetchData(nango: NangoSync) { - const responses = await getAllChannels(nango, 'conversations.list'); + const responses = await getAllPages(nango, 'conversations.list'); const mappedChannels: SlackChannel[] = responses.map((record: any) => { return { @@ -27,7 +27,6 @@ export default async function fetchData(nango: NangoSync) { // Now let's also join all public channels where we are not yet a member await joinPublicChannels(nango, mappedChannels); - // console.log(mappedChannels) // Save channels await nango.batchSave(mappedChannels, 'SlackChannel'); } @@ -35,7 +34,7 @@ export default async function fetchData(nango: NangoSync) { // Checks for public channels where the bot is not a member yet and joins them async function joinPublicChannels(nango: NangoSync, channels: SlackChannel[]) { // Get ID of all channels where we are already a member - const joinedChannelsResponse = await getAllChannels(nango, 'users.conversations'); + const joinedChannelsResponse = await getAllPages(nango, 'users.conversations'); const channelIds = joinedChannelsResponse.map((record: any) => { return record.id; }); @@ -53,18 +52,27 @@ async function joinPublicChannels(nango: NangoSync, channels: SlackChannel[]) { } } -async function getAllChannels(nango: NangoSync, endpoint: string) { - const channels: any[] = []; +async function getAllPages(nango: NangoSync, endpoint: string) { + var nextCursor = 'x'; + var responses: any[] = []; - const proxyConfig = { - endpoint, - paginate: { - response_data_path: 'channels' + while (nextCursor !== '') { + const response = await nango.get({ + endpoint: endpoint, + params: { + limit: '200', + cursor: nextCursor !== 'x' ? nextCursor : '' + } + }); + + if (!response.data.ok) { + await nango.log(`Received a Slack API error (for ${endpoint}): ${JSON.stringify(response.data, null, 2)}`); } - }; - for await (const channelBatch of nango.paginate(proxyConfig)) { - channels.push(...channelBatch); + + const { channels, response_metadata } = response.data; + responses = responses.concat(channels); + nextCursor = response_metadata.next_cursor; } - return channels; + return responses; } diff --git a/integration-templates/slack/slack-messages.ts b/integration-templates/slack/slack-messages.ts index 997461f585a..65b059ff625 100644 --- a/integration-templates/slack/slack-messages.ts +++ b/integration-templates/slack/slack-messages.ts @@ -3,7 +3,7 @@ import { createHash } from 'crypto'; export default async function fetchData(nango: NangoSync) { // Get all channels we are part of - let channels = await getAllRecords(nango, 'users.conversations', {}, 'channels'); + let channels = await getAllPages(nango, 'users.conversations', {}, 'channels'); await nango.log(`Bot is part of ${channels.length} channels`); @@ -16,7 +16,7 @@ export default async function fetchData(nango: NangoSync) { // For every channel read messages, replies & reactions for (let channel of channels) { - let allMessages = await getAllRecords(nango, 'conversations.history', { channel: channel.id, oldest: oldestTimestamp.toString() }, 'messages'); + let allMessages = await getAllPages(nango, 'conversations.history', { channel: channel.id, oldest: oldestTimestamp.toString() }, 'messages'); for (let message of allMessages) { const mappedMessage: SlackMessage = { @@ -71,7 +71,7 @@ export default async function fetchData(nango: NangoSync) { // Replies to fetch? if (message.reply_count > 0) { - const allReplies = await getAllRecords(nango, 'conversations.replies', { channel: channel.id, ts: message.thread_ts }, 'messages'); + const allReplies = await getAllPages(nango, 'conversations.replies', { channel: channel.id, ts: message.thread_ts }, 'messages'); for (let reply of allReplies) { if (reply.ts === message.ts) { @@ -141,21 +141,31 @@ export default async function fetchData(nango: NangoSync) { await nango.batchSave(batchReactions, 'SlackMessageReaction'); } -async function getAllRecords(nango: NangoSync, endpoint: string, params: Record, resultsKey: string) { - let records: any[] = []; - - const proxyConfig = { - endpoint: endpoint, - params, - retries: 10, - paginate: { - response_data_path: resultsKey +async function getAllPages(nango: NangoSync, endpoint: string, params: Record, resultsKey: string) { + let nextCursor = 'x'; + let responses: any[] = []; + + while (nextCursor !== '') { + const response = await nango.get({ + endpoint: endpoint, + params: { + limit: '200', + cursor: nextCursor !== 'x' ? nextCursor : '', + ...params + }, + retries: 10 + }); + + if (!response.data.ok) { + await nango.log(`Received a Slack API error (for ${endpoint}): ${JSON.stringify(response.data, null, 2)}`); } - }; - for await (const recordBatch of nango.paginate(proxyConfig)) { - records.push(...recordBatch); + const results = response.data[resultsKey]; + const response_metadata = response.data.response_metadata; + + responses = responses.concat(results); + nextCursor = response_metadata && response_metadata.next_cursor ? response_metadata.next_cursor : ''; } - return records; + return responses; } diff --git a/integration-templates/slack/slack-users.ts b/integration-templates/slack/slack-users.ts index 175f1b4d233..9cc49d6be1d 100644 --- a/integration-templates/slack/slack-users.ts +++ b/integration-templates/slack/slack-users.ts @@ -2,10 +2,30 @@ import type { SlackUser, NangoSync } from './models'; export default async function fetchData(nango: NangoSync) { // Fetch all users (paginated) - const users: any[] = await getAllUsers(nango); + let nextCursor = 'x'; + let responses: any[] = []; + + while (nextCursor !== '') { + const response = await nango.get({ + endpoint: 'users.list', + retries: 10, + params: { + limit: '200', + cursor: nextCursor !== 'x' ? nextCursor : '' + } + }); + + if (!response.data.ok) { + await nango.log(`Received a Slack API error: ${JSON.stringify(response.data, null, 2)}`); + } + + const { members, response_metadata } = response.data; + responses = responses.concat(members); + nextCursor = response_metadata.next_cursor; + } // Transform users into our data model - const mappedUsers: SlackUser[] = users.map((record: any) => { + const users: SlackUser[] = responses.map((record: any) => { return { id: record.id, team_id: record.team_id, @@ -35,22 +55,5 @@ export default async function fetchData(nango: NangoSync) { }; }); - await nango.batchSave(mappedUsers, 'SlackUser'); -} -async function getAllUsers(nango: NangoSync) { - const users: any[] = []; - - const proxyConfig = { - endpoint: 'users.list', - retries: 10, - paginate: { - response_data_path: 'members' - } - }; - - for await (const userBatch of nango.paginate(proxyConfig)) { - users.push(...userBatch); - } - - return users; + await nango.batchSave(users, 'SlackUser'); } From ffde95d1229e167803321239d6f6c44a65876ad2 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Thu, 12 Oct 2023 18:42:39 +0300 Subject: [PATCH 39/50] Revert accidental changes --- package-lock.json | 16 ++++++++++++++++ packages/shared/lib/models/Provider.ts | 4 ++-- packages/shared/lib/sdk/sync.ts | 17 ++++++++--------- packages/shared/package.json | 4 ++-- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index ff506de5eae..f9a001ee7bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5243,6 +5243,12 @@ "resolved": "https://registry.npmjs.org/@types/pako/-/pako-1.0.4.tgz", "integrity": "sha512-Z+5bJSm28EXBSUJEgx29ioWeEEHUh6TiMkZHDhLwjc9wVFH+ressbkmX6waUZc5R3Gobn4Qu5llGxaoflZ+yhA==" }, + "node_modules/@types/parse-link-header": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/parse-link-header/-/parse-link-header-2.0.1.tgz", + "integrity": "sha512-BrKNSrRTqn3UkMXvdVtr/znJch0PMBpEvEP8oBkxDx7eEGntuFLI+WpA5HGsNHK4SlqyhaMa+Ks0ViwyixQB5w==", + "dev": true + }, "node_modules/@types/passport": { "version": "1.0.12", "dev": true, @@ -10294,6 +10300,14 @@ "node": ">=6" } }, + "node_modules/parse-link-header": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-link-header/-/parse-link-header-2.0.0.tgz", + "integrity": "sha512-xjU87V0VyHZybn2RrCX5TIFGxTVZE6zqqZWMPlIKiSKuWh/X5WZdt+w1Ki1nXB+8L/KtL+nZ4iq+sfI6MrhhMw==", + "dependencies": { + "xtend": "~4.0.1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "license": "MIT", @@ -13874,6 +13888,7 @@ "lodash": "^4.17.21", "md5": "^2.3.0", "ms": "^2.1.3", + "parse-link-header": "^2.0.0", "pg": "^8.8.0", "posthog-node": "^2.2.3", "rimraf": "^5.0.1", @@ -13894,6 +13909,7 @@ "@types/js-yaml": "^4.0.5", "@types/lodash": "^4.14.195", "@types/node": "^18.7.6", + "@types/parse-link-header": "^2.0.0", "@types/uuid": "^9.0.0", "typescript": "^4.7.4" }, diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index b9386101b3f..04af00a3b76 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,4 +1,4 @@ -import type { CursorPagination, NextUrlPagination } from '../sdk/sync.js'; +import type { CursorPagination, NextUrlPagination, OffsetPagination } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -28,7 +28,7 @@ export interface Template { after?: string; }; decompress?: boolean; - paginate?: NextUrlPagination | CursorPagination; + paginate?: NextUrlPagination | CursorPagination | OffsetPagination; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index e8e411814a8..25f30059e70 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -102,7 +102,7 @@ interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; - paginate?: Partial | Partial | Partial; // Supported only by Syncs and Actions ATM + paginate?: Partial | Partial | Partial; } enum AuthModes { @@ -361,7 +361,6 @@ export class NangoAction { } if (!config.method) { - // default to GET if user doesn't specify a different method themselves config.method = 'GET'; } @@ -469,7 +468,7 @@ export class NangoAction { } } default: - throw Error(`'${paginationConfig.type} ' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)} `); + throw Error(`'${paginationConfig.type} ' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)}`); } } @@ -570,7 +569,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'error', activity_log_id: this.activityLogId as number, - content: `There was an issue with the batch save.${error?.message} `, + content: `There was an issue with the batch save.${error?.message}`, timestamp: Date.now() }); } @@ -613,7 +612,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'info', activity_log_id: this.activityLogId as number, - content: `Batch save was a success and resulted in ${JSON.stringify(updatedResults, null, 2)} `, + content: `Batch save was a success and resulted in ${JSON.stringify(updatedResults, null, 2)}`, timestamp: Date.now() }); @@ -621,7 +620,7 @@ export class NangoSync extends NangoAction { return true; } else { - const content = `There was an issue with the batch save.${responseResults?.error} `; + const content = `There was an issue with the batch save.${responseResults?.error}`; if (!this.dryRun) { await createActivityLogMessage({ @@ -672,7 +671,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'error', activity_log_id: this.activityLogId as number, - content: `There was an issue with the batch delete.${error?.message} `, + content: `There was an issue with the batch delete.${error?.message}`, timestamp: Date.now() }); } @@ -716,7 +715,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'info', activity_log_id: this.activityLogId as number, - content: `Batch delete was a success and resulted in ${JSON.stringify(updatedResults, null, 2)} `, + content: `Batch delete was a success and resulted in ${JSON.stringify(updatedResults, null, 2)}`, timestamp: Date.now() }); @@ -724,7 +723,7 @@ export class NangoSync extends NangoAction { return true; } else { - const content = `There was an issue with the batch delete.${responseResults?.error} `; + const content = `There was an issue with the batch delete.${responseResults?.error}`; if (!this.dryRun) { await createActivityLogMessage({ diff --git a/packages/shared/package.json b/packages/shared/package.json index 5f72d8f7d41..a7ccbafdafc 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -63,9 +63,9 @@ "@types/debug": "^4.1.7", "@types/human-to-cron": "^0.3.0", "@types/js-yaml": "^4.0.5", - "@types/lodash": "^4.14.199", + "@types/lodash": "^4.14.195", "@types/node": "^18.7.6", - "@types/parse-link-header": "^2.0.1", + "@types/parse-link-header": "^2.0.0", "@types/uuid": "^9.0.0", "typescript": "^4.7.4" } From bab9e62cea14e07f909de884bf3ef7f2b0c73078 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Fri, 13 Oct 2023 10:44:44 +0300 Subject: [PATCH 40/50] Update the interface for link pagination --- packages/shared/lib/sdk/sync.ts | 14 +++++++------- packages/shared/providers.yaml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 25f30059e70..15a96f37f95 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -65,7 +65,7 @@ interface DataResponse { export enum PaginationType { CURSOR = 'cursor', - NEXT_URL = 'next_url', + LINK = 'link', OFFSET = 'offset' } @@ -83,7 +83,7 @@ export interface CursorPagination extends Pagination { export interface NextUrlPagination extends Pagination { link_rel?: string; - next_url_body_parameter_path?: string; + link_body_parameter_path?: string; } export interface OffsetPagination extends Pagination { @@ -346,7 +346,7 @@ export class NangoAction { const templatePaginationConfig: Pagination | undefined = template.proxy?.paginate; if (!templatePaginationConfig) { - throw Error(`Pagination is not supported for ${providerConfigKey}. Please, add pagination config to 'providers.yaml' file`); + throw Error(`Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`); } let paginationConfig: Pagination = templatePaginationConfig; @@ -406,7 +406,7 @@ export class NangoAction { } } } - case PaginationType.NEXT_URL: { + case PaginationType.LINK: { const nextUrlPagination: NextUrlPagination = paginationConfig as NextUrlPagination; this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); @@ -476,11 +476,11 @@ export class NangoAction { if (nextUrlPagination.link_rel) { const linkHeader = parseLinksHeader(response.headers['link']); return linkHeader?.[nextUrlPagination.link_rel]?.url; - } else if (nextUrlPagination.next_url_body_parameter_path) { - return this.getNestedField(response.data, nextUrlPagination.next_url_body_parameter_path); + } else if (nextUrlPagination.link_body_parameter_path) { + return this.getNestedField(response.data, nextUrlPagination.link_body_parameter_path); } - throw Error(`Either 'link_rel' or 'next_url_body_parameter_path' should be specified for '${paginationConfig.type}' pagination`); + throw Error(`Either 'link_rel' or 'link_body_parameter_path' should be specified for '${paginationConfig.type}' pagination`); } private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index aa592c1aad7..88c44534b20 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -368,7 +368,7 @@ github: retry: at: 'x-ratelimit-reset' paginate: - type: next_url + type: link limit_parameter_name: per_page link_rel: next docs: https://docs.github.com/en/rest @@ -522,11 +522,11 @@ jira: proxy: base_url: https://api.atlassian.com paginate: - type: next_url + type: link link_rel: next limit_parameter_name: limit response_data_path: results - next_url_body_parameter_path: _links.next + link_body_parameter_path: _links.next keap: auth_mode: OAUTH2 authorization_url: https://accounts.infusionsoft.com/app/oauth/authorize From 3336fc912a65d37771a29054b5745cd619655aa5 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Fri, 13 Oct 2023 10:45:00 +0300 Subject: [PATCH 41/50] Cover most of pagination with unit tests --- packages/shared/lib/sdk/sync.unit.test.ts | 316 ++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 packages/shared/lib/sdk/sync.unit.test.ts diff --git a/packages/shared/lib/sdk/sync.unit.test.ts b/packages/shared/lib/sdk/sync.unit.test.ts new file mode 100644 index 00000000000..5e861aada90 --- /dev/null +++ b/packages/shared/lib/sdk/sync.unit.test.ts @@ -0,0 +1,316 @@ +import { Nango } from '@nangohq/node'; +import type { NextUrlPagination, OffsetPagination } from 'nango'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { AuthModes, type Template } from '../models/index.js'; +import configService from '../services/config.service.js'; +import { CursorPagination, NangoAction } from './sync.js'; +import { isValidHttpUrl } from '../utils/utils.js'; + +vi.mock('@nangohq/node', () => { + const Nango = vi.fn(); + return { Nango }; +}); + +describe('Pagination', () => { + const providerConfigKey = 'github'; + + const cursorPagination: CursorPagination = { + type: 'cursor', + next_cursor_parameter_path: 'metadata.next_cursor', + cursor_parameter_name: 'cursor', + limit_parameter_name: 'limit', + response_data_path: 'issues' + }; + const offsetPagination: OffsetPagination = { + type: 'offset', + limit_parameter_name: 'per_page', + offset_parameter_name: 'offset', + response_data_path: 'issues' + }; + const nextUrlPagination: NextUrlPagination = { + type: 'link', + response_data_path: 'issues', + limit_parameter_name: 'limit', + link_body_parameter_path: 'metadata.next_cursor' + }; + + const paginationConfigs = [cursorPagination, offsetPagination, nextUrlPagination]; + + let nangoAction: NangoAction; + let nango: Nango; + + beforeEach(() => { + const config: any = { + secretKey: 'encrypted', + serverUrl: 'https://example.com', + providerConfigKey + }; + nangoAction = new NangoAction(config); + nango = new Nango({ secretKey: config.secretKey }); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('Throws error if there is no pagination config in provider template', async () => { + const template: Template = { + auth_mode: AuthModes.OAuth2, + proxy: { base_url: '' }, + authorization_url: '', + token_url: '' + }; + vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); + + const expectedErrorMessage: string = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; + await expect(() => nangoAction.paginate({ endpoint: '' }).next()).rejects.toThrowError(expectedErrorMessage); + }); + + it('Sends pagination params in body for POST HTTP method', async () => { + stubProviderTemplate(cursorPagination); + + // TODO: mock to return at least one more page to check that cursor is passed in body too + (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValue({ data: { issues: [] } }); + + const endpoint: string = '/issues'; + + await nangoAction.paginate({ endpoint, method: 'POST', paginate: { limit: 2 } }).next(); + + expect(nango.proxy).toHaveBeenCalledWith({ + method: 'POST', + endpoint, + data: { limit: 2 }, + paginate: { limit: 2 } + }); + }); + + it('Overrides template pagination params with ones passed in the proxy config', async () => { + stubProviderTemplate(cursorPagination); + + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ data: { issues: [{}, {}, {}] } }) + .mockReturnValueOnce({ data: { issues: [] } }); + + const endpoint: string = '/issues'; + const paginationConfigOverride: OffsetPagination = { + type: 'offset', + limit_parameter_name: 'per_page', + limit: 3, + offset_parameter_name: 'offset', + response_data_path: 'issues' + }; + + const generator = nangoAction.paginate({ endpoint, paginate: paginationConfigOverride }); + for await (const batch of generator) { + console.log(batch); + } + + expect(nango.proxy).toHaveBeenLastCalledWith({ + method: 'GET', + endpoint, + params: { offset: '3', per_page: 3 }, + paginate: paginationConfigOverride + }); + }); + + it('Paginates using offset', async () => { + stubProviderTemplate(offsetPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ data: { issues: firstBatch } }) + .mockReturnValueOnce({ data: { issues: secondBatch } }) + .mockReturnValueOnce({ data: { issues: [] } }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + const expectedRecords = [...firstBatch, ...secondBatch]; + + expect(actualRecords).toStrictEqual(expectedRecords); + }); + + it('Paginates using cursor', async () => { + stubProviderTemplate(cursorPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: '2' + } + } + }) + .mockReturnValueOnce({ + data: { + issues: secondBatch, + metadata: { + next_cursor: '2' + } + } + }) + .mockReturnValueOnce({ data: { issues: thirdBatch } }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; + + expect(actualRecords).toStrictEqual(expectedRecords); + }); + + it('Stops pagination if cursor is empty', async () => { + stubProviderTemplate(cursorPagination); + + const onlyBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValueOnce({ + data: { + issues: onlyBatch, + metadata: { + next_cursor: '' + } + } + }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + expect(actualRecords).toStrictEqual(onlyBatch); + }); + + it.each(paginationConfigs)( + 'Extracts records from nested body param for $type pagination type', + async (paginationConfig: CursorPagination | OffsetPagination | NextUrlPagination) => { + stubProviderTemplate(paginationConfig); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const emptyBatch: any[] = []; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: '' + } + } + }) + + .mockReturnValueOnce({ + data: { + issues: emptyBatch, + metadata: { + next_cursor: '' + } + } + }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + expect(actualRecords).toStrictEqual(firstBatch); + } + ); + + it.each([ + // TODO: validate proper config is passed to proxy + ['https://api.gihub.com/issues?page=2', 'https://api.gihub.com/issues?page=3'], + ['/issues?page=2', '/issues?page=3'] + ])('Paginates using next URL/path %s from body', async (nextUrlOrPathValue, anotherNextUrlOrPathValue) => { + stubProviderTemplate(nextUrlPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: nextUrlOrPathValue + } + } + }) + .mockReturnValueOnce({ + data: { + issues: secondBatch, + metadata: { + next_cursor: anotherNextUrlOrPathValue + } + } + }) + .mockReturnValueOnce({ data: { issues: thirdBatch } }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; + let expectedEndpoint: string; + if (isValidHttpUrl(anotherNextUrlOrPathValue)) { + const url: URL = new URL(anotherNextUrlOrPathValue); + expectedEndpoint = url.pathname + url.search; + } else { + expectedEndpoint = anotherNextUrlOrPathValue; + } + + expect(actualRecords).toStrictEqual(expectedRecords); + expect(nango.proxy).toHaveBeenCalledWith( + expect.objectContaining({ + endpoint: expectedEndpoint + }) + ); + }); + + const stubProviderTemplate = (paginationConfig: CursorPagination | OffsetPagination | NextUrlPagination) => { + const template: Template = buildTemplate(paginationConfig); + vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); + }; + + const buildTemplate = (paginationConfig: CursorPagination | OffsetPagination | NextUrlPagination): Template => { + return { + auth_mode: AuthModes.OAuth2, + proxy: { base_url: 'https://api.github.com/', paginate: paginationConfig }, + authorization_url: '', + token_url: '' + }; + }; +}); From 05d4b2d7f00b21796cef4ca98d614ef73059a852 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Fri, 13 Oct 2023 14:25:07 +0300 Subject: [PATCH 42/50] Update pagination interface --- packages/shared/lib/models/Provider.ts | 4 +- packages/shared/lib/sdk/sync.ts | 54 +-- packages/shared/lib/sdk/sync.unit.test.ts | 545 +++++++++++----------- packages/shared/providers.yaml | 34 +- 4 files changed, 318 insertions(+), 319 deletions(-) diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index 04af00a3b76..2fd91159f1b 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -1,4 +1,4 @@ -import type { CursorPagination, NextUrlPagination, OffsetPagination } from '../sdk/sync.js'; +import type { CursorPagination, LinkPagination, OffsetPagination } from '../sdk/sync.js'; import type { AuthModes } from './Auth.js'; import type { TimestampsAndDeleted } from './Generic.js'; @@ -28,7 +28,7 @@ export interface Template { after?: string; }; decompress?: boolean; - paginate?: NextUrlPagination | CursorPagination | OffsetPagination; + paginate?: LinkPagination | CursorPagination | OffsetPagination; }; authorization_url: string; authorization_params?: Record; diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 15a96f37f95..a3053dd4ce9 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -72,22 +72,22 @@ export enum PaginationType { interface Pagination { type: string; limit?: number; - response_data_path?: string; - limit_parameter_name: string; + response_path?: string; + limit_name_in_request: string; } export interface CursorPagination extends Pagination { - next_cursor_parameter_path: string; - cursor_parameter_name: string; + cursor_path_in_response: string; + cursor_name_in_request: string; } -export interface NextUrlPagination extends Pagination { - link_rel?: string; - link_body_parameter_path?: string; +export interface LinkPagination extends Pagination { + link_rel_in_response_header?: string; + link_path_in_response_body?: string; } export interface OffsetPagination extends Pagination { - offset_parameter_name: string; + offset_name_in_request: string; } interface ProxyConfiguration { @@ -102,7 +102,7 @@ interface ProxyConfiguration { data?: unknown; retries?: number; baseUrlOverride?: string; - paginate?: Partial | Partial | Partial; + paginate?: Partial | Partial | Partial; } enum AuthModes { @@ -368,7 +368,7 @@ export class NangoAction { const passPaginationParamsInBody: boolean = ['post', 'put', 'patch'].includes(configMethod); const updatedBodyOrParams: Record = ((passPaginationParamsInBody ? config.data : config.params) as Record) ?? {}; - const limitParameterName: string = paginationConfig.limit_parameter_name; + const limitParameterName: string = paginationConfig.limit_name_in_request; if (paginationConfig['limit']) { updatedBodyOrParams[limitParameterName] = paginationConfig['limit']; @@ -382,15 +382,15 @@ export class NangoAction { let nextCursor: string | undefined; while (true) { if (nextCursor) { - updatedBodyOrParams[cursorPagination.cursor_parameter_name] = nextCursor; + updatedBodyOrParams[cursorPagination.cursor_name_in_request] = nextCursor; } this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = cursorPagination.response_data_path - ? this.getNestedField(response.data, cursorPagination.response_data_path) + const responseData: T[] = cursorPagination.response_path + ? this.getNestedField(response.data, cursorPagination.response_path) : response.data; if (!responseData.length) { @@ -399,7 +399,7 @@ export class NangoAction { yield responseData; - nextCursor = this.getNestedField(response.data, cursorPagination.next_cursor_parameter_path); + nextCursor = this.getNestedField(response.data, cursorPagination.cursor_path_in_response); if (!nextCursor || nextCursor.trim().length === 0) { return; @@ -407,14 +407,14 @@ export class NangoAction { } } case PaginationType.LINK: { - const nextUrlPagination: NextUrlPagination = paginationConfig as NextUrlPagination; + const linkPagination: LinkPagination = paginationConfig as LinkPagination; this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); while (true) { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_data_path - ? this.getNestedField(response.data, paginationConfig.response_data_path) + const responseData: T[] = paginationConfig.response_path + ? this.getNestedField(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; @@ -422,7 +422,7 @@ export class NangoAction { yield responseData; - let nextPageUrl: string | undefined = this.getNextPageUrlFromBodyOrHeaders(nextUrlPagination, response, paginationConfig); + let nextPageUrl: string | undefined = this.getNextPageUrlFromBodyOrHeaders(linkPagination, response, paginationConfig); if (!nextPageUrl) { return; @@ -440,7 +440,7 @@ export class NangoAction { } case PaginationType.OFFSET: { const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; - const offsetParameterName: string = offsetPagination.offset_parameter_name; + const offsetParameterName: string = offsetPagination.offset_name_in_request; let offset: number = 0; while (true) { @@ -450,8 +450,8 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_data_path - ? this.getNestedField(response.data, paginationConfig.response_data_path) + const responseData: T[] = paginationConfig.response_path + ? this.getNestedField(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; @@ -472,15 +472,15 @@ export class NangoAction { } } - private getNextPageUrlFromBodyOrHeaders(nextUrlPagination: NextUrlPagination, response: AxiosResponse, paginationConfig: Pagination) { - if (nextUrlPagination.link_rel) { + private getNextPageUrlFromBodyOrHeaders(linkPagination: LinkPagination, response: AxiosResponse, paginationConfig: Pagination) { + if (linkPagination.link_rel_in_response_header) { const linkHeader = parseLinksHeader(response.headers['link']); - return linkHeader?.[nextUrlPagination.link_rel]?.url; - } else if (nextUrlPagination.link_body_parameter_path) { - return this.getNestedField(response.data, nextUrlPagination.link_body_parameter_path); + return linkHeader?.[linkPagination.link_rel_in_response_header]?.url; + } else if (linkPagination.link_path_in_response_body) { + return this.getNestedField(response.data, linkPagination.link_path_in_response_body); } - throw Error(`Either 'link_rel' or 'link_body_parameter_path' should be specified for '${paginationConfig.type}' pagination`); + throw Error(`Either 'link_rel_in_response_header' or 'link_path_in_response_body' should be specified for '${paginationConfig.type}' pagination`); } private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { diff --git a/packages/shared/lib/sdk/sync.unit.test.ts b/packages/shared/lib/sdk/sync.unit.test.ts index 5e861aada90..3c7023c3002 100644 --- a/packages/shared/lib/sdk/sync.unit.test.ts +++ b/packages/shared/lib/sdk/sync.unit.test.ts @@ -1,316 +1,315 @@ import { Nango } from '@nangohq/node'; -import type { NextUrlPagination, OffsetPagination } from 'nango'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { AuthModes, type Template } from '../models/index.js'; import configService from '../services/config.service.js'; -import { CursorPagination, NangoAction } from './sync.js'; +import { CursorPagination, NangoAction, LinkPagination, OffsetPagination } from './sync.js'; import { isValidHttpUrl } from '../utils/utils.js'; vi.mock('@nangohq/node', () => { - const Nango = vi.fn(); - return { Nango }; + const Nango = vi.fn(); + return { Nango }; }); describe('Pagination', () => { - const providerConfigKey = 'github'; - - const cursorPagination: CursorPagination = { - type: 'cursor', - next_cursor_parameter_path: 'metadata.next_cursor', - cursor_parameter_name: 'cursor', - limit_parameter_name: 'limit', - response_data_path: 'issues' + const providerConfigKey = 'github'; + + const cursorPagination: CursorPagination = { + type: 'cursor', + cursor_path_in_response: 'metadata.next_cursor', + cursor_name_in_request: 'cursor', + limit_name_in_request: 'limit', + response_path: 'issues' + }; + const offsetPagination: OffsetPagination = { + type: 'offset', + limit_name_in_request: 'per_page', + offset_name_in_request: 'offset', + response_path: 'issues' + }; + const linkPagination: LinkPagination = { + type: 'link', + response_path: 'issues', + limit_name_in_request: 'limit', + link_path_in_response_body: 'metadata.next_cursor' + }; + + const paginationConfigs = [cursorPagination, offsetPagination, linkPagination]; + + let nangoAction: NangoAction; + let nango: Nango; + + beforeEach(() => { + const config: any = { + secretKey: 'encrypted', + serverUrl: 'https://example.com', + providerConfigKey }; - const offsetPagination: OffsetPagination = { - type: 'offset', - limit_parameter_name: 'per_page', - offset_parameter_name: 'offset', - response_data_path: 'issues' - }; - const nextUrlPagination: NextUrlPagination = { - type: 'link', - response_data_path: 'issues', - limit_parameter_name: 'limit', - link_body_parameter_path: 'metadata.next_cursor' + nangoAction = new NangoAction(config); + nango = new Nango({ secretKey: config.secretKey }); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('Throws error if there is no pagination config in provider template', async () => { + const template: Template = { + auth_mode: AuthModes.OAuth2, + proxy: { base_url: '' }, + authorization_url: '', + token_url: '' }; + vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); - const paginationConfigs = [cursorPagination, offsetPagination, nextUrlPagination]; + const expectedErrorMessage: string = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; + await expect(() => nangoAction.paginate({ endpoint: '' }).next()).rejects.toThrowError(expectedErrorMessage); + }); - let nangoAction: NangoAction; - let nango: Nango; + it('Sends pagination params in body for POST HTTP method', async () => { + stubProviderTemplate(cursorPagination); - beforeEach(() => { - const config: any = { - secretKey: 'encrypted', - serverUrl: 'https://example.com', - providerConfigKey - }; - nangoAction = new NangoAction(config); - nango = new Nango({ secretKey: config.secretKey }); - }); + // TODO: mock to return at least one more page to check that cursor is passed in body too + (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValue({ data: { issues: [] } }); - afterEach(() => { - vi.clearAllMocks(); - }); - - it('Throws error if there is no pagination config in provider template', async () => { - const template: Template = { - auth_mode: AuthModes.OAuth2, - proxy: { base_url: '' }, - authorization_url: '', - token_url: '' - }; - vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); - - const expectedErrorMessage: string = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; - await expect(() => nangoAction.paginate({ endpoint: '' }).next()).rejects.toThrowError(expectedErrorMessage); - }); + const endpoint: string = '/issues'; - it('Sends pagination params in body for POST HTTP method', async () => { - stubProviderTemplate(cursorPagination); + await nangoAction.paginate({ endpoint, method: 'POST', paginate: { limit: 2 } }).next(); - // TODO: mock to return at least one more page to check that cursor is passed in body too - (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValue({ data: { issues: [] } }); - - const endpoint: string = '/issues'; - - await nangoAction.paginate({ endpoint, method: 'POST', paginate: { limit: 2 } }).next(); - - expect(nango.proxy).toHaveBeenCalledWith({ - method: 'POST', - endpoint, - data: { limit: 2 }, - paginate: { limit: 2 } - }); + expect(nango.proxy).toHaveBeenCalledWith({ + method: 'POST', + endpoint, + data: { limit: 2 }, + paginate: { limit: 2 } }); + }); + + it('Overrides template pagination params with ones passed in the proxy config', async () => { + stubProviderTemplate(cursorPagination); + + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ data: { issues: [{}, {}, {}] } }) + .mockReturnValueOnce({ data: { issues: [] } }); + + const endpoint: string = '/issues'; + const paginationConfigOverride: OffsetPagination = { + type: 'offset', + limit_name_in_request: 'per_page', + limit: 3, + offset_name_in_request: 'offset', + response_path: 'issues' + }; - it('Overrides template pagination params with ones passed in the proxy config', async () => { - stubProviderTemplate(cursorPagination); - - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ data: { issues: [{}, {}, {}] } }) - .mockReturnValueOnce({ data: { issues: [] } }); - - const endpoint: string = '/issues'; - const paginationConfigOverride: OffsetPagination = { - type: 'offset', - limit_parameter_name: 'per_page', - limit: 3, - offset_parameter_name: 'offset', - response_data_path: 'issues' - }; - - const generator = nangoAction.paginate({ endpoint, paginate: paginationConfigOverride }); - for await (const batch of generator) { - console.log(batch); - } + const generator = nangoAction.paginate({ endpoint, paginate: paginationConfigOverride }); + for await (const batch of generator) { + console.log(batch); + } - expect(nango.proxy).toHaveBeenLastCalledWith({ - method: 'GET', - endpoint, - params: { offset: '3', per_page: 3 }, - paginate: paginationConfigOverride - }); + expect(nango.proxy).toHaveBeenLastCalledWith({ + method: 'GET', + endpoint, + params: { offset: '3', per_page: 3 }, + paginate: paginationConfigOverride }); + }); + + it('Paginates using offset', async () => { + stubProviderTemplate(offsetPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ data: { issues: firstBatch } }) + .mockReturnValueOnce({ data: { issues: secondBatch } }) + .mockReturnValueOnce({ data: { issues: [] } }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + const expectedRecords = [...firstBatch, ...secondBatch]; + + expect(actualRecords).toStrictEqual(expectedRecords); + }); + + it('Paginates using cursor', async () => { + stubProviderTemplate(cursorPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: '2' + } + } + }) + .mockReturnValueOnce({ + data: { + issues: secondBatch, + metadata: { + next_cursor: '2' + } + } + }) + .mockReturnValueOnce({ data: { issues: thirdBatch } }); - it('Paginates using offset', async () => { - stubProviderTemplate(offsetPagination); - - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ data: { issues: firstBatch } }) - .mockReturnValueOnce({ data: { issues: secondBatch } }) - .mockReturnValueOnce({ data: { issues: [] } }); + const endpoint: string = '/issues'; - const endpoint: string = '/issues'; + const generator = nangoAction.paginate({ endpoint }); - const generator = nangoAction.paginate({ endpoint }); + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); - } + const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; - const expectedRecords = [...firstBatch, ...secondBatch]; + expect(actualRecords).toStrictEqual(expectedRecords); + }); - expect(actualRecords).toStrictEqual(expectedRecords); - }); + it('Stops pagination if cursor is empty', async () => { + stubProviderTemplate(cursorPagination); - it('Paginates using cursor', async () => { - stubProviderTemplate(cursorPagination); - - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; - const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ - data: { - issues: firstBatch, - metadata: { - next_cursor: '2' - } - } - }) - .mockReturnValueOnce({ - data: { - issues: secondBatch, - metadata: { - next_cursor: '2' - } - } - }) - .mockReturnValueOnce({ data: { issues: thirdBatch } }); - - const endpoint: string = '/issues'; - - const generator = nangoAction.paginate({ endpoint }); - - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); + const onlyBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValueOnce({ + data: { + issues: onlyBatch, + metadata: { + next_cursor: '' } - - const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; - - expect(actualRecords).toStrictEqual(expectedRecords); + } }); - it('Stops pagination if cursor is empty', async () => { - stubProviderTemplate(cursorPagination); + const endpoint: string = '/issues'; - const onlyBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValueOnce({ - data: { - issues: onlyBatch, - metadata: { - next_cursor: '' - } - } - }); + const generator = nangoAction.paginate({ endpoint }); - const endpoint: string = '/issues'; + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } - const generator = nangoAction.paginate({ endpoint }); + expect(actualRecords).toStrictEqual(onlyBatch); + }); - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); - } + it.each(paginationConfigs)( + 'Extracts records from nested body param for $type pagination type', + async (paginationConfig: CursorPagination | OffsetPagination | LinkPagination) => { + stubProviderTemplate(paginationConfig); - expect(actualRecords).toStrictEqual(onlyBatch); - }); - - it.each(paginationConfigs)( - 'Extracts records from nested body param for $type pagination type', - async (paginationConfig: CursorPagination | OffsetPagination | NextUrlPagination) => { - stubProviderTemplate(paginationConfig); - - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const emptyBatch: any[] = []; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ - data: { - issues: firstBatch, - metadata: { - next_cursor: '' - } - } - }) - - .mockReturnValueOnce({ - data: { - issues: emptyBatch, - metadata: { - next_cursor: '' - } - } - }); - - const endpoint: string = '/issues'; - - const generator = nangoAction.paginate({ endpoint }); - - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const emptyBatch: any[] = []; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: '' } + } + }) + + .mockReturnValueOnce({ + data: { + issues: emptyBatch, + metadata: { + next_cursor: '' + } + } + }); - expect(actualRecords).toStrictEqual(firstBatch); - } - ); - - it.each([ - // TODO: validate proper config is passed to proxy - ['https://api.gihub.com/issues?page=2', 'https://api.gihub.com/issues?page=3'], - ['/issues?page=2', '/issues?page=3'] - ])('Paginates using next URL/path %s from body', async (nextUrlOrPathValue, anotherNextUrlOrPathValue) => { - stubProviderTemplate(nextUrlPagination); - - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; - const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ - data: { - issues: firstBatch, - metadata: { - next_cursor: nextUrlOrPathValue - } - } - }) - .mockReturnValueOnce({ - data: { - issues: secondBatch, - metadata: { - next_cursor: anotherNextUrlOrPathValue - } - } - }) - .mockReturnValueOnce({ data: { issues: thirdBatch } }); - - const endpoint: string = '/issues'; - - const generator = nangoAction.paginate({ endpoint }); - - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + expect(actualRecords).toStrictEqual(firstBatch); + } + ); + + it.each([ + // TODO: validate proper config is passed to proxy + ['https://api.gihub.com/issues?page=2', 'https://api.gihub.com/issues?page=3'], + ['/issues?page=2', '/issues?page=3'] + ])('Paginates using next URL/path %s from body', async (nextUrlOrPathValue, anotherNextUrlOrPathValue) => { + stubProviderTemplate(linkPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: nextUrlOrPathValue + } } - - const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; - let expectedEndpoint: string; - if (isValidHttpUrl(anotherNextUrlOrPathValue)) { - const url: URL = new URL(anotherNextUrlOrPathValue); - expectedEndpoint = url.pathname + url.search; - } else { - expectedEndpoint = anotherNextUrlOrPathValue; + }) + .mockReturnValueOnce({ + data: { + issues: secondBatch, + metadata: { + next_cursor: anotherNextUrlOrPathValue + } } - - expect(actualRecords).toStrictEqual(expectedRecords); - expect(nango.proxy).toHaveBeenCalledWith( - expect.objectContaining({ - endpoint: expectedEndpoint - }) - ); - }); - - const stubProviderTemplate = (paginationConfig: CursorPagination | OffsetPagination | NextUrlPagination) => { - const template: Template = buildTemplate(paginationConfig); - vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); - }; - - const buildTemplate = (paginationConfig: CursorPagination | OffsetPagination | NextUrlPagination): Template => { - return { - auth_mode: AuthModes.OAuth2, - proxy: { base_url: 'https://api.github.com/', paginate: paginationConfig }, - authorization_url: '', - token_url: '' - }; + }) + .mockReturnValueOnce({ data: { issues: thirdBatch } }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; + let expectedEndpoint: string; + if (isValidHttpUrl(anotherNextUrlOrPathValue)) { + const url: URL = new URL(anotherNextUrlOrPathValue); + expectedEndpoint = url.pathname + url.search; + } else { + expectedEndpoint = anotherNextUrlOrPathValue; + } + + expect(actualRecords).toStrictEqual(expectedRecords); + expect(nango.proxy).toHaveBeenCalledWith( + expect.objectContaining({ + endpoint: expectedEndpoint + }) + ); + }); + + const stubProviderTemplate = (paginationConfig: CursorPagination | OffsetPagination | LinkPagination) => { + const template: Template = buildTemplate(paginationConfig); + vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); + }; + + const buildTemplate = (paginationConfig: CursorPagination | OffsetPagination | LinkPagination): Template => { + return { + auth_mode: AuthModes.OAuth2, + proxy: { base_url: 'https://api.github.com/', paginate: paginationConfig }, + authorization_url: '', + token_url: '' }; + }; }); diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 88c44534b20..8b476fb50b9 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -72,10 +72,10 @@ asana: base_url: https://app.asana.com paginate: type: cursor - next_cursor_parameter_path: next_page.offset - cursor_parameter_name: offset - response_data_path: data - limit_parameter_name: limit + cursor_path_in_response: next_page.offset + cursor_name_in_request: offset + response_path: data + limit_name_in_request: limit docs: https://developers.asana.com/reference ashby: auth_mode: BASIC @@ -369,8 +369,8 @@ github: at: 'x-ratelimit-reset' paginate: type: link - limit_parameter_name: per_page - link_rel: next + limit_name_in_request: per_page + link_rel_in_response_header: next docs: https://docs.github.com/en/rest github-app: alias: github @@ -489,10 +489,10 @@ hubspot: decompress: true paginate: type: cursor - next_cursor_parameter_path: paging.next.after - limit_parameter_name: limit - cursor_parameter_name: after - response_data_path: results + cursor_path_in_response: paging.next.after + limit_name_in_request: limit + cursor_name_in_request: after + response_path: results docs: https://developers.hubspot.com/docs/api/overview instagram: auth_mode: OAUTH2 @@ -523,10 +523,10 @@ jira: base_url: https://api.atlassian.com paginate: type: link - link_rel: next - limit_parameter_name: limit - response_data_path: results - link_body_parameter_path: _links.next + link_rel_in_response_header: next + limit_name_in_request: limit + response_path: results + link_path_in_response_body: _links.next keap: auth_mode: OAUTH2 authorization_url: https://accounts.infusionsoft.com/app/oauth/authorize @@ -878,9 +878,9 @@ slack: base_url: https://slack.com/api paginate: type: cursor - next_cursor_parameter_path: response_metadata.next_cursor - cursor_parameter_name: cursor - limit_parameter_name: limit + cursor_path_in_response: response_metadata.next_cursor + cursor_name_in_request: cursor + limit_name_in_request: limit docs: https://api.slack.com/apis smugmug: auth_mode: OAUTH1 From 220c233422ce2e6696f97a0328f048aa3b361215 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Fri, 13 Oct 2023 14:27:03 +0300 Subject: [PATCH 43/50] Slightly optimize offset pagination --- packages/shared/lib/sdk/sync.ts | 4 + packages/shared/lib/sdk/sync.unit.test.ts | 542 +++++++++++----------- 2 files changed, 275 insertions(+), 271 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index a3053dd4ce9..57613254330 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -459,6 +459,10 @@ export class NangoAction { yield responseData; + if (paginationConfig['limit'] && responseData.length < paginationConfig['limit']) { + return; + } + if (responseData.length < 1) { // Last page was empty so no need to fetch further return; diff --git a/packages/shared/lib/sdk/sync.unit.test.ts b/packages/shared/lib/sdk/sync.unit.test.ts index 3c7023c3002..6432f4e3eef 100644 --- a/packages/shared/lib/sdk/sync.unit.test.ts +++ b/packages/shared/lib/sdk/sync.unit.test.ts @@ -6,310 +6,310 @@ import { CursorPagination, NangoAction, LinkPagination, OffsetPagination } from import { isValidHttpUrl } from '../utils/utils.js'; vi.mock('@nangohq/node', () => { - const Nango = vi.fn(); - return { Nango }; + const Nango = vi.fn(); + return { Nango }; }); describe('Pagination', () => { - const providerConfigKey = 'github'; - - const cursorPagination: CursorPagination = { - type: 'cursor', - cursor_path_in_response: 'metadata.next_cursor', - cursor_name_in_request: 'cursor', - limit_name_in_request: 'limit', - response_path: 'issues' - }; - const offsetPagination: OffsetPagination = { - type: 'offset', - limit_name_in_request: 'per_page', - offset_name_in_request: 'offset', - response_path: 'issues' - }; - const linkPagination: LinkPagination = { - type: 'link', - response_path: 'issues', - limit_name_in_request: 'limit', - link_path_in_response_body: 'metadata.next_cursor' - }; - - const paginationConfigs = [cursorPagination, offsetPagination, linkPagination]; - - let nangoAction: NangoAction; - let nango: Nango; - - beforeEach(() => { - const config: any = { - secretKey: 'encrypted', - serverUrl: 'https://example.com', - providerConfigKey + const providerConfigKey = 'github'; + + const cursorPagination: CursorPagination = { + type: 'cursor', + cursor_path_in_response: 'metadata.next_cursor', + cursor_name_in_request: 'cursor', + limit_name_in_request: 'limit', + response_path: 'issues' }; - nangoAction = new NangoAction(config); - nango = new Nango({ secretKey: config.secretKey }); - }); - - afterEach(() => { - vi.clearAllMocks(); - }); - - it('Throws error if there is no pagination config in provider template', async () => { - const template: Template = { - auth_mode: AuthModes.OAuth2, - proxy: { base_url: '' }, - authorization_url: '', - token_url: '' + const offsetPagination: OffsetPagination = { + type: 'offset', + limit_name_in_request: 'per_page', + offset_name_in_request: 'offset', + response_path: 'issues' + }; + const linkPagination: LinkPagination = { + type: 'link', + response_path: 'issues', + limit_name_in_request: 'limit', + link_path_in_response_body: 'metadata.next_cursor' }; - vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); - - const expectedErrorMessage: string = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; - await expect(() => nangoAction.paginate({ endpoint: '' }).next()).rejects.toThrowError(expectedErrorMessage); - }); - it('Sends pagination params in body for POST HTTP method', async () => { - stubProviderTemplate(cursorPagination); + const paginationConfigs = [cursorPagination, offsetPagination, linkPagination]; - // TODO: mock to return at least one more page to check that cursor is passed in body too - (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValue({ data: { issues: [] } }); + let nangoAction: NangoAction; + let nango: Nango; - const endpoint: string = '/issues'; + beforeEach(() => { + const config: any = { + secretKey: 'encrypted', + serverUrl: 'https://example.com', + providerConfigKey + }; + nangoAction = new NangoAction(config); + nango = new Nango({ secretKey: config.secretKey }); + }); - await nangoAction.paginate({ endpoint, method: 'POST', paginate: { limit: 2 } }).next(); + afterEach(() => { + vi.clearAllMocks(); + }); - expect(nango.proxy).toHaveBeenCalledWith({ - method: 'POST', - endpoint, - data: { limit: 2 }, - paginate: { limit: 2 } + it('Throws error if there is no pagination config in provider template', async () => { + const template: Template = { + auth_mode: AuthModes.OAuth2, + proxy: { base_url: '' }, + authorization_url: '', + token_url: '' + }; + vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); + + const expectedErrorMessage: string = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; + await expect(() => nangoAction.paginate({ endpoint: '' }).next()).rejects.toThrowError(expectedErrorMessage); }); - }); - - it('Overrides template pagination params with ones passed in the proxy config', async () => { - stubProviderTemplate(cursorPagination); - - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ data: { issues: [{}, {}, {}] } }) - .mockReturnValueOnce({ data: { issues: [] } }); - - const endpoint: string = '/issues'; - const paginationConfigOverride: OffsetPagination = { - type: 'offset', - limit_name_in_request: 'per_page', - limit: 3, - offset_name_in_request: 'offset', - response_path: 'issues' - }; - const generator = nangoAction.paginate({ endpoint, paginate: paginationConfigOverride }); - for await (const batch of generator) { - console.log(batch); - } + it('Sends pagination params in body for POST HTTP method', async () => { + stubProviderTemplate(cursorPagination); - expect(nango.proxy).toHaveBeenLastCalledWith({ - method: 'GET', - endpoint, - params: { offset: '3', per_page: 3 }, - paginate: paginationConfigOverride + // TODO: mock to return at least one more page to check that cursor is passed in body too + (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValue({ data: { issues: [] } }); + + const endpoint: string = '/issues'; + + await nangoAction.paginate({ endpoint, method: 'POST', paginate: { limit: 2 } }).next(); + + expect(nango.proxy).toHaveBeenCalledWith({ + method: 'POST', + endpoint, + data: { limit: 2 }, + paginate: { limit: 2 } + }); }); - }); - - it('Paginates using offset', async () => { - stubProviderTemplate(offsetPagination); - - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ data: { issues: firstBatch } }) - .mockReturnValueOnce({ data: { issues: secondBatch } }) - .mockReturnValueOnce({ data: { issues: [] } }); - - const endpoint: string = '/issues'; - - const generator = nangoAction.paginate({ endpoint }); - - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); - } - - const expectedRecords = [...firstBatch, ...secondBatch]; - - expect(actualRecords).toStrictEqual(expectedRecords); - }); - - it('Paginates using cursor', async () => { - stubProviderTemplate(cursorPagination); - - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; - const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ - data: { - issues: firstBatch, - metadata: { - next_cursor: '2' - } - } - }) - .mockReturnValueOnce({ - data: { - issues: secondBatch, - metadata: { - next_cursor: '2' - } - } - }) - .mockReturnValueOnce({ data: { issues: thirdBatch } }); - const endpoint: string = '/issues'; + it('Overrides template pagination params with ones passed in the proxy config', async () => { + stubProviderTemplate(cursorPagination); + + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ data: { issues: [{}, {}, {}] } }) + .mockReturnValueOnce({ data: { issues: [] } }); + + const endpoint: string = '/issues'; + const paginationConfigOverride: OffsetPagination = { + type: 'offset', + limit_name_in_request: 'per_page', + limit: 3, + offset_name_in_request: 'offset', + response_path: 'issues' + }; + + const generator = nangoAction.paginate({ endpoint, paginate: paginationConfigOverride }); + for await (const batch of generator) { + console.log(batch); + } - const generator = nangoAction.paginate({ endpoint }); + expect(nango.proxy).toHaveBeenLastCalledWith({ + method: 'GET', + endpoint, + params: { offset: '3', per_page: 3 }, + paginate: paginationConfigOverride + }); + }); - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); - } + it('Paginates using offset', async () => { + stubProviderTemplate(offsetPagination); - const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ data: { issues: firstBatch } }) + .mockReturnValueOnce({ data: { issues: secondBatch } }) + .mockReturnValueOnce({ data: { issues: [] } }); - expect(actualRecords).toStrictEqual(expectedRecords); - }); + const endpoint: string = '/issues'; - it('Stops pagination if cursor is empty', async () => { - stubProviderTemplate(cursorPagination); + const generator = nangoAction.paginate({ endpoint }); - const onlyBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValueOnce({ - data: { - issues: onlyBatch, - metadata: { - next_cursor: '' + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); } - } - }); - const endpoint: string = '/issues'; + const expectedRecords = [...firstBatch, ...secondBatch]; + + expect(actualRecords).toStrictEqual(expectedRecords); + }); - const generator = nangoAction.paginate({ endpoint }); + it('Paginates using cursor', async () => { + stubProviderTemplate(cursorPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: '2' + } + } + }) + .mockReturnValueOnce({ + data: { + issues: secondBatch, + metadata: { + next_cursor: '2' + } + } + }) + .mockReturnValueOnce({ data: { issues: thirdBatch } }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); - } + const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; - expect(actualRecords).toStrictEqual(onlyBatch); - }); + expect(actualRecords).toStrictEqual(expectedRecords); + }); - it.each(paginationConfigs)( - 'Extracts records from nested body param for $type pagination type', - async (paginationConfig: CursorPagination | OffsetPagination | LinkPagination) => { - stubProviderTemplate(paginationConfig); + it('Stops pagination if cursor is empty', async () => { + stubProviderTemplate(cursorPagination); - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const emptyBatch: any[] = []; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ - data: { - issues: firstBatch, - metadata: { - next_cursor: '' - } - } - }) - - .mockReturnValueOnce({ - data: { - issues: emptyBatch, - metadata: { - next_cursor: '' + const onlyBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValueOnce({ + data: { + issues: onlyBatch, + metadata: { + next_cursor: '' + } } - } }); - const endpoint: string = '/issues'; - - const generator = nangoAction.paginate({ endpoint }); - - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); - } - - expect(actualRecords).toStrictEqual(firstBatch); - } - ); - - it.each([ - // TODO: validate proper config is passed to proxy - ['https://api.gihub.com/issues?page=2', 'https://api.gihub.com/issues?page=3'], - ['/issues?page=2', '/issues?page=3'] - ])('Paginates using next URL/path %s from body', async (nextUrlOrPathValue, anotherNextUrlOrPathValue) => { - stubProviderTemplate(linkPagination); - - const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; - const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; - const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; - (await import('@nangohq/node')).Nango.prototype.proxy = vi - .fn() - .mockReturnValueOnce({ - data: { - issues: firstBatch, - metadata: { - next_cursor: nextUrlOrPathValue - } + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); } - }) - .mockReturnValueOnce({ - data: { - issues: secondBatch, - metadata: { - next_cursor: anotherNextUrlOrPathValue - } + + expect(actualRecords).toStrictEqual(onlyBatch); + }); + + it.each(paginationConfigs)( + 'Extracts records from nested body param for $type pagination type', + async (paginationConfig: CursorPagination | OffsetPagination | LinkPagination) => { + stubProviderTemplate(paginationConfig); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const emptyBatch: any[] = []; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: '' + } + } + }) + + .mockReturnValueOnce({ + data: { + issues: emptyBatch, + metadata: { + next_cursor: '' + } + } + }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + expect(actualRecords).toStrictEqual(firstBatch); } - }) - .mockReturnValueOnce({ data: { issues: thirdBatch } }); - - const endpoint: string = '/issues'; - - const generator = nangoAction.paginate({ endpoint }); - - let actualRecords: any[] = []; - for await (const batch of generator) { - actualRecords.push(...batch); - } - - const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; - let expectedEndpoint: string; - if (isValidHttpUrl(anotherNextUrlOrPathValue)) { - const url: URL = new URL(anotherNextUrlOrPathValue); - expectedEndpoint = url.pathname + url.search; - } else { - expectedEndpoint = anotherNextUrlOrPathValue; - } - - expect(actualRecords).toStrictEqual(expectedRecords); - expect(nango.proxy).toHaveBeenCalledWith( - expect.objectContaining({ - endpoint: expectedEndpoint - }) ); - }); - - const stubProviderTemplate = (paginationConfig: CursorPagination | OffsetPagination | LinkPagination) => { - const template: Template = buildTemplate(paginationConfig); - vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); - }; - - const buildTemplate = (paginationConfig: CursorPagination | OffsetPagination | LinkPagination): Template => { - return { - auth_mode: AuthModes.OAuth2, - proxy: { base_url: 'https://api.github.com/', paginate: paginationConfig }, - authorization_url: '', - token_url: '' + + it.each([ + // TODO: validate proper config is passed to proxy + ['https://api.gihub.com/issues?page=2', 'https://api.gihub.com/issues?page=3'], + ['/issues?page=2', '/issues?page=3'] + ])('Paginates using next URL/path %s from body', async (nextUrlOrPathValue, anotherNextUrlOrPathValue) => { + stubProviderTemplate(linkPagination); + + const firstBatch: any[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const secondBatch: any[] = [{ id: 4 }, { id: 5 }, { id: 6 }]; + const thirdBatch: any[] = [{ id: 7 }, { id: 8 }, { id: 9 }]; + (await import('@nangohq/node')).Nango.prototype.proxy = vi + .fn() + .mockReturnValueOnce({ + data: { + issues: firstBatch, + metadata: { + next_cursor: nextUrlOrPathValue + } + } + }) + .mockReturnValueOnce({ + data: { + issues: secondBatch, + metadata: { + next_cursor: anotherNextUrlOrPathValue + } + } + }) + .mockReturnValueOnce({ data: { issues: thirdBatch } }); + + const endpoint: string = '/issues'; + + const generator = nangoAction.paginate({ endpoint }); + + let actualRecords: any[] = []; + for await (const batch of generator) { + actualRecords.push(...batch); + } + + const expectedRecords = [...firstBatch, ...secondBatch, ...thirdBatch]; + let expectedEndpoint: string; + if (isValidHttpUrl(anotherNextUrlOrPathValue)) { + const url: URL = new URL(anotherNextUrlOrPathValue); + expectedEndpoint = url.pathname + url.search; + } else { + expectedEndpoint = anotherNextUrlOrPathValue; + } + + expect(actualRecords).toStrictEqual(expectedRecords); + expect(nango.proxy).toHaveBeenCalledWith( + expect.objectContaining({ + endpoint: expectedEndpoint + }) + ); + }); + + const stubProviderTemplate = (paginationConfig: CursorPagination | OffsetPagination | LinkPagination) => { + const template: Template = buildTemplate(paginationConfig); + vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); + }; + + const buildTemplate = (paginationConfig: CursorPagination | OffsetPagination | LinkPagination): Template => { + return { + auth_mode: AuthModes.OAuth2, + proxy: { base_url: 'https://api.github.com/', paginate: paginationConfig }, + authorization_url: '', + token_url: '' + }; }; - }; }); From aff3f00b8a5403e24e8e2b84fe58506d371b3bbe Mon Sep 17 00:00:00 2001 From: omotnyk Date: Fri, 13 Oct 2023 14:28:41 +0300 Subject: [PATCH 44/50] Replace mentions of `url` with `link` --- packages/shared/lib/sdk/sync.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 57613254330..b77bb1d9761 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -422,17 +422,17 @@ export class NangoAction { yield responseData; - let nextPageUrl: string | undefined = this.getNextPageUrlFromBodyOrHeaders(linkPagination, response, paginationConfig); + let nextPageLink: string | undefined = this.getNextPageLinkFromBodyOrHeaders(linkPagination, response, paginationConfig); - if (!nextPageUrl) { + if (!nextPageLink) { return; } - if (!isValidHttpUrl(nextPageUrl)) { + if (!isValidHttpUrl(nextPageLink)) { // some providers only send path+query params in the link so we can immediately assign those to the endpoint - config.endpoint = nextPageUrl; + config.endpoint = nextPageLink; } else { - const url: URL = new URL(nextPageUrl); + const url: URL = new URL(nextPageLink); config.endpoint = url.pathname + url.search; } delete config.params; @@ -476,7 +476,7 @@ export class NangoAction { } } - private getNextPageUrlFromBodyOrHeaders(linkPagination: LinkPagination, response: AxiosResponse, paginationConfig: Pagination) { + private getNextPageLinkFromBodyOrHeaders(linkPagination: LinkPagination, response: AxiosResponse, paginationConfig: Pagination) { if (linkPagination.link_rel_in_response_header) { const linkHeader = parseLinksHeader(response.headers['link']); return linkHeader?.[linkPagination.link_rel_in_response_header]?.url; From 5129e1b46d098708d96e0fbd7e9e20c5e2e0d387 Mon Sep 17 00:00:00 2001 From: omotnyk Date: Fri, 13 Oct 2023 15:48:11 +0300 Subject: [PATCH 45/50] Use lodash --- packages/shared/lib/sdk/sync.ts | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index b77bb1d9761..7db42bd05de 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -12,6 +12,7 @@ import configService from '../services/config.service.js'; import type { Template } from '../models/index.js'; import { isValidHttpUrl } from '../utils/utils.js'; import parseLinksHeader from 'parse-link-header'; +import * as _ from 'lodash'; type LogLevel = 'info' | 'debug' | 'error' | 'warn' | 'http' | 'verbose' | 'silly'; @@ -390,7 +391,7 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); const responseData: T[] = cursorPagination.response_path - ? this.getNestedField(response.data, cursorPagination.response_path) + ? _.get(response.data, cursorPagination.response_path) : response.data; if (!responseData.length) { @@ -399,7 +400,7 @@ export class NangoAction { yield responseData; - nextCursor = this.getNestedField(response.data, cursorPagination.cursor_path_in_response); + nextCursor = _.get(response.data, cursorPagination.cursor_path_in_response); if (!nextCursor || nextCursor.trim().length === 0) { return; @@ -414,7 +415,7 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); const responseData: T[] = paginationConfig.response_path - ? this.getNestedField(response.data, paginationConfig.response_path) + ? _.get(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; @@ -451,7 +452,7 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); const responseData: T[] = paginationConfig.response_path - ? this.getNestedField(response.data, paginationConfig.response_path) + ? _.get(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; @@ -481,7 +482,7 @@ export class NangoAction { const linkHeader = parseLinksHeader(response.headers['link']); return linkHeader?.[linkPagination.link_rel_in_response_header]?.url; } else if (linkPagination.link_path_in_response_body) { - return this.getNestedField(response.data, linkPagination.link_path_in_response_body); + return _.get(response.data, linkPagination.link_path_in_response_body); } throw Error(`Either 'link_rel_in_response_header' or 'link_path_in_response_body' should be specified for '${paginationConfig.type}' pagination`); @@ -494,22 +495,6 @@ export class NangoAction { config.params = updatedBodyOrParams; } } - - private getNestedField(object: any, path: string, defaultValue?: any): any { - // TODO: extract to util or figure out how to use lodash - const keys = path.split('.'); - let result = object; - - for (const key of keys) { - if (result && typeof result === 'object' && key in result) { - result = result[key]; - } else { - return defaultValue; - } - } - - return result !== undefined ? result : defaultValue; - } } export class NangoSync extends NangoAction { From 29f3df466f928068aabc351aa30039dc90cf17f2 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 19 Oct 2023 11:22:28 +0200 Subject: [PATCH 46/50] formatting --- packages/shared/lib/sdk/sync.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 40f47d0a5a0..ef75eddc7af 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -390,9 +390,7 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = cursorPagination.response_path - ? _.get(response.data, cursorPagination.response_path) - : response.data; + const responseData: T[] = cursorPagination.response_path ? _.get(response.data, cursorPagination.response_path) : response.data; if (!responseData.length) { return; @@ -414,9 +412,7 @@ export class NangoAction { while (true) { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_path - ? _.get(response.data, paginationConfig.response_path) - : response.data; + const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; } @@ -451,9 +447,7 @@ export class NangoAction { const response: AxiosResponse = await this.proxy(config); - const responseData: T[] = paginationConfig.response_path - ? _.get(response.data, paginationConfig.response_path) - : response.data; + const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; } From ec7c58b1ce590be230da6da459691da78d5d79c0 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 19 Oct 2023 11:39:34 +0200 Subject: [PATCH 47/50] initial minor clean up --- packages/shared/lib/sdk/sync.ts | 7 +++---- packages/shared/lib/sdk/sync.unit.test.ts | 16 ++++++++-------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index ef75eddc7af..7aec5949930 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -9,7 +9,6 @@ import { LogActionEnum } from '../models/Activity.js'; import { Nango } from '@nangohq/node'; import configService from '../services/config.service.js'; -import type { Template } from '../models/index.js'; import { isValidHttpUrl } from '../utils/utils.js'; import parseLinksHeader from 'parse-link-header'; import * as _ from 'lodash'; @@ -343,7 +342,7 @@ export class NangoAction { public async *paginate(config: ProxyConfiguration): AsyncGenerator { const providerConfigKey: string = this.providerConfigKey as string; - const template: Template = configService.getTemplate(providerConfigKey); + const template = configService.getTemplate(providerConfigKey); const templatePaginationConfig: Pagination | undefined = template.proxy?.paginate; if (!templatePaginationConfig) { @@ -419,7 +418,7 @@ export class NangoAction { yield responseData; - let nextPageLink: string | undefined = this.getNextPageLinkFromBodyOrHeaders(linkPagination, response, paginationConfig); + const nextPageLink: string | undefined = this.getNextPageLinkFromBodyOrHeaders(linkPagination, response, paginationConfig); if (!nextPageLink) { return; @@ -438,7 +437,7 @@ export class NangoAction { case PaginationType.OFFSET: { const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; const offsetParameterName: string = offsetPagination.offset_name_in_request; - let offset: number = 0; + let offset = 0; while (true) { updatedBodyOrParams[offsetParameterName] = `${offset}`; diff --git a/packages/shared/lib/sdk/sync.unit.test.ts b/packages/shared/lib/sdk/sync.unit.test.ts index 6432f4e3eef..e4811a4ed40 100644 --- a/packages/shared/lib/sdk/sync.unit.test.ts +++ b/packages/shared/lib/sdk/sync.unit.test.ts @@ -61,7 +61,7 @@ describe('Pagination', () => { }; vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); - const expectedErrorMessage: string = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; + const expectedErrorMessage = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; await expect(() => nangoAction.paginate({ endpoint: '' }).next()).rejects.toThrowError(expectedErrorMessage); }); @@ -71,7 +71,7 @@ describe('Pagination', () => { // TODO: mock to return at least one more page to check that cursor is passed in body too (await import('@nangohq/node')).Nango.prototype.proxy = vi.fn().mockReturnValue({ data: { issues: [] } }); - const endpoint: string = '/issues'; + const endpoint = '/issues'; await nangoAction.paginate({ endpoint, method: 'POST', paginate: { limit: 2 } }).next(); @@ -91,7 +91,7 @@ describe('Pagination', () => { .mockReturnValueOnce({ data: { issues: [{}, {}, {}] } }) .mockReturnValueOnce({ data: { issues: [] } }); - const endpoint: string = '/issues'; + const endpoint = '/issues'; const paginationConfigOverride: OffsetPagination = { type: 'offset', limit_name_in_request: 'per_page', @@ -124,7 +124,7 @@ describe('Pagination', () => { .mockReturnValueOnce({ data: { issues: secondBatch } }) .mockReturnValueOnce({ data: { issues: [] } }); - const endpoint: string = '/issues'; + const endpoint = '/issues'; const generator = nangoAction.paginate({ endpoint }); @@ -164,7 +164,7 @@ describe('Pagination', () => { }) .mockReturnValueOnce({ data: { issues: thirdBatch } }); - const endpoint: string = '/issues'; + const endpoint = '/issues'; const generator = nangoAction.paginate({ endpoint }); @@ -191,7 +191,7 @@ describe('Pagination', () => { } }); - const endpoint: string = '/issues'; + const endpoint = '/issues'; const generator = nangoAction.paginate({ endpoint }); @@ -230,7 +230,7 @@ describe('Pagination', () => { } }); - const endpoint: string = '/issues'; + const endpoint = '/issues'; const generator = nangoAction.paginate({ endpoint }); @@ -273,7 +273,7 @@ describe('Pagination', () => { }) .mockReturnValueOnce({ data: { issues: thirdBatch } }); - const endpoint: string = '/issues'; + const endpoint = '/issues'; const generator = nangoAction.paginate({ endpoint }); From 8307e09366b09a3a57504fbadb5745d6b23cc622 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 19 Oct 2023 11:48:59 +0200 Subject: [PATCH 48/50] remove formatting update --- packages/shared/lib/sdk/sync.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 7aec5949930..cd1e64e9934 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -559,7 +559,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'error', activity_log_id: this.activityLogId as number, - content: `There was an issue with the batch save.${error?.message}`, + content: `There was an issue with the batch save. ${error?.message}`, timestamp: Date.now() }); } @@ -610,7 +610,7 @@ export class NangoSync extends NangoAction { return true; } else { - const content = `There was an issue with the batch save.${responseResults?.error}`; + const content = `There was an issue with the batch save. ${responseResults?.error}`; if (!this.dryRun) { await createActivityLogMessage({ @@ -670,7 +670,7 @@ export class NangoSync extends NangoAction { await createActivityLogMessage({ level: 'error', activity_log_id: this.activityLogId as number, - content: `There was an issue with the batch delete.${error?.message}`, + content: `There was an issue with the batch delete. ${error?.message}`, timestamp: Date.now() }); } @@ -722,7 +722,7 @@ export class NangoSync extends NangoAction { return true; } else { - const content = `There was an issue with the batch delete.${responseResults?.error}`; + const content = `There was an issue with the batch delete. ${responseResults?.error}`; if (!this.dryRun) { await createActivityLogMessage({ From b77b03005f117d4c550646e33cae0c315159d733 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 19 Oct 2023 12:58:52 +0200 Subject: [PATCH 49/50] organizational refactor --- packages/shared/lib/sdk/sync.ts | 138 +++--------------- .../shared/lib/services/paginate.service.ts | 135 +++++++++++++++++ scripts/validation/providers/schema.json | 3 +- 3 files changed, 160 insertions(+), 116 deletions(-) create mode 100644 packages/shared/lib/services/paginate.service.ts diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index cd1e64e9934..df24e5d9d6f 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -9,8 +9,7 @@ import { LogActionEnum } from '../models/Activity.js'; import { Nango } from '@nangohq/node'; import configService from '../services/config.service.js'; -import { isValidHttpUrl } from '../utils/utils.js'; -import parseLinksHeader from 'parse-link-header'; +import paginateService from '../services/paginate.service.js'; import * as _ from 'lodash'; type LogLevel = 'info' | 'debug' | 'error' | 'warn' | 'http' | 'verbose' | 'silly'; @@ -49,7 +48,7 @@ interface ParamsSerializerOptions extends SerializerOptions { serialize?: CustomParamsSerializer; } -interface AxiosResponse { +export interface AxiosResponse { data: T; status: number; statusText: string; @@ -69,7 +68,7 @@ export enum PaginationType { OFFSET = 'offset' } -interface Pagination { +export interface Pagination { type: string; limit?: number; response_path?: string; @@ -90,7 +89,7 @@ export interface OffsetPagination extends Pagination { offset_name_in_request: string; } -interface ProxyConfiguration { +export interface ProxyConfiguration { endpoint: string; providerConfigKey?: string; connectionId?: string; @@ -349,7 +348,7 @@ export class NangoAction { throw Error(`Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`); } - let paginationConfig: Pagination = templatePaginationConfig; + let paginationConfig: Pagination = templatePaginationConfig as Pagination; delete paginationConfig.limit; if (config.paginate) { @@ -374,120 +373,29 @@ export class NangoAction { updatedBodyOrParams[limitParameterName] = paginationConfig['limit']; } - // TODO: Consider creating 'Paginator' interface and moving the case block to specific implementations of 'Paginator' switch (paginationConfig.type) { - case PaginationType.CURSOR: { - const cursorPagination: CursorPagination = paginationConfig as CursorPagination; - - let nextCursor: string | undefined; - while (true) { - if (nextCursor) { - updatedBodyOrParams[cursorPagination.cursor_name_in_request] = nextCursor; - } - - this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - - const response: AxiosResponse = await this.proxy(config); - - const responseData: T[] = cursorPagination.response_path ? _.get(response.data, cursorPagination.response_path) : response.data; - - if (!responseData.length) { - return; - } - - yield responseData; - - nextCursor = _.get(response.data, cursorPagination.cursor_path_in_response); - - if (!nextCursor || nextCursor.trim().length === 0) { - return; - } - } - } - case PaginationType.LINK: { - const linkPagination: LinkPagination = paginationConfig as LinkPagination; - - this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - while (true) { - const response: AxiosResponse = await this.proxy(config); - - const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; - if (!responseData.length) { - return; - } - - yield responseData; - - const nextPageLink: string | undefined = this.getNextPageLinkFromBodyOrHeaders(linkPagination, response, paginationConfig); - - if (!nextPageLink) { - return; - } - - if (!isValidHttpUrl(nextPageLink)) { - // some providers only send path+query params in the link so we can immediately assign those to the endpoint - config.endpoint = nextPageLink; - } else { - const url: URL = new URL(nextPageLink); - config.endpoint = url.pathname + url.search; - } - delete config.params; - } - } - case PaginationType.OFFSET: { - const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; - const offsetParameterName: string = offsetPagination.offset_name_in_request; - let offset = 0; - - while (true) { - updatedBodyOrParams[offsetParameterName] = `${offset}`; - - this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); - - const response: AxiosResponse = await this.proxy(config); - - const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; - if (!responseData.length) { - return; - } - - yield responseData; - - if (paginationConfig['limit'] && responseData.length < paginationConfig['limit']) { - return; - } - - if (responseData.length < 1) { - // Last page was empty so no need to fetch further - return; - } - - offset += responseData.length; - } - } + case PaginationType.CURSOR: + return yield* paginateService.cursor( + config, + paginationConfig as CursorPagination, + updatedBodyOrParams, + passPaginationParamsInBody, + this.proxy + ); + case PaginationType.LINK: + return yield* paginateService.link(config, paginationConfig, updatedBodyOrParams, passPaginationParamsInBody, this.proxy); + case PaginationType.OFFSET: + return yield* paginateService.offset( + config, + paginationConfig as OffsetPagination, + updatedBodyOrParams, + passPaginationParamsInBody, + this.proxy + ); default: throw Error(`'${paginationConfig.type} ' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)}`); } } - - private getNextPageLinkFromBodyOrHeaders(linkPagination: LinkPagination, response: AxiosResponse, paginationConfig: Pagination) { - if (linkPagination.link_rel_in_response_header) { - const linkHeader = parseLinksHeader(response.headers['link']); - return linkHeader?.[linkPagination.link_rel_in_response_header]?.url; - } else if (linkPagination.link_path_in_response_body) { - return _.get(response.data, linkPagination.link_path_in_response_body); - } - - throw Error(`Either 'link_rel_in_response_header' or 'link_path_in_response_body' should be specified for '${paginationConfig.type}' pagination`); - } - - private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { - if (passPaginationParamsInBody) { - config.data = updatedBodyOrParams; - } else { - config.params = updatedBodyOrParams; - } - } } export class NangoSync extends NangoAction { diff --git a/packages/shared/lib/services/paginate.service.ts b/packages/shared/lib/services/paginate.service.ts new file mode 100644 index 00000000000..73efaa1bd04 --- /dev/null +++ b/packages/shared/lib/services/paginate.service.ts @@ -0,0 +1,135 @@ +import parseLinksHeader from 'parse-link-header'; +import * as _ from 'lodash'; +import type { Pagination, AxiosResponse, ProxyConfiguration, CursorPagination, OffsetPagination, LinkPagination } from '../sdk/sync.js'; +import { isValidHttpUrl } from '../utils/utils.js'; + +class PaginationService { + public async *cursor( + config: ProxyConfiguration, + paginationConfig: CursorPagination, + updatedBodyOrParams: Record, + passPaginationParamsInBody: boolean, + proxy: (config: ProxyConfiguration) => Promise + ): AsyncGenerator { + const cursorPagination: CursorPagination = paginationConfig as CursorPagination; + + let nextCursor: string | undefined; + + while (true) { + if (nextCursor) { + updatedBodyOrParams[cursorPagination.cursor_name_in_request] = nextCursor; + } + + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); + + const response: AxiosResponse = await proxy(config); + + const responseData: T[] = cursorPagination.response_path ? _.get(response.data, cursorPagination.response_path) : response.data; + + if (!responseData.length) { + return; + } + + yield responseData; + + nextCursor = _.get(response.data, cursorPagination.cursor_path_in_response); + + if (!nextCursor || nextCursor.trim().length === 0) { + return; + } + } + } + + public async *link( + config: ProxyConfiguration, + paginationConfig: LinkPagination, + updatedBodyOrParams: Record, + passPaginationParamsInBody: boolean, + proxy: (config: ProxyConfiguration) => Promise + ): AsyncGenerator { + const linkPagination: LinkPagination = paginationConfig as LinkPagination; + + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); + + while (true) { + const response: AxiosResponse = await proxy(config); + + const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; + if (!responseData.length) { + return; + } + + yield responseData; + + const nextPageLink: string | undefined = this.getNextPageLinkFromBodyOrHeaders(linkPagination, response, paginationConfig); + + if (!nextPageLink) { + return; + } + + if (!isValidHttpUrl(nextPageLink)) { + // some providers only send path+query params in the link so we can immediately assign those to the endpoint + config.endpoint = nextPageLink; + } else { + const url: URL = new URL(nextPageLink); + config.endpoint = url.pathname + url.search; + } + delete config.params; + } + } + + public async *offset( + config: ProxyConfiguration, + paginationConfig: OffsetPagination, + updatedBodyOrParams: Record, + passPaginationParamsInBody: boolean, + proxy: (config: ProxyConfiguration) => Promise + ): AsyncGenerator { + const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; + const offsetParameterName: string = offsetPagination.offset_name_in_request; + let offset = 0; + + while (true) { + updatedBodyOrParams[offsetParameterName] = `${offset}`; + + this.updateConfigBodyOrParams(passPaginationParamsInBody, config, updatedBodyOrParams); + + const response: AxiosResponse = await proxy(config); + + const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; + if (!responseData.length) { + return; + } + + yield responseData; + + if (paginationConfig['limit'] && responseData.length < paginationConfig['limit']) { + return; + } + + if (responseData.length < 1) { + // Last page was empty so no need to fetch further + return; + } + + offset += responseData.length; + } + } + + private updateConfigBodyOrParams(passPaginationParamsInBody: boolean, config: ProxyConfiguration, updatedBodyOrParams: Record) { + passPaginationParamsInBody ? (config.data = updatedBodyOrParams) : (config.params = updatedBodyOrParams); + } + + private getNextPageLinkFromBodyOrHeaders(linkPagination: LinkPagination, response: AxiosResponse, paginationConfig: Pagination) { + if (linkPagination.link_rel_in_response_header) { + const linkHeader = parseLinksHeader(response.headers['link']); + return linkHeader?.[linkPagination.link_rel_in_response_header]?.url; + } else if (linkPagination.link_path_in_response_body) { + return _.get(response.data, linkPagination.link_path_in_response_body); + } + + throw Error(`Either 'link_rel_in_response_header' or 'link_path_in_response_body' should be specified for '${paginationConfig.type}' pagination`); + } +} + +export default new PaginationService(); diff --git a/scripts/validation/providers/schema.json b/scripts/validation/providers/schema.json index c2429d6c5f5..4bcfadd569f 100644 --- a/scripts/validation/providers/schema.json +++ b/scripts/validation/providers/schema.json @@ -6,7 +6,7 @@ "properties": { "auth_mode": { "type": "string", - "enum": [ "API_KEY", "APP", "BASIC", "NONE", "OAUTH1", "OAUTH2"] + "enum": ["API_KEY", "APP", "BASIC", "NONE", "OAUTH1", "OAUTH2"] }, "authorization_url": { "type": "string" }, "token_url": { "type": "string" }, @@ -20,6 +20,7 @@ "type": "object", "properties": { "base_url": { "type": "string" }, + "paginate": { "type": "object" }, "headers": { "type": "object", "patternProperties": { From e02e11d8d913adc24b6504375d2df3c03cebfccb Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 19 Oct 2023 15:21:05 +0200 Subject: [PATCH 50/50] add notion and add validation --- package-lock.json | 17 ++++++- packages/shared/lib/sdk/sync.ts | 31 +++++------- packages/shared/lib/sdk/sync.unit.test.ts | 2 +- .../shared/lib/services/paginate.service.ts | 47 ++++++++++++++++--- packages/shared/package.json | 4 +- packages/shared/providers.yaml | 6 +++ 6 files changed, 77 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index 93f0b01a8c2..759880504c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4906,9 +4906,16 @@ }, "node_modules/@types/lodash": { "version": "4.14.195", - "dev": true, "license": "MIT" }, + "node_modules/@types/lodash-es": { + "version": "4.17.10", + "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.10.tgz", + "integrity": "sha512-YJP+w/2khSBwbUSFdGsSqmDvmnN3cCKoPOL7Zjle6s30ZtemkkqhjVfFqGwPN7ASil5VyjE2GtyU/yqYY6mC0A==", + "dependencies": { + "@types/lodash": "*" + } + }, "node_modules/@types/long": { "version": "4.0.2", "license": "MIT" @@ -8797,6 +8804,11 @@ "version": "4.17.21", "license": "MIT" }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" + }, "node_modules/lodash.camelcase": { "version": "4.3.0", "license": "MIT" @@ -12725,6 +12737,7 @@ "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", + "@types/lodash-es": "^4.17.10", "amqplib": "^0.10.3", "archiver": "^6.0.1", "axios": "^1.3.4", @@ -12739,7 +12752,7 @@ "js-yaml": "^4.1.0", "jsonwebtoken": "^9.0.2", "knex": "^2.3.0", - "lodash": "^4.17.21", + "lodash-es": "^4.17.21", "md5": "^2.3.0", "ms": "^2.1.3", "parse-link-header": "^2.0.0", diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index df24e5d9d6f..e9e607ab0aa 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -10,7 +10,6 @@ import { LogActionEnum } from '../models/Activity.js'; import { Nango } from '@nangohq/node'; import configService from '../services/config.service.js'; import paginateService from '../services/paginate.service.js'; -import * as _ from 'lodash'; type LogLevel = 'info' | 'debug' | 'error' | 'warn' | 'http' | 'verbose' | 'silly'; @@ -344,24 +343,18 @@ export class NangoAction { const template = configService.getTemplate(providerConfigKey); const templatePaginationConfig: Pagination | undefined = template.proxy?.paginate; - if (!templatePaginationConfig) { - throw Error(`Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`); + if (!templatePaginationConfig && (!config.paginate || !config.paginate.type)) { + throw Error('There was no pagination configuration for this integration or configuration passed in.'); } - let paginationConfig: Pagination = templatePaginationConfig as Pagination; - delete paginationConfig.limit; + const paginationConfig: Pagination = { + ...(templatePaginationConfig || {}), + ...(config.paginate || {}) + } as Pagination; - if (config.paginate) { - const paginationConfigOverride: Record = config.paginate as Record; + paginateService.validateConfiguration(paginationConfig); - if (paginationConfigOverride) { - paginationConfig = { ...paginationConfig, ...paginationConfigOverride }; - } - } - - if (!config.method) { - config.method = 'GET'; - } + config.method = config.method || 'GET'; const configMethod: string = config.method.toLocaleLowerCase(); const passPaginationParamsInBody: boolean = ['post', 'put', 'patch'].includes(configMethod); @@ -373,24 +366,24 @@ export class NangoAction { updatedBodyOrParams[limitParameterName] = paginationConfig['limit']; } - switch (paginationConfig.type) { + switch (paginationConfig.type.toLowerCase()) { case PaginationType.CURSOR: return yield* paginateService.cursor( config, paginationConfig as CursorPagination, updatedBodyOrParams, passPaginationParamsInBody, - this.proxy + this.proxy.bind(this) ); case PaginationType.LINK: - return yield* paginateService.link(config, paginationConfig, updatedBodyOrParams, passPaginationParamsInBody, this.proxy); + return yield* paginateService.link(config, paginationConfig, updatedBodyOrParams, passPaginationParamsInBody, this.proxy.bind(this)); case PaginationType.OFFSET: return yield* paginateService.offset( config, paginationConfig as OffsetPagination, updatedBodyOrParams, passPaginationParamsInBody, - this.proxy + this.proxy.bind(this) ); default: throw Error(`'${paginationConfig.type} ' pagination is not supported. Please, make sure it's one of ${Object.values(PaginationType)}`); diff --git a/packages/shared/lib/sdk/sync.unit.test.ts b/packages/shared/lib/sdk/sync.unit.test.ts index e4811a4ed40..df8f26430e2 100644 --- a/packages/shared/lib/sdk/sync.unit.test.ts +++ b/packages/shared/lib/sdk/sync.unit.test.ts @@ -61,7 +61,7 @@ describe('Pagination', () => { }; vi.spyOn(configService, 'getTemplate').mockImplementation(() => template); - const expectedErrorMessage = `Pagination is not supported for '${providerConfigKey}'. Please, add pagination config to 'providers.yaml' file`; + const expectedErrorMessage = 'There was no pagination configuration for this integration or configuration passed in'; await expect(() => nangoAction.paginate({ endpoint: '' }).next()).rejects.toThrowError(expectedErrorMessage); }); diff --git a/packages/shared/lib/services/paginate.service.ts b/packages/shared/lib/services/paginate.service.ts index 73efaa1bd04..dfd5d83ca5b 100644 --- a/packages/shared/lib/services/paginate.service.ts +++ b/packages/shared/lib/services/paginate.service.ts @@ -1,9 +1,44 @@ import parseLinksHeader from 'parse-link-header'; -import * as _ from 'lodash'; +import get from 'lodash-es/get.js'; import type { Pagination, AxiosResponse, ProxyConfiguration, CursorPagination, OffsetPagination, LinkPagination } from '../sdk/sync.js'; +import { PaginationType } from '../sdk/sync.js'; import { isValidHttpUrl } from '../utils/utils.js'; class PaginationService { + public validateConfiguration(paginationConfig: Pagination): void { + if (!paginationConfig.type) { + throw new Error('Pagination type is required'); + } + const { type } = paginationConfig; + if (type.toLowerCase() === PaginationType.CURSOR) { + const cursorPagination: CursorPagination = paginationConfig as CursorPagination; + if (!cursorPagination.cursor_name_in_request) { + throw new Error('Param cursor_name_in_request is required for cursor pagination'); + } + if (!cursorPagination.cursor_path_in_response) { + throw new Error('Param cursor_path_in_response is required for cursor pagination'); + } + + if (paginationConfig.limit && !paginationConfig.limit_name_in_request) { + throw new Error('Param limit_name_in_request is required for cursor pagination when limit is set'); + } + } else if (type.toLowerCase() === PaginationType.LINK) { + const linkPagination: LinkPagination = paginationConfig as LinkPagination; + if (!linkPagination.link_rel_in_response_header && !linkPagination.link_path_in_response_body) { + throw new Error('Either param link_rel_in_response_header or link_path_in_response_body is required for link pagination'); + } + } else if (type.toLowerCase() === PaginationType.OFFSET) { + const offsetPagination: OffsetPagination = paginationConfig as OffsetPagination; + if (!offsetPagination.offset_name_in_request) { + throw new Error('Param offset_name_in_request is required for offset pagination'); + } + } else { + throw new Error( + `Pagination type ${type} is not supported. Only ${PaginationType.CURSOR}, ${PaginationType.LINK}, and ${PaginationType.OFFSET} pagination types are supported.` + ); + } + } + public async *cursor( config: ProxyConfiguration, paginationConfig: CursorPagination, @@ -24,7 +59,7 @@ class PaginationService { const response: AxiosResponse = await proxy(config); - const responseData: T[] = cursorPagination.response_path ? _.get(response.data, cursorPagination.response_path) : response.data; + const responseData: T[] = cursorPagination.response_path ? get(response.data, cursorPagination.response_path) : response.data; if (!responseData.length) { return; @@ -32,7 +67,7 @@ class PaginationService { yield responseData; - nextCursor = _.get(response.data, cursorPagination.cursor_path_in_response); + nextCursor = get(response.data, cursorPagination.cursor_path_in_response); if (!nextCursor || nextCursor.trim().length === 0) { return; @@ -54,7 +89,7 @@ class PaginationService { while (true) { const response: AxiosResponse = await proxy(config); - const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; + const responseData: T[] = paginationConfig.response_path ? get(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; } @@ -96,7 +131,7 @@ class PaginationService { const response: AxiosResponse = await proxy(config); - const responseData: T[] = paginationConfig.response_path ? _.get(response.data, paginationConfig.response_path) : response.data; + const responseData: T[] = paginationConfig.response_path ? get(response.data, paginationConfig.response_path) : response.data; if (!responseData.length) { return; } @@ -125,7 +160,7 @@ class PaginationService { const linkHeader = parseLinksHeader(response.headers['link']); return linkHeader?.[linkPagination.link_rel_in_response_header]?.url; } else if (linkPagination.link_path_in_response_body) { - return _.get(response.data, linkPagination.link_path_in_response_body); + return get(response.data, linkPagination.link_path_in_response_body); } throw Error(`Either 'link_rel_in_response_header' or 'link_path_in_response_body' should be specified for '${paginationConfig.type}' pagination`); diff --git a/packages/shared/package.json b/packages/shared/package.json index 287d77dbfae..8dac778082b 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -37,7 +37,7 @@ "js-yaml": "^4.1.0", "jsonwebtoken": "^9.0.2", "knex": "^2.3.0", - "lodash": "^4.17.21", + "lodash-es": "^4.17.21", "md5": "^2.3.0", "ms": "^2.1.3", "parse-link-header": "^2.0.0", @@ -61,8 +61,8 @@ "@types/cors": "^2.8.12", "@types/debug": "^4.1.7", "@types/human-to-cron": "^0.3.0", + "@types/lodash-es": "^4.17.10", "@types/js-yaml": "^4.0.5", - "@types/lodash": "^4.14.195", "@types/node": "^18.7.6", "@types/parse-link-header": "^2.0.0", "@types/uuid": "^9.0.0", diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 87ae2f4a54c..a3b595b8708 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -674,6 +674,12 @@ notion: base_url: https://api.notion.com headers: 'Notion-Version': '2022-06-28' + paginate: + type: cursor + cursor_path_in_response: start_cursor + cursor_name_in_request: start_cursor + limit_name_in_request: page_size + response_path: results docs: https://developers.notion.com/reference one-drive: alias: microsoft-teams