From cbb9e818056b19cc42b2cabf2684ea91d8504639 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:46:26 -0500 Subject: [PATCH] feat: return records count in /sync/status endpoint/sdk (#2961) Screenshot 2024-11-07 at 14 34 38 ## Issue ticket number and link https://linear.app/nango/issue/NAN-1928/return-sync-object-count-from-api ## Checklist before requesting a review (skip if just adding/editing APIs & templates) - [ ] I added tests, otherwise the reason is: - [ ] I added observability, otherwise the reason is: - [ ] I added analytics, otherwise the reason is: --- docs-v2/reference/sdks/node.mdx | 12 +++++++- docs-v2/spec.yaml | 3 ++ packages/node-client/lib/types.ts | 1 + .../lib/controllers/onboarding.controller.ts | 10 ++++++- .../server/lib/controllers/sync.controller.ts | 1 + packages/shared/lib/clients/orchestrator.ts | 2 ++ packages/shared/lib/models/Sync.ts | 11 ++++---- .../lib/services/sync/manager.service.ts | 28 +++++++++++++++---- 8 files changed, 56 insertions(+), 12 deletions(-) diff --git a/docs-v2/reference/sdks/node.mdx b/docs-v2/reference/sdks/node.mdx index 49880fe20b5..32c622a9b77 100644 --- a/docs-v2/reference/sdks/node.mdx +++ b/docs-v2/reference/sdks/node.mdx @@ -993,7 +993,17 @@ await nango.syncStatus('', ['SYNC_NAME1', 'SYNC_NAME2'], '", "nextScheduledSyncAt": "", "frequency": "", - "latestResult": {} + "latestResult": { + "": { + "added": , + "updated": , + "deleted": , + } + }, + "recordCount": { + "": + ... + } } ] } diff --git a/docs-v2/spec.yaml b/docs-v2/spec.yaml index 8971ce025f9..ce2267b657d 100644 --- a/docs-v2/spec.yaml +++ b/docs-v2/spec.yaml @@ -1275,6 +1275,9 @@ paths: latestResult: type: object description: Additional information regarding the latest result of the sync. Contains a model name with added, updated and deleted records + recordCount: + type: object + description: Total count of records for each model synced by the sync '400': description: Invalid request content: diff --git a/packages/node-client/lib/types.ts b/packages/node-client/lib/types.ts index 84b8732683c..ea6ebe63f2a 100644 --- a/packages/node-client/lib/types.ts +++ b/packages/node-client/lib/types.ts @@ -217,6 +217,7 @@ export interface SyncStatus { status: 'RUNNING' | 'SUCCESS' | 'ERROR' | 'PAUSED' | 'STOPPED'; frequency: string; latestResult: Record; + recordCount: Record; } export interface StatusAction { diff --git a/packages/server/lib/controllers/onboarding.controller.ts b/packages/server/lib/controllers/onboarding.controller.ts index e4e24a23f25..ff915837c11 100644 --- a/packages/server/lib/controllers/onboarding.controller.ts +++ b/packages/server/lib/controllers/onboarding.controller.ts @@ -244,7 +244,15 @@ class OnboardingController { success, error, response: status - } = await syncManager.getSyncStatus(environment.id, DEMO_GITHUB_CONFIG_KEY, [DEMO_SYNC_NAME], orchestrator, req.body.connectionId, true); + } = await syncManager.getSyncStatus( + environment.id, + DEMO_GITHUB_CONFIG_KEY, + [DEMO_SYNC_NAME], + orchestrator, + recordsService, + req.body.connectionId, + true + ); if (!success || !status) { void analytics.track(AnalyticsTypes.DEMO_4_ERR, account.id, { user_id: user.id }); diff --git a/packages/server/lib/controllers/sync.controller.ts b/packages/server/lib/controllers/sync.controller.ts index b3e703a8c66..187f60a9273 100644 --- a/packages/server/lib/controllers/sync.controller.ts +++ b/packages/server/lib/controllers/sync.controller.ts @@ -523,6 +523,7 @@ class SyncController { provider_config_key as string, syncNames, orchestrator, + recordsService, connection_id as string, false, connection diff --git a/packages/shared/lib/clients/orchestrator.ts b/packages/shared/lib/clients/orchestrator.ts index a2ceac90373..1db038a9c1d 100644 --- a/packages/shared/lib/clients/orchestrator.ts +++ b/packages/shared/lib/clients/orchestrator.ts @@ -30,6 +30,7 @@ import { isSyncJobRunning, updateSyncJobStatus } from '../services/sync/job.serv import { getSyncConfigRaw, getSyncConfigBySyncId } from '../services/sync/config/config.service.js'; import environmentService from '../services/environment.service.js'; import type { DBEnvironment, DBTeam } from '@nangohq/types'; +import type { RecordCount } from '@nangohq/records'; export interface RecordsServiceInterface { deleteRecordsBySyncId({ @@ -43,6 +44,7 @@ export interface RecordsServiceInterface { model: string; syncId: string; }): Promise<{ totalDeletedRecords: number }>; + getRecordCountsByModel({ connectionId, environmentId }: { connectionId: number; environmentId: number }): Promise>>; } export interface OrchestratorClientInterface { diff --git a/packages/shared/lib/models/Sync.ts b/packages/shared/lib/models/Sync.ts index d2d43960205..6df667bffd7 100644 --- a/packages/shared/lib/models/Sync.ts +++ b/packages/shared/lib/models/Sync.ts @@ -59,15 +59,16 @@ export interface Job extends TimestampsAndDeleted { export interface ReportedSyncJobStatus { id?: string; - type: SyncType; + type: SyncType | 'INITIAL'; name?: string; status: SyncStatus; - latestResult?: SyncResultByModel; + latestResult?: SyncResultByModel | undefined; jobStatus?: SyncStatus; - frequency: string; - finishedAt: Date; + frequency: string | null; + finishedAt: Date | undefined; nextScheduledSyncAt: Date | null; - latestExecutionStatus: SyncStatus; + latestExecutionStatus: SyncStatus | undefined; + recordCount: Record; } // TODO: change that to use Parsed type diff --git a/packages/shared/lib/services/sync/manager.service.ts b/packages/shared/lib/services/sync/manager.service.ts index f5ef0b10416..9c6760835f5 100644 --- a/packages/shared/lib/services/sync/manager.service.ts +++ b/packages/shared/lib/services/sync/manager.service.ts @@ -311,6 +311,7 @@ export class SyncManagerService { providerConfigKey: string, syncNames: string[], orchestrator: Orchestrator, + recordsService: RecordsServiceInterface, connectionId?: string, includeJobStatus = false, optionalConnection?: Connection | null @@ -333,7 +334,7 @@ export class SyncManagerService { continue; } - const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator }); + const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator, recordsService }); syncsWithStatus.push(reportedStatus); } @@ -348,7 +349,7 @@ export class SyncManagerService { } for (const sync of syncs) { - const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator }); + const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator, recordsService }); syncsWithStatus.push(reportedStatus); } @@ -417,13 +418,15 @@ export class SyncManagerService { environmentId, providerConfigKey, includeJobStatus, - orchestrator + orchestrator, + recordsService }: { sync: Sync; environmentId: number; providerConfigKey: string; includeJobStatus: boolean; orchestrator: Orchestrator; + recordsService: RecordsServiceInterface; }): Promise { const latestJob = await getLatestSyncJob(sync.id); const schedules = await orchestrator.searchSchedules([{ syncId: sync.id, environmentId }]); @@ -432,13 +435,27 @@ export class SyncManagerService { } const schedule = schedules.value.get(sync.id); let frequency = sync.frequency; + const syncConfig = await getSyncConfigByParams(environmentId, sync.name, providerConfigKey); if (!frequency) { - const syncConfig = await getSyncConfigByParams(environmentId, sync.name, providerConfigKey); frequency = syncConfig?.runs || null; } if (!schedule) { throw new Error(`Schedule for sync ${sync.id} and environment ${environmentId} not found`); } + + const countRes = await recordsService.getRecordCountsByModel({ connectionId: sync.nango_connection_id, environmentId }); + if (countRes.isErr()) { + throw new Error(`Failed to get records count for sync ${sync.id} in environment ${environmentId}: ${stringifyError(countRes.error)}`); + } + const recordCount: Record = + syncConfig?.models.reduce( + (acc, model) => { + acc[model] = countRes.isOk() ? countRes.value[model]?.count || 0 : 0; + return acc; + }, + {} as Record + ) || {}; + return { id: sync.id, type: latestJob?.type === SyncType.INCREMENTAL ? latestJob.type : 'INITIAL', @@ -449,8 +466,9 @@ export class SyncManagerService { frequency, latestResult: latestJob?.result, latestExecutionStatus: latestJob?.status, + recordCount, ...(includeJobStatus ? { jobStatus: latestJob?.status as SyncStatus } : {}) - } as ReportedSyncJobStatus; + }; } }