feat: return records count in /sync/status endpoint/sdk (#2961)

<img width="528" alt="Screenshot 2024-11-07 at 14 34 38" src="https://github.com/user-attachments/assets/825bb1de-dd9a-4a76-865a-77a2d745054f"> ## Issue ticket number and link https://linear.app/nango/issue/NAN-1928/return-sync-object-count-from-api ## Checklist before requesting a review (skip if just adding/editing APIs & templates) - [ ] I added tests, otherwise the reason is: - [ ] I added observability, otherwise the reason is: - [ ] I added analytics, otherwise the reason is:
NangoHQ · Nov 12, 2024 · cbb9e81 · cbb9e81
1 parent 8ea1142
commit cbb9e81
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 12 deletions.
diff --git a/docs-v2/reference/sdks/node.mdx b/docs-v2/reference/sdks/node.mdx
@@ -993,7 +993,17 @@ await nango.syncStatus('<INTEGRATION-ID>', ['SYNC_NAME1', 'SYNC_NAME2'], '<CONNE
             "finishedAt": "<string>",
             "nextScheduledSyncAt": "<string>",
             "frequency": "<string>",
-            "latestResult": {}
+            "latestResult": {
+                "<string>": {
+                    "added": <number>,
+                    "updated": <number>,
+                    "deleted": <number>,
+                }
+            },
+            "recordCount": {
+                "<string>": <number>
+                ...
+            }
         }
     ]
 }

diff --git a/docs-v2/spec.yaml b/docs-v2/spec.yaml
@@ -1275,6 +1275,9 @@ paths:
                                                 latestResult:
                                                     type: object
                                                     description: Additional information regarding the latest result of the sync. Contains a model name with added, updated and deleted records
+                                                recordCount:
+                                                    type: object
+                                                    description: Total count of records for each model synced by the sync
                 '400':
                     description: Invalid request
                     content:

diff --git a/packages/node-client/lib/types.ts b/packages/node-client/lib/types.ts
@@ -217,6 +217,7 @@ export interface SyncStatus {
     status: 'RUNNING' | 'SUCCESS' | 'ERROR' | 'PAUSED' | 'STOPPED';
     frequency: string;
     latestResult: Record<string, StatusAction>;
+    recordCount: Record<string, number>;
 }
 
 export interface StatusAction {

diff --git a/packages/server/lib/controllers/onboarding.controller.ts b/packages/server/lib/controllers/onboarding.controller.ts
@@ -244,7 +244,15 @@ class OnboardingController {
                 success,
                 error,
                 response: status
-            } = await syncManager.getSyncStatus(environment.id, DEMO_GITHUB_CONFIG_KEY, [DEMO_SYNC_NAME], orchestrator, req.body.connectionId, true);
+            } = await syncManager.getSyncStatus(
+                environment.id,
+                DEMO_GITHUB_CONFIG_KEY,
+                [DEMO_SYNC_NAME],
+                orchestrator,
+                recordsService,
+                req.body.connectionId,
+                true
+            );
 
             if (!success || !status) {
                 void analytics.track(AnalyticsTypes.DEMO_4_ERR, account.id, { user_id: user.id });

diff --git a/packages/server/lib/controllers/sync.controller.ts b/packages/server/lib/controllers/sync.controller.ts
@@ -523,6 +523,7 @@ class SyncController {
                 provider_config_key as string,
                 syncNames,
                 orchestrator,
+                recordsService,
                 connection_id as string,
                 false,
                 connection

diff --git a/packages/shared/lib/clients/orchestrator.ts b/packages/shared/lib/clients/orchestrator.ts
@@ -30,6 +30,7 @@ import { isSyncJobRunning, updateSyncJobStatus } from '../services/sync/job.serv
 import { getSyncConfigRaw, getSyncConfigBySyncId } from '../services/sync/config/config.service.js';
 import environmentService from '../services/environment.service.js';
 import type { DBEnvironment, DBTeam } from '@nangohq/types';
+import type { RecordCount } from '@nangohq/records';
 
 export interface RecordsServiceInterface {
     deleteRecordsBySyncId({
@@ -43,6 +44,7 @@ export interface RecordsServiceInterface {
         model: string;
         syncId: string;
     }): Promise<{ totalDeletedRecords: number }>;
+    getRecordCountsByModel({ connectionId, environmentId }: { connectionId: number; environmentId: number }): Promise<Result<Record<string, RecordCount>>>;
 }
 
 export interface OrchestratorClientInterface {

diff --git a/packages/shared/lib/models/Sync.ts b/packages/shared/lib/models/Sync.ts
@@ -59,15 +59,16 @@ export interface Job extends TimestampsAndDeleted {
 
 export interface ReportedSyncJobStatus {
     id?: string;
-    type: SyncType;
+    type: SyncType | 'INITIAL';
     name?: string;
     status: SyncStatus;
-    latestResult?: SyncResultByModel;
+    latestResult?: SyncResultByModel | undefined;
     jobStatus?: SyncStatus;
-    frequency: string;
-    finishedAt: Date;
+    frequency: string | null;
+    finishedAt: Date | undefined;
     nextScheduledSyncAt: Date | null;
-    latestExecutionStatus: SyncStatus;
+    latestExecutionStatus: SyncStatus | undefined;
+    recordCount: Record<string, number>;
 }
 
 // TODO: change that to use Parsed type

diff --git a/packages/shared/lib/services/sync/manager.service.ts b/packages/shared/lib/services/sync/manager.service.ts
@@ -311,6 +311,7 @@ export class SyncManagerService {
         providerConfigKey: string,
         syncNames: string[],
         orchestrator: Orchestrator,
+        recordsService: RecordsServiceInterface,
         connectionId?: string,
         includeJobStatus = false,
         optionalConnection?: Connection | null
@@ -333,7 +334,7 @@ export class SyncManagerService {
                     continue;
                 }
 
-                const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator });
+                const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator, recordsService });
 
                 syncsWithStatus.push(reportedStatus);
             }
@@ -348,7 +349,7 @@ export class SyncManagerService {
             }
 
             for (const sync of syncs) {
-                const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator });
+                const reportedStatus = await this.syncStatus({ sync, environmentId, providerConfigKey, includeJobStatus, orchestrator, recordsService });
 
                 syncsWithStatus.push(reportedStatus);
             }
@@ -417,13 +418,15 @@ export class SyncManagerService {
         environmentId,
         providerConfigKey,
         includeJobStatus,
-        orchestrator
+        orchestrator,
+        recordsService
     }: {
         sync: Sync;
         environmentId: number;
         providerConfigKey: string;
         includeJobStatus: boolean;
         orchestrator: Orchestrator;
+        recordsService: RecordsServiceInterface;
     }): Promise<ReportedSyncJobStatus> {
         const latestJob = await getLatestSyncJob(sync.id);
         const schedules = await orchestrator.searchSchedules([{ syncId: sync.id, environmentId }]);
@@ -432,13 +435,27 @@ export class SyncManagerService {
         }
         const schedule = schedules.value.get(sync.id);
         let frequency = sync.frequency;
+        const syncConfig = await getSyncConfigByParams(environmentId, sync.name, providerConfigKey);
         if (!frequency) {
-            const syncConfig = await getSyncConfigByParams(environmentId, sync.name, providerConfigKey);
             frequency = syncConfig?.runs || null;
         }
         if (!schedule) {
             throw new Error(`Schedule for sync ${sync.id} and environment ${environmentId} not found`);
         }
+
+        const countRes = await recordsService.getRecordCountsByModel({ connectionId: sync.nango_connection_id, environmentId });
+        if (countRes.isErr()) {
+            throw new Error(`Failed to get records count for sync ${sync.id} in environment ${environmentId}: ${stringifyError(countRes.error)}`);
+        }
+        const recordCount: Record<string, number> =
+            syncConfig?.models.reduce(
+                (acc, model) => {
+                    acc[model] = countRes.isOk() ? countRes.value[model]?.count || 0 : 0;
+                    return acc;
+                },
+                {} as Record<string, number>
+            ) || {};
+
         return {
             id: sync.id,
             type: latestJob?.type === SyncType.INCREMENTAL ? latestJob.type : 'INITIAL',
@@ -449,8 +466,9 @@ export class SyncManagerService {
             frequency,
             latestResult: latestJob?.result,
             latestExecutionStatus: latestJob?.status,
+            recordCount,
             ...(includeJobStatus ? { jobStatus: latestJob?.status as SyncStatus } : {})
-        } as ReportedSyncJobStatus;
+        };
     }
 }