[Github #1022] dryrun log messages (#1187)

* [gh-#1022] display log messages at the end and prompt to show more

* [gh-#1022] [gh-#1020] allow metadata to be stubbed in a dryRun

packages/cli/lib/index.ts

@@ -100,6 +100,7 @@ program
     .option('-e [environment]', 'The Nango environment, defaults to dev.', 'dev')
     .option('-l, --lastSyncDate [lastSyncDate]', 'Optional (for syncs only): last sync date to retrieve records greater than this date')
     .option('-i, --input [input]', 'Optional (for actions only): input to pass to the action script')
+    .option('-m, --metadata [metadata]', 'Optional (for syncs only): metadata to stub for the sync script')
     .action(async function (this: Command, sync: string, connectionId: string) {
         const { autoConfirm, debug, e: environment } = this.opts();
         await verifyNecessaryFiles(autoConfirm, debug);

packages/cli/lib/sync.ts

@@ -24,6 +24,7 @@ import type {
     NangoIntegration,
     NangoIntegrationData,
     SimplifiedNangoIntegration,
+    Metadata,
     NangoConfigMetadata
 } from '@nangohq/shared';
 import {
@@ -67,6 +68,7 @@ interface RunArgs extends GlobalOptions {
     lastSyncDate?: string;
     useServerLastSyncDate?: boolean;
     input?: object;
+    metadata?: Metadata;
 }
 
 const exampleSyncName = 'github-issue-example';
@@ -577,7 +579,7 @@ export const adminDeploy = async (environmentName: string, debug = false) => {
 
 export const dryRun = async (options: RunArgs, environment: string, debug = false) => {
     let syncName = '';
-    let connectionId, suppliedLastSyncDate, actionInput;
+    let connectionId, suppliedLastSyncDate, actionInput, stubbedMetadata;
 
     await parseSecretKey(environment, debug);
 
@@ -593,7 +595,7 @@ export const dryRun = async (options: RunArgs, environment: string, debug = fals
     }
 
     if (Object.keys(options).length > 0) {
-        ({ sync: syncName, connectionId, lastSyncDate: suppliedLastSyncDate, input: actionInput } = options);
+        ({ sync: syncName, connectionId, lastSyncDate: suppliedLastSyncDate, input: actionInput, metadata: stubbedMetadata } = options);
     }
 
     if (!syncName) {
@@ -676,6 +678,8 @@ export const dryRun = async (options: RunArgs, environment: string, debug = fals
         normalizedInput = actionInput;
     }
 
+    const logMessages: string[] = [];
+
     const syncRun = new syncRunService({
         integrationService,
         writeToDb: false,
@@ -688,7 +692,9 @@ export const dryRun = async (options: RunArgs, environment: string, debug = fals
         syncName,
         syncType: SyncType.INITIAL,
         loadLocation: './',
-        debug
+        debug,
+        logMessages,
+        stubbedMetadata
     });
 
     try {
@@ -698,6 +704,36 @@ export const dryRun = async (options: RunArgs, environment: string, debug = fals
         if (results) {
             console.log(JSON.stringify(results, null, 2));
         }
+
+        if (syncRun.logMessages && syncRun.logMessages.length > 0) {
+            const logMessages = syncRun.logMessages as unknown[];
+            let index = 0;
+            const batchCount = 10;
+
+            const displayBatch = () => {
+                for (let i = 0; i < batchCount && index < logMessages.length; i++, index++) {
+                    const logs = logMessages[index];
+                    console.log(chalk.yellow(JSON.stringify(logs, null, 2)));
+                }
+            };
+
+            console.log(chalk.yellow('The following log messages were generated:'));
+
+            displayBatch();
+
+            while (index < syncRun.logMessages.length) {
+                const remaining = syncRun.logMessages.length - index;
+                const confirmation = await promptly.confirm(
+                    `There are ${remaining} logs messages remaining. Would you like to see the next 10 log messages? (y/n)`
+                );
+                if (confirmation) {
+                    displayBatch();
+                } else {
+                    break;
+                }
+            }
+        }
+
         process.exit(0);
     } catch (e) {
         process.exit(1);

packages/node-client/lib/types.ts

@@ -79,7 +79,7 @@ export interface ApiKeyCredentials {
 type AuthCredentials = OAuth2Credentials | OAuth1Credentials | BasicApiCredentials | ApiKeyCredentials;
 
 export interface Metadata {
-    [key: string]: string | Record<string, string>;
+    [key: string]: string | Record<string, any>;
 }
 
 export interface Connection {

packages/shared/lib/models/Connection.ts

@@ -2,8 +2,7 @@ import type { AuthCredentials, ApiKeyCredentials, BasicApiCredentials, AppCreden
 import type { TimestampsAndDeleted } from './Generic.js';
 
 export interface Metadata {
-    fieldMapping: Record<string, string>;
-    [key: string]: string | Record<string, string>;
+    [key: string]: string | Record<string, any>;
 }
 
 export interface BaseConnection extends TimestampsAndDeleted {

packages/shared/lib/sdk/sync.ts

@@ -151,7 +151,7 @@ interface OAuth1Credentials extends CredentialsCommon {
 type AuthCredentials = OAuth2Credentials | OAuth1Credentials | BasicApiCredentials | ApiKeyCredentials | AppCredentials;
 
 interface Metadata {
-    [key: string]: string | Record<string, string>;
+    [key: string]: string | Record<string, any>;
 }
 
 interface Connection {
@@ -182,6 +182,9 @@ interface NangoProps {
     dryRun?: boolean;
     track_deletes?: boolean;
     attributes?: object | undefined;
+
+    logMessages?: unknown[] | undefined;
+    stubbedMetadata?: Metadata | undefined;
 }
 
 interface UserLogParameters {
@@ -395,6 +398,8 @@ export class NangoAction {
 export class NangoSync extends NangoAction {
     lastSyncDate?: Date;
     track_deletes = false;
+    logMessages?: unknown[] | undefined = [];
+    stubbedMetadata?: Metadata | undefined = {};
 
     constructor(config: NangoProps) {
         super(config);
@@ -406,6 +411,14 @@ export class NangoSync extends NangoAction {
         if (config.track_deletes) {
             this.track_deletes = config.track_deletes;
         }
+
+        if (config.logMessages) {
+            this.logMessages = config.logMessages;
+        }
+
+        if (config.stubbedMetadata) {
+            this.stubbedMetadata = config.stubbedMetadata;
+        }
     }
 
     /**
@@ -471,8 +484,8 @@ export class NangoSync extends NangoAction {
         }
 
         if (this.dryRun) {
-            console.log('A batch save call would save following data:');
-            console.log(JSON.stringify(results, null, 2));
+            this.logMessages?.push(`A batch save call would delete the following data`);
+            this.logMessages?.push(...results);
             return null;
         }
 
@@ -586,8 +599,8 @@ export class NangoSync extends NangoAction {
         }
 
         if (this.dryRun) {
-            console.log('A batch delete call would delete the following data:');
-            console.log(JSON.stringify(results, null, 2));
+            this.logMessages?.push(`A batch delete call would delete the following data`);
+            this.logMessages?.push(...results);
             return null;
         }
 
@@ -659,4 +672,11 @@ export class NangoSync extends NangoAction {
             throw new Error(responseResults?.error);
         }
     }
+    public override async getMetadata<T = Metadata>(): Promise<T> {
+        if (this.dryRun && this.stubbedMetadata) {
+            return this.stubbedMetadata as T;
+        }
+
+        return super.getMetadata();
+    }
 }

packages/shared/lib/services/sync/run.service.ts

@@ -21,6 +21,7 @@ import type { NangoIntegrationData, NangoIntegration } from '../../integrations/
 import type { UpsertResponse, UpsertSummary } from '../../models/Data.js';
 import { LogActionEnum } from '../../models/Activity.js';
 import type { Environment } from '../../models/Environment';
+import type { Metadata } from '../../models/Connection';
 
 interface SyncRunConfig {
     integrationService: IntegrationServiceInterface;
@@ -38,6 +39,9 @@ interface SyncRunConfig {
     loadLocation?: string;
     debug?: boolean;
     input?: object;
+
+    logMessages?: unknown[] | undefined;
+    stubbedMetadata?: Metadata | undefined;
 }
 
 export default class SyncRun {
@@ -56,6 +60,9 @@ export default class SyncRun {
     debug?: boolean;
     input?: object;
 
+    logMessages?: unknown[] | undefined = [];
+    stubbedMetadata?: Metadata | undefined = {};
+
     constructor(config: SyncRunConfig) {
         this.integrationService = config.integrationService;
         this.writeToDb = config.writeToDb;
@@ -91,6 +98,14 @@ export default class SyncRun {
         if (config.provider) {
             this.provider = config.provider;
         }
+
+        if (config.logMessages) {
+            this.logMessages = config.logMessages;
+        }
+
+        if (config.stubbedMetadata) {
+            this.stubbedMetadata = config.stubbedMetadata;
+        }
     }
 
     async run(
@@ -239,7 +254,9 @@ export default class SyncRun {
                 lastSyncDate: lastSyncDate as Date,
                 dryRun: !this.writeToDb,
                 attributes: syncData.attributes,
-                track_deletes: trackDeletes as boolean
+                track_deletes: trackDeletes as boolean,
+                logMessages: this.logMessages,
+                stubbedMetadata: this.stubbedMetadata
             });
 
             if (this.debug) {