Adding a different concurrency count for snapshot processing for Cs (#…

…1814)

* updated snapshot concurrency

* Removing the build script

* lint fix

* Fixing few cases

* 🔖 v1.31.0-alpha.0

* 🔖 v1.31.0-alpha.0 lerna.json

* Adding Error Capturing on Finalize Builds (#1818)

* Adding Error Capturing on Finalize Builds

* 🔖 v1.31.0-alpha.1

* 🔖 v1.31.0-alpha.2

* Adding prodBuild param for scanner alt env (#1830)

* Adding prodBuild param for scanner alt env

* renaming config

---------

Co-authored-by: rishigupta1599 <114384996+rishigupta1599@users.noreply.github.com>

* 🔖 v1.31.0-alpha.0 lerna.json

* Removing unwanted files

* 🔖 v1.31.0-alpha.3

* Add tests + fix concurrency

* Undo package json changes

---------

Co-authored-by: amit3200 <tusharamit@yahoo.com>
Co-authored-by: rishigupta1599 <114384996+rishigupta1599@users.noreply.github.com>
Co-authored-by: rishigupta1599 <rishi@browserstack.com>
Co-authored-by: Chinmay Maheshwari <chinmay.m@browserstack.com>

packages/cli-build/src/finalize.js

@@ -20,8 +20,11 @@ export const finalize = command('finalize', {
 
   // rely on the parallel nonce to cause the API to return the current running build for the nonce
   let { data: build } = await percy.client.createBuild({ cliStartTime: percy.cliStartTime });
-  await percy.client.finalizeBuild(build.id, { all: true });
-
+  try {
+    await percy.client.finalizeBuild(build.id, { all: true });
+  } catch (error) {
+    exit(1, 'Percy build failed during finalize', error.message);
+  }
   let { 'build-number': number, 'web-url': url } = build.attributes;
   log.info(`Finalized build #${number}: ${url}`);
 });

packages/cli-build/test/finalize.test.js

@@ -1,4 +1,5 @@
 import { logger, setupTest } from '@percy/cli-command/test/helpers';
+import api from '@percy/client/test/helpers';
 import { finalize } from '@percy/cli-build';
 
 describe('percy build:finalize', () => {
@@ -50,4 +51,13 @@ describe('percy build:finalize', () => {
       '[percy] Finalized build #1: https://percy.io/test/test/123'
     ]);
   });
+
+  it('should reject promise if finalize fails', async () => {
+    process.env.PERCY_TOKEN = '<<PERCY_TOKEN>>';
+    api.reply('/builds/123/finalize?all-shards=true', () => [500, new Error('Failed')]);
+
+    await expectAsync(finalize()).toBeRejected();
+
+    expect(logger.stderr).toEqual(['[percy] Error: Percy build failed during finalize']);
+  });
 });

packages/client/src/client.js

@@ -172,7 +172,8 @@ export class PercyClient {
           partial: this.env.partial,
           tags: tagsArr,
           'cli-start-time': cliStartTime,
-          source: source
+          source: source,
+          'skip-base-build': this.config.percy?.skipBaseBuild
         },
         relationships: {
           resources: {
@@ -367,6 +368,7 @@ export class PercyClient {
     validateId('build', buildId);
     this.log.debug(`Uploading resources for ${buildId}...`, meta);
 
+    const uploadConcurrency = parseInt(process.env.PERCY_RESOURCE_UPLOAD_CONCURRENCY) || 2;
     return pool(function*() {
       for (let resource of resources) {
         let resourceMeta = {
@@ -377,7 +379,7 @@ export class PercyClient {
         yield this.uploadResource(buildId, resource, resourceMeta);
         this.log.debug(`Uploaded resource ${resource.url}`, resourceMeta);
       }
-    }, this, 2);
+    }, this, uploadConcurrency);
   }
 
   // Creates a snapshot for the active build using the provided attributes.

packages/client/test/client.test.js

@@ -150,6 +150,10 @@ describe('PercyClient', () => {
 
   describe('#createBuild()', () => {
     let cliStartTime = new Date().toISOString();
+    beforeEach(() => {
+      delete process.env.PERCY_AUTO_ENABLED_GROUP_BUILD;
+    });
+
     it('creates a new build', async () => {
       await expectAsync(client.createBuild()).toBeResolvedTo({
         data: {
@@ -384,6 +388,47 @@ describe('PercyClient', () => {
           }
         }));
     });
+
+    it('creates a new build with skipBaseBuild config', async () => {
+      client = new PercyClient({
+        token: 'PERCY_TOKEN',
+        config: { percy: { skipBaseBuild: true } }
+      });
+      await expectAsync(client.createBuild({ projectType: 'web' })).toBeResolvedTo({
+        data: {
+          id: '123',
+          attributes: {
+            'build-number': 1,
+            'web-url': 'https://percy.io/test/test/123'
+          }
+        }
+      });
+
+      expect(api.requests['/builds'][0].body.data)
+        .toEqual(jasmine.objectContaining({
+          attributes: {
+            branch: client.env.git.branch,
+            type: 'web',
+            'target-branch': client.env.target.branch,
+            'target-commit-sha': client.env.target.commit,
+            'commit-sha': client.env.git.sha,
+            'commit-committed-at': client.env.git.committedAt,
+            'commit-author-name': client.env.git.authorName,
+            'commit-author-email': client.env.git.authorEmail,
+            'commit-committer-name': client.env.git.committerName,
+            'commit-committer-email': client.env.git.committerEmail,
+            'commit-message': client.env.git.message,
+            'pull-request-number': client.env.pullRequest,
+            'parallel-nonce': client.env.parallel.nonce,
+            'parallel-total-shards': client.env.parallel.total,
+            'cli-start-time': null,
+            source: 'user_created',
+            partial: client.env.partial,
+            'skip-base-build': true,
+            tags: []
+          }
+        }));
+    });
   });
 
   describe('#getBuild()', () => {

packages/core/src/config.js

@@ -16,6 +16,10 @@ export const configSchema = {
       },
       labels: {
         type: 'string'
+      },
+      skipBaseBuild: {
+        type: 'boolean',
+        default: false
       }
     }
   },
@@ -259,6 +263,10 @@ export const configSchema = {
         type: 'integer',
         minimum: 1
       },
+      snapshotConcurrency: {
+        type: 'integer',
+        minimum: 1
+      },
       retry: {
         type: 'boolean',
         default: false

packages/core/src/discovery.js

@@ -441,7 +441,12 @@ export function createDiscoveryQueue(percy) {
                 }
                 return resource;
               },
-              saveResource: r => { snapshot.resources.set(r.url, r); cache.set(r.url, r); }
+              saveResource: r => {
+                snapshot.resources.set(r.url, r);
+                if (!snapshot.discovery.disableCache) {
+                  cache.set(r.url, r);
+                }
+              }
             }
           });
 

packages/core/src/percy.js

@@ -154,11 +154,10 @@ export class Percy {
       // replace arrays instead of merging
       return Array.isArray(next) && [path, next];
     });
-
-    // adjust queue concurrency
-    let { concurrency } = this.config.discovery;
+    const concurrency = this.config.discovery.concurrency;
+    const snapshotConcurrency = parseInt(process.env.PERCY_SNAPSHOT_UPLOAD_CONCURRENCY) || concurrency;
     this.#discovery.set({ concurrency });
-    this.#snapshots.set({ concurrency });
+    this.#snapshots.set({ concurrency: snapshotConcurrency });
 
     return this.config;
   }