From c6816da956b4af0c58495f195770178f4bb9f55f Mon Sep 17 00:00:00 2001 From: Andrew Henry Date: Thu, 6 Jul 2023 17:06:18 -0700 Subject: [PATCH 1/2] Implement persistence batching for Couch DB --- .../persistence/couch/CouchObjectProvider.js | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/src/plugins/persistence/couch/CouchObjectProvider.js b/src/plugins/persistence/couch/CouchObjectProvider.js index 8982d97bd84..ada7c804851 100644 --- a/src/plugins/persistence/couch/CouchObjectProvider.js +++ b/src/plugins/persistence/couch/CouchObjectProvider.js @@ -24,6 +24,7 @@ import CouchDocument from './CouchDocument'; import CouchObjectQueue from './CouchObjectQueue'; import { PENDING, CONNECTED, DISCONNECTED, UNKNOWN } from './CouchStatusIndicator'; import { isNotebookOrAnnotationType } from '../../notebook/notebook-constants.js'; +import _ from 'lodash'; const REV = '_rev'; const ID = '_id'; @@ -42,6 +43,8 @@ class CouchObjectProvider { this.batchIds = []; this.onEventMessage = this.onEventMessage.bind(this); this.onEventError = this.onEventError.bind(this); + this.flushPersistenceQueue = _.debounce(this.flushPersistenceQueue.bind(this)); + this.persistenceQueue = []; } /** @@ -668,9 +671,12 @@ class CouchObjectProvider { if (!this.objectQueue[key].pending) { this.objectQueue[key].pending = true; const queued = this.objectQueue[key].dequeue(); - let document = new CouchDocument(key, queued.model); - document.metadata.created = Date.now(); - this.request(key, 'PUT', document) + let couchDocument = new CouchDocument(key, queued.model); + couchDocument.metadata.created = Date.now(); + this.#enqueueForPersistence({ + key, + document: couchDocument + }) .then((response) => { this.#checkResponse(response, queued.intermediateResponse, key); }) @@ -683,6 +689,42 @@ class CouchObjectProvider { return intermediateResponse.promise; } + #enqueueForPersistence({ key, document }) { + return new Promise((resolve, reject) => { + this.persistenceQueue.push({ + key, + document, + resolve, + reject + }); + this.flushPersistenceQueue(); + }); + } + + async flushPersistenceQueue() { + if (this.persistenceQueue.length > 1) { + const batch = { + docs: this.persistenceQueue.map((queued) => queued.document) + }; + const response = await this.request('_bulk_docs', 'POST', batch); + response.forEach((responseMetadatum) => { + const queued = this.persistenceQueue.find( + (queuedMetadatum) => queuedMetadatum.key === responseMetadatum.id + ); + if (responseMetadatum.ok) { + queued.resolve(responseMetadatum); + } else { + queued.reject(responseMetadatum); + } + }); + } else if (this.persistenceQueue.length === 1) { + const { key, document, resolve, reject } = this.persistenceQueue[0]; + + this.request(key, 'PUT', document).then(resolve).catch(reject); + } + this.persistenceQueue = []; + } + /** * @private */ From 447d9fadbca2e94de318755dab77f60672a411a5 Mon Sep 17 00:00:00 2001 From: Andrew Henry Date: Thu, 6 Jul 2023 17:06:28 -0700 Subject: [PATCH 2/2] Add tests for persistence batching --- src/plugins/persistence/couch/pluginSpec.js | 129 ++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/src/plugins/persistence/couch/pluginSpec.js b/src/plugins/persistence/couch/pluginSpec.js index cd522f71fd6..39893372f45 100644 --- a/src/plugins/persistence/couch/pluginSpec.js +++ b/src/plugins/persistence/couch/pluginSpec.js @@ -243,6 +243,135 @@ describe('the plugin', () => { expect(requestMethod).toEqual('GET'); }); }); + describe('batches persistence', () => { + let successfulMockPromise; + let partialFailureMockPromise; + let objectsToPersist; + + beforeEach(() => { + successfulMockPromise = Promise.resolve({ + json: () => { + return [ + { + id: 'object-1', + ok: true + }, + { + id: 'object-2', + ok: true + }, + { + id: 'object-3', + ok: true + } + ]; + } + }); + + partialFailureMockPromise = Promise.resolve({ + json: () => { + return [ + { + id: 'object-1', + ok: true + }, + { + id: 'object-2', + ok: false + }, + { + id: 'object-3', + ok: true + } + ]; + } + }); + + objectsToPersist = [ + { + identifier: { + namespace: '', + key: 'object-1' + }, + name: 'object-1', + type: 'folder', + modified: 0 + }, + { + identifier: { + namespace: '', + key: 'object-2' + }, + name: 'object-2', + type: 'folder', + modified: 0 + }, + { + identifier: { + namespace: '', + key: 'object-3' + }, + name: 'object-3', + type: 'folder', + modified: 0 + } + ]; + }); + it('for multiple simultaneous successful saves', async () => { + fetch.and.returnValue(successfulMockPromise); + + await Promise.all( + objectsToPersist.map((objectToPersist) => openmct.objects.save(objectToPersist)) + ); + + const requestUrl = fetch.calls.mostRecent().args[0]; + const requestMethod = fetch.calls.mostRecent().args[1].method; + const requestBody = JSON.parse(fetch.calls.mostRecent().args[1].body); + + expect(fetch).toHaveBeenCalledTimes(1); + expect(requestUrl.includes('_bulk_docs')).toBeTrue(); + expect(requestMethod).toEqual('POST'); + expect( + objectsToPersist.every( + (object, index) => object.identifier.key === requestBody.docs[index]._id + ) + ).toBeTrue(); + }); + it('for multiple simultaneous saves with partial failure', async () => { + fetch.and.returnValue(partialFailureMockPromise); + + let saveResults = await Promise.all( + objectsToPersist.map((objectToPersist) => + openmct.objects + .save(objectToPersist) + .then(() => true) + .catch(() => false) + ) + ); + expect(saveResults[0]).toBeTrue(); + expect(saveResults[1]).toBeFalse(); + expect(saveResults[2]).toBeTrue(); + }); + it('except for a single save', async () => { + fetch.and.returnValue({ + json: () => { + return { + id: 'object-1', + ok: true + }; + } + }); + await openmct.objects.save(objectsToPersist[0]); + + const requestUrl = fetch.calls.mostRecent().args[0]; + const requestMethod = fetch.calls.mostRecent().args[1].method; + + expect(fetch).toHaveBeenCalledTimes(1); + expect(requestUrl.includes('_bulk_docs')).toBeFalse(); + expect(requestUrl.endsWith('object-1')).toBeTrue(); + expect(requestMethod).toEqual('PUT'); + }); + }); describe('implements server-side search', () => { let mockPromise; beforeEach(() => {