From a5296a6dcaf02840ae9914c2a22090f0c6da017a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 23 Nov 2022 16:51:36 +0000 Subject: [PATCH] feat: support pagination (#204) Closes #201 --- packages/upload-client/package.json | 23 ++-- packages/upload-client/src/store.js | 8 +- packages/upload-client/src/types.ts | 31 ++++-- packages/upload-client/src/upload.js | 8 +- packages/upload-client/test/store.test.js | 113 +++++++++++++++++-- packages/upload-client/test/upload.test.js | 123 ++++++++++++++++++--- 6 files changed, 258 insertions(+), 48 deletions(-) diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 960be52d5..d8f377b84 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -101,20 +101,21 @@ "project": "./tsconfig.json" }, "rules": { - "unicorn/prefer-number-properties": "off", - "unicorn/no-null": "off", - "unicorn/prefer-set-has": "off", - "unicorn/no-array-for-each": "off", - "unicorn/prefer-export-from": "off", - "unicorn/catch-error-name": "off", - "unicorn/explicit-length-check": "off", - "unicorn/prefer-type-error": "off", "eqeqeq": "off", - "no-void": "off", + "jsdoc/check-indentation": "off", + "jsdoc/require-hyphen-before-param-description": "off", "no-console": "off", "no-continue": "off", - "jsdoc/check-indentation": "off", - "jsdoc/require-hyphen-before-param-description": "off" + "no-void": "off", + "unicorn/catch-error-name": "off", + "unicorn/explicit-length-check": "off", + "unicorn/no-array-for-each": "off", + "unicorn/no-await-expression-member": "off", + "unicorn/no-null": "off", + "unicorn/prefer-export-from": "off", + "unicorn/prefer-number-properties": "off", + "unicorn/prefer-set-has": "off", + "unicorn/prefer-type-error": "off" }, "env": { "mocha": true diff --git a/packages/upload-client/src/store.js b/packages/upload-client/src/store.js index c3dbd752c..f6729b72b 100644 --- a/packages/upload-client/src/store.js +++ b/packages/upload-client/src/store.js @@ -116,7 +116,8 @@ export async function add( * has the capability to perform the action. * * The issuer needs the `store/list` delegated capability. - * @param {import('./types').RequestOptions} [options] + * @param {import('./types').ListRequestOptions} [options] + * @returns {Promise>} */ export async function list( { issuer, with: resource, proofs, audience = servicePrincipal }, @@ -130,7 +131,10 @@ export async function list( audience, with: resource, proofs, - nb: {}, + nb: { + cursor: options.cursor, + size: options.size, + }, }) .execute(conn) diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 247dbe921..5ee58b14c 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -47,22 +47,23 @@ export interface StoreAddResponse { } export interface ListResponse { - count: number - page: number - pageSize: number + cursor?: string + size: number results?: R[] } export interface StoreListResult { - payloadCID: CARLink + payloadCID: string + origin?: string size: number - uploadedAt: number + uploadedAt: string } export interface UploadListResult { - carCID: CARLink - dataCID: Link - uploadedAt: number + uploaderDID: string + dataCID: string + carCID: string + uploadedAt: string } export interface InvocationConfig { @@ -150,8 +151,22 @@ export interface Connectable { connection?: ConnectionView } +export interface Pageable { + /** + * Opaque string specifying where to start retrival of the next page of + * results. + */ + cursor?: string + /** + * Maximum number of items to return. + */ + size?: number +} + export interface RequestOptions extends Retryable, Abortable, Connectable {} +export interface ListRequestOptions extends RequestOptions, Pageable {} + export interface ShardingOptions { /** * The target shard size. Actual size of CAR output may be bigger due to CAR diff --git a/packages/upload-client/src/upload.js b/packages/upload-client/src/upload.js index c1f1ddcdc..af03d9fb5 100644 --- a/packages/upload-client/src/upload.js +++ b/packages/upload-client/src/upload.js @@ -75,7 +75,8 @@ export async function add( * has the capability to perform the action. * * The issuer needs the `upload/list` delegated capability. - * @param {import('./types').RequestOptions} [options] + * @param {import('./types').ListRequestOptions} [options] + * @returns {Promise>} */ export async function list( { issuer, with: resource, proofs, audience = servicePrincipal }, @@ -90,7 +91,10 @@ export async function list( audience, with: resource, proofs, - nb: {}, + nb: { + cursor: options.cursor, + size: options.size, + }, }) .execute(conn) diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js index c5651a5ee..b8eb190d5 100644 --- a/packages/upload-client/test/store.test.js +++ b/packages/upload-client/test/store.test.js @@ -290,14 +290,13 @@ describe('Store.list', () => { it('lists stored CAR files', async () => { const car = await randomCAR(128) const res = { - page: 1, - pageSize: 1000, - count: 1, + cursor: 'test', + size: 1000, results: [ { - payloadCID: car.cid, + payloadCID: car.cid.toString(), size: 123, - uploadedAt: Date.now(), + uploadedAt: new Date().toISOString(), }, ], } @@ -348,21 +347,111 @@ describe('Store.list', () => { assert(service.store.list.called) assert.equal(service.store.list.callCount, 1) - assert.equal(list.count, res.count) - assert.equal(list.page, res.page) - assert.equal(list.pageSize, res.pageSize) + assert.equal(list.cursor, res.cursor) + assert.equal(list.size, res.size) assert(list.results) assert.equal(list.results.length, res.results.length) list.results.forEach((r, i) => { - assert.equal( - r.payloadCID.toString(), - res.results[i].payloadCID.toString() - ) + assert.equal(r.payloadCID, res.results[i].payloadCID) assert.equal(r.size, res.results[i].size) assert.equal(r.uploadedAt, res.results[i].uploadedAt) }) }) + it('paginates', async () => { + const cursor = 'test' + const page0 = { + cursor, + size: 1, + results: [ + { + payloadCID: (await randomCAR(128)).cid.toString(), + size: 123, + uploadedAt: new Date().toISOString(), + }, + ], + } + const page1 = { + size: 1, + results: [ + { + payloadCID: (await randomCAR(128)).cid.toString(), + size: 123, + uploadedAt: new Date().toISOString(), + }, + ], + } + + const space = await Signer.generate() + const agent = await Signer.generate() + + const proofs = [ + await StoreCapabilities.list.delegate({ + issuer: space, + audience: agent, + with: space.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + store: { + list: provide(StoreCapabilities.list, ({ invocation }) => { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, StoreCapabilities.list.can) + assert.equal(invCap.with, space.did()) + assert.equal(invCap.nb?.size, 1) + return invCap.nb?.cursor === cursor ? page1 : page0 + }), + }, + }) + + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) + const connection = Client.connect({ + id: serviceSigner, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + const results0 = await Store.list( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + { size: 1, connection } + ) + const results1 = await Store.list( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + { size: 1, cursor: results0.cursor, connection } + ) + + assert(service.store.list.called) + assert.equal(service.store.list.callCount, 2) + + assert.equal(results0.cursor, cursor) + assert(results0.results) + assert.equal(results0.results.length, page0.results.length) + results0.results.forEach((r, i) => { + assert.equal(r.payloadCID, page0.results[i].payloadCID) + assert.equal(r.size, page0.results[i].size) + assert.equal(r.uploadedAt, page0.results[i].uploadedAt) + }) + + assert(results1.results) + assert.equal(results1.cursor, undefined) + assert.equal(results1.results.length, page1.results.length) + results1.results.forEach((r, i) => { + assert.equal(r.payloadCID, page1.results[i].payloadCID) + assert.equal(r.size, page1.results[i].size) + assert.equal(r.uploadedAt, page1.results[i].uploadedAt) + }) + }) + it('throws on service error', async () => { const space = await Signer.generate() const agent = await Signer.generate() diff --git a/packages/upload-client/test/upload.test.js b/packages/upload-client/test/upload.test.js index bf6bea5d6..7d52fabd4 100644 --- a/packages/upload-client/test/upload.test.js +++ b/packages/upload-client/test/upload.test.js @@ -118,23 +118,22 @@ describe('Upload.add', () => { describe('Upload.list', () => { it('lists uploads', async () => { + const space = await Signer.generate() + const agent = await Signer.generate() + const car = await randomCAR(128) const res = { - page: 1, - pageSize: 1000, - count: 1, + cursor: 'test', + size: 1000, results: [ { - carCID: car.cid, - dataCID: car.roots[0], - uploadedAt: Date.now(), + uploaderDID: agent.did(), + carCID: car.cid.toString(), + dataCID: car.roots[0].toString(), + uploadedAt: new Date().toISOString(), }, ], } - - const space = await Signer.generate() - const agent = await Signer.generate() - const proofs = [ await UploadCapabilities.list.delegate({ issuer: space, @@ -178,9 +177,8 @@ describe('Upload.list', () => { assert(service.upload.list.called) assert.equal(service.upload.list.callCount, 1) - assert.equal(list.count, res.count) - assert.equal(list.page, res.page) - assert.equal(list.pageSize, res.pageSize) + assert.equal(list.cursor, res.cursor) + assert.equal(list.size, res.size) assert(list.results) assert.equal(list.results.length, res.results.length) list.results.forEach((r, i) => { @@ -190,6 +188,105 @@ describe('Upload.list', () => { }) }) + it('paginates', async () => { + const space = await Signer.generate() + const agent = await Signer.generate() + + const cursor = 'test' + const car0 = await randomCAR(128) + const page0 = { + cursor, + size: 1, + results: [ + { + uploaderDID: agent.did(), + carCID: car0.cid.toString(), + dataCID: car0.roots[0].toString(), + uploadedAt: new Date().toISOString(), + }, + ], + } + const car1 = await randomCAR(128) + const page1 = { + size: 1, + results: [ + { + uploaderDID: agent.did(), + carCID: car1.cid.toString(), + dataCID: car1.roots[0].toString(), + uploadedAt: new Date().toISOString(), + }, + ], + } + const proofs = [ + await UploadCapabilities.list.delegate({ + issuer: space, + audience: agent, + with: space.did(), + expiration: Infinity, + }), + ] + + const service = mockService({ + upload: { + list: provide(UploadCapabilities.list, ({ invocation }) => { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.equal(invCap.can, UploadCapabilities.list.can) + assert.equal(invCap.with, space.did()) + assert.equal(invCap.nb?.size, 1) + return invCap.nb?.cursor === cursor ? page1 : page0 + }), + }, + }) + + const server = Server.create({ + id: serviceSigner, + service, + decoder: CAR, + encoder: CBOR, + }) + const connection = Client.connect({ + id: serviceSigner, + encoder: CAR, + decoder: CBOR, + channel: server, + }) + + const results0 = await Upload.list( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + { size: 1, connection } + ) + const results1 = await Upload.list( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + { size: 1, cursor: results0.cursor, connection } + ) + + assert(service.upload.list.called) + assert.equal(service.upload.list.callCount, 2) + + assert.equal(results0.cursor, page0.cursor) + assert.equal(results0.size, page0.size) + assert(results0.results) + assert.equal(results0.results.length, page0.results.length) + results0.results.forEach((r, i) => { + assert.equal(r.carCID.toString(), page0.results[i].carCID.toString()) + assert.equal(r.dataCID.toString(), page0.results[i].dataCID.toString()) + assert.equal(r.uploadedAt, page0.results[i].uploadedAt) + }) + + assert.equal(results1.cursor, undefined) + assert.equal(results1.size, page1.size) + assert(results1.results) + assert.equal(results1.results.length, page1.results.length) + results1.results.forEach((r, i) => { + assert.equal(r.carCID.toString(), page1.results[i].carCID.toString()) + assert.equal(r.dataCID.toString(), page1.results[i].dataCID.toString()) + assert.equal(r.uploadedAt, page1.results[i].uploadedAt) + }) + }) + it('throws on service error', async () => { const space = await Signer.generate() const agent = await Signer.generate()