feat: support pre caveat in store/list and upload/list (#139)

Implement support for the `pre` caveat introduced in storacha/w3up#423 Also introduce `startCursor` and `endCursor` with the intention to deprecate `cursor`. By providing cursors for the beginning and end of the range, we enable clients to page forward or backward without saving state anywhere. Example usage: Given uploads like: ```bash $ w3 ls bafybeia7tr4dgyln7zeyyyzmkppkcts6azdssykuluwzmmswysieyadcbm bafybeibvbxjeodaa6hdqlgbwmv4qzdp3bxnwdoukay4dpl7aemkiwc2eje bafybeid2fbwlsb7euczxw7ig3q3fyezw7tfkcikrer75w4ucfawh5vqfdm bafybeidibxjgswcu2q4gx7j5mzq26qtcczt3nczg6k4ybqct2rm567kpoy bafybeie7vavzb2ojufppuaxer3kfu4yxi36z7ozdz7wnhtylnd7rl5p6ji bafybeiefiddap3p3z3p3obatviipgda2us3umt4uxeh2trrni5pt6bfc6e bafybeifohbynkellbberk3hi447dntungwgicmsvxuwdttk5k5k535mw2a bafybeigkeedelu7fkqgeafzh7zrtwmcn747clncxuufl2dzu46mww735qq bafybeign7hb7ynhc6o3sevas365xuzuqzrhpfzgksi4dzpcbphl4xrerye bafybeignyyeaqn6b2fzqswrozt2txexjbwesrt4dbxlquoac4nn7mbi2gm bafybeigz7o2ibtv6k5pnklfza3ebsl6jeveguoyxwvrmmtvp2lkvqqjyem ``` You can now page forward like: ```bash $ w3 can upload ls --size 4 bafybeia7tr4dgyln7zeyyyzmkppkcts6azdssykuluwzmmswysieyadcbm bafybeibvbxjeodaa6hdqlgbwmv4qzdp3bxnwdoukay4dpl7aemkiwc2eje bafybeid2fbwlsb7euczxw7ig3q3fyezw7tfkcikrer75w4ucfawh5vqfdm bafybeidibxjgswcu2q4gx7j5mzq26qtcczt3nczg6k4ybqct2rm567kpoy ``` ```bash $ w3 can upload ls --size 4 --cursor bafybeidibxjgswcu2q4gx7j5mzq26qtcczt3nczg6k4ybqct2rm567kpoy bafybeie7vavzb2ojufppuaxer3kfu4yxi36z7ozdz7wnhtylnd7rl5p6ji bafybeiefiddap3p3z3p3obatviipgda2us3umt4uxeh2trrni5pt6bfc6e bafybeifohbynkellbberk3hi447dntungwgicmsvxuwdttk5k5k535mw2a bafybeigkeedelu7fkqgeafzh7zrtwmcn747clncxuufl2dzu46mww735qq ``` ```bash $ w3 can upload ls --size 4 --cursor bafybeigkeedelu7fkqgeafzh7zrtwmcn747clncxuufl2dzu46mww735qq bafybeign7hb7ynhc6o3sevas365xuzuqzrhpfzgksi4dzpcbphl4xrerye bafybeignyyeaqn6b2fzqswrozt2txexjbwesrt4dbxlquoac4nn7mbi2gm bafybeigz7o2ibtv6k5pnklfza3ebsl6jeveguoyxwvrmmtvp2lkvqqjyem ``` and page backward like: ```bash $ w3 can upload ls --size 4 --cursor bafybeign7hb7ynhc6o3sevas365xuzuqzrhpfzgksi4dzpcbphl4xrerye --pre bafybeigkeedelu7fkqgeafzh7zrtwmcn747clncxuufl2dzu46mww735qq bafybeifohbynkellbberk3hi447dntungwgicmsvxuwdttk5k5k535mw2a bafybeiefiddap3p3z3p3obatviipgda2us3umt4uxeh2trrni5pt6bfc6e bafybeie7vavzb2ojufppuaxer3kfu4yxi36z7ozdz7wnhtylnd7rl5p6ji ``` ```bash $ w3 can upload ls --size 4 --cursor bafybeie7vavzb2ojufppuaxer3kfu4yxi36z7ozdz7wnhtylnd7rl5p6ji --pre bafybeidibxjgswcu2q4gx7j5mzq26qtcczt3nczg6k4ybqct2rm567kpoy bafybeid2fbwlsb7euczxw7ig3q3fyezw7tfkcikrer75w4ucfawh5vqfdm bafybeibvbxjeodaa6hdqlgbwmv4qzdp3bxnwdoukay4dpl7aemkiwc2eje bafybeia7tr4dgyln7zeyyyzmkppkcts6azdssykuluwzmmswysieyadcbm ``` Note that the cursor values here happen to match the CID strings - this is an implementation detail and should not be relied upon. Instead, cursor values should be taken from `startCursor` and `endCursor` in the list response: ```bash $ w3 can upload ls --size 4 --cursor bafybeidibxjgswcu2q4gx7j5mzq26qtcczt3nczg6k4ybqct2rm567kpoy --raw {"size":4,"cursor":"bafybeigkeedelu7fkqgeafzh7zrtwmcn747clncxuufl2dzu46mww735qq","results":[{"root":{"/":"bafybeie7vavzb2ojufppuaxer3kfu4yxi36z7ozdz7wnhtylnd7rl5p6ji"},"shards":[{"/":"bagbaieracmkgwrw6rowsk5jse5eihyhszyrq5w23aqosajyckn2tfbotdcqq"}],"updatedAt":"2023-02-13T16:30:04.172Z","insertedAt":"2023-02-13T16:30:04.172Z"},{"root":{"/":"bafybeiefiddap3p3z3p3obatviipgda2us3umt4uxeh2trrni5pt6bfc6e"},"shards":[{"/":"bagbaieraz3iyicghnyzpjld7yxf7rmntf5cdbewo4pmuck6kkkc6f7z3sznq"}],"updatedAt":"2023-02-13T16:29:51.864Z","insertedAt":"2023-02-13T10:57:39.017Z"},{"root":{"/":"bafybeifohbynkellbberk3hi447dntungwgicmsvxuwdttk5k5k535mw2a"},"shards":[{"/":"bagbaierasu5cufqagk34dkmhszbwuhiydjjjp3pl4bacyvskljqtkcxhgbta"}],"updatedAt":"2023-02-13T16:30:11.648Z","insertedAt":"2023-02-13T10:57:16.264Z"},{"root":{"/":"bafybeigkeedelu7fkqgeafzh7zrtwmcn747clncxuufl2dzu46mww735qq"},"shards":[{"/":"bagbaieralqhbrq2mqupqijkoylagbams263jwq6helsl6kbdpivupjm7egxq"}],"updatedAt":"2023-02-13T16:30:18.403Z","insertedAt":"2023-02-13T10:57:49.695Z"}], "endCursor":"bafybeigkeedelu7fkqgeafzh7zrtwmcn747clncxuufl2dzu46mww735qq","startCursor":"bafybeie7vavzb2ojufppuaxer3kfu4yxi36z7ozdz7wnhtylnd7rl5p6ji"} ``` nb that these w3 commands require the changes I implemented in storacha/w3cli#45 --------- Co-authored-by: Benjamin Goering <171782+gobengo@users.noreply.github.com>
storacha · Feb 16, 2023 · 7ae751d · 7ae751d
1 parent 984222a
commit 7ae751d
Show file tree

Hide file tree

Showing 14 changed files with 602 additions and 230 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -21,21 +21,22 @@
     "@serverless-stack/resources": "^1.18.0",
     "@tsconfig/node16": "^1.0.3",
     "@types/git-rev-sync": "^2.0.0",
-    "@ucanto/core": "^4.0.3",
-    "@ucanto/client": "^4.0.3",
-    "@ucanto/principal": "^4.0.3",
-    "@ucanto/transport": "^4.0.3",
-    "@web3-storage/access": "^9.2.0",
-    "@web3-storage/w3up-client": "^4.0.1",
+    "@ucanto/client": "^4.2.3",
+    "@ucanto/core": "^4.2.3",
+    "@ucanto/principal": "^4.2.3",
+    "@ucanto/transport": "^4.2.3",
     "@web-std/blob": "^3.0.4",
     "@web-std/fetch": "^4.1.0",
+    "@web3-storage/access": "^9.2.0",
+    "@web3-storage/w3up-client": "^4.1.0",
     "ava": "^4.3.3",
     "dotenv": "^16.0.3",
     "git-rev-sync": "^3.0.2",
     "hd-scripts": "^3.0.2",
     "lint-staged": "^13.0.3",
     "p-wait-for": "^5.0.0",
-    "typescript": "^4.9.3"
+    "typescript": "^4.9.3",
+    "multiformats": "^11.0.1"
   },
   "dependencies": {
     "@serverless-stack/node": "^1.18.0",

diff --git a/replicator/package.json b/replicator/package.json
@@ -13,7 +13,7 @@
     "@ipld/car": "^5.0.1",
     "@ipld/dag-pb": "^3.0.0",
     "ava": "^4.3.3",
-    "multiformats": "^10.0.2",
+    "multiformats": "^11.0.0",
     "nanoid": "^4.0.0",
     "testcontainers": "^8.13.0",
     "uint8arrays": "^4.0.2"

diff --git a/satnav/package.json b/satnav/package.json
@@ -11,7 +11,7 @@
     "@sentry/serverless": "^7.22.0",
     "@ipld/car": "^5.0.1",
     "cardex": "^1.0.0",
-    "multiformats": "^10.0.2",
+    "multiformats": "^11.0.1",
     "uint8arrays": "^4.0.2"
   },
   "devDependencies": {

diff --git a/upload-api/package.json b/upload-api/package.json
@@ -14,15 +14,15 @@
     "@ipld/dag-ucan": "^3.0.1",
     "@sentry/serverless": "^7.22.0",
     "@serverless-stack/node": "^1.18.2",
-    "@ucanto/client": "^4.0.3",
-    "@ucanto/interface": "^4.0.3",
-    "@ucanto/principal": "^4.0.3",
-    "@ucanto/server": "^4.0.3",
-    "@ucanto/transport": "^4.0.3",
+    "@ucanto/client": "^4.2.3",
+    "@ucanto/interface": "^4.2.3",
+    "@ucanto/principal": "^4.2.3",
+    "@ucanto/server": "^4.2.3",
+    "@ucanto/transport": "^4.2.3",
     "@web-std/fetch": "^4.1.0",
     "@web3-storage/access": "^9.0.0",
     "@web3-storage/capabilities": "^2.0.0",
-    "multiformats": "^10.0.2",
+    "multiformats": "^11.0.1",
     "p-retry": "^5.1.2",
     "uint8arrays": "^4.0.2"
   },

diff --git a/upload-api/service/store/list.js b/upload-api/service/store/list.js
@@ -17,12 +17,13 @@ export function storeListProvider(context) {
   return Server.provide(
     Store.list,
     async ({ capability }) => {
-      const { cursor, size } = capability.nb
+      const { cursor, size, pre } = capability.nb
       const space = Server.DID.parse(capability.with).did()
 
       return await context.storeTable.list(space, {
         size,
-        cursor
+        cursor,
+        pre
       })
     }
   )

diff --git a/upload-api/service/types.ts b/upload-api/service/types.ts
@@ -116,11 +116,15 @@ export interface UploadListItem extends UploadAddResult {
 
 export interface ListOptions {
   size?: number,
-  cursor?: string
+  cursor?: string,
+  pre?: boolean
 }
 
 export interface ListResponse<R> {
+  // cursor is deprecated in favor of endCursor and will be removed in a future version
   cursor?: string,
+  startCursor?: string,
+  endCursor?: string,
   size: number,
   results: R[]
 }

diff --git a/upload-api/service/upload/list.js b/upload-api/service/upload/list.js
@@ -16,12 +16,13 @@ export function uploadListProvider(context) {
   return Server.provide(
     Upload.list,
     async ({ capability }) => {
-      const { cursor, size } = capability.nb
+      const { cursor, size, pre } = capability.nb
       const space = Server.DID.parse(capability.with).did()
 
       return await context.uploadTable.list(space, {
         size,
-        cursor
+        cursor,
+        pre
       })
   })
 }
diff --git a/upload-api/tables/store.js b/upload-api/tables/store.js
@@ -119,20 +119,25 @@ export function createStoreTable (region, tableName, options = {}) {
             AttributeValueList: [{ S: space }],
           },
         },
+        ScanIndexForward: ! options.pre,
         ExclusiveStartKey: exclusiveStartKey,
         AttributesToGet: ['link', 'size', 'origin', 'insertedAt'],
       })
       const response = await dynamoDb.send(cmd)
 
       const results = response.Items?.map(i => toStoreListResult(unmarshall(i))) ?? []
+      const startCursor = results[0] ? results[0].link.toString() : undefined
       // Get cursor of the item where list operation stopped (inclusive).
       // This value can be used to start a new operation to continue listing.
       const lastKey = response.LastEvaluatedKey && unmarshall(response.LastEvaluatedKey)
-      const cursor = lastKey ? lastKey.link : undefined
+      const endCursor = lastKey ? lastKey.link : undefined
 
       return {
         size: results.length,
-        cursor,
+        // cursor is deprecated and will be removed in a future version
+        cursor: endCursor,
+        startCursor,
+        endCursor,
         results
       }
     }

diff --git a/upload-api/tables/upload.js b/upload-api/tables/upload.js
@@ -143,23 +143,28 @@ export function createUploadTable (region, tableName, options = {}) {
             AttributeValueList: [{ S: space }],
           },
         },
+        ScanIndexForward: ! options.pre,
         ExclusiveStartKey: exclusiveStartKey,
         AttributesToGet: ['space', 'root', 'shards', 'insertedAt', 'updatedAt'],
       })
       const response = await dynamoDb.send(cmd)
 
       /** @type {UploadListItem[]} */
       const results = response.Items?.map(i => toUploadListItem(unmarshall(i))) || []
+      const startCursor = results[0] ? results[0].root.toString() : undefined
 
       // Get cursor of the item where list operation stopped (inclusive).
       // This value can be used to start a new operation to continue listing.
       const lastKey = response.LastEvaluatedKey && unmarshall(response.LastEvaluatedKey)
-      const cursor = lastKey ? lastKey.root : undefined
+      const endCursor = lastKey ? lastKey.root : undefined
 
       return {
         size: results.length,
-        results,
-        cursor
+        // cursor is deprecated and will be removed in a future version
+        cursor: endCursor,
+        startCursor,
+        endCursor,
+        results
       }
     },
   }

diff --git a/upload-api/test/helpers/random.js b/upload-api/test/helpers/random.js
@@ -1,6 +1,5 @@
 import { webcrypto } from 'crypto'
 import { Blob } from '@web-std/blob'
-
 import { CarWriter } from '@ipld/car'
 import { CID } from 'multiformats/cid'
 import * as raw from 'multiformats/codecs/raw'