test: add tests

.gitignore

@@ -1,5 +1,5 @@
 node_modules
 coverage
-pail.car
+*.car
 dist
 .clinic

bench/put-x10_000.js

@@ -4,7 +4,7 @@ import { put } from '../src/index.js'
 import { ShardBlock } from '../src/shard.js'
 import { MemoryBlockstore } from '../src/block.js'
 import { randomCID, randomString, randomInteger } from '../test/helpers.js'
-import { collectMetrics, verify, writePail } from './util.js'
+import { collectMetrics, writePail } from './util.js'
 
 const NUM = 10_000
 
@@ -50,7 +50,6 @@ async function main () {
     console.timeEnd(`put x${NUM}`)
     await writePail(blocks, root)
     console.log(await collectMetrics(blocks, root))
-    await verify(blocks, root, new Map(kvs))
   }
 }
 

bench/util.js

@@ -2,10 +2,10 @@ import fs from 'fs'
 import { Readable } from 'stream'
 import { CarWriter } from '@ipld/car'
 // eslint-disable-next-line no-unused-vars
-import * as API from '../src/v1/api.js'
-import { get, entries } from '../src/v1/index.js'
+import * as API from '../src/api.js'
+import { get, entries } from '../src/index.js'
 import { MemoryBlockstore } from '../src/block.js'
-import { ShardFetcher } from '../src/v1/shard.js'
+import { ShardFetcher } from '../src/shard.js'
 
 /**
  * @param {MemoryBlockstore} blocks
@@ -75,20 +75,3 @@ export const collectMetrics = async (blocks, root) => {
     totalSize
   }
 }
-
-/**
- * @param {MemoryBlockstore} blocks
- * @param {API.ShardLink} root
- * @param {Map<string, API.UnknownLink>} data
- */
-export const verify = async (blocks, root, data) => {
-  for (const [k, v] of data) {
-    const result = await get(blocks, root, k)
-    if (!result) throw new Error(`missing item: "${k}": ${v}`)
-    if (result.toString() !== v.toString()) throw new Error(`incorrect value for ${k}: ${result} !== ${v}`)
-  }
-  let total = 0
-  for await (const _ of entries(blocks, root)) total++
-  if (data.size !== total) throw new Error(`incorrect entry count: ${total} !== ${data.size}`)
-  console.log(`✅ verified correct`)
-}

src/api.ts

@@ -23,9 +23,7 @@ export interface Shard extends ShardConfig {
 
 export type ShardLink = Link<Shard, typeof dagCBOR.code, typeof sha256.code, 1>
 
-export interface ShardBlockView extends BlockView<Shard, typeof dagCBOR.code, typeof sha256.code, 1> {
-  prefix: string
-}
+export interface ShardBlockView extends BlockView<Shard, typeof dagCBOR.code, typeof sha256.code, 1> {}
 
 export interface ShardDiff {
   additions: ShardBlockView[]
@@ -38,10 +36,17 @@ export interface BlockFetcher {
 }
 
 export interface ShardConfig {
-  /** Max encoded shard size in bytes - default 512 KiB. */
-  maxSize: number
-  /** Max key length (in UTF-8 encoded characters) - default 64. */
-  maxKeyLength: number
+  /** Shard compatibility version. */
+  version: number
+  /**
+   * Characters allowed in keys, referring to a known character set.
+   * e.g. "ascii" refers to the printable ASCII characters in the code range 32-126.
+   */
+  keyChars: string
+  /** Max key size in bytes - default 4096 bytes. */
+  maxKeySize: number
+  /** The key prefix from the root to this shard. */
+  prefix: string
 }
 
 export type ShardOptions = Partial<ShardConfig>

src/diff.js

@@ -18,11 +18,11 @@ import { ShardFetcher } from './shard.js'
  * @param {API.ShardLink} b Comparison DAG.
  * @returns {Promise<CombinedDiff>}
  */
-export const difference = async (blocks, a, b, prefix = '') => {
+export const difference = async (blocks, a, b) => {
   if (isEqual(a, b)) return { keys: [], shards: { additions: [], removals: [] } }
 
   const shards = new ShardFetcher(blocks)
-  const [ashard, bshard] = await Promise.all([shards.get(a, prefix), shards.get(b, prefix)])
+  const [ashard, bshard] = await Promise.all([shards.get(a), shards.get(b)])
 
   const aents = new Map(ashard.value.entries)
   const bents = new Map(bshard.value.entries)
@@ -36,19 +36,19 @@ export const difference = async (blocks, a, b, prefix = '') => {
     const bval = bents.get(akey)
     if (bval) continue
     if (!Array.isArray(aval)) {
-      keys.set(`${ashard.prefix}${akey}`, [aval, null])
+      keys.set(`${ashard.value.prefix}${akey}`, [aval, null])
       continue
     }
     // if shard link _with_ value
     if (aval[1] != null) {
-      keys.set(`${ashard.prefix}${akey}`, [aval[1], null])
+      keys.set(`${ashard.value.prefix}${akey}`, [aval[1], null])
     }
-    for await (const s of collect(shards, aval[0], `${ashard.prefix}${akey}`)) {
+    for await (const s of collect(shards, aval[0])) {
       for (const [k, v] of s.value.entries) {
         if (!Array.isArray(v)) {
-          keys.set(`${s.prefix}${k}`, [v, null])
+          keys.set(`${s.value.prefix}${k}`, [v, null])
         } else if (v[1] != null) {
-          keys.set(`${s.prefix}${k}`, [v[1], null])
+          keys.set(`${s.value.prefix}${k}`, [v[1], null])
         }
       }
       removals.set(s.cid.toString(), s)
@@ -60,22 +60,22 @@ export const difference = async (blocks, a, b, prefix = '') => {
     const aval = aents.get(bkey)
     if (!Array.isArray(bval)) {
       if (!aval) {
-        keys.set(`${bshard.prefix}${bkey}`, [null, bval])
+        keys.set(`${bshard.value.prefix}${bkey}`, [null, bval])
       } else if (Array.isArray(aval)) {
-        keys.set(`${bshard.prefix}${bkey}`, [aval[1] ?? null, bval])
+        keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval])
       } else if (!isEqual(aval, bval)) {
-        keys.set(`${bshard.prefix}${bkey}`, [aval, bval])
+        keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval])
       }
       continue
     }
     if (aval && Array.isArray(aval)) { // updated in B
       if (isEqual(aval[0], bval[0])) {
         if (bval[1] != null && (aval[1] == null || !isEqual(aval[1], bval[1]))) {
-          keys.set(`${bshard.prefix}${bkey}`, [aval[1] ?? null, bval[1]])
+          keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval[1]])
         }
         continue // updated value?
       }
-      const res = await difference(blocks, aval[0], bval[0], `${bshard.prefix}${bkey}`)
+      const res = await difference(blocks, aval[0], bval[0])
       for (const shard of res.shards.additions) {
         additions.set(shard.cid.toString(), shard)
       }
@@ -87,28 +87,28 @@ export const difference = async (blocks, a, b, prefix = '') => {
       }
     } else if (aval) { // updated in B value => link+value
       if (bval[1] == null) {
-        keys.set(`${bshard.prefix}${bkey}`, [aval, null])
+        keys.set(`${bshard.value.prefix}${bkey}`, [aval, null])
       } else if (!isEqual(aval, bval[1])) {
-        keys.set(`${bshard.prefix}${bkey}`, [aval, bval[1]])
+        keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval[1]])
       }
-      for await (const s of collect(shards, bval[0], `${bshard.prefix}${bkey}`)) {
+      for await (const s of collect(shards, bval[0])) {
         for (const [k, v] of s.value.entries) {
           if (!Array.isArray(v)) {
-            keys.set(`${s.prefix}${k}`, [null, v])
+            keys.set(`${s.value.prefix}${k}`, [null, v])
           } else if (v[1] != null) {
-            keys.set(`${s.prefix}${k}`, [null, v[1]])
+            keys.set(`${s.value.prefix}${k}`, [null, v[1]])
           }
         }
         additions.set(s.cid.toString(), s)
       }
     } else { // added in B
-      keys.set(`${bshard.prefix}${bkey}`, [null, bval[0]])
-      for await (const s of collect(shards, bval[0], `${bshard.prefix}${bkey}`)) {
+      keys.set(`${bshard.value.prefix}${bkey}`, [null, bval[0]])
+      for await (const s of collect(shards, bval[0])) {
         for (const [k, v] of s.value.entries) {
           if (!Array.isArray(v)) {
-            keys.set(`${s.prefix}${k}`, [null, v])
+            keys.set(`${s.value.prefix}${k}`, [null, v])
           } else if (v[1] != null) {
-            keys.set(`${s.prefix}${k}`, [null, v[1]])
+            keys.set(`${s.value.prefix}${k}`, [null, v[1]])
           }
         }
         additions.set(s.cid.toString(), s)
@@ -141,11 +141,11 @@ const isEqual = (a, b) => a.toString() === b.toString()
  * @param {API.ShardLink} root
  * @returns {AsyncIterableIterator<API.ShardBlockView>}
  */
-async function * collect (shards, root, prefix = '') {
-  const shard = await shards.get(root, prefix)
+async function * collect (shards, root) {
+  const shard = await shards.get(root)
   yield shard
-  for (const [k, v] of shard.value.entries) {
+  for (const [, v] of shard.value.entries) {
     if (!Array.isArray(v)) continue
-    yield * collect(shards, v[0], `${prefix}${k}`)
+    yield * collect(shards, v[0])
   }
 }