From f3085f97af4486dad205636a3d851d0187981ad7 Mon Sep 17 00:00:00 2001 From: Matt Ober <7811297+obo20@users.noreply.github.com> Date: Fri, 17 Sep 2021 13:03:59 -0500 Subject: [PATCH 1/6] added shardSplitThreshold added shardSplitThreshold to the car pack index --- src/pack/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pack/index.ts b/src/pack/index.ts index d6535d4..e4b2da0 100644 --- a/src/pack/index.ts +++ b/src/pack/index.ts @@ -20,10 +20,11 @@ export type PackProperties = { maxChunkSize?: number, maxChildrenPerNode?: number, wrapWithDirectory?: boolean, - hasher?: MultihashHasher + hasher?: MultihashHasher, + shardSplitThreshold?: number } -export async function pack ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackProperties) { +export async function pack ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackProperties) { if (!input || (Array.isArray(input) && !input.length)) { throw new Error('missing input file(s)') } @@ -38,7 +39,8 @@ export async function pack ({ input, blockstore: userBlockstore, hasher, maxChun hasher: hasher || unixfsImporterOptionsDefault.hasher, maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize, maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode, - wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory + wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory, + shardSplitThreshold: shardSplitThreshold || undefined }) )) From 468829641797c7c9e0b011b8e6747d9d24eb0d12 Mon Sep 17 00:00:00 2001 From: Matt Ober <7811297+obo20@users.noreply.github.com> Date: Fri, 17 Sep 2021 13:07:17 -0500 Subject: [PATCH 2/6] Update blob.ts --- src/pack/blob.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pack/blob.ts b/src/pack/blob.ts index 27b38c3..4b8e96d 100644 --- a/src/pack/blob.ts +++ b/src/pack/blob.ts @@ -8,7 +8,7 @@ import { MemoryBlockStore } from '../blockstore/memory' import { pack } from './index' import type { PackProperties } from './index' -export async function packToBlob ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackProperties) { +export async function packToBlob ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackProperties) { const blockstore = userBlockstore ? userBlockstore : new MemoryBlockStore() const { root, out } = await pack({ input, @@ -16,7 +16,8 @@ export async function packToBlob ({ input, blockstore: userBlockstore, hasher, m hasher, maxChunkSize, maxChildrenPerNode, - wrapWithDirectory + wrapWithDirectory, + shardSplitThreshold }) const carParts = await all(out) From 4cc413d07fd31ab7628b7cf1071dd21226586808 Mon Sep 17 00:00:00 2001 From: Matt Ober <7811297+obo20@users.noreply.github.com> Date: Fri, 17 Sep 2021 13:07:36 -0500 Subject: [PATCH 3/6] Update fs.ts --- src/pack/fs.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pack/fs.ts b/src/pack/fs.ts index 8c3c1e5..ae4a291 100644 --- a/src/pack/fs.ts +++ b/src/pack/fs.ts @@ -12,7 +12,7 @@ export type PackToFsProperties = PackProperties & { output?: string } -export async function packToFs ({ input, output, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackToFsProperties) { +export async function packToFs ({ input, output, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackToFsProperties) { const blockstore = userBlockstore ? userBlockstore : new FsBlockStore() const location = output || `${os.tmpdir()}/${(parseInt(String(Math.random() * 1e9), 10)).toString() + Date.now()}` const writable = fs.createWriteStream(location) @@ -24,7 +24,8 @@ export async function packToFs ({ input, output, blockstore: userBlockstore, has hasher, maxChunkSize, maxChildrenPerNode, - wrapWithDirectory + wrapWithDirectory, + shardSplitThreshold }) if (!userBlockstore) { From 53052c5cbc4718eee7070113de5fbb8b252b86f9 Mon Sep 17 00:00:00 2001 From: Matt Ober <7811297+obo20@users.noreply.github.com> Date: Fri, 17 Sep 2021 13:08:28 -0500 Subject: [PATCH 4/6] Update stream.ts --- src/pack/stream.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pack/stream.ts b/src/pack/stream.ts index 4ef2bcb..094573c 100644 --- a/src/pack/stream.ts +++ b/src/pack/stream.ts @@ -20,7 +20,7 @@ export type PackToStreamProperties = PackProperties & { } // Node version of toCar with Node Stream Writable -export async function packToStream ({ input, writable, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackToStreamProperties) { +export async function packToStream ({ input, writable, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackToStreamProperties) { if (!input || (Array.isArray(input) && !input.length)) { throw new Error('given input could not be parsed correctly') } @@ -37,7 +37,8 @@ export async function packToStream ({ input, writable, blockstore: userBlockstor hasher: hasher || unixfsImporterOptionsDefault.hasher, maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize, maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode, - wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory + wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory, + shardSplitThreshold: shardSplitThreshold || undefined }) )) From dd95f4f19ffc6e145b878564c35aea96d3e3d193 Mon Sep 17 00:00:00 2001 From: Matt Ober <7811297+obo20@users.noreply.github.com> Date: Fri, 17 Sep 2021 13:13:54 -0500 Subject: [PATCH 5/6] Update index.ts --- src/pack/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pack/index.ts b/src/pack/index.ts index e4b2da0..ca53b9a 100644 --- a/src/pack/index.ts +++ b/src/pack/index.ts @@ -40,7 +40,7 @@ export async function pack ({ input, blockstore: userBlockstore, hasher, maxChun maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize, maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode, wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory, - shardSplitThreshold: shardSplitThreshold || undefined + ...(shardSplitThreshold) && { shardSplitThreshold : shardSplitThreshold } }) )) From 3fe4d35bdcd09047333fcc477eeae5432c34139a Mon Sep 17 00:00:00 2001 From: Matt Ober <7811297+obo20@users.noreply.github.com> Date: Fri, 17 Sep 2021 13:22:28 -0500 Subject: [PATCH 6/6] Update stream.ts --- src/pack/stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pack/stream.ts b/src/pack/stream.ts index 094573c..d21d3ad 100644 --- a/src/pack/stream.ts +++ b/src/pack/stream.ts @@ -38,7 +38,7 @@ export async function packToStream ({ input, writable, blockstore: userBlockstor maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize, maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode, wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory, - shardSplitThreshold: shardSplitThreshold || undefined + ...(shardSplitThreshold) && { shardSplitThreshold : shardSplitThreshold } }) ))