Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Add the shardSplitThreshold option to this library #92

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Add the shardSplitThreshold option to this library #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f3085f9
added shardSplitThreshold
obo20 Sep 17, 2021
4688296
Update blob.ts
obo20 Sep 17, 2021
4cc413d
Update fs.ts
obo20 Sep 17, 2021
53052c5
Update stream.ts
obo20 Sep 17, 2021
dd95f4f
Update index.ts
obo20 Sep 17, 2021
3fe4d35
Update stream.ts
obo20 Sep 17, 2021
e351a59
Merge pull request #1 from obo20/obo20-add-shardSplitThreshold
obo20 Sep 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/pack/blob.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,16 @@ import { MemoryBlockStore } from '../blockstore/memory'
import { pack } from './index'
import type { PackProperties } from './index'

export async function packToBlob ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackProperties) {
export async function packToBlob ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackProperties) {
const blockstore = userBlockstore ? userBlockstore : new MemoryBlockStore()
const { root, out } = await pack({
input,
blockstore,
hasher,
maxChunkSize,
maxChildrenPerNode,
wrapWithDirectory
wrapWithDirectory,
shardSplitThreshold
})

const carParts = await all(out)
Expand Down
5 changes: 3 additions & 2 deletions src/pack/fs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export type PackToFsProperties = PackProperties & {
output?: string
}

export async function packToFs ({ input, output, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackToFsProperties) {
export async function packToFs ({ input, output, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackToFsProperties) {
const blockstore = userBlockstore ? userBlockstore : new FsBlockStore()
const location = output || `${os.tmpdir()}/${(parseInt(String(Math.random() * 1e9), 10)).toString() + Date.now()}`
const writable = fs.createWriteStream(location)
Expand All @@ -24,7 +24,8 @@ export async function packToFs ({ input, output, blockstore: userBlockstore, has
hasher,
maxChunkSize,
maxChildrenPerNode,
wrapWithDirectory
wrapWithDirectory,
shardSplitThreshold
})

if (!userBlockstore) {
Expand Down
8 changes: 5 additions & 3 deletions src/pack/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,11 @@ export type PackProperties = {
maxChunkSize?: number,
maxChildrenPerNode?: number,
wrapWithDirectory?: boolean,
hasher?: MultihashHasher
hasher?: MultihashHasher,
shardSplitThreshold?: number
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

linter failing here with:

src/pack/index.ts:24:31
ERROR: 24:31  no-trailing-whitespace    trailing whitespace
Suggested change
shardSplitThreshold?: number
shardSplitThreshold?: number

}

export async function pack ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackProperties) {
export async function pack ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackProperties) {
if (!input || (Array.isArray(input) && !input.length)) {
throw new Error('missing input file(s)')
}
Expand All @@ -38,7 +39,8 @@ export async function pack ({ input, blockstore: userBlockstore, hasher, maxChun
hasher: hasher || unixfsImporterOptionsDefault.hasher,
maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize,
maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode,
wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory
wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory,
...(shardSplitThreshold) && { shardSplitThreshold : shardSplitThreshold }
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tslint job is failing here with:

src/pack/index.ts:24:31
ERROR: 43:37  object-literal-shorthand  Expected property shorthand in object literal ('{shardSplitThreshold}').
Suggested change
...(shardSplitThreshold) && { shardSplitThreshold : shardSplitThreshold }
...(shardSplitThreshold) && { shardSplitThreshold }

})
))

Expand Down
5 changes: 3 additions & 2 deletions src/pack/stream.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export type PackToStreamProperties = PackProperties & {
}

// Node version of toCar with Node Stream Writable
export async function packToStream ({ input, writable, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackToStreamProperties) {
export async function packToStream ({ input, writable, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, shardSplitThreshold }: PackToStreamProperties) {
if (!input || (Array.isArray(input) && !input.length)) {
throw new Error('given input could not be parsed correctly')
}
Expand All @@ -37,7 +37,8 @@ export async function packToStream ({ input, writable, blockstore: userBlockstor
hasher: hasher || unixfsImporterOptionsDefault.hasher,
maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize,
maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode,
wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory
wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory,
...(shardSplitThreshold) && { shardSplitThreshold : shardSplitThreshold }
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
...(shardSplitThreshold) && { shardSplitThreshold : shardSplitThreshold }
...(shardSplitThreshold) && { shardSplitThreshold }

})
))

Expand Down