diff --git a/examples/react/file-upload/src/ContentPage.js b/examples/react/file-upload/src/ContentPage.js index 71c364e8..99a18696 100644 --- a/examples/react/file-upload/src/ContentPage.js +++ b/examples/react/file-upload/src/ContentPage.js @@ -17,8 +17,8 @@ export function ContentPage () { try { // Build a DAG from the file data to obtain the root CID. setStatus('encoding') - const { root, car } = await uploader.encodeFile(file) - setRootCid(root.toString()) + const { cid, car } = await uploader.encodeFile(file) + setRootCid(cid.toString()) // Upload the DAG to the service. setStatus('uploading') diff --git a/examples/react/multi-file-upload/src/ContentPage.js b/examples/react/multi-file-upload/src/ContentPage.js index 2472b8d8..af89490d 100644 --- a/examples/react/multi-file-upload/src/ContentPage.js +++ b/examples/react/multi-file-upload/src/ContentPage.js @@ -19,13 +19,13 @@ export function ContentPage () { try { // Build a DAG from the file data to obtain the root CID. setStatus('encoding') - const { root, car } = files.length > 1 + const { cid, car } = files.length > 1 ? await uploader.encodeDirectory(files) : wrapInDirectory ? await uploader.encodeDirectory(files) : await uploader.encodeFile(files[0]) - setRootCid(root.toString()) + setRootCid(cid.toString()) // Upload the DAG to the service. setStatus('uploading') diff --git a/packages/uploader-core/src/streams.ts b/packages/uploader-core/src/streams.ts index fcccd5b3..7de0de34 100644 --- a/packages/uploader-core/src/streams.ts +++ b/packages/uploader-core/src/streams.ts @@ -1,9 +1,9 @@ -export function toIterable (readable: ReadableStream): AsyncIterable { +export function toIterable (readable: ReadableStream | NodeJS.ReadableStream): AsyncIterable { // @ts-expect-error if (readable[Symbol.asyncIterator] != null) return readable // Browser ReadableStream - if (readable.getReader != null) { + if ('getReader' in readable) { return (async function * () { const reader = readable.getReader() diff --git a/packages/uploader-core/src/unixfs-car.ts b/packages/uploader-core/src/unixfs-car.ts index 38cf7ffb..3315cf0d 100644 --- a/packages/uploader-core/src/unixfs-car.ts +++ b/packages/uploader-core/src/unixfs-car.ts @@ -7,7 +7,7 @@ import { toIterable } from './streams' const queuingStrategy = UnixFS.withCapacity(1048576 * 175) export interface EncodeResult { - root: CID + cid: CID car: AsyncIterable } @@ -16,83 +16,45 @@ export async function encodeFile (blob: Blob): Promise { const unixfsWriter = UnixFS.createWriter({ writable }) const unixfsFileWriter = UnixFS.createFileWriter(unixfsWriter) - await unixfsFileWriter.write(new Uint8Array(await blob.arrayBuffer())) + const stream = toIterable(blob.stream()) + for await (const chunk of stream) { + await unixfsFileWriter.write(chunk) + } const { cid } = await unixfsFileWriter.close() - // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 - const { writer: carBlockWriter, out } = CarWriter.create(cid) - - let error: Error - - void (async () => { - try { - await unixfsWriter.close() - } catch (err) { - // @ts-expect-error - error = new Error('failed to close UnixFS writer stream', { cause: err }) - } - })() - - void (async () => { - try { - for await (const block of toIterable(readable)) { - // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 - await carBlockWriter.put(block) - } - } catch (err: any) { - error = err - } finally { - try { - await carBlockWriter.close() - } catch (err: any) { - error = err - } - } - })() + unixfsWriter.close().catch(err => console.error('failed to close UnixFS writer stream', err)) - return { - // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 - root: cid, - car: (async function * () { - yield * out - // @ts-expect-error Variable 'error' is used before being assigned. - if (error != null) throw error - })() - } + // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 + return { cid, car: createCar(cid, readable) } } -class TreeFile { +class UnixFsFileBuilder { #file: File - #writer: UnixFS.View - constructor (file: File, writer: UnixFS.View) { + constructor (file: File) { this.#file = file - this.#writer = writer } // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - async finalize () { - const unixfsFileWriter = UnixFS.createFileWriter(this.#writer) - await unixfsFileWriter.write(new Uint8Array(await this.#file.arrayBuffer())) + async finalize (writer: UnixFS.View) { + const unixfsFileWriter = UnixFS.createFileWriter(writer) + const stream = toIterable(this.#file.stream()) + for await (const chunk of stream) { + await unixfsFileWriter.write(chunk) + } return await unixfsFileWriter.close() } } -class TreeDirectory { - entries: Map - #writer: UnixFS.View - - constructor (writer: UnixFS.View) { - this.#writer = writer - this.entries = new Map() - } +class UnixFSDirectoryBuilder { + entries: Map = new Map() // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - async finalize () { - const dirWriter = UnixFS.createDirectoryWriter(this.#writer) + async finalize (writer: UnixFS.View) { + const dirWriter = UnixFS.createDirectoryWriter(writer) for (const [name, entry] of this.entries) { - const link = await entry.finalize() + const link = await entry.finalize(writer) dirWriter.set(name, link) } return await dirWriter.close() @@ -100,74 +62,65 @@ class TreeDirectory { } export async function encodeDirectory (files: Iterable): Promise { - const { readable, writable } = new TransformStream({}, queuingStrategy) - const unixfsWriter = UnixFS.createWriter({ writable }) - - const root = new TreeDirectory(unixfsWriter) + const rootDir = new UnixFSDirectoryBuilder() for (const file of files) { const path = file.name.split('/') if (path[0] === '' || path[0] === '.') { path.shift() } - let dir = root + let dir = rootDir for (const [i, name] of path.entries()) { if (i === path.length - 1) { - dir.entries.set(name, new TreeFile(file, unixfsWriter)) + dir.entries.set(name, new UnixFsFileBuilder(file)) break } - let treeDir = dir.entries.get(name) - if (treeDir == null) { - treeDir = new TreeDirectory(unixfsWriter) - dir.entries.set(name, treeDir) + let dirBuilder = dir.entries.get(name) + if (dirBuilder == null) { + dirBuilder = new UnixFSDirectoryBuilder() + dir.entries.set(name, dirBuilder) } - if (!(treeDir instanceof TreeDirectory)) { + if (!(dirBuilder instanceof UnixFSDirectoryBuilder)) { throw new Error(`"${name}" cannot be a file and a directory`) } - dir = treeDir + dir = dirBuilder } } - const { cid } = await root.finalize() + const { readable, writable } = new TransformStream({}, queuingStrategy) + const unixfsWriter = UnixFS.createWriter({ writable }) + const { cid } = await rootDir.finalize(unixfsWriter) + + unixfsWriter.close().catch(err => console.error('failed to close UnixFS writer stream', err)) // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 - const { writer: carBlockWriter, out } = CarWriter.create(cid) + return { cid, car: createCar(cid, readable) } +} - let error: Error - - void (async () => { - try { - await unixfsWriter.close() - } catch (err) { - // @ts-expect-error - error = new Error('failed to close UnixFS writer stream', { cause: err }) - } - })() +function createCar (rootCid: CID, blocks: ReadableStream): AsyncIterable { + const { writer, out } = CarWriter.create(rootCid) + let error: Error void (async () => { try { - for await (const block of toIterable(readable)) { + for await (const block of toIterable(blocks)) { // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 - await carBlockWriter.put(block) + await writer.put(block) } } catch (err: any) { error = err } finally { try { - await carBlockWriter.close() + await writer.close() } catch (err: any) { error = err } } })() - return { - // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30 - root: cid, - car: (async function * () { - yield * out - // @ts-expect-error Variable 'error' is used before being assigned. - if (error != null) throw error - })() - } + return (async function * () { + yield * out + // @ts-expect-error Variable 'error' is used before being assigned. + if (error != null) throw error + })() }