fix: simplify encoder

examples/react/file-upload/src/ContentPage.js

@@ -17,8 +17,8 @@ export function ContentPage () {
     try {
       // Build a DAG from the file data to obtain the root CID.
       setStatus('encoding')
-      const { root, car } = await uploader.encodeFile(file)
-      setRootCid(root.toString())
+      const { cid, car } = await uploader.encodeFile(file)
+      setRootCid(cid.toString())
 
       // Upload the DAG to the service.
       setStatus('uploading')

examples/react/multi-file-upload/src/ContentPage.js

@@ -19,13 +19,13 @@ export function ContentPage () {
     try {
       // Build a DAG from the file data to obtain the root CID.
       setStatus('encoding')
-      const { root, car } = files.length > 1
+      const { cid, car } = files.length > 1
         ? await uploader.encodeDirectory(files)
         : wrapInDirectory
           ? await uploader.encodeDirectory(files)
           : await uploader.encodeFile(files[0])
 
-      setRootCid(root.toString())
+      setRootCid(cid.toString())
 
       // Upload the DAG to the service.
       setStatus('uploading')

packages/uploader-core/src/streams.ts

@@ -1,9 +1,9 @@
-export function toIterable<T> (readable: ReadableStream<T>): AsyncIterable<T> {
+export function toIterable<T> (readable: ReadableStream<T> | NodeJS.ReadableStream): AsyncIterable<T> {
   // @ts-expect-error
   if (readable[Symbol.asyncIterator] != null) return readable
 
   // Browser ReadableStream
-  if (readable.getReader != null) {
+  if ('getReader' in readable) {
     return (async function * () {
       const reader = readable.getReader()
 

packages/uploader-core/src/unixfs-car.ts

@@ -7,7 +7,7 @@ import { toIterable } from './streams'
 const queuingStrategy = UnixFS.withCapacity(1048576 * 175)
 
 export interface EncodeResult {
-  root: CID
+  cid: CID
   car: AsyncIterable<Uint8Array>
 }
 
@@ -16,158 +16,111 @@ export async function encodeFile (blob: Blob): Promise<EncodeResult> {
   const unixfsWriter = UnixFS.createWriter({ writable })
 
   const unixfsFileWriter = UnixFS.createFileWriter(unixfsWriter)
-  await unixfsFileWriter.write(new Uint8Array(await blob.arrayBuffer()))
+  const stream = toIterable<Uint8Array>(blob.stream())
+  for await (const chunk of stream) {
+    await unixfsFileWriter.write(chunk)
+  }
 
   const { cid } = await unixfsFileWriter.close()
 
-  // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30
-  const { writer: carBlockWriter, out } = CarWriter.create(cid)
-
-  let error: Error
-
-  void (async () => {
-    try {
-      await unixfsWriter.close()
-    } catch (err) {
-      // @ts-expect-error
-      error = new Error('failed to close UnixFS writer stream', { cause: err })
-    }
-  })()
-
-  void (async () => {
-    try {
-      for await (const block of toIterable(readable)) {
-        // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30
-        await carBlockWriter.put(block)
-      }
-    } catch (err: any) {
-      error = err
-    } finally {
-      try {
-        await carBlockWriter.close()
-      } catch (err: any) {
-        error = err
-      }
-    }
-  })()
+  unixfsWriter.close().catch(err => console.error('failed to close UnixFS writer stream', err))
 
-  return {
-    // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30
-    root: cid,
-    car: (async function * () {
-      yield * out
-      // @ts-expect-error Variable 'error' is used before being assigned.
-      if (error != null) throw error
-    })()
-  }
+  // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30
+  return { cid, car: createCar(cid, readable) }
 }
 
-class TreeFile {
+class UnixFsFileBuilder {
   #file: File
-  #writer: UnixFS.View
 
-  constructor (file: File, writer: UnixFS.View) {
+  constructor (file: File) {
     this.#file = file
-    this.#writer = writer
   }
 
   // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-  async finalize () {
-    const unixfsFileWriter = UnixFS.createFileWriter(this.#writer)
-    await unixfsFileWriter.write(new Uint8Array(await this.#file.arrayBuffer()))
+  async finalize (writer: UnixFS.View) {
+    const unixfsFileWriter = UnixFS.createFileWriter(writer)
+    const stream = toIterable<Uint8Array>(this.#file.stream())
+    for await (const chunk of stream) {
+      await unixfsFileWriter.write(chunk)
+    }
     return await unixfsFileWriter.close()
   }
 }
 
-class TreeDirectory {
-  entries: Map<string, TreeFile | TreeDirectory>
-  #writer: UnixFS.View
-
-  constructor (writer: UnixFS.View) {
-    this.#writer = writer
-    this.entries = new Map()
-  }
+class UnixFSDirectoryBuilder {
+  entries: Map<string, UnixFsFileBuilder | UnixFSDirectoryBuilder> = new Map()
 
   // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-  async finalize () {
-    const dirWriter = UnixFS.createDirectoryWriter(this.#writer)
+  async finalize (writer: UnixFS.View) {
+    const dirWriter = UnixFS.createDirectoryWriter(writer)
     for (const [name, entry] of this.entries) {
-      const link = await entry.finalize()
+      const link = await entry.finalize(writer)
       dirWriter.set(name, link)
     }
     return await dirWriter.close()
   }
 }
 
 export async function encodeDirectory (files: Iterable<File>): Promise<EncodeResult> {
-  const { readable, writable } = new TransformStream<Block, Block>({}, queuingStrategy)
-  const unixfsWriter = UnixFS.createWriter({ writable })
-
-  const root = new TreeDirectory(unixfsWriter)
+  const rootDir = new UnixFSDirectoryBuilder()
 
   for (const file of files) {
     const path = file.name.split('/')
     if (path[0] === '' || path[0] === '.') {
       path.shift()
     }
-    let dir = root
+    let dir = rootDir
     for (const [i, name] of path.entries()) {
       if (i === path.length - 1) {
-        dir.entries.set(name, new TreeFile(file, unixfsWriter))
+        dir.entries.set(name, new UnixFsFileBuilder(file))
         break
       }
-      let treeDir = dir.entries.get(name)
-      if (treeDir == null) {
-        treeDir = new TreeDirectory(unixfsWriter)
-        dir.entries.set(name, treeDir)
+      let dirBuilder = dir.entries.get(name)
+      if (dirBuilder == null) {
+        dirBuilder = new UnixFSDirectoryBuilder()
+        dir.entries.set(name, dirBuilder)
       }
-      if (!(treeDir instanceof TreeDirectory)) {
+      if (!(dirBuilder instanceof UnixFSDirectoryBuilder)) {
         throw new Error(`"${name}" cannot be a file and a directory`)
       }
-      dir = treeDir
+      dir = dirBuilder
     }
   }
 
-  const { cid } = await root.finalize()
+  const { readable, writable } = new TransformStream<Block, Block>({}, queuingStrategy)
+  const unixfsWriter = UnixFS.createWriter({ writable })
+  const { cid } = await rootDir.finalize(unixfsWriter)
+
+  unixfsWriter.close().catch(err => console.error('failed to close UnixFS writer stream', err))
 
   // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30
-  const { writer: carBlockWriter, out } = CarWriter.create(cid)
+  return { cid, car: createCar(cid, readable) }
+}
 
-  let error: Error
-
-  void (async () => {
-    try {
-      await unixfsWriter.close()
-    } catch (err) {
-      // @ts-expect-error
-      error = new Error('failed to close UnixFS writer stream', { cause: err })
-    }
-  })()
+function createCar (rootCid: CID, blocks: ReadableStream<UnixFS.Block>): AsyncIterable<Uint8Array> {
+  const { writer, out } = CarWriter.create(rootCid)
 
+  let error: Error
   void (async () => {
     try {
-      for await (const block of toIterable(readable)) {
+      for await (const block of toIterable(blocks)) {
         // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30
-        await carBlockWriter.put(block)
+        await writer.put(block)
       }
     } catch (err: any) {
       error = err
     } finally {
       try {
-        await carBlockWriter.close()
+        await writer.close()
       } catch (err: any) {
         error = err
       }
     }
   })()
 
-  return {
-    // @ts-expect-error https://github.com/ipld/js-unixfs/issues/30
-    root: cid,
-    car: (async function * () {
-      yield * out
-      // @ts-expect-error Variable 'error' is used before being assigned.
-      if (error != null) throw error
-    })()
-  }
+  return (async function * () {
+    yield * out
+    // @ts-expect-error Variable 'error' is used before being assigned.
+    if (error != null) throw error
+  })()
 }