file inference improvements for .tbr and .tgz

When unpacking, only infer brotli compression from the filename if the
first 512 bytes are an invalid tar header (or the stream is less than
512 bytes)

While Brotli doesn't give us magic header bytes like gzip, we can be
reasonably sure that a .tbr file starting with 512 bytes of valid tar
data is almost certainly not a brotli compressed archive.

And a .tbr file starting with the magic gzip bytes is almost certainly a
gzip archive, and not brotli, despite what the filename says.

In all cases, if explicit boolean or object values appear in the options
for either gzip or brotli, we respect that, and ignore the filename.

Author: isaacs

lib/pack.js

@@ -79,6 +79,7 @@ const Pack = warner(class Pack extends Minipass {
 
     this.portable = !!opt.portable
     this.zip = null
+
     if (opt.gzip || opt.brotli) {
       if (opt.gzip && opt.brotli) {
         throw new TypeError('gzip and brotli are mutually exclusive')

lib/parse.js

@@ -100,7 +100,13 @@ module.exports = warner(class Parser extends EE {
     // Unlike gzip, brotli doesn't have any magic bytes to identify it
     // Users need to explicitly tell us they're extracting a brotli file
     // Or we infer from the file extension
-    this.brotli = opt.brotli || (opt.file && (opt.file.endsWith('.tar.br') || opt.file.endsWith('.tbr')))
+    const isTBR = (opt.file && (
+        opt.file.endsWith('.tar.br') || opt.file.endsWith('.tbr')))
+    // if it's a tbr file it MIGHT be brotli, but we don't know until
+    // we look at it and verify it's not a valid tar file.
+    this.brotli = !opt.gzip && opt.brotli !== undefined ? opt.brotli
+      : isTBR ? undefined
+      : false
 
     // have to set this so that streams are ok piping into it
     this.writable = true
@@ -351,7 +357,9 @@ module.exports = warner(class Parser extends EE {
     }
 
     // first write, might be gzipped
-    if (this[UNZIP] === null && chunk) {
+    const needSniff = this[UNZIP] === null ||
+      this.brotli === undefined && this[UNZIP] === false
+    if (needSniff && chunk) {
       if (this[BUFFER]) {
         chunk = Buffer.concat([this[BUFFER], chunk])
         this[BUFFER] = null
@@ -360,15 +368,45 @@ module.exports = warner(class Parser extends EE {
         this[BUFFER] = chunk
         return true
       }
+
+      // look for gzip header
       for (let i = 0; this[UNZIP] === null && i < gzipHeader.length; i++) {
         if (chunk[i] !== gzipHeader[i]) {
           this[UNZIP] = false
         }
       }
+
+      const maybeBrotli = this.brotli === undefined
+      if (this[UNZIP] === false && maybeBrotli) {
+        // read the first header to see if it's a valid tar file. If so,
+        // we can safely assume that it's not actually brotli, despite the
+        // .tbr or .tar.br file extension.
+        // if we ended before getting a full chunk, yes, def brotli
+        if (chunk.length < 512) {
+          if (this[ENDED]) {
+            this.brotli = true
+          } else {
+            this[BUFFER] = chunk
+            return true
+          }
+        } else {
+          // if it's tar, it's pretty reliably not brotli, chances of
+          // that happening are astronomical.
+          try {
+            new Header(chunk.slice(0, 512))
+            this.brotli = false
+          } catch (_) {
+            this.brotli = true
+          }
+        }
+      }
+
       if (this[UNZIP] === null || (this[UNZIP] === false && this.brotli)) {
         const ended = this[ENDED]
         this[ENDED] = false
-        this[UNZIP] = this.brotli ? new zlib.BrotliDecompress() : new zlib.Unzip()
+        this[UNZIP] = this[UNZIP] === null
+          ? new zlib.Unzip()
+          : new zlib.BrotliDecompress()
         this[UNZIP].on('data', chunk => this[CONSUMECHUNK](chunk))
         this[UNZIP].on('error', er => this.abort(er))
         this[UNZIP].on('end', _ => {
@@ -506,6 +544,7 @@ module.exports = warner(class Parser extends EE {
         this[UNZIP].end(chunk)
       } else {
         this[ENDED] = true
+        if (this.brotli === undefined) chunk = chunk || Buffer.alloc(0)
         this.write(chunk)
       }
     }

test/parse.js

@@ -80,7 +80,7 @@ t.test('fixture tests', t => {
       const eventsFile = parsedir + '/' + base + tail
       const expect = require(eventsFile)
 
-      t.test('one byte at a time', t => {
+      t.test('uncompressed one byte at a time', t => {
         const bs = new ByteStream()
         const opt = (maxMeta || filter || strict) ? {
           maxMetaEntrySize: maxMeta,
@@ -93,7 +93,7 @@ t.test('fixture tests', t => {
         bs.end(tardata)
       })
 
-      t.test('all at once', t => {
+      t.test('uncompressed all at once', t => {
         const p = new Parse({
           maxMetaEntrySize: maxMeta,
           filter: filter ? (path, entry) => entry.size % 2 !== 0 : null,
@@ -103,6 +103,31 @@ t.test('fixture tests', t => {
         p.end(tardata)
       })
 
+      t.test('uncompressed one byte at a time, filename .tbr', t => {
+        const bs = new ByteStream()
+        const opt = (maxMeta || filter || strict) ? {
+          maxMetaEntrySize: maxMeta,
+          filter: filter ? (path, entry) => entry.size % 2 !== 0 : null,
+          strict: strict,
+          file: 'example.tbr',
+        } : null
+        const bp = new Parse(opt)
+        trackEvents(t, expect, bp)
+        bs.pipe(bp)
+        bs.end(tardata)
+      })
+
+      t.test('uncompressed all at once, filename .tar.br', t => {
+        const p = new Parse({
+          maxMetaEntrySize: maxMeta,
+          filter: filter ? (path, entry) => entry.size % 2 !== 0 : null,
+          strict: strict,
+          file: 'example.tar.br',
+        })
+        trackEvents(t, expect, p)
+        p.end(tardata)
+      })
+
       t.test('gzipped all at once', t => {
         const p = new Parse({
           maxMetaEntrySize: maxMeta,
@@ -113,6 +138,17 @@ t.test('fixture tests', t => {
         p.end(zlib.gzipSync(tardata))
       })
 
+      t.test('gzipped all at once, filename .tbr', t => {
+        const p = new Parse({
+          maxMetaEntrySize: maxMeta,
+          filter: filter ? (path, entry) => entry.size % 2 !== 0 : null,
+          strict: strict,
+          file: 'example.tbr',
+        })
+        trackEvents(t, expect, p)
+        p.end(zlib.gzipSync(tardata))
+      })
+
       t.test('gzipped byte at a time', t => {
         const bs = new ByteStream()
         const bp = new Parse({
@@ -171,6 +207,19 @@ t.test('fixture tests', t => {
         bs.end(zlib.brotliCompressSync(tardata))
       })
 
+      t.test('compress with brotli .tbr byte at a time', t => {
+        const bs = new ByteStream()
+        const bp = new Parse({
+          maxMetaEntrySize: maxMeta,
+          filter: filter ? (path, entry) => entry.size % 2 !== 0 : null,
+          strict: strict,
+          file: 'example.tbr',
+        })
+        trackEvents(t, expect, bp)
+        bs.pipe(bp)
+        bs.end(zlib.brotliCompressSync(tardata))
+      })
+
       t.test('async chunks', t => {
         const p = new Parse({
           maxMetaEntrySize: maxMeta,