From ef8d4397cc909557ea229dfcf9f3285753ab7872 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 6 Oct 2023 15:48:46 +0200 Subject: [PATCH] Enable automatic compression format detection --- .../HTTPDecompression.swift | 45 ++++++++++++++----- .../HTTPRequestDecompressor.swift | 2 +- .../HTTPResponseDecompressor.swift | 2 +- .../HTTPRequestDecompressorTest.swift | 15 +++++-- .../HTTPResponseDecompressorTest.swift | 31 +++++++++---- 5 files changed, 70 insertions(+), 25 deletions(-) diff --git a/Sources/NIOHTTPCompression/HTTPDecompression.swift b/Sources/NIOHTTPCompression/HTTPDecompression.swift index d99b65b0..5e11c3c5 100644 --- a/Sources/NIOHTTPCompression/HTTPDecompression.swift +++ b/Sources/NIOHTTPCompression/HTTPDecompression.swift @@ -94,18 +94,41 @@ public enum NIOHTTPDecompression { return nil } } - - var window: CInt { - switch self { - case .deflate: - return 15 - case .gzip: - return 15 + 16 - } - } } struct Decompressor { + /// `15` is the base two logarithm of the maximum window size (the size of the history buffer). It should be in the range 8..15 for this version of the library. + /// `32` enables automatic detection of gzip or zlib compression format with automatic header detection. + /// + /// Documentation from https://www.zlib.net/manual.html: + /// The windowBits parameter is the base two logarithm of the maximum window size (the size of the history buffer). + /// It should be in the range 8..15 for this version of the library. + /// The default value is 15 if inflateInit is used instead. + /// windowBits must be greater than or equal to the windowBits value provided to deflateInit2() while compressing, + /// or it must be equal to 15 if deflateInit2() was not used. + /// If a compressed stream with a larger window size is given as input, + /// inflate() will return with the error code Z_DATA_ERROR instead of trying to allocate a larger window. + /// windowBits can also be zero to request that inflate use the window size in the zlib header of the compressed stream. + /// windowBits can also be –8..–15 for raw inflate. + /// In this case, -windowBits determines the window size. + /// inflate() will then process raw deflate data, not looking for a zlib or gzip header, not generating a check value, + /// and not looking for any check values for comparison at the end of the stream. + /// This is for use with other formats that use the deflate compressed data format such as zip. + /// Those formats provide their own check values. + /// If a custom format is developed using the raw deflate format for compressed data, + /// it is recommended that a check value such as an Adler-32 or a CRC-32 be applied to the uncompressed data as is done in the zlib, + /// gzip, and zip formats. For most applications, the zlib format should be used as is. + /// Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits. + /// windowBits can also be greater than 15 for optional gzip decoding. + /// Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection, + /// or add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR). + /// If a gzip stream is being decoded, strm->adler is a CRC-32 instead of an Adler-32. + /// Unlike the gunzip utility and gzread() (see below), inflate() will not automatically decode concatenated gzip members. + /// inflate() will return Z_STREAM_END at the end of the gzip member. + /// The state would need to be reset to continue decoding a subsequent gzip member. + /// This must be done if there is more data after a gzip member, in order for the decompression to be compliant with the gzip standard (RFC 1952). + static let windowBitsWithAutomaticCompressionFormatDetection: Int32 = 15 + 32 + private let limit: NIOHTTPDecompression.DecompressionLimit private var stream = z_stream() private var inflated = 0 @@ -125,12 +148,12 @@ public enum NIOHTTPDecompression { return result } - mutating func initializeDecoder(encoding: NIOHTTPDecompression.CompressionAlgorithm) throws { + mutating func initializeDecoder() throws { self.stream.zalloc = nil self.stream.zfree = nil self.stream.opaque = nil - let rc = CNIOExtrasZlib_inflateInit2(&self.stream, encoding.window) + let rc = CNIOExtrasZlib_inflateInit2(&self.stream, Self.windowBitsWithAutomaticCompressionFormatDetection) guard rc == Z_OK else { throw NIOHTTPDecompression.DecompressionError.initializationError(Int(rc)) } diff --git a/Sources/NIOHTTPCompression/HTTPRequestDecompressor.swift b/Sources/NIOHTTPCompression/HTTPRequestDecompressor.swift index aa3c5c2f..0e6d6307 100644 --- a/Sources/NIOHTTPCompression/HTTPRequestDecompressor.swift +++ b/Sources/NIOHTTPCompression/HTTPRequestDecompressor.swift @@ -55,7 +55,7 @@ public final class NIOHTTPRequestDecompressor: ChannelDuplexHandler, RemovableCh let length = head.headers[canonicalForm: "Content-Length"].first.flatMap({ Int($0) }) { do { - try self.decompressor.initializeDecoder(encoding: algorithm) + try self.decompressor.initializeDecoder() self.compression = Compression(algorithm: algorithm, contentLength: length) } catch let error { context.fireErrorCaught(error) diff --git a/Sources/NIOHTTPCompression/HTTPResponseDecompressor.swift b/Sources/NIOHTTPCompression/HTTPResponseDecompressor.swift index 5df5d341..e4f40178 100644 --- a/Sources/NIOHTTPCompression/HTTPResponseDecompressor.swift +++ b/Sources/NIOHTTPCompression/HTTPResponseDecompressor.swift @@ -71,7 +71,7 @@ public final class NIOHTTPResponseDecompressor: ChannelDuplexHandler, RemovableC do { if let algorithm = algorithm { self.compression = Compression(algorithm: algorithm, compressedLength: 0) - try self.decompressor.initializeDecoder(encoding: algorithm) + try self.decompressor.initializeDecoder() } context.fireChannelRead(data) diff --git a/Tests/NIOHTTPCompressionTests/HTTPRequestDecompressorTest.swift b/Tests/NIOHTTPCompressionTests/HTTPRequestDecompressorTest.swift index 8e035785..2da83713 100644 --- a/Tests/NIOHTTPCompressionTests/HTTPRequestDecompressorTest.swift +++ b/Tests/NIOHTTPCompressionTests/HTTPRequestDecompressorTest.swift @@ -103,13 +103,20 @@ class HTTPRequestDecompressorTest: XCTestCase { try channel.pipeline.addHandler(NIOHTTPRequestDecompressor(limit: .none)).wait() let body = Array(repeating: testString, count: 1000).joined() - - for algorithm in [nil, "gzip", "deflate"] { + let algorithms: [(actual: String, announced: String)?] = [ + nil, + (actual: "gzip", announced: "gzip"), + (actual: "deflate", announced: "deflate"), + (actual: "gzip", announced: "deflate"), + (actual: "deflate", announced: "gzip"), + ] + + for algorithm in algorithms { let compressed: ByteBuffer var headers = HTTPHeaders() if let algorithm = algorithm { - headers.add(name: "Content-Encoding", value: algorithm) - compressed = compress(ByteBuffer.of(string: body), algorithm) + headers.add(name: "Content-Encoding", value: algorithm.announced) + compressed = compress(ByteBuffer.of(string: body), algorithm.actual) } else { compressed = ByteBuffer.of(string: body) } diff --git a/Tests/NIOHTTPCompressionTests/HTTPResponseDecompressorTest.swift b/Tests/NIOHTTPCompressionTests/HTTPResponseDecompressorTest.swift index b42e629f..7e7ef1c9 100644 --- a/Tests/NIOHTTPCompressionTests/HTTPResponseDecompressorTest.swift +++ b/Tests/NIOHTTPCompressionTests/HTTPResponseDecompressorTest.swift @@ -151,13 +151,20 @@ class HTTPResponseDecompressorTest: XCTestCase { for _ in 1...1000 { body += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." } - - for algorithm in [nil, "gzip", "deflate"] { + let algorithms: [(actual: String, announced: String)?] = [ + nil, + (actual: "gzip", announced: "gzip"), + (actual: "deflate", announced: "deflate"), + (actual: "gzip", announced: "deflate"), + (actual: "deflate", announced: "gzip"), + ] + + for algorithm in algorithms { let compressed: ByteBuffer var headers = HTTPHeaders() if let algorithm = algorithm { - headers.add(name: "Content-Encoding", value: algorithm) - compressed = compress(ByteBuffer.of(string: body), algorithm) + headers.add(name: "Content-Encoding", value: algorithm.announced) + compressed = compress(ByteBuffer.of(string: body), algorithm.actual) } else { compressed = ByteBuffer.of(string: body) } @@ -198,13 +205,21 @@ class HTTPResponseDecompressorTest: XCTestCase { for _ in 1...1000 { body += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." } - - for algorithm in [nil, "gzip", "deflate"] { + + let algorithms: [(actual: String, announced: String)?] = [ + nil, + (actual: "gzip", announced: "gzip"), + (actual: "deflate", announced: "deflate"), + (actual: "gzip", announced: "deflate"), + (actual: "deflate", announced: "gzip"), + ] + + for algorithm in algorithms { let compressed: ByteBuffer var headers = HTTPHeaders() if let algorithm = algorithm { - headers.add(name: "Content-Encoding", value: algorithm) - compressed = compress(ByteBuffer.of(string: body), algorithm) + headers.add(name: "Content-Encoding", value: algorithm.announced) + compressed = compress(ByteBuffer.of(string: body), algorithm.actual) } else { compressed = ByteBuffer.of(string: body) }