Skip to content

Commit

Permalink
Add TryCompress and TryDecompress (#118)
Browse files Browse the repository at this point in the history
These methods handle insufficient output buffer sizes without throwing
an exception, instead returning false.
  • Loading branch information
brantburnett authored Dec 22, 2024
1 parent f0e5154 commit 67dd494
Show file tree
Hide file tree
Showing 7 changed files with 195 additions and 27 deletions.
2 changes: 2 additions & 0 deletions Snappier.Benchmarks/BlockCompressHtml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ public int Compress()
{
using var compressor = new SnappyCompressor();

#pragma warning disable CS0618 // Type or member is obsolete
return compressor.Compress(_input.Span, _output.Span);
#pragma warning restore CS0618 // Type or member is obsolete
}
}
}
2 changes: 2 additions & 0 deletions Snappier.Benchmarks/Overview.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ public int BlockCompress64KbHtml()
{
using var compressor = new SnappyCompressor();

#pragma warning disable CS0618 // Type or member is obsolete
return compressor.Compress(_htmlMemory.Span, _outputBuffer.Span);
#pragma warning restore CS0618 // Type or member is obsolete
}

[Benchmark]
Expand Down
92 changes: 92 additions & 0 deletions Snappier.Tests/SnappyTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,60 @@ public void CompressAndDecompressFile_LimitedOutputBuffer()
Assert.Equal(input, output);
}

[Fact]
public void Compress_InsufficientOutputBuffer()
{
using var resource =
typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.alice29.txt");
Assert.NotNull(resource);

var input = new byte[65536];
var bytesRead = resource.Read(input, 0, input.Length);

var compressed = new byte[1024];
Assert.Throws<ArgumentException>(() => Snappy.Compress(input.AsSpan(0, bytesRead), compressed));
}

[Fact]
public void TryCompressAndDecompress()
{
using var resource =
typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.alice29.txt");
Assert.NotNull(resource);

var input = new byte[65536];
var bytesRead = resource.Read(input, 0, input.Length);

var compressed = new byte[Snappy.GetMaxCompressedLength(bytesRead)];
var result = Snappy.TryCompress(input.AsSpan(0, bytesRead), compressed, out var compressedLength);
Assert.True(result);

var compressedSpan = compressed.AsSpan(0, compressedLength);

var output = new byte[Snappy.GetUncompressedLength(compressedSpan)];
result = Snappy.TryDecompress(compressedSpan, output, out var outputLength);
Assert.True(result);

Assert.Equal(input.Length, outputLength);
Assert.Equal(input, output);
}

[Fact]
public void TryCompress_InsufficientOutputBuffer()
{
using var resource =
typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.alice29.txt");
Assert.NotNull(resource);

var input = new byte[65536];
var bytesRead = resource.Read(input, 0, input.Length);

var compressed = new byte[1024];
var result = Snappy.TryCompress(input.AsSpan(0, bytesRead), compressed, out _);

Assert.False(result);
}

#if NET6_0_OR_GREATER

[Theory]
Expand Down Expand Up @@ -167,6 +221,21 @@ public void BadData_InsufficentOutputBuffer_ThrowsArgumentException()
});
}

[Fact]
public void TryDecompress_InsufficentOutputBuffer_ReturnsFalse()
{
var input = new byte[100000];
ArrayFill(input, (byte)'A');

var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)];
var compressedLength = Snappy.Compress(input, compressed);

var output = new byte[100];
var result = Snappy.TryDecompress(compressed.AsSpan(0, compressedLength), output, out _);

Assert.False(result);
}

[Fact]
public void BadData_SimpleCorruption_ThrowsInvalidDataException()
{
Expand Down Expand Up @@ -234,6 +303,29 @@ public void BadData_FromFile_ThrowsInvalidDataException(string filename)
});
}

[Theory]
[InlineData("baddata1.snappy")]
[InlineData("baddata2.snappy")]
[InlineData("baddata3.snappy")]
public void BadData_TryDecompress_ThrowsInvalidDataException(string filename)
{
using var resource =
typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}");
Assert.NotNull(resource);

var input = new byte[resource.Length];
var bytesRead = resource.Read(input, 0, input.Length);

Assert.Throws<InvalidDataException>(() =>
{
var length = Snappy.GetUncompressedLength(input.AsSpan(0, bytesRead));
Assert.InRange(length, 0, 1 << 20);

var output = new byte[length];
Snappy.TryDecompress(input.AsSpan(0, bytesRead), output, out _);
});
}

[Fact]
public void DecompressToMemory()
{
Expand Down
31 changes: 21 additions & 10 deletions Snappier/Internal/SnappyCompressor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,18 @@ internal class SnappyCompressor : IDisposable
{
private HashTable? _workingMemory = new();

[Obsolete("Retained for benchmark comparisons to previous versions")]
public int Compress(ReadOnlySpan<byte> input, Span<byte> output)
{
if (!TryCompress(input, output, out int bytesWritten))
{
ThrowHelper.ThrowArgumentExceptionInsufficientOutputBuffer(nameof(output));
}

return bytesWritten;
}

public bool TryCompress(ReadOnlySpan<byte> input, Span<byte> output, out int bytesWritten)
{
if (_workingMemory == null)
{
Expand All @@ -18,7 +29,7 @@ public int Compress(ReadOnlySpan<byte> input, Span<byte> output)

_workingMemory.EnsureCapacity(input.Length);

int bytesWritten = VarIntEncoding.Write(output, (uint)input.Length);
bytesWritten = VarIntEncoding.Write(output, (uint)input.Length);
output = output.Slice(bytesWritten);

while (input.Length > 0)
Expand All @@ -29,15 +40,13 @@ public int Compress(ReadOnlySpan<byte> input, Span<byte> output)

int maxOutput = Helpers.MaxCompressedLength(fragment.Length);

int written;
if (output.Length >= maxOutput)
{
// The output span is large enough to hold the maximum possible compressed output,
// compress directly to that span.

int written = CompressFragment(fragment, output, hashTable);

output = output.Slice(written);
bytesWritten += written;
written = CompressFragment(fragment, output, hashTable);
}
else
{
Expand All @@ -47,26 +56,28 @@ public int Compress(ReadOnlySpan<byte> input, Span<byte> output)
byte[] scratch = ArrayPool<byte>.Shared.Rent(maxOutput);
try
{
int written = CompressFragment(fragment, scratch.AsSpan(), hashTable);
written = CompressFragment(fragment, scratch.AsSpan(), hashTable);
if (output.Length < written)
{
ThrowHelper.ThrowArgumentException("Insufficient output buffer", nameof(output));
bytesWritten = 0;
return false;
}

scratch.AsSpan(0, written).CopyTo(output);
output = output.Slice(written);
bytesWritten += written;
}
finally
{
ArrayPool<byte>.Shared.Return(scratch);
}
}

output = output.Slice(written);
bytesWritten += written;

input = input.Slice(fragment.Length);
}

return bytesWritten;
return true;
}

public void Compress(ReadOnlySequence<byte> input, IBufferWriter<byte> bufferWriter)
Expand Down
6 changes: 5 additions & 1 deletion Snappier/Internal/SnappyStreamCompressor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,11 @@ private void CompressBlock(ReadOnlySpan<byte> input)
// Make room for the header and CRC
Span<byte> blockBody = output.Slice(headerSize);

int bytesWritten = _compressor.Compress(input, blockBody);
if (!_compressor.TryCompress(input, blockBody, out int bytesWritten))
{
// Should be unreachable since we're allocating a buffer of the correct size.
ThrowHelper.ThrowInvalidOperationException();
}

if (bytesWritten < input.Length)
{
Expand Down
12 changes: 11 additions & 1 deletion Snappier/Internal/ThrowHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,22 @@ public static void ThrowIfNull([NotNull] object? argument, [CallerArgumentExpres
}
#endif

[DoesNotReturn]
[MethodImpl(MethodImplOptions.NoInlining)] // Avoid inlining to reduce code size for a cold path
public static void ThrowArgumentExceptionInsufficientOutputBuffer(string? paramName) =>
ThrowArgumentException("Output buffer is too small.", paramName);

[DoesNotReturn]
public static void ThrowInvalidDataException(string? message) =>
throw new InvalidDataException(message);

[DoesNotReturn]
public static void ThrowInvalidOperationException(string? message) =>
[MethodImpl(MethodImplOptions.NoInlining)] // Avoid inlining to reduce code size for a cold path
public static void ThrowInvalidDataExceptionIncompleteSnappyBlock() =>
throw new InvalidDataException("Incomplete Snappy block.");

[DoesNotReturn]
public static void ThrowInvalidOperationException(string? message = null) =>
throw new InvalidOperationException(message);

[DoesNotReturn]
Expand Down
77 changes: 62 additions & 15 deletions Snappier/Snappy.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,53 @@ public static int GetMaxCompressedLength(int inputLength) =>
/// <param name="input">Data to compress.</param>
/// <param name="output">Buffer to receive the compressed data.</param>
/// <returns>Number of bytes written to <paramref name="output"/>.</returns>
/// <exception cref="ArgumentException">Output buffer is too small.</exception>
/// <remarks>
/// The output buffer must be large enough to contain the compressed output.
/// </remarks>
public static int Compress(ReadOnlySpan<byte> input, Span<byte> output)
{
if (!TryCompress(input, output, out var bytesWritten))
{
ThrowHelper.ThrowArgumentExceptionInsufficientOutputBuffer(nameof(output));
}

return bytesWritten;
}

/// <summary>
/// Attempt to compress the input data into the output buffer.
/// </summary>
/// <param name="input">Data to compress.</param>
/// <param name="output">Buffer to receive the compressed data.</param>
/// <param name="bytesWritten">Number of bytes written to the <paramref name="output"/>.</param>
/// <returns><c>true</c> if the compression was successful, <c>false</c> if the output buffer is too small.</returns>
public static bool TryCompress(ReadOnlySpan<byte> input, Span<byte> output, out int bytesWritten)
{
if (output.IsEmpty)
{
// Minimum of 1 byte is required to store a zero-length block, short circuit.
bytesWritten = 0;
return false;
}

using var compressor = new SnappyCompressor();

return compressor.Compress(input, output);
return compressor.TryCompress(input, output, out bytesWritten);
}

/// <summary>
/// Compress a block of Snappy data.
/// </summary>
/// <param name="input">Data to compress.</param>
/// <param name="output">Buffer writer to receive the compressed data.</param>
/// <exception cref="ArgumentNullException"><paramref name="output"/> is null.</exception>
/// <exception cref="ArgumentException"><paramref name="input"/> is larger than the maximum of 4,294,967,295 bytes.</exception>
/// <remarks>
/// For the best performance the input sequence should be comprised of segments some multiple of 64KB
/// in size or a single <see cref="ReadOnlySpan{T}"/> wrapped in a sequence.
/// <para>
/// For the best performance, sequences with more than one segement should be comprised of segments some multiple of 64KB
/// in size (i.e. 64KB or 128KB or 256KB each) with only the final segment varying.
/// </para>
/// </remarks>
public static void Compress(ReadOnlySequence<byte> input, IBufferWriter<byte> output)
{
Expand All @@ -73,7 +102,11 @@ public static IMemoryOwner<byte> CompressToMemory(ReadOnlySpan<byte> input)

try
{
int length = Compress(input, buffer);
if (!TryCompress(input, buffer, out int length))
{
// Should be unreachable since we're allocating a buffer of the correct size.
ThrowHelper.ThrowInvalidOperationException();
}

return new ByteArrayPoolMemoryOwner(buffer, length);
}
Expand Down Expand Up @@ -123,24 +156,38 @@ public static int GetUncompressedLength(ReadOnlySpan<byte> input) =>
/// <exception cref="InvalidDataException">Invalid Snappy block.</exception>
/// <exception cref="ArgumentException">Output buffer is too small.</exception>
public static int Decompress(ReadOnlySpan<byte> input, Span<byte> output)
{
bool result = TryDecompress(input, output, out int bytesWritten);
if (!result)
{
ThrowHelper.ThrowArgumentExceptionInsufficientOutputBuffer(nameof(output));
}

return bytesWritten;
}

/// <summary>
/// Decompress a block of Snappy data. This must be an entire block.
/// </summary>
/// <param name="input">Data to decompress.</param>
/// <param name="output">Buffer to receive the decompressed data.</param>
/// <param name="bytesWritten">Number of bytes written to the <paramref name="output"/>.</param>
/// <returns><c>true</c> if the compression was successful, <c>false</c> if the output buffer is too small.</returns>
/// <exception cref="InvalidDataException">Invalid Snappy block.</exception>
public static bool TryDecompress(ReadOnlySpan<byte> input, Span<byte> output, out int bytesWritten)
{
using var decompressor = new SnappyDecompressor();

decompressor.Decompress(input);

if (!decompressor.AllDataDecompressed)
{
ThrowHelper.ThrowInvalidDataException("Incomplete Snappy block.");
ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock();
}

int read = decompressor.Read(output);

if (!decompressor.EndOfFile)
{
ThrowHelper.ThrowArgumentException("Output buffer is too small.", nameof(output));
}
bytesWritten = decompressor.Read(output);

return read;
return decompressor.EndOfFile;
}

/// <summary>
Expand All @@ -165,7 +212,7 @@ public static void Decompress(ReadOnlySequence<byte> input, IBufferWriter<byte>

if (!decompressor.AllDataDecompressed)
{
ThrowHelper.ThrowInvalidDataException("Incomplete Snappy block.");
ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock();
}
}

Expand All @@ -186,7 +233,7 @@ public static IMemoryOwner<byte> DecompressToMemory(ReadOnlySpan<byte> input)

if (!decompressor.AllDataDecompressed)
{
ThrowHelper.ThrowInvalidDataException("Incomplete Snappy block.");
ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock();
}

return decompressor.ExtractData();
Expand All @@ -212,7 +259,7 @@ public static IMemoryOwner<byte> DecompressToMemory(ReadOnlySequence<byte> input

if (!decompressor.AllDataDecompressed)
{
ThrowHelper.ThrowInvalidDataException("Incomplete Snappy block.");
ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock();
}

return decompressor.ExtractData();
Expand Down

0 comments on commit 67dd494

Please # to comment.