Skip to content

fix: decompress Gzipped data #317

New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Merged
merged 5 commits into from
May 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ jobs:
- restore_cache:
name: Restoring reSharper Cache
keys:
- &cache-key reSharper-cache-2021_3_3
- &cache-key reSharper-cache-2022_1_0
- run:
name: Check code formatting
command: |
Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

### Bug Fixes
1. [#313](https://github.com/influxdata/influxdb-client-csharp/pull/313): Using default `org` and `bucket` in `WriteApiAsync`
1. [#317](https://github.com/influxdata/influxdb-client-csharp/pull/317): Decompress Gzipped data
1. [#317](https://github.com/influxdata/influxdb-client-csharp/pull/317): Logging HTTP headers from streaming request

### Documentation
1. [#315](https://github.com/influxdata/influxdb-client-csharp/pull/315): Clarify `timeout` option
Expand Down
19 changes: 15 additions & 4 deletions Client.Core/Internal/AbstractQueryClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
Expand Down Expand Up @@ -112,7 +114,8 @@ private async Task Query(Func<Func<HttpResponseMessage, RestResponse>, RestReque
var query = queryFn.Invoke(response =>
{
var result = GetStreamFromResponse(response, cancellationToken);
result = AfterIntercept((int)response.StatusCode, () => response.Headers.ToHeaderParameters(),
result = AfterIntercept((int)response.StatusCode,
() => response.Headers.ToHeaderParameters(response.Content.Headers),
result);

RaiseForInfluxError(response, result);
Expand Down Expand Up @@ -155,7 +158,8 @@ private void QuerySync(Func<Func<HttpResponseMessage, RestResponse>, RestRequest
var query = queryFn.Invoke(response =>
{
var result = GetStreamFromResponse(response, cancellationToken);
result = AfterIntercept((int)response.StatusCode, () => response.Headers.ToHeaderParameters(),
result = AfterIntercept((int)response.StatusCode,
() => response.Headers.ToHeaderParameters(response.Content.Headers),
result);

RaiseForInfluxError(response, result);
Expand Down Expand Up @@ -194,7 +198,8 @@ protected async IAsyncEnumerable<T> QueryEnumerable<T>(
var query = queryFn.Invoke(response =>
{
stream = GetStreamFromResponse(response, cancellationToken);
stream = AfterIntercept((int)response.StatusCode, () => response.Headers.ToHeaderParameters(), stream);
stream = AfterIntercept((int)response.StatusCode,
() => response.Headers.ToHeaderParameters(response.Content.Headers), stream);

RaiseForInfluxError(response, stream);

Expand Down Expand Up @@ -394,7 +399,13 @@ private Stream GetStreamFromResponse(HttpResponseMessage response, CancellationT
# else
var readAsStreamAsync = response.Content.ReadAsStreamAsync();
#endif
return readAsStreamAsync.ConfigureAwait(false).GetAwaiter().GetResult();
var streamFromResponse = readAsStreamAsync.ConfigureAwait(false).GetAwaiter().GetResult();
if (response.Content.Headers.ContentEncoding.Any(x => "gzip".Equals(x, StringComparison.OrdinalIgnoreCase)))
{
streamFromResponse = new GZipStream(streamFromResponse, CompressionMode.Decompress);
}

return streamFromResponse;
}
}
}
10 changes: 9 additions & 1 deletion Client.Core/Internal/RestSharpExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,17 @@ namespace InfluxDB.Client.Core.Internal
{
internal static class RestSharpExtensions
{
internal static IEnumerable<HeaderParameter> ToHeaderParameters(this HttpHeaders httpHeaders)
/// <summary>
/// Transform `HttpHeaders` to `HeaderParameter` type.
/// </summary>
/// <param name="httpHeaders"></param>
/// <param name="httpContentHeaders">Additionally content Headers</param>
/// <returns>IEnumerable&lt;HeaderParameter&gt;</returns>
internal static IEnumerable<HeaderParameter> ToHeaderParameters(this HttpHeaders httpHeaders,
HttpContentHeaders httpContentHeaders = null)
{
return httpHeaders
.Concat(httpContentHeaders ?? Enumerable.Empty<KeyValuePair<string, IEnumerable<string>>>())
.SelectMany(x => x.Value.Select(y => (x.Key, y)))
.Select(x => new HeaderParameter(x.Key, x.y));
}
Expand Down
16 changes: 16 additions & 0 deletions Client.Test/ItWriteQueryApiTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -920,5 +920,21 @@ public async Task WriteTooManyData()
foreach (var successEvent in successEvents)
Assert.AreEqual(50_000, successEvent.LineProtocol.Split("\n").Length);
}

[Test]
public async Task GzipWithLargeAmountOfData()
{
Client.EnableGzip();

var records = new List<string>();
for (var i = 0; i < 1000; i++) records.Add($"mem{i},tag=a value={i}i {i}");
await Client.GetWriteApiAsync().WriteRecordsAsync(records);

var tables = await _queryApi.QueryAsync(
$"from(bucket:\"{_bucket.Name}\") |> range(start: 0)");

Assert.AreEqual(1000, tables.Count);
Assert.AreEqual(1, tables[0].Records.Count);
}
}
}
18 changes: 18 additions & 0 deletions Client.Test/QueryApiTest.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using InfluxDB.Client.Api.Domain;
Expand Down Expand Up @@ -127,6 +128,23 @@ public void RequiredOrgQueryAsync()
ae.Message);
}

[Test]
public async Task LoggedContentType()
{
var writer = new StringWriter();
Trace.Listeners.Add(new TextWriterTraceListener(writer));

_influxDbClient.SetLogLevel(LogLevel.Headers);

MockServer
.Given(Request.Create().WithPath("/api/v2/query").UsingPost())
.RespondWith(CreateResponse(Data));

await _queryApi.QueryAsync("from(...");

StringAssert.Contains("Content-Type=text/csv; charset=utf-8", writer.ToString());
}

private class SyncPoco
{
[Column(IsMeasurement = true)] public string Measurement { get; set; }
Expand Down
2 changes: 1 addition & 1 deletion Scripts/reformat-code.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ cd "$SCRIPT_PATH"/..
#
# Install ReSharper command line tools
#
dotnet tool install --tool-path="./reSharperCLI" JetBrains.ReSharper.GlobalTools --version 2021.3.3 || true
dotnet tool install --tool-path="./reSharperCLI" JetBrains.ReSharper.GlobalTools --version 2022.1.0 || true

#
# Reformat code
Expand Down