From 94bd5f2063f40f6d095fa66d8ea1ee4d779b0ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20St=C3=B6ger?= Date: Tue, 11 Feb 2020 14:15:40 +0100 Subject: [PATCH 1/6] Added compression --- .../Client/LineProtocolClient.cs | 42 ++++++++++++++++--- .../InfluxDB.LineProtocol.csproj | 1 + 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs b/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs index 279962c..d95a612 100644 --- a/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs +++ b/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs @@ -1,8 +1,10 @@ -using InfluxDB.LineProtocol.Payload; +using ICSharpCode.SharpZipLib.GZip; +using InfluxDB.LineProtocol.Payload; using System; using System.IO; -using System.Net; +using System.IO.Compression; using System.Net.Http; +using System.Net.Http.Headers; using System.Net.Sockets; using System.Text; using System.Threading; @@ -13,9 +15,10 @@ namespace InfluxDB.LineProtocol.Client public class LineProtocolClient : LineProtocolClientBase { private readonly HttpClient _httpClient; + private readonly bool _enableCompression; - public LineProtocolClient(Uri serverBaseAddress, string database, string username = null, string password = null) - : this(new HttpClientHandler(), serverBaseAddress, database, username, password) + public LineProtocolClient(Uri serverBaseAddress, string database, string username = null, string password = null, bool enableCompression = false) + : this(new HttpClientHandler(), serverBaseAddress, database, username, password, enableCompression) { } @@ -24,7 +27,8 @@ protected LineProtocolClient( Uri serverBaseAddress, string database, string username, - string password) + string password, + bool enableCompression) :base(serverBaseAddress, database, username, password) { if (serverBaseAddress == null) @@ -34,6 +38,7 @@ protected LineProtocolClient( // Overload that allows injecting handler is protected to avoid HttpMessageHandler being part of our public api which would force clients to reference System.Net.Http when using the lib. _httpClient = new HttpClient(handler) { BaseAddress = serverBaseAddress }; + _enableCompression = enableCompression; } protected override async Task OnSendAsync( @@ -64,7 +69,21 @@ protected override async Task OnSendAsync( break; } - var content = new StringContent(payload, Encoding.UTF8); + HttpContent content; + + if (_enableCompression) + { + var compressed = Compress(Encoding.UTF8.GetBytes(payload)); + + content = new ByteArrayContent(compressed); + content.Headers.ContentEncoding.Add("gzip"); + content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8" }; + } + else + { + content = new StringContent(payload, Encoding.UTF8); + } + var response = await _httpClient.PostAsync(endpoint, content, cancellationToken).ConfigureAwait(false); if (response.IsSuccessStatusCode) { @@ -80,5 +99,16 @@ protected override async Task OnSendAsync( return new LineProtocolWriteResult(false, $"{response.StatusCode} {response.ReasonPhrase} {body}"); } + + private byte[] Compress(byte[] input) + { + using (var ms = new MemoryStream()) + { + using (var gz = new GZipStream(ms, CompressionLevel.Fastest)) + gz.Write(input, 0, input.Length); + + return ms.ToArray(); + } + } } } diff --git a/src/InfluxDB.LineProtocol/InfluxDB.LineProtocol.csproj b/src/InfluxDB.LineProtocol/InfluxDB.LineProtocol.csproj index 5b7facb..fa87428 100644 --- a/src/InfluxDB.LineProtocol/InfluxDB.LineProtocol.csproj +++ b/src/InfluxDB.LineProtocol/InfluxDB.LineProtocol.csproj @@ -18,6 +18,7 @@ + From df523f9db2408ced56022b03aad3a89dc49349ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20St=C3=B6ger?= Date: Tue, 11 Feb 2020 14:36:52 +0100 Subject: [PATCH 2/6] Added compression --- src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs | 3 +-- .../Client/MockLineProtocolClient.cs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs b/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs index d95a612..aefeaaf 100644 --- a/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs +++ b/src/InfluxDB.LineProtocol/Client/LineProtocolClient.cs @@ -1,5 +1,4 @@ -using ICSharpCode.SharpZipLib.GZip; -using InfluxDB.LineProtocol.Payload; +using InfluxDB.LineProtocol.Payload; using System; using System.IO; using System.IO.Compression; diff --git a/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs b/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs index 47cab05..54172ca 100644 --- a/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs +++ b/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs @@ -11,7 +11,7 @@ public class MockLineProtocolClient : LineProtocolClient { } - private MockLineProtocolClient(MockHttpMessageHandler handler, Uri serverBaseAddress, string database) : base(handler, serverBaseAddress, database, null, null) + private MockLineProtocolClient(MockHttpMessageHandler handler, Uri serverBaseAddress, string database) : base(handler, serverBaseAddress, database, null, null, false) { Handler = handler; BaseAddress = serverBaseAddress; From 58c44e7120246a1cc11a3b39487c7f661684b40a Mon Sep 17 00:00:00 2001 From: stmax82 Date: Wed, 3 Jun 2020 22:22:29 +0200 Subject: [PATCH 3/6] Added unit tests for LineProtocolClient with and without compression --- .../Client/LineProtocolClientTests.cs | 70 +++++++++++++++++++ .../Client/MockLineProtocolClient.cs | 4 +- 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 test/InfluxDB.LineProtocol.Tests/Client/LineProtocolClientTests.cs diff --git a/test/InfluxDB.LineProtocol.Tests/Client/LineProtocolClientTests.cs b/test/InfluxDB.LineProtocol.Tests/Client/LineProtocolClientTests.cs new file mode 100644 index 0000000..2e1bd7f --- /dev/null +++ b/test/InfluxDB.LineProtocol.Tests/Client/LineProtocolClientTests.cs @@ -0,0 +1,70 @@ +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using InfluxDB.LineProtocol.Payload; +using RichardSzalay.MockHttp; +using Xunit; + +namespace InfluxDB.LineProtocol.Tests.Client +{ + public class LineProtocolClientTests + { + [Fact] + public async Task Sending_uncompressed_data() + { + var client = new MockLineProtocolClient("foo", false); + + var payload = new LineProtocolPayload(); + + payload.Add(new LineProtocolPoint("bar", new Dictionary { { "baz", 42 } })); + + client.Handler + .Expect($"{client.BaseAddress}write?db=foo") + .WithContent("bar baz=42i\n") + .Respond(HttpStatusCode.NoContent); + + var result = await client.WriteAsync(payload); + + Assert.True(result.Success); + } + + [Fact] + public async Task Sending_compressed_data() + { + var client = new MockLineProtocolClient("foo", true); + + var payload = new LineProtocolPayload(); + + payload.Add(new LineProtocolPoint("bar", new Dictionary { { "baz", 42 } })); + + client.Handler + .Expect($"{client.BaseAddress}write?db=foo") + .WithHeaders("Content-Encoding", "gzip") + .With(req => + { + var expected = "bar baz=42i\n"; + var actual = Gunzip(req.Content.ReadAsStreamAsync().Result); + + return expected == actual; + }) + .Respond(HttpStatusCode.NoContent); + + var result = await client.WriteAsync(payload); + + Assert.True(result.Success); + } + + private string Gunzip(Stream compressed) + { + using (var decompressed = new MemoryStream()) + using (var gunzip = new GZipStream(compressed, CompressionMode.Decompress)) + { + gunzip.CopyTo(decompressed); + return Encoding.UTF8.GetString(decompressed.ToArray()); + } + } + } +} diff --git a/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs b/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs index 54172ca..5377353 100644 --- a/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs +++ b/test/InfluxDB.LineProtocol.Tests/Client/MockLineProtocolClient.cs @@ -7,11 +7,11 @@ namespace InfluxDB.LineProtocol.Tests.Client { public class MockLineProtocolClient : LineProtocolClient { - public MockLineProtocolClient(string database) : this(new MockHttpMessageHandler(), new Uri("http://localhost:8086"), database) + public MockLineProtocolClient(string database, bool enableCompression = false) : this(new MockHttpMessageHandler(), new Uri("http://localhost:8086"), database, enableCompression) { } - private MockLineProtocolClient(MockHttpMessageHandler handler, Uri serverBaseAddress, string database) : base(handler, serverBaseAddress, database, null, null, false) + private MockLineProtocolClient(MockHttpMessageHandler handler, Uri serverBaseAddress, string database, bool enableCompression) : base(handler, serverBaseAddress, database, null, null, enableCompression) { Handler = handler; BaseAddress = serverBaseAddress; From 1be5833d5d11325a77005f62eb7e9035155b0cb8 Mon Sep 17 00:00:00 2001 From: stmax82 Date: Wed, 3 Jun 2020 22:29:31 +0200 Subject: [PATCH 4/6] Updated the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c99dad2..2d839b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,4 +3,4 @@ ### Features 1. [#52](https://github.com/influxdata/influxdb-csharp/pull/52): Basic Changes and Move to .NET Standard 2.0 / Framework 4.6.1 1. [#59](https://github.com/influxdata/influxdb-csharp/pull/59): Add maxBatchSize parameter to IntervalBatcher - +1. [#81](https://github.com/influxdata/influxdb-csharp/pull/81): Add compression to LineProtocolClient From 2a84db6a4844b625a66fd84d8abe5576ab09a675 Mon Sep 17 00:00:00 2001 From: stmax82 Date: Wed, 3 Jun 2020 22:43:16 +0200 Subject: [PATCH 5/6] Oops, fixed the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c59716..66a357b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 1.2.0 [unreleased] ### CI +1. [#81](https://github.com/influxdata/influxdb-csharp/pull/81): Add compression to LineProtocolClient 1. [#86](https://github.com/influxdata/influxdb-csharp/pull/86): AppVeyor deploy preview releases ## 1.1.1 [2020-04-14] @@ -8,4 +9,3 @@ ### Features 1. [#52](https://github.com/influxdata/influxdb-csharp/pull/52): Basic Changes and Move to .NET Standard 2.0 / Framework 4.6.1 1. [#59](https://github.com/influxdata/influxdb-csharp/pull/59): Add maxBatchSize parameter to IntervalBatcher -1. [#81](https://github.com/influxdata/influxdb-csharp/pull/81): Add compression to LineProtocolClient From e10b0fcbd412bc6e501d4454edafb8f43e5dfbce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Thu, 4 Jun 2020 08:50:45 +0200 Subject: [PATCH 6/6] Update CHANGELOG.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66a357b..2c8d903 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ ## 1.2.0 [unreleased] -### CI +### Features 1. [#81](https://github.com/influxdata/influxdb-csharp/pull/81): Add compression to LineProtocolClient + +### CI 1. [#86](https://github.com/influxdata/influxdb-csharp/pull/86): AppVeyor deploy preview releases ## 1.1.1 [2020-04-14]