From 0a61090fed8aeec5e6126f1748b3fbc0a76118a0 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 2 May 2022 14:55:43 +0200 Subject: [PATCH 1/6] feat: Optionally align limit() and tail() before pivot() function --- Client.Linq.Test/InfluxDBQueryVisitorTest.cs | 26 ++++++++++++ Client.Linq/InfluxDBQueryable.cs | 7 ++++ Client.Linq/Internal/QueryAggregator.cs | 42 ++++++++++++++------ Client.Test/AbstractItClientTest.cs | 2 - 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/Client.Linq.Test/InfluxDBQueryVisitorTest.cs b/Client.Linq.Test/InfluxDBQueryVisitorTest.cs index 0d01d2cac..6ad1d1bbc 100644 --- a/Client.Linq.Test/InfluxDBQueryVisitorTest.cs +++ b/Client.Linq.Test/InfluxDBQueryVisitorTest.cs @@ -1107,6 +1107,32 @@ public void FilterByTimeAndTagWithAnds() Assert.AreEqual(expected, visitor.BuildFluxQuery()); } + + [Test] + public void AlignLimitFunctionBeforePivot() + { + var query = from s in InfluxDBQueryable.Queryable("my-bucket", "my-org", _queryApi, + new QueryableOptimizerSettings { AlignLimitFunctionAfterPivot = false }) + select s; + + var visitor = BuildQueryVisitor(query, MakeExpression(query, q => q.TakeLast(10))); + var expected = "start_shifted = int(v: time(v: p2))\n\n" + + "from(bucket: p1) " + + "|> range(start: time(v: start_shifted)) " + + "|> tail(n: p3) " + + "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\") " + + "|> drop(columns: [\"_start\", \"_stop\", \"_measurement\"])"; + Assert.AreEqual(expected, visitor.BuildFluxQuery()); + + visitor = BuildQueryVisitor(query, MakeExpression(query, q => q.Take(10))); + expected = "start_shifted = int(v: time(v: p2))\n\n" + + "from(bucket: p1) " + + "|> range(start: time(v: start_shifted)) " + + "|> limit(n: p3) " + + "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\") " + + "|> drop(columns: [\"_start\", \"_stop\", \"_measurement\"])"; + Assert.AreEqual(expected, visitor.BuildFluxQuery()); + } private InfluxDBQueryVisitor BuildQueryVisitor(IQueryable queryable, Expression expression = null) { diff --git a/Client.Linq/InfluxDBQueryable.cs b/Client.Linq/InfluxDBQueryable.cs index bccaf15ae..cd37c72d4 100644 --- a/Client.Linq/InfluxDBQueryable.cs +++ b/Client.Linq/InfluxDBQueryable.cs @@ -21,6 +21,7 @@ public QueryableOptimizerSettings() { QueryMultipleTimeSeries = false; AlignFieldsWithPivot = true; + AlignLimitFunctionAfterPivot = true; DropMeasurementColumn = true; DropStartColumn = true; DropStopColumn = true; @@ -41,6 +42,12 @@ public QueryableOptimizerSettings() /// to align fields to tabular way. /// public bool AlignFieldsWithPivot { get; set; } + + /// + /// Gets or set whether the drive should align limit() and tail() functions + /// after pivot() function. + /// + public bool AlignLimitFunctionAfterPivot { get; set; } /// /// Gets or sets whether the _measurement column will be dropped from query results. diff --git a/Client.Linq/Internal/QueryAggregator.cs b/Client.Linq/Internal/QueryAggregator.cs index 02acd5ff0..bb74a0981 100644 --- a/Client.Linq/Internal/QueryAggregator.cs +++ b/Client.Linq/Internal/QueryAggregator.cs @@ -166,12 +166,19 @@ internal string BuildFluxQuery(QueryableOptimizerSettings settings) BuildOperator("from", "bucket", _bucketAssignment), BuildRange(transforms), BuildFilter(_filterByTags), - BuildAggregateWindow(_aggregateWindow), - settings.AlignFieldsWithPivot - ? "pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")" - : "" + BuildAggregateWindow(_aggregateWindow) }; + if (!settings.AlignLimitFunctionAfterPivot) + { + AddLimitFunctions(parts); + } + + if (settings.AlignFieldsWithPivot) + { + parts.Add("pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")"); + } + var drop = BuildDrop(settings); if (!string.IsNullOrEmpty(drop)) { @@ -195,14 +202,10 @@ internal string BuildFluxQuery(QueryableOptimizerSettings settings) descendingVariable)); } - // https://docs.influxdata.com/flux/v0.x/stdlib/universe/limit/ - foreach (var limitNOffsetAssignment in _limitTailNOffsetAssignments) - if (limitNOffsetAssignment.N != null) - { - parts.Add(BuildOperator(limitNOffsetAssignment.FluxFunction, - "n", limitNOffsetAssignment.N, - "offset", limitNOffsetAssignment.Offset)); - } + if (settings.AlignLimitFunctionAfterPivot) + { + AddLimitFunctions(parts); + } if (_resultFunction != ResultFunction.None) { @@ -223,6 +226,21 @@ internal string BuildFluxQuery(QueryableOptimizerSettings settings) return query.ToString(); } + private void AddLimitFunctions(List parts) + { + // https://docs.influxdata.com/flux/latest/stdlib/universe/limit/ + // https://docs.influxdata.com/flux/latest/stdlib/universe/tail/ + foreach (var limitNOffsetAssignment in _limitTailNOffsetAssignments) + { + if (limitNOffsetAssignment.N != null) + { + parts.Add(BuildOperator(limitNOffsetAssignment.FluxFunction, + "n", limitNOffsetAssignment.N, + "offset", limitNOffsetAssignment.Offset)); + } + } + } + private string BuildAggregateWindow((string Every, string Period, string Fn)? aggregateWindow) { if (aggregateWindow == null) diff --git a/Client.Test/AbstractItClientTest.cs b/Client.Test/AbstractItClientTest.cs index c8a90d869..a81fe090d 100644 --- a/Client.Test/AbstractItClientTest.cs +++ b/Client.Test/AbstractItClientTest.cs @@ -3,10 +3,8 @@ using System.Linq; using System.Threading.Tasks; using InfluxDB.Client.Api.Domain; -using InfluxDB.Client.Core; using InfluxDB.Client.Core.Test; using NUnit.Framework; -using Task = System.Threading.Tasks.Task; namespace InfluxDB.Client.Test { From 1ceb1e8c6f32e9fb9c28668aa09abc735e3f2d90 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 2 May 2022 15:03:30 +0200 Subject: [PATCH 2/6] docs: update README.md --- Client.Linq/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Client.Linq/README.md b/Client.Linq/README.md index 96609083f..f26547b5a 100644 --- a/Client.Linq/README.md +++ b/Client.Linq/README.md @@ -851,6 +851,16 @@ from(bucket: "my-bucket") |> limit(n: 10) ``` +**_Note:_** the `limit()` function can be align before `pivot()` function by: + +```c# +var optimizerSettings = + new QueryableOptimizerSettings + { + AlignLimitFunctionAfterPivot = false + }; +``` + ### TakeLast ```c# @@ -869,6 +879,16 @@ from(bucket: "my-bucket") |> tail(n: 10) ``` +**_Note:_** the `tail()` function can be align before `pivot()` function by: + +```c# +var optimizerSettings = + new QueryableOptimizerSettings + { + AlignLimitFunctionAfterPivot = false + }; +``` + ### Skip ```c# From 9cb77baabd64268a3bbbadb0c76148991ba01355 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 2 May 2022 15:06:57 +0200 Subject: [PATCH 3/6] fix: code style --- Client.Linq.Test/InfluxDBQueryVisitorTest.cs | 12 ++++++------ Client.Linq/InfluxDBQueryable.cs | 2 +- Client.Linq/Internal/QueryAggregator.cs | 8 +++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Client.Linq.Test/InfluxDBQueryVisitorTest.cs b/Client.Linq.Test/InfluxDBQueryVisitorTest.cs index 6ad1d1bbc..3f3006d70 100644 --- a/Client.Linq.Test/InfluxDBQueryVisitorTest.cs +++ b/Client.Linq.Test/InfluxDBQueryVisitorTest.cs @@ -1107,7 +1107,7 @@ public void FilterByTimeAndTagWithAnds() Assert.AreEqual(expected, visitor.BuildFluxQuery()); } - + [Test] public void AlignLimitFunctionBeforePivot() { @@ -1126,11 +1126,11 @@ public void AlignLimitFunctionBeforePivot() visitor = BuildQueryVisitor(query, MakeExpression(query, q => q.Take(10))); expected = "start_shifted = int(v: time(v: p2))\n\n" + - "from(bucket: p1) " + - "|> range(start: time(v: start_shifted)) " + - "|> limit(n: p3) " + - "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\") " + - "|> drop(columns: [\"_start\", \"_stop\", \"_measurement\"])"; + "from(bucket: p1) " + + "|> range(start: time(v: start_shifted)) " + + "|> limit(n: p3) " + + "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\") " + + "|> drop(columns: [\"_start\", \"_stop\", \"_measurement\"])"; Assert.AreEqual(expected, visitor.BuildFluxQuery()); } diff --git a/Client.Linq/InfluxDBQueryable.cs b/Client.Linq/InfluxDBQueryable.cs index cd37c72d4..d03e9f5b4 100644 --- a/Client.Linq/InfluxDBQueryable.cs +++ b/Client.Linq/InfluxDBQueryable.cs @@ -42,7 +42,7 @@ public QueryableOptimizerSettings() /// to align fields to tabular way. /// public bool AlignFieldsWithPivot { get; set; } - + /// /// Gets or set whether the drive should align limit() and tail() functions /// after pivot() function. diff --git a/Client.Linq/Internal/QueryAggregator.cs b/Client.Linq/Internal/QueryAggregator.cs index bb74a0981..fb5286c68 100644 --- a/Client.Linq/Internal/QueryAggregator.cs +++ b/Client.Linq/Internal/QueryAggregator.cs @@ -169,16 +169,16 @@ internal string BuildFluxQuery(QueryableOptimizerSettings settings) BuildAggregateWindow(_aggregateWindow) }; - if (!settings.AlignLimitFunctionAfterPivot) + if (!settings.AlignLimitFunctionAfterPivot) { AddLimitFunctions(parts); } if (settings.AlignFieldsWithPivot) { - parts.Add("pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")"); + parts.Add("pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")"); } - + var drop = BuildDrop(settings); if (!string.IsNullOrEmpty(drop)) { @@ -231,14 +231,12 @@ private void AddLimitFunctions(List parts) // https://docs.influxdata.com/flux/latest/stdlib/universe/limit/ // https://docs.influxdata.com/flux/latest/stdlib/universe/tail/ foreach (var limitNOffsetAssignment in _limitTailNOffsetAssignments) - { if (limitNOffsetAssignment.N != null) { parts.Add(BuildOperator(limitNOffsetAssignment.FluxFunction, "n", limitNOffsetAssignment.N, "offset", limitNOffsetAssignment.Offset)); } - } } private string BuildAggregateWindow((string Every, string Period, string Fn)? aggregateWindow) From 50bd474447f5e5e7f1e5bc8b8cea55fa601ac137 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 2 May 2022 15:08:31 +0200 Subject: [PATCH 4/6] docs: update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd9f1dbf0..8c70ae8f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 4.2.0 [unreleased] +### Features +1. [#319](https://github.com/influxdata/influxdb-client-csharp/pull/319): Optionally align `limit()` and `tail()` before `pivot()` function [LINQ] + ### Breaking Changes 1. [#316](https://github.com/influxdata/influxdb-client-csharp/pull/316): Rename `InvocableScripts` to `InvokableScripts` From ce42de43337851b9fef931efbafc80b451cd30d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bedn=C3=A1=C5=99?= Date: Fri, 6 May 2022 11:09:55 +0200 Subject: [PATCH 5/6] docs: update Client.Linq/README.md Co-authored-by: Robert Hajek --- Client.Linq/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Client.Linq/README.md b/Client.Linq/README.md index f26547b5a..c0fa7dce6 100644 --- a/Client.Linq/README.md +++ b/Client.Linq/README.md @@ -888,7 +888,7 @@ var optimizerSettings = AlignLimitFunctionAfterPivot = false }; ``` - +**_Performance note:_** The `pivot()` is a [“heavy” function](https://docs.influxdata.com/influxdb/cloud/query-data/optimize-queries/#use-heavy-functions-sparingly). Using `limit()` or `tail()` before `pivot()` is much faster but works only if you have consistent data series. See #318 for more details. ### Skip ```c# From 5a4d7525507b53ec27db0bc0f60928609cdfcd5b Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 6 May 2022 11:20:55 +0200 Subject: [PATCH 6/6] docs: add performance note --- Client.Linq/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Client.Linq/README.md b/Client.Linq/README.md index c0fa7dce6..dfe0d4432 100644 --- a/Client.Linq/README.md +++ b/Client.Linq/README.md @@ -861,6 +861,8 @@ var optimizerSettings = }; ``` +**_Performance:_** The `pivot()` is a [“heavy” function](https://docs.influxdata.com/influxdb/cloud/query-data/optimize-queries/#use-heavy-functions-sparingly). Using `limit()` before `pivot()` is much faster but works only if you have consistent data series. See [#318](https://github.com/influxdata/influxdb-client-csharp/issues/318) for more details. + ### TakeLast ```c# @@ -888,7 +890,8 @@ var optimizerSettings = AlignLimitFunctionAfterPivot = false }; ``` -**_Performance note:_** The `pivot()` is a [“heavy” function](https://docs.influxdata.com/influxdb/cloud/query-data/optimize-queries/#use-heavy-functions-sparingly). Using `limit()` or `tail()` before `pivot()` is much faster but works only if you have consistent data series. See #318 for more details. +**_Performance:_** The `pivot()` is a [“heavy” function](https://docs.influxdata.com/influxdb/cloud/query-data/optimize-queries/#use-heavy-functions-sparingly). Using `tail()` before `pivot()` is much faster but works only if you have consistent data series. See [#318](https://github.com/influxdata/influxdb-client-csharp/issues/318) for more details. + ### Skip ```c#