Fix #82: QueryMultiSeriesAsync should return "partial" (#89)

Fix #82: QueryMultiSeriesAsync should return "partial" if responses are truncated by InfluxDB
Update test script to add max-row-limit

CHANGELOG.md

@@ -1,3 +1,28 @@
+## v0.11.0 [02/01/2020]
+
+### Release Notes
+Option to add a rentention policy with infinite duration - Thanks to @jasase 
+Handle truncated responses due to "max-row-limit", (flag "partial=true"). Thanks to @tbraun-hk 
+
+### Bugfixes
+
+- [#82](https://github.com/AdysTech/InfluxDB.Client.Net/issues/82): QueryMultiSeriesAsync should return "partial" if responses are truncated by InfluxDB
+- [#83](https://github.com/AdysTech/InfluxDB.Client.Net/issues/83): Create Infinite Retention Policy
+
+## v0.9.0 [10/12/2019]
+
+### Release Notes
+Add  MeasurementHierarchy to IInfluxDatabase, SeriesCount and PointsCount properties to IInfluxMeasurement
+Now calling `GetInfluxDBStructureAsync` populates the structure with retention policies, and also gives the unique series and point counts for each of the measurements
+Allow for deleting/dropping the Influx database by calling `DropDatabaseAsync`, allow delete entire measurement via	`DropMeasurementAsync` as well as specific data points with and where clause using `TestDeletePointsAsync`
+
+### Bugfixes
+
+- [#72](https://github.com/AdysTech/InfluxDB.Client.Net/issues/72): Performance improvements
+- [#69](https://github.com/AdysTech/InfluxDB.Client.Net/issues/69): easiest way to count the entries in the measurement
+- [#67](https://github.com/AdysTech/InfluxDB.Client.Net/issues/67): delete an existing database
+- [#41](https://github.com/AdysTech/InfluxDB.Client.Net/issues/41): DELETE FROM {Measurement} WHERE
+
 ## v0.9.0 [10/12/2019]
 
 ### Release Notes

src/AdysTech.InfluxDB.Client.Net.Core/AdysTech.InfluxDB.Client.Net.Core.csproj

@@ -6,7 +6,7 @@
     <Product>AdysTech.InfluxDB.Client.Net</Product>
     <Company>AdysTech</Company>
     <Authors>AdysTech;mvadu</Authors>
-    <Version>0.9.0.0</Version>
+    <Version>0.11.1.0</Version>
     <PackageId>AdysTech.InfluxDB.Client.Net.Core</PackageId>
     <Copyright>© AdysTech 2016-2019</Copyright>
     <PackageProjectUrl>https://github.com/AdysTech/InfluxDB.Client.Net</PackageProjectUrl>
@@ -29,8 +29,8 @@ It currently supports
 10. Chunking Support in queries
 11. Drop databases, measurements or points
 12. Get series count or points count for a measurement</PackageReleaseNotes>
-    <AssemblyVersion>0.10.0.0</AssemblyVersion>
-    <FileVersion>0.10.0.0</FileVersion>
+    <AssemblyVersion>0.11.1.0</AssemblyVersion>
+    <FileVersion>0.11.1.0</FileVersion>
     <PackageTags>InfluxDB Influx TSDB TimeSeries InfluxData Chunking retention RetentionPolicy</PackageTags>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
   </PropertyGroup>

src/AdysTech.InfluxDB.Client.Net/Properties/AssemblyInfo.cs

@@ -31,5 +31,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("0.10.0.0")]
-[assembly: AssemblyFileVersion("0.10.0.0")]
+[assembly: AssemblyVersion("0.11.1.0")]
+[assembly: AssemblyFileVersion("0.11.1.0")]

src/DataStructures/InfluxDBClient.cs

@@ -574,6 +574,7 @@ public async Task<List<IInfluxSeries>> QueryMultiSeriesAsync(string dbName, stri
             {
                 var results = new List<IInfluxSeries>();
                 var rawResult = JsonConvert.DeserializeObject<InfluxResponse>(await response.Content.ReadAsStringAsync());
+                var partialResult = rawResult.Results?.Any(r => r.Partial == true);
 
                 if (rawResult?.Results?.Count > 1)
                     throw new ArgumentException("The query is resulting in a format, which is not supported by this method yet");
@@ -582,7 +583,7 @@ public async Task<List<IInfluxSeries>> QueryMultiSeriesAsync(string dbName, stri
                 {
                     foreach (var series in rawResult?.Results[0]?.Series)
                     {
-                        InfluxSeries result = GetInfluxSeries(precision, series);
+                        InfluxSeries result = GetInfluxSeries(precision, series, partialResult);
                         results.Add(result);
                     }
                 }
@@ -629,11 +630,12 @@ public async Task<List<IInfluxSeries>> QueryMultiSeriesAsync(string dbName, stri
                     {
                         var str = await reader.ReadLineAsync();
                         var rawResult = JsonConvert.DeserializeObject<InfluxResponse>(str);
+                        var partialResult = rawResult?.Results?.Any(r => r.Partial == true);
                         if (rawResult?.Results[0]?.Series != null)
                         {
                             foreach (var series in rawResult?.Results[0]?.Series)
                             {
-                                InfluxSeries result = GetInfluxSeries(precision, series);
+                                InfluxSeries result = GetInfluxSeries(precision, series, partialResult);
                                 results.Add(result);
                             }
                         }
@@ -650,9 +652,11 @@ public async Task<List<IInfluxSeries>> QueryMultiSeriesAsync(string dbName, stri
         /// </summary>
         /// <param name="precision"></param>
         /// <param name="series"></param>
+        /// <param name="partialResult"></param>
         /// <param name="SafePropertyNames">If true the first letter of each property name will be Capital, making them safer to use in C#</param>
         /// <returns></returns>
-        private static InfluxSeries GetInfluxSeries(TimePrecision precision, Series series, bool SafePropertyNames = true)
+        private static InfluxSeries GetInfluxSeries(TimePrecision precision, Series series, bool? partialResult, 
+            bool SafePropertyNames = true)
         {
             var result = new InfluxSeries()
             {
@@ -661,6 +665,8 @@ private static InfluxSeries GetInfluxSeries(TimePrecision precision, Series seri
 
             result.SeriesName = series.Name;
             result.Tags = series.Tags;
+            result.Partial = partialResult ?? false;
+
             var entries = new List<dynamic>();
             for (var row = 0; row < series?.Values?.Count; row++)
             {

src/DataStructures/InfluxSeries.cs

@@ -33,5 +33,12 @@ public class InfluxSeries : IInfluxSeries
         /// The objects will have columns as Peoperties with their current values
         /// </summary>
         public IReadOnlyList<dynamic> Entries { get; internal set; }
+
+        /// <summary>
+        /// True if the influx query was answered with a partial response due to e.g. exceeding a configured
+        /// max-row-limit in the InfluxDB. As we don't know which series was truncated by InfluxDB, all series
+        /// of the response will be flagged with Partial=true.
+        /// </summary>
+        public bool Partial { get; set; }
     }
 }

src/Interfaces/IInfluxSeries.cs

@@ -27,5 +27,12 @@ public interface IInfluxSeries
         /// Dictionary of tags, and their respective values.
         /// </summary>
         IReadOnlyDictionary<string, string> Tags { get; }
+
+        /// <summary>
+        /// True if the influx query was answered with a partial response due to e.g. exceeding a configured
+        /// max-row-limit in the InfluxDB. As we don't know which series was truncated by InfluxDB, all series
+        /// of the response will be flagged with Partial=true.
+        /// </summary>
+        bool Partial { get; set; }
     }
 }

tests/InfluxDB.Client.Net.TestSetup.ps1

@@ -4,8 +4,10 @@
 #executed before CreateDatabase test
 #
 ##################################################################################################
-Stop-Process -name influxd -Force -ErrorAction SilentlyContinue
-
+if( $null -ne (Get-Process -name influxd -ErrorAction SilentlyContinue)){
+    Stop-Process -name influxd -Force -ErrorAction SilentlyContinue
+    Start-Sleep -Seconds 10
+}
 
 $nightlies = $true
 if ($nightlies -ne $true) {
@@ -47,8 +49,8 @@ else {
 }
 
 
-$cred = Get-Content .\tests\cred.json -Encoding UTF8 | ConvertFrom-Json
-#$cred = @{User="admin"; Password="test123$€₹#₳₷ȅ"}
+#$cred = Get-Content .\tests\cred.json -Encoding UTF8 | ConvertFrom-Json
+$cred = @{User="admin"; Password="test123$€₹#₳₷ȅ"}
 $authHeader = @{Authorization = "Basic $([System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("$($cred.User):$($cred.Password)")))"}
 
 #$x = 7z e $archivepath -o"$env:Temp/influxdb" -y
@@ -89,6 +91,9 @@ $proc.Kill()
 write-output "enable auth in config"
 (get-content $influxconf | foreach-object {$_ -replace "auth-enabled = false" , "auth-enabled = true" })  | Set-Content $influxconf	
 
+write-output "max-row-limit to 100000"
+(get-content $influxconf | foreach-object {$_ -replace "[# ]?\s*max-row-limit = 0" , "  max-row-limit = 100000" })  | Set-Content $influxconf	
+
 write-output "start influxdb with auth enabled"
 $proc = Start-Process -FilePath $influxd -ArgumentList "-config $influxconf" -PassThru
 Start-Sleep -s 5

tests/InfluxDBClientTest.cs

@@ -830,6 +830,58 @@ public async Task TestPerformance()
             }
         }
 
+        /// <summary>
+        /// You need to configure your test InfluxDB with a max-row-limit of 100.000 for this test to succeed!
+        /// </summary>
+        /// <returns></returns>
+        [TestMethod, TestCategory("Perf")]
+        public async Task TestPartialResponseBecauseOfMaxRowLimit()
+        {
+            try
+            {
+                var client = new InfluxDBClient(influxUrl, dbUName, dbpwd);
+
+                var points = new List<IInfluxDatapoint>();
+
+                var today = DateTime.Now.ToShortDateString();
+                var now = DateTime.UtcNow;
+                var nowString = DateTime.Now.ToShortTimeString();
+                var measurement = "Partialtest";
+
+                for (int i = 0; i < 200000; i++)
+                {
+                    var valMixed = new InfluxDatapoint<InfluxValueField>();
+                    valMixed.Tags.Add("TestDate", today);
+                    valMixed.Tags.Add("TestTime", nowString);
+                    valMixed.UtcTimestamp = now + TimeSpan.FromMilliseconds(i * 100);
+                    valMixed.Fields.Add("Open", new InfluxValueField(DataGen.RandomDouble()));
+                    valMixed.Fields.Add("High", new InfluxValueField(DataGen.RandomDouble()));
+                    valMixed.Fields.Add("Low", new InfluxValueField(DataGen.RandomDouble()));
+                    valMixed.Fields.Add("Close", new InfluxValueField(DataGen.RandomDouble()));
+                    valMixed.Fields.Add("Volume", new InfluxValueField(DataGen.RandomDouble()));
+
+                    valMixed.MeasurementName = measurement;
+                    valMixed.Precision = TimePrecision.Nanoseconds;
+                    points.Add(valMixed);
+                }
+
+
+                Assert.IsTrue(await client.PostPointsAsync(dbName, points, 25000), "PostPointsAsync retunred false");
+
+                var r = await client.QueryMultiSeriesAsync(dbName, "SELECT * FROM Partialtest");
+                Assert.IsTrue(r.All(s => s.Partial), "Not all influx series returned by the query contained the flag 'partial=true'");
+
+                r = await client.QueryMultiSeriesAsync(dbName, "SELECT * FROM Partialtest limit 50000");
+                Assert.IsTrue(!r.Any(s => s.Partial), "At least one of the influx series returned by the query contained the flag 'partial=true'");
+            }
+            catch (Exception e)
+            {
+
+                Assert.Fail($"Unexpected exception of type {e.GetType()} caught: {e.Message}");
+                return;
+            }
+        }
+
         [TestMethod, TestCategory("Drop")]
         public async Task TestDropDatabaseAsync()
         {