From a49fc639c33b47096b95a96a2fcf67abcc6c817b Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 4 Jul 2019 14:57:12 +1000 Subject: [PATCH 01/10] Serialize only script in script_score function (#3909) This commit updates the ScoreFunctionJsonFormatter to serialize only the script inside of the script_score JSON object. Fixes #3904 (cherry picked from commit ac604726caa9897c47d8caead2b1296ae99b830f) --- .../Functions/ScoreFunctionJsonFormatter.cs | 7 ++- .../FunctionScoreQueryUsageTests.cs | 60 ++++++++++++++++--- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/Nest/QueryDsl/Compound/FunctionScore/Functions/ScoreFunctionJsonFormatter.cs b/src/Nest/QueryDsl/Compound/FunctionScore/Functions/ScoreFunctionJsonFormatter.cs index 71ca0ec758f..85167ca50f8 100644 --- a/src/Nest/QueryDsl/Compound/FunctionScore/Functions/ScoreFunctionJsonFormatter.cs +++ b/src/Nest/QueryDsl/Compound/FunctionScore/Functions/ScoreFunctionJsonFormatter.cs @@ -158,8 +158,11 @@ public void Serialize(ref JsonWriter writer, IScoreFunction value, IJsonFormatte private static void WriteScriptScore(ref JsonWriter writer, IScriptScoreFunction value, IJsonFormatterResolver formatterResolver) { writer.WritePropertyName("script_score"); - var scriptFormatter = formatterResolver.GetFormatter(); - scriptFormatter.Serialize(ref writer, value, formatterResolver); + writer.WriteBeginObject(); + writer.WritePropertyName("script"); + var scriptFormatter = formatterResolver.GetFormatter(); + scriptFormatter.Serialize(ref writer, value?.Script, formatterResolver); + writer.WriteEndObject(); } private static void WriteRandomScore(ref JsonWriter writer, IRandomScoreFunction value, IJsonFormatterResolver formatterResolver) diff --git a/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs b/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs index 52a46f11f11..c447de19448 100644 --- a/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs +++ b/src/Tests/Tests/QueryDsl/Compound/FunctionScore/FunctionScoreQueryUsageTests.cs @@ -37,7 +37,19 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba MinScore = 1.0, Functions = new List { - new ExponentialDecayFunction { Origin = 1.0, Decay = 0.5, Field = Field(p => p.NumberOfCommits), Scale = 0.1, Weight = 2.1 }, + new ExponentialDecayFunction + { + Origin = 1.0, + Decay = 0.5, + Field = Field(p => p.NumberOfCommits), + Scale = 0.1, + Weight = 2.1, + Filter = new NumericRangeQuery + { + Field = Field(f => f.NumberOfContributors), + GreaterThan = 10 + } + }, new GaussDateDecayFunction { Origin = DateMath.Now, Field = Field(p => p.LastActivity), Decay = 0.5, Scale = TimeSpan.FromDays(1) }, new LinearGeoDecayFunction @@ -52,7 +64,7 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba new RandomScoreFunction { Seed = 1337, Field = "_seq_no" }, new RandomScoreFunction { Seed = "randomstring", Field = "_seq_no" }, new WeightFunction { Weight = 1.0 }, - new ScriptScoreFunction { Script = new InlineScript("Math.log(2 + doc['numberOfCommits'].value)") } + new ScriptScoreFunction { Script = new InlineScript("Math.log(2 + doc['numberOfCommits'].value)"), Weight = 2.0 } } }; @@ -76,7 +88,17 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba decay = 0.5 } }, - weight = 2.1 + weight = 2.1, + filter = new + { + range = new + { + numberOfContributors = new + { + gt = 10.0 + } + } + } }, new { @@ -127,7 +149,8 @@ public FunctionScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : ba { source = "Math.log(2 + doc['numberOfCommits'].value)" } - } + }, + weight = 2.0 } }, max_boost = 20.0, @@ -150,15 +173,36 @@ protected override QueryContainer QueryFluent(QueryContainerDescriptor .MaxBoost(20.0) .MinScore(1.0) .Functions(f => f - .Exponential(b => b.Field(p => p.NumberOfCommits).Decay(0.5).Origin(1.0).Scale(0.1).Weight(2.1)) + .Exponential(b => b + .Field(p => p.NumberOfCommits) + .Decay(0.5) + .Origin(1.0) + .Scale(0.1) + .Weight(2.1) + .Filter(fi => fi + .Range(r => r + .Field(p => p.NumberOfContributors) + .GreaterThan(10) + ) + ) + ) .GaussDate(b => b.Field(p => p.LastActivity).Origin(DateMath.Now).Decay(0.5).Scale("1d")) - .LinearGeoLocation(b => - b.Field(p => p.LocationPoint).Origin(new GeoLocation(70, -70)).Scale(Distance.Miles(1)).MultiValueMode(MultiValueMode.Average)) + .LinearGeoLocation(b => b + .Field(p => p.LocationPoint) + .Origin(new GeoLocation(70, -70)) + .Scale(Distance.Miles(1)) + .MultiValueMode(MultiValueMode.Average) + ) .FieldValueFactor(b => b.Field(p => p.NumberOfContributors).Factor(1.1).Missing(0.1).Modifier(FieldValueFactorModifier.Square)) .RandomScore(r => r.Seed(1337).Field("_seq_no")) .RandomScore(r => r.Seed("randomstring").Field("_seq_no")) .Weight(1.0) - .ScriptScore(s => s.Script(ss => ss.Source("Math.log(2 + doc['numberOfCommits'].value)"))) + .ScriptScore(s => s + .Script(ss => ss + .Source("Math.log(2 + doc['numberOfCommits'].value)") + ) + .Weight(2) + ) ) ); } From 6fe6961a311b889ff70591ff6fe08c0cd704ddf9 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 4 Jul 2019 14:58:25 +1000 Subject: [PATCH 02/10] Include time zone information for Local and Utc DateTimeKind (#3910) This commit adds the time zone information for a DateTime within a DateMath type, when the DateTimeKind is Local or Utc. This behaviour aligns 7.x with 6.x. Introduce StringBuilderCache for caching a StringBuilder per thread. Use of this can be extended in a later commit. Fixes #3899 --- .../date-math/date-math-expressions.asciidoc | 30 ++++++--- .../DocGenerator/StringExtensions.cs | 2 +- .../DocGenerator/SyntaxNodeExtensions.cs | 7 +- .../Extensions/StringBuilderCache.cs | 64 +++++++++++++++++++ src/Nest/CommonOptions/DateMath/DateMath.cs | 42 ++++++++++-- .../DateMath/DateMathExpressions.doc.cs | 32 +++++++++- 6 files changed, 157 insertions(+), 20 deletions(-) create mode 100644 src/Elasticsearch.Net/Extensions/StringBuilderCache.cs diff --git a/docs/common-options/date-math/date-math-expressions.asciidoc b/docs/common-options/date-math/date-math-expressions.asciidoc index 0315dd2eac5..075531c08b7 100644 --- a/docs/common-options/date-math/date-math-expressions.asciidoc +++ b/docs/common-options/date-math/date-math-expressions.asciidoc @@ -83,14 +83,28 @@ anchor will be an actual `DateTime`, even after a serialization/deserialization [source,csharp] ---- var date = new DateTime(2015, 05, 05); -Expect("2015-05-05T00:00:00") - .WhenSerializing(date) - .AssertSubject(dateMath => ((IDateMath)dateMath) - .Anchor.Match( - d => d.Should().Be(date), - s => s.Should().BeNull() - ) - ); +---- + +will serialize to + +[source,javascript] +---- +"2015-05-05T00:00:00" +---- + +When the `DateTime` is local or UTC, the time zone information is included. +For example, for a UTC `DateTime` + +[source,csharp] +---- +var utcDate = new DateTime(2015, 05, 05, 0, 0, 0, DateTimeKind.Utc); +---- + +will serialize to + +[source,javascript] +---- +"2015-05-05T00:00:00Z" ---- ==== Complex expressions diff --git a/src/CodeGeneration/DocGenerator/StringExtensions.cs b/src/CodeGeneration/DocGenerator/StringExtensions.cs index 0f61e90ff8a..0e461d1ffff 100644 --- a/src/CodeGeneration/DocGenerator/StringExtensions.cs +++ b/src/CodeGeneration/DocGenerator/StringExtensions.cs @@ -213,7 +213,7 @@ public static string RemoveNumberOfLeadingTabsOrSpacesAfterNewline(this string i public static string[] SplitOnNewLines(this string input, StringSplitOptions options) => input.Split(new[] { "\r\n", "\n" }, options); - public static bool TryGetJsonForAnonymousType(this string anonymousTypeString, out string json) + public static bool TryGetJsonForExpressionSyntax(this string anonymousTypeString, out string json) { json = null; diff --git a/src/CodeGeneration/DocGenerator/SyntaxNodeExtensions.cs b/src/CodeGeneration/DocGenerator/SyntaxNodeExtensions.cs index e57c2bf3ea6..28316f49a48 100644 --- a/src/CodeGeneration/DocGenerator/SyntaxNodeExtensions.cs +++ b/src/CodeGeneration/DocGenerator/SyntaxNodeExtensions.cs @@ -72,11 +72,10 @@ public static bool TryGetJsonForSyntaxNode(this SyntaxNode node, out string json json = null; // find the first anonymous object or new object expression - var creationExpressionSyntax = node.DescendantNodes() - .FirstOrDefault(n => n is AnonymousObjectCreationExpressionSyntax || n is ObjectCreationExpressionSyntax); + var syntax = node.DescendantNodes() + .FirstOrDefault(n => n is AnonymousObjectCreationExpressionSyntax || n is ObjectCreationExpressionSyntax || n is LiteralExpressionSyntax); - return creationExpressionSyntax != null && - creationExpressionSyntax.ToFullString().TryGetJsonForAnonymousType(out json); + return syntax != null && syntax.ToFullString().TryGetJsonForExpressionSyntax(out json); } /// diff --git a/src/Elasticsearch.Net/Extensions/StringBuilderCache.cs b/src/Elasticsearch.Net/Extensions/StringBuilderCache.cs new file mode 100644 index 00000000000..873dbd9f8d0 --- /dev/null +++ b/src/Elasticsearch.Net/Extensions/StringBuilderCache.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Text; + +namespace Elasticsearch.Net.Extensions +{ + /// Provide a cached reusable instance of stringbuilder per thread. + internal static class StringBuilderCache + { + private const int DefaultCapacity = 16; // == StringBuilder.DefaultCapacity + + // The value 360 was chosen in discussion with performance experts as a compromise between using + // as little memory per thread as possible and still covering a large part of short-lived + // StringBuilder creations on the startup path of VS designers. + private const int MaxBuilderSize = 360; + + // WARNING: We allow diagnostic tools to directly inspect this member (t_cachedInstance). + // See https://github.com/dotnet/corert/blob/master/Documentation/design-docs/diagnostics/diagnostics-tools-contract.md for more details. + // Please do not change the type, the name, or the semantic usage of this member without understanding the implication for tools. + // Get in touch with the diagnostics team if you have questions. + [ThreadStatic] + private static StringBuilder _cachedInstance; + + /// Get a StringBuilder for the specified capacity. + /// If a StringBuilder of an appropriate size is cached, it will be returned and the cache emptied. + public static StringBuilder Acquire(int capacity = DefaultCapacity) + { + if (capacity <= MaxBuilderSize) + { + var sb = _cachedInstance; + if (sb != null) + { + // Avoid stringbuilder block fragmentation by getting a new StringBuilder + // when the requested size is larger than the current capacity + if (capacity <= sb.Capacity) + { + _cachedInstance = null; + sb.Clear(); + return sb; + } + } + } + + return new StringBuilder(capacity); + } + + /// Place the specified builder in the cache if it is not too big. + public static void Release(StringBuilder sb) + { + if (sb.Capacity <= MaxBuilderSize) _cachedInstance = sb; + } + + /// ToString() the stringbuilder, Release it to the cache, and return the resulting string. + public static string GetStringAndRelease(StringBuilder sb) + { + var result = sb.ToString(); + Release(sb); + return result; + } + } +} diff --git a/src/Nest/CommonOptions/DateMath/DateMath.cs b/src/Nest/CommonOptions/DateMath/DateMath.cs index ee32d7d0b8e..c08436564d5 100644 --- a/src/Nest/CommonOptions/DateMath/DateMath.cs +++ b/src/Nest/CommonOptions/DateMath/DateMath.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Text; using System.Text.RegularExpressions; using Elasticsearch.Net.Extensions; @@ -109,21 +110,50 @@ public override string ToString() } /// - /// Formats a to have a minimum of 3 decimal places if there - /// are sub second values + /// Formats a to have a minimum of 3 decimal places if there are sub second values /// - /// Fixes bug in Elasticsearch: https://github.com/elastic/elasticsearch/pull/41871 private static string ToMinThreeDecimalPlaces(DateTime dateTime) { - var format = dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFF"); + var builder = StringBuilderCache.Acquire(33); + var format = dateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss.FFFFFFF", CultureInfo.InvariantCulture); + builder.Append(format); + // Fixes bug in Elasticsearch: https://github.com/elastic/elasticsearch/pull/41871 if (format.Length > 20 && format.Length < 23) { var diff = 23 - format.Length; - return $"{format}{new string('0', diff)}"; + for (int i = 0; i < diff; i++) + builder.Append('0'); } - return format; + switch (dateTime.Kind) + { + case DateTimeKind.Local: + var offset = TimeZoneInfo.Local.GetUtcOffset(dateTime); + if (offset >= TimeSpan.Zero) + builder.Append('+'); + else + { + builder.Append('-'); + offset = offset.Negate(); + } + + AppendTwoDigitNumber(builder, offset.Hours); + builder.Append(':'); + AppendTwoDigitNumber(builder, offset.Minutes); + break; + case DateTimeKind.Utc: + builder.Append('Z'); + break; + } + + return StringBuilderCache.GetStringAndRelease(builder); + } + + private static void AppendTwoDigitNumber(StringBuilder result, int val) + { + result.Append((char)('0' + (val / 10))); + result.Append((char)('0' + (val % 10))); } } diff --git a/src/Tests/Tests/CommonOptions/DateMath/DateMathExpressions.doc.cs b/src/Tests/Tests/CommonOptions/DateMath/DateMathExpressions.doc.cs index 144e06d54b7..3d039b4fe36 100644 --- a/src/Tests/Tests/CommonOptions/DateMath/DateMathExpressions.doc.cs +++ b/src/Tests/Tests/CommonOptions/DateMath/DateMathExpressions.doc.cs @@ -61,7 +61,15 @@ [U] public void SimpleExpressions() * anchor will be an actual `DateTime`, even after a serialization/deserialization round trip */ var date = new DateTime(2015, 05, 05); - Expect("2015-05-05T00:00:00") + + /** + * will serialize to + */ + //json + var expected = "2015-05-05T00:00:00"; + + // hide + Expect(expected) .WhenSerializing(date) .AssertSubject(dateMath => ((IDateMath)dateMath) .Anchor.Match( @@ -69,6 +77,28 @@ [U] public void SimpleExpressions() s => s.Should().BeNull() ) ); + + /** + * When the `DateTime` is local or UTC, the time zone information is included. + * For example, for a UTC `DateTime` + */ + var utcDate = new DateTime(2015, 05, 05, 0, 0, 0, DateTimeKind.Utc); + + /** + * will serialize to + */ + //json + expected = "2015-05-05T00:00:00Z"; + + // hide + Expect(expected) + .WhenSerializing(utcDate) + .AssertSubject(dateMath => ((IDateMath)dateMath) + .Anchor.Match( + d => d.Should().Be(utcDate), + s => s.Should().BeNull() + ) + ); } [U] public void ComplexExpressions() From 329e03df8ecd9e73f27615f4d37592df3f72bc5a Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Mon, 8 Jul 2019 11:48:13 +1000 Subject: [PATCH 03/10] Do not set HttpRequestMessage content when no PostData (#3925) This commit changes the HttpConnection to not set Content on HttpRequestMessage when there is no PostData. The check is performed inside of Request and RequestAsync to avoid creating any content instance and allocations. Pass RequestData through to _onStreamAvailable delegates so that HttpCompression, ConnectionSettings and PostData can be accessed on the passed instance. Fixes #3907 (cherry picked from commit 90c2bf70b7c059508457c7e0ce3f08de6a314b04) --- .../Connection/Content/RequestDataContent.cs | 32 ++++++++++--------- .../Connection/HttpConnection.cs | 12 +++++-- src/Tests/Tests.Reproduce/GithubIssue3907.cs | 32 +++++++++++++++++++ 3 files changed, 58 insertions(+), 18 deletions(-) create mode 100644 src/Tests/Tests.Reproduce/GithubIssue3907.cs diff --git a/src/Elasticsearch.Net/Connection/Content/RequestDataContent.cs b/src/Elasticsearch.Net/Connection/Content/RequestDataContent.cs index d624e90f4cc..a85afcce514 100644 --- a/src/Elasticsearch.Net/Connection/Content/RequestDataContent.cs +++ b/src/Elasticsearch.Net/Connection/Content/RequestDataContent.cs @@ -25,8 +25,7 @@ namespace Elasticsearch.Net internal class RequestDataContent : HttpContent { private readonly RequestData _requestData; - private readonly Func _onStreamAvailable; - + private readonly Func _onStreamAvailable; public RequestDataContent(RequestData requestData) { @@ -35,12 +34,17 @@ public RequestDataContent(RequestData requestData) if (requestData.HttpCompression) Headers.ContentEncoding.Add("gzip"); - Task OnStreamAvailable(PostData data, Stream stream, HttpContent content, TransportContext context) + Task OnStreamAvailable(RequestData data, Stream stream, HttpContent content, TransportContext context) { + if (data.HttpCompression) + stream = new GZipStream(stream, CompressionMode.Compress, false); + using(stream) - data.Write(stream, requestData.ConnectionSettings); + data.PostData.Write(stream, data.ConnectionSettings); + return Task.CompletedTask; } + _onStreamAvailable = OnStreamAvailable; } public RequestDataContent(RequestData requestData, CancellationToken token) @@ -50,11 +54,15 @@ public RequestDataContent(RequestData requestData, CancellationToken token) if (requestData.HttpCompression) Headers.ContentEncoding.Add("gzip"); - async Task OnStreamAvailable(PostData data, Stream stream, HttpContent content, TransportContext context) + async Task OnStreamAvailable(RequestData data, Stream stream, HttpContent content, TransportContext context) { + if (data.HttpCompression) + stream = new GZipStream(stream, CompressionMode.Compress, false); + using (stream) - await data.WriteAsync(stream, requestData.ConnectionSettings, token).ConfigureAwait(false); + await data.PostData.WriteAsync(stream, data.ConnectionSettings, token).ConfigureAwait(false); } + _onStreamAvailable = OnStreamAvailable; } @@ -69,16 +77,9 @@ async Task OnStreamAvailable(PostData data, Stream stream, HttpContent content, [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Exception is passed as task result.")] protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context) { - - var data = _requestData.PostData; - if (data == null) return; - var serializeToStreamTask = new TaskCompletionSource(); - - if (_requestData.HttpCompression) - stream = new GZipStream(stream, CompressionMode.Compress, false); var wrappedStream = new CompleteTaskOnCloseStream(stream, serializeToStreamTask); - await _onStreamAvailable(data, wrappedStream, this, context).ConfigureAwait(false); + await _onStreamAvailable(_requestData, wrappedStream, this, context).ConfigureAwait(false); await serializeToStreamTask.Task.ConfigureAwait(false); } @@ -111,7 +112,6 @@ protected override void Dispose(bool disposing) base.Dispose(); } - public override void Close() => _serializeToStreamTask.TrySetResult(true); } @@ -193,6 +193,8 @@ public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, As public override void EndWrite(IAsyncResult asyncResult) => _innerStream.EndWrite(asyncResult); public override void WriteByte(byte value) => _innerStream.WriteByte(value); + + public override void Close() => _innerStream.Close(); } } } diff --git a/src/Elasticsearch.Net/Connection/HttpConnection.cs b/src/Elasticsearch.Net/Connection/HttpConnection.cs index 734e7e6d3b0..c9b36d209b0 100644 --- a/src/Elasticsearch.Net/Connection/HttpConnection.cs +++ b/src/Elasticsearch.Net/Connection/HttpConnection.cs @@ -57,7 +57,10 @@ public virtual TResponse Request(RequestData requestData) try { var requestMessage = CreateHttpRequestMessage(requestData); - SetContent(requestMessage, requestData); + + if (requestData.PostData != null) + SetContent(requestMessage, requestData); + using(requestMessage?.Content ?? (IDisposable)Stream.Null) using (var d = DiagnosticSource.Diagnose(DiagnosticSources.HttpConnection.SendAndReceiveHeaders, requestData)) { @@ -107,8 +110,11 @@ public virtual async Task RequestAsync(RequestData request try { var requestMessage = CreateHttpRequestMessage(requestData); - SetAsyncContent(requestMessage, requestData, cancellationToken); - using(requestMessage?.Content ?? (IDisposable)Stream.Null) + + if (requestData.PostData != null) + SetAsyncContent(requestMessage, requestData, cancellationToken); + + using(requestMessage?.Content ?? (IDisposable)Stream.Null) using (var d = DiagnosticSource.Diagnose(DiagnosticSources.HttpConnection.SendAndReceiveHeaders, requestData)) { responseMessage = await client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); diff --git a/src/Tests/Tests.Reproduce/GithubIssue3907.cs b/src/Tests/Tests.Reproduce/GithubIssue3907.cs new file mode 100644 index 00000000000..e958e5f7f34 --- /dev/null +++ b/src/Tests/Tests.Reproduce/GithubIssue3907.cs @@ -0,0 +1,32 @@ +using System; +using System.Net; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; + +namespace Tests.Reproduce +{ + public class GithubIssue3907 : IClusterFixture + { + private readonly IntrusiveOperationCluster _cluster; + + // use intrusive operation cluster because we're changing the underlying http handler + // and this cluster runs with a max concurrency of 1, so changing http handler + // will not affect other integration tests + public GithubIssue3907(IntrusiveOperationCluster cluster) => _cluster = cluster; + + [I] + public void NotUsingSocketsHttpHandlerDoesNotCauseException() + { + AppContext.SetSwitch("System.Net.Http.UseSocketsHttpHandler", false); + + var response = _cluster.Client.Indices.Exists("non_existent_index"); + response.ApiCall.HttpStatusCode.Should().Be(404); + response.OriginalException.Should().BeNull(); + + AppContext.SetSwitch("System.Net.Http.UseSocketsHttpHandler", true); + } + } +} From 865e1b97d67d567327254545a9b565bf59f43d86 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Mon, 8 Jul 2019 12:04:31 +1000 Subject: [PATCH 04/10] Support numbers with exponent in WKT (#3922) This commit provides support for numbers with an exponent when reading WKT. Fixes #3920 (cherry picked from commit 8386d4ce39d1da988c4380a2bcdfb3397e4841e7) --- src/Nest/QueryDsl/Geo/WKT/GeoWKTReader.cs | 54 ++++--------------- .../Tests/QueryDsl/Geo/Shape/GeoWKTTests.cs | 16 ++++++ 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/src/Nest/QueryDsl/Geo/WKT/GeoWKTReader.cs b/src/Nest/QueryDsl/Geo/WKT/GeoWKTReader.cs index 924304b2294..b5f551b26cc 100644 --- a/src/Nest/QueryDsl/Geo/WKT/GeoWKTReader.cs +++ b/src/Nest/QueryDsl/Geo/WKT/GeoWKTReader.cs @@ -259,14 +259,14 @@ private static TokenType NextCloserOrComma(WellKnownTextTokenizer tokenizer) private static double NextNumber(WellKnownTextTokenizer tokenizer) { - if (tokenizer.NextToken() == TokenType.Number) + if (tokenizer.NextToken() == TokenType.Word) { if (string.Equals(tokenizer.TokenValue, WellKnownTextTokenizer.NaN, StringComparison.OrdinalIgnoreCase)) return double.NaN; if (double.TryParse( tokenizer.TokenValue, - NumberStyles.AllowDecimalPoint | NumberStyles.AllowLeadingSign, + NumberStyles.AllowDecimalPoint | NumberStyles.AllowLeadingSign | NumberStyles.AllowExponent, CultureInfo.InvariantCulture, out var d)) return d; } @@ -278,7 +278,7 @@ private static double NextNumber(WellKnownTextTokenizer tokenizer) private static bool IsNumberNext(WellKnownTextTokenizer tokenizer) { var token = tokenizer.PeekToken(); - return token == TokenType.Number; + return token == TokenType.Word; } } @@ -288,7 +288,6 @@ private static bool IsNumberNext(WellKnownTextTokenizer tokenizer) internal enum CharacterType : byte { Whitespace, - Digit, Alpha, Comment } @@ -300,7 +299,6 @@ internal enum TokenType : byte { None, Word, - Number, LParen, RParen, Comma @@ -339,15 +337,14 @@ static WellKnownTextTokenizer() // build a map of ASCII chars and their types // Any unmapped ASCII will be considered whitespace // and anything > 0 outside of ASCII will be considered alpha. - // Treat + - and . as digit characters to make parsing numbers easier. Chars('a', 'z', CharacterType.Alpha); Chars('A', 'Z', CharacterType.Alpha); Chars(128 + 32, 255, CharacterType.Alpha); - Chars('0', '9', CharacterType.Digit); + Chars('0', '9', CharacterType.Alpha); Chars(LParen, RParen, CharacterType.Alpha); - Chars(Plus, Plus, CharacterType.Digit); + Chars(Plus, Plus, CharacterType.Alpha); Chars(Comma, Comma, CharacterType.Alpha); - Chars(Minus, Dot, CharacterType.Digit); + Chars(Minus, Dot, CharacterType.Alpha); Chars(Comment, Comment, CharacterType.Comment); } @@ -399,7 +396,6 @@ public string TokenString() switch (TokenType) { case TokenType.Word: - case TokenType.Number: return TokenValue; case TokenType.None: return "END-OF-STREAM"; @@ -514,33 +510,6 @@ public TokenType NextToken() { var i = 0; - do - { - _buffer.Insert(i++, (char)c); - c = Read(); - - if (c < 0) - characterType = CharacterType.Whitespace; - else if (c < CharacterTypesLength) - characterType = CharacterTypes[c]; - else - characterType = CharacterType.Alpha; - } while (characterType == CharacterType.Alpha); - - _peekChar = c; - TokenValue = new string(_buffer.ToArray(), 0, i); - - // special case for NaN - if (string.Equals(TokenValue, NaN, StringComparison.OrdinalIgnoreCase)) - return TokenType = TokenType.Number; - - return TokenType = TokenType.Word; - } - - if (characterType == CharacterType.Digit) - { - var i = 0; - var dots = 0; do { _buffer.Insert(i++, (char)c); @@ -550,20 +519,19 @@ public TokenType NextToken() characterType = CharacterType.Whitespace; else if (c < CharacterTypesLength) { + if (c == LParen || c == RParen || c == Comma) + break; + characterType = CharacterTypes[c]; - if (c == Dot) - dots++; } else characterType = CharacterType.Alpha; - } while (characterType == CharacterType.Digit); + } while (characterType == CharacterType.Alpha); _peekChar = c; TokenValue = new string(_buffer.ToArray(), 0, i); - return dots > 1 - ? TokenType = TokenType.Word - : TokenType = TokenType.Number; + return TokenType = TokenType.Word; } if (characterType == CharacterType.Comment) diff --git a/src/Tests/Tests/QueryDsl/Geo/Shape/GeoWKTTests.cs b/src/Tests/Tests/QueryDsl/Geo/Shape/GeoWKTTests.cs index b941cf94ae7..dbbbecce7f8 100644 --- a/src/Tests/Tests/QueryDsl/Geo/Shape/GeoWKTTests.cs +++ b/src/Tests/Tests/QueryDsl/Geo/Shape/GeoWKTTests.cs @@ -23,6 +23,22 @@ public void ReadAndWritePoint() GeoWKTWriter.Write(point).Should().Be(wkt); } + [U] + public void ReadAndWritePointWithExponent() + { + var wkt = "POINT (1.2E2 -2.5E-05)"; + var shape = GeoWKTReader.Read(wkt); + + shape.Should().BeOfType(); + var point = (PointGeoShape)shape; + + point.Coordinates.Latitude.Should().Be(-0.000025); + point.Coordinates.Longitude.Should().Be(120); + + // 1.2E2 will be expanded + GeoWKTWriter.Write(point).Should().Be("POINT (120 -2.5E-05)"); + } + [U] public void ReadAndWriteMultiPoint() { From dd6c997820dd62f460d7d7483a5dcc8b3769cac9 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 9 Jul 2019 13:58:49 +1000 Subject: [PATCH 05/10] Add script_score query support (#3919) This commit adds script_score query support to the high level client Closes #3843 (cherry picked from commit a4741949bd50da2210d7699f0485961cf09fecd8) --- docs/query-dsl.asciidoc | 4 + .../function-score-query-usage.asciidoc | 57 ++++++-- .../script-score-query-usage.asciidoc | 102 +++++++++++++++ .../DocGenerator/StringExtensions.cs | 3 +- .../Abstractions/Container/IQueryContainer.cs | 4 + .../Container/QueryContainer-Assignments.cs | 7 + .../Container/QueryContainerDescriptor.cs | 3 + src/Nest/QueryDsl/Query.cs | 4 + .../ScriptScore/ScriptScoreQuery.cs | 77 +++++++++++ .../QueryDsl/Visitor/DslPrettyPrintVisitor.cs | 2 + src/Nest/QueryDsl/Visitor/QueryVisitor.cs | 4 + src/Nest/QueryDsl/Visitor/QueryWalker.cs | 1 + .../ScriptScore/ScriptScoreQueryUsageTests.cs | 122 ++++++++++++++++++ 13 files changed, 381 insertions(+), 9 deletions(-) create mode 100644 docs/query-dsl/specialized/script-score/script-score-query-usage.asciidoc create mode 100644 src/Nest/QueryDsl/Specialized/ScriptScore/ScriptScoreQuery.cs create mode 100644 src/Tests/Tests/QueryDsl/Specialized/ScriptScore/ScriptScoreQueryUsageTests.cs diff --git a/docs/query-dsl.asciidoc b/docs/query-dsl.asciidoc index f007d3898e2..cd7bbc48004 100644 --- a/docs/query-dsl.asciidoc +++ b/docs/query-dsl.asciidoc @@ -275,6 +275,8 @@ Specialized types of queries that do not fit into other groups * <> +* <> + See the Elasticsearch documentation on {ref_current}/specialized-queries.html[Specialized queries] for more details. :includes-from-dirs: query-dsl/specialized @@ -287,6 +289,8 @@ include::query-dsl/specialized/percolate/percolate-query-usage.asciidoc[] include::query-dsl/specialized/script/script-query-usage.asciidoc[] +include::query-dsl/specialized/script-score/script-score-query-usage.asciidoc[] + [[span-queries]] == Span queries diff --git a/docs/query-dsl/compound/function-score/function-score-query-usage.asciidoc b/docs/query-dsl/compound/function-score/function-score-query-usage.asciidoc index 0a35c4916b3..7cd30fba882 100644 --- a/docs/query-dsl/compound/function-score/function-score-query-usage.asciidoc +++ b/docs/query-dsl/compound/function-score/function-score-query-usage.asciidoc @@ -29,15 +29,36 @@ q .MaxBoost(20.0) .MinScore(1.0) .Functions(f => f - .Exponential(b => b.Field(p => p.NumberOfCommits).Decay(0.5).Origin(1.0).Scale(0.1).Weight(2.1)) + .Exponential(b => b + .Field(p => p.NumberOfCommits) + .Decay(0.5) + .Origin(1.0) + .Scale(0.1) + .Weight(2.1) + .Filter(fi => fi + .Range(r => r + .Field(p => p.NumberOfContributors) + .GreaterThan(10) + ) + ) + ) .GaussDate(b => b.Field(p => p.LastActivity).Origin(DateMath.Now).Decay(0.5).Scale("1d")) - .LinearGeoLocation(b => - b.Field(p => p.LocationPoint).Origin(new GeoLocation(70, -70)).Scale(Distance.Miles(1)).MultiValueMode(MultiValueMode.Average)) + .LinearGeoLocation(b => b + .Field(p => p.LocationPoint) + .Origin(new GeoLocation(70, -70)) + .Scale(Distance.Miles(1)) + .MultiValueMode(MultiValueMode.Average) + ) .FieldValueFactor(b => b.Field(p => p.NumberOfContributors).Factor(1.1).Missing(0.1).Modifier(FieldValueFactorModifier.Square)) .RandomScore(r => r.Seed(1337).Field("_seq_no")) .RandomScore(r => r.Seed("randomstring").Field("_seq_no")) .Weight(1.0) - .ScriptScore(s => s.Script(ss => ss.Source("Math.log(2 + doc['numberOfCommits'].value)"))) + .ScriptScore(s => s + .Script(ss => ss + .Source("Math.log(2 + doc['numberOfCommits'].value)") + ) + .Weight(2) + ) ) ) ---- @@ -57,7 +78,19 @@ new FunctionScoreQuery() MinScore = 1.0, Functions = new List { - new ExponentialDecayFunction { Origin = 1.0, Decay = 0.5, Field = Field(p => p.NumberOfCommits), Scale = 0.1, Weight = 2.1 }, + new ExponentialDecayFunction + { + Origin = 1.0, + Decay = 0.5, + Field = Field(p => p.NumberOfCommits), + Scale = 0.1, + Weight = 2.1, + Filter = new NumericRangeQuery + { + Field = Field(f => f.NumberOfContributors), + GreaterThan = 10 + } + }, new GaussDateDecayFunction { Origin = DateMath.Now, Field = Field(p => p.LastActivity), Decay = 0.5, Scale = TimeSpan.FromDays(1) }, new LinearGeoDecayFunction @@ -72,7 +105,7 @@ new FunctionScoreQuery() new RandomScoreFunction { Seed = 1337, Field = "_seq_no" }, new RandomScoreFunction { Seed = "randomstring", Field = "_seq_no" }, new WeightFunction { Weight = 1.0 }, - new ScriptScoreFunction { Script = new InlineScript("Math.log(2 + doc['numberOfCommits'].value)") } + new ScriptScoreFunction { Script = new InlineScript("Math.log(2 + doc['numberOfCommits'].value)"), Weight = 2.0 } } } ---- @@ -94,7 +127,14 @@ new FunctionScoreQuery() "decay": 0.5 } }, - "weight": 2.1 + "weight": 2.1, + "filter": { + "range": { + "numberOfContributors": { + "gt": 10.0 + } + } + } }, { "gauss": { @@ -145,7 +185,8 @@ new FunctionScoreQuery() "script": { "source": "Math.log(2 + doc['numberOfCommits'].value)" } - } + }, + "weight": 2.0 } ], "max_boost": 20.0, diff --git a/docs/query-dsl/specialized/script-score/script-score-query-usage.asciidoc b/docs/query-dsl/specialized/script-score/script-score-query-usage.asciidoc new file mode 100644 index 00000000000..c99e29fb06a --- /dev/null +++ b/docs/query-dsl/specialized/script-score/script-score-query-usage.asciidoc @@ -0,0 +1,102 @@ +:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/7.0 + +:github: https://github.com/elastic/elasticsearch-net + +:nuget: https://www.nuget.org/packages + +//// +IMPORTANT NOTE +============== +This file has been generated from https://github.com/elastic/elasticsearch-net/tree/master/src/Tests/Tests/QueryDsl/Specialized/ScriptScore/ScriptScoreQueryUsageTests.cs. +If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file, +please modify the original csharp file found at the link and submit the PR with that change. Thanks! +//// + +[[script-score-query-usage]] +=== Script Score Query Usage + +A query allowing you to modify the score of documents that are retrieved by a query. +This can be useful if, for example, a score function is computationally expensive and +it is sufficient to compute the score on a filtered set of documents. + +See the Elasticsearch documentation on {ref_current}/query-dsl-script-score-query.html[script_score query] for more details. + +==== Fluent DSL example + +[source,csharp] +---- +q +.ScriptScore(sn => sn + .Name("named_query") + .Boost(1.1) + .Query(qq => qq + .Range(r => r + .Field(f => f.NumberOfCommits) + .GreaterThan(50) + ) + ) + .Script(s => s + .Source(_scriptScoreSource) + .Params(p => p + .Add("origin", 100) + .Add("scale", 10) + .Add("decay", 0.5) + .Add("offset", 0) + ) + ) +) +---- + +==== Object Initializer syntax example + +[source,csharp] +---- +new ScriptScoreQuery +{ + Name = "named_query", + Boost = 1.1, + Query = new NumericRangeQuery + { + Field = Infer.Field(f => f.NumberOfCommits), + GreaterThan = 50 + }, + Script = new InlineScript(_scriptScoreSource) + { + Params = new Dictionary + { + { "origin", 100 }, + { "scale", 10 }, + { "decay", 0.5 }, + { "offset", 0 } + } + }, +} +---- + +[source,javascript] +.Example json output +---- +{ + "script_score": { + "_name": "named_query", + "boost": 1.1, + "query": { + "range": { + "numberOfCommits": { + "gt": 50.0 + } + } + }, + "script": { + "source": "decayNumericLinear(params.origin, params.scale, params.offset, params.decay, doc['numberOfCommits'].value)", + "params": { + "origin": 100, + "scale": 10, + "decay": 0.5, + "offset": 0 + } + } + } +} +---- + diff --git a/src/CodeGeneration/DocGenerator/StringExtensions.cs b/src/CodeGeneration/DocGenerator/StringExtensions.cs index 0e461d1ffff..25fe8da0987 100644 --- a/src/CodeGeneration/DocGenerator/StringExtensions.cs +++ b/src/CodeGeneration/DocGenerator/StringExtensions.cs @@ -114,7 +114,8 @@ public static class StringExtensions new []{ new [] {8.2, 18.2}, new [] {8.2, -18.8}, new [] {-8.8, -10.8}, new [] {8.8, 18.2}} }" }, - { "ProjectFilterExpectedJson", "new {term = new {type = new {value = \"project\"}}}" } + { "ProjectFilterExpectedJson", "new {term = new {type = new {value = \"project\"}}}" }, + { "_scriptScoreSource", "\"decayNumericLinear(params.origin, params.scale, params.offset, params.decay, doc['numberOfCommits'].value)\""} }; private static readonly Regex LeadingSpacesAndAsterisk = new Regex(@"^(?[ \t]*\*\s?).*", RegexOptions.Compiled); diff --git a/src/Nest/QueryDsl/Abstractions/Container/IQueryContainer.cs b/src/Nest/QueryDsl/Abstractions/Container/IQueryContainer.cs index 0a28e8d6181..81b7aa43690 100644 --- a/src/Nest/QueryDsl/Abstractions/Container/IQueryContainer.cs +++ b/src/Nest/QueryDsl/Abstractions/Container/IQueryContainer.cs @@ -115,6 +115,10 @@ public interface IQueryContainer [DataMember(Name ="script")] IScriptQuery Script { get; set; } + /// + [DataMember(Name ="script_score")] + IScriptScoreQuery ScriptScore { get; set; } + [DataMember(Name ="simple_query_string")] ISimpleQueryStringQuery SimpleQueryString { get; set; } diff --git a/src/Nest/QueryDsl/Abstractions/Container/QueryContainer-Assignments.cs b/src/Nest/QueryDsl/Abstractions/Container/QueryContainer-Assignments.cs index 15024c1fa55..0481cd3efeb 100644 --- a/src/Nest/QueryDsl/Abstractions/Container/QueryContainer-Assignments.cs +++ b/src/Nest/QueryDsl/Abstractions/Container/QueryContainer-Assignments.cs @@ -38,6 +38,7 @@ public partial class QueryContainer : IQueryContainer, IDescriptor private IRawQuery _raw; private IRegexpQuery _regexp; private IScriptQuery _script; + private IScriptScoreQuery _scriptScore; private ISimpleQueryStringQuery _simpleQueryString; private ISpanContainingQuery _spanContaining; private ISpanFieldMaskingQuery _spanFieldMasking; @@ -251,6 +252,12 @@ IScriptQuery IQueryContainer.Script set => _script = Set(value); } + IScriptScoreQuery IQueryContainer.ScriptScore + { + get => _scriptScore; + set => _scriptScore = Set(value); + } + ISimpleQueryStringQuery IQueryContainer.SimpleQueryString { get => _simpleQueryString; diff --git a/src/Nest/QueryDsl/Abstractions/Container/QueryContainerDescriptor.cs b/src/Nest/QueryDsl/Abstractions/Container/QueryContainerDescriptor.cs index 2cd57ad0b19..ef57d8f59ae 100644 --- a/src/Nest/QueryDsl/Abstractions/Container/QueryContainerDescriptor.cs +++ b/src/Nest/QueryDsl/Abstractions/Container/QueryContainerDescriptor.cs @@ -437,6 +437,9 @@ public QueryContainer FunctionScore(Func, IFunct public QueryContainer Script(Func, IScriptQuery> selector) => WrapInContainer(selector, (query, container) => container.Script = query); + public QueryContainer ScriptScore(Func, IScriptScoreQuery> selector) => + WrapInContainer(selector, (query, container) => container.ScriptScore = query); + public QueryContainer Exists(Func, IExistsQuery> selector) => WrapInContainer(selector, (query, container) => container.Exists = query); diff --git a/src/Nest/QueryDsl/Query.cs b/src/Nest/QueryDsl/Query.cs index 048f1f41716..9917545ad76 100644 --- a/src/Nest/QueryDsl/Query.cs +++ b/src/Nest/QueryDsl/Query.cs @@ -117,6 +117,10 @@ public static QueryContainer Regexp(Func, IRegexpQuery> public static QueryContainer Script(Func, IScriptQuery> selector) => new QueryContainerDescriptor().Script(selector); + /// + public static QueryContainer ScriptScore(Func, IScriptScoreQuery> selector) => + new QueryContainerDescriptor().ScriptScore(selector); + public static QueryContainer SimpleQueryString(Func, ISimpleQueryStringQuery> selector) => new QueryContainerDescriptor().SimpleQueryString(selector); diff --git a/src/Nest/QueryDsl/Specialized/ScriptScore/ScriptScoreQuery.cs b/src/Nest/QueryDsl/Specialized/ScriptScore/ScriptScoreQuery.cs new file mode 100644 index 00000000000..b0e8d73bb1d --- /dev/null +++ b/src/Nest/QueryDsl/Specialized/ScriptScore/ScriptScoreQuery.cs @@ -0,0 +1,77 @@ +using System; +using System.Runtime.Serialization; +using Elasticsearch.Net.Utf8Json; + +namespace Nest +{ + /// + /// A query allowing you to modify the score of documents that are retrieved by a query. + /// This can be useful if, for example, a score function is computationally expensive and it is sufficient to + /// compute the score on a filtered set of documents. + /// + [ReadAs(typeof(ScriptScoreQuery))] + [InterfaceDataContract] + public interface IScriptScoreQuery : IQuery + { + /// + /// The query to execute + /// + [DataMember(Name = "query")] + QueryContainer Query { get; set; } + + /// + /// The script to execute + /// + [DataMember(Name = "script")] + IScript Script { get; set; } + } + + /// + public class ScriptScoreQuery : QueryBase, IScriptScoreQuery + { + /// + public QueryContainer Query { get; set; } + + /// + public IScript Script { get; set; } + + protected override bool Conditionless => IsConditionless(this); + + internal override void InternalWrapInContainer(IQueryContainer c) => c.ScriptScore = this; + + internal static bool IsConditionless(IScriptScoreQuery q) + { + if (q.Script == null || q.Query.IsConditionless()) + return true; + + switch (q.Script) + { + case IInlineScript inlineScript: + return inlineScript.Source.IsNullOrEmpty(); + case IIndexedScript indexedScript: + return indexedScript.Id.IsNullOrEmpty(); + } + + return false; + } + } + + /// + public class ScriptScoreQueryDescriptor + : QueryDescriptorBase, IScriptScoreQuery> + , IScriptScoreQuery where T : class + { + protected override bool Conditionless => ScriptScoreQuery.IsConditionless(this); + QueryContainer IScriptScoreQuery.Query { get; set; } + + IScript IScriptScoreQuery.Script { get; set; } + + /// + public ScriptScoreQueryDescriptor Query(Func, QueryContainer> selector) => + Assign(selector, (a, v) => a.Query = v?.Invoke(new QueryContainerDescriptor())); + + /// + public ScriptScoreQueryDescriptor Script(Func selector) => + Assign(selector, (a, v) => a.Script = v?.Invoke(new ScriptDescriptor())); + } +} diff --git a/src/Nest/QueryDsl/Visitor/DslPrettyPrintVisitor.cs b/src/Nest/QueryDsl/Visitor/DslPrettyPrintVisitor.cs index b06a76780e7..68f0c599872 100644 --- a/src/Nest/QueryDsl/Visitor/DslPrettyPrintVisitor.cs +++ b/src/Nest/QueryDsl/Visitor/DslPrettyPrintVisitor.cs @@ -187,6 +187,8 @@ public virtual void Visit(IGeoShapeQuery query) public virtual void Visit(IScriptQuery query) => Write("script"); + public virtual void Visit(IScriptScoreQuery query) => Write("script_score"); + public virtual void Visit(IRawQuery query) => Write("raw"); public virtual void Visit(IPercolateQuery query) => Write("percolate"); diff --git a/src/Nest/QueryDsl/Visitor/QueryVisitor.cs b/src/Nest/QueryDsl/Visitor/QueryVisitor.cs index dd5f757cb9e..492f2bc7bae 100644 --- a/src/Nest/QueryDsl/Visitor/QueryVisitor.cs +++ b/src/Nest/QueryDsl/Visitor/QueryVisitor.cs @@ -86,6 +86,8 @@ public interface IQueryVisitor void Visit(IScriptQuery query); + void Visit(IScriptScoreQuery query); + void Visit(IGeoPolygonQuery query); void Visit(IGeoDistanceQuery query); @@ -237,6 +239,8 @@ public virtual void Visit(ITermsQuery query) { } public virtual void Visit(IScriptQuery query) { } + public virtual void Visit(IScriptScoreQuery query) { } + public virtual void Visit(IGeoPolygonQuery query) { } public virtual void Visit(IGeoDistanceQuery query) { } diff --git a/src/Nest/QueryDsl/Visitor/QueryWalker.cs b/src/Nest/QueryDsl/Visitor/QueryWalker.cs index dc7e8ab7834..0af0664ffec 100644 --- a/src/Nest/QueryDsl/Visitor/QueryWalker.cs +++ b/src/Nest/QueryDsl/Visitor/QueryWalker.cs @@ -44,6 +44,7 @@ public void Walk(IQueryContainer qd, IQueryVisitor visitor) VisitQuery(qd.MatchPhrase, visitor, (v, d) => v.Visit(d)); VisitQuery(qd.MatchPhrasePrefix, visitor, (v, d) => v.Visit(d)); VisitQuery(qd.Script, visitor, (v, d) => v.Visit(d)); + VisitQuery(qd.ScriptScore, visitor, (v, d) => v.Visit(d)); VisitQuery(qd.Exists, visitor, (v, d) => v.Visit(d)); VisitQuery(qd.GeoPolygon, visitor, (v, d) => v.Visit(d)); VisitQuery(qd.GeoDistance, visitor, (v, d) => v.Visit(d)); diff --git a/src/Tests/Tests/QueryDsl/Specialized/ScriptScore/ScriptScoreQueryUsageTests.cs b/src/Tests/Tests/QueryDsl/Specialized/ScriptScore/ScriptScoreQueryUsageTests.cs new file mode 100644 index 00000000000..66ba1f04a48 --- /dev/null +++ b/src/Tests/Tests/QueryDsl/Specialized/ScriptScore/ScriptScoreQueryUsageTests.cs @@ -0,0 +1,122 @@ +using System.Collections.Generic; +using Nest; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.QueryDsl.Specialized.ScriptScore +{ + /** + * A query allowing you to modify the score of documents that are retrieved by a query. + * This can be useful if, for example, a score function is computationally expensive and + * it is sufficient to compute the score on a filtered set of documents. + * + * See the Elasticsearch documentation on {ref_current}/query-dsl-script-score-query.html[script_score query] for more details. + */ + public class ScriptScoreQueryUsageTests : QueryDslUsageTestsBase + { + private static readonly string _scriptScoreSource = "decayNumericLinear(params.origin, params.scale, params.offset, params.decay, doc['numberOfCommits'].value)"; + + public ScriptScoreQueryUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } + + protected override ConditionlessWhen ConditionlessWhen => new ConditionlessWhen(a => a.ScriptScore) + { + q => + { + q.Query = null; + }, + q => + { + q.Script = null; + }, + q => + { + q.Script = new InlineScript(null); + }, + q => + { + q.Script = new InlineScript(""); + }, + q => + { + q.Script = new IndexedScript(null); + }, + q => + { + q.Script = new IndexedScript(""); + } + }; + + protected override QueryContainer QueryInitializer => new ScriptScoreQuery + { + Name = "named_query", + Boost = 1.1, + Query = new NumericRangeQuery + { + Field = Infer.Field(f => f.NumberOfCommits), + GreaterThan = 50 + }, + Script = new InlineScript(_scriptScoreSource) + { + Params = new Dictionary + { + { "origin", 100 }, + { "scale", 10 }, + { "decay", 0.5 }, + { "offset", 0 } + } + }, + }; + + protected override object QueryJson => new + { + script_score = new + { + _name = "named_query", + boost = 1.1, + query = new + { + range = new + { + numberOfCommits = new + { + gt = 50.0 + } + } + }, + script = new + { + source = _scriptScoreSource, + @params = new + { + origin = 100, + scale = 10, + decay = 0.5, + offset = 0 + } + } + } + }; + + protected override QueryContainer QueryFluent(QueryContainerDescriptor q) => q + .ScriptScore(sn => sn + .Name("named_query") + .Boost(1.1) + .Query(qq => qq + .Range(r => r + .Field(f => f.NumberOfCommits) + .GreaterThan(50) + ) + ) + .Script(s => s + .Source(_scriptScoreSource) + .Params(p => p + .Add("origin", 100) + .Add("scale", 10) + .Add("decay", 0.5) + .Add("offset", 0) + ) + ) + ); + } +} From 7b814e0343dcccb2fa355ef291d76ee0a0e1adf8 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 9 Jul 2019 14:03:02 +1000 Subject: [PATCH 06/10] Add track_total_hits to search request body (#3917) This commit removes track_total_hits as a query string parameter on a search request and instead defines it in the body, allowing track_total_hits to work with individual search requests in multi search API. Closes #3906 (cherry picked from commit 75317ecc215980dac685fec836bd0dc6b8d08e63) --- .../Overrides/GlobalOverrides.cs | 3 +- .../RequestParameters.NoNamespace.cs | 7 - src/Nest/Descriptors.NoNamespace.cs | 2 - src/Nest/Requests.NoNamespace.cs | 7 - ...onConverter.cs => MultiSearchFormatter.cs} | 2 +- .../Search/MultiSearch/MultiSearchRequest.cs | 2 +- src/Nest/Search/Search/SearchRequest.cs | 258 +++++++++++------- .../Search/MultiSearch/MultiSearchApiTests.cs | 9 +- 8 files changed, 171 insertions(+), 119 deletions(-) rename src/Nest/Search/MultiSearch/{MultiSearchJsonConverter.cs => MultiSearchFormatter.cs} (95%) diff --git a/src/CodeGeneration/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs b/src/CodeGeneration/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs index 1b83c798db5..66bff45327b 100644 --- a/src/CodeGeneration/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs +++ b/src/CodeGeneration/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs @@ -40,7 +40,8 @@ public class GlobalOverrides : EndpointOverridesBase "copy_settings", //this still needs a PR? "source", // allows the body to be specified as a request param, we do not want to advertise this with a strongly typed method "timestamp", - "_source_include", "_source_exclude" // can be removed once https://github.com/elastic/elasticsearch/pull/41439 is in + "_source_include", "_source_exclude", // can be removed once https://github.com/elastic/elasticsearch/pull/41439 is in + "track_total_hits" }; } } diff --git a/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.NoNamespace.cs b/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.NoNamespace.cs index d8423db8444..3342d8e688c 100644 --- a/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.NoNamespace.cs +++ b/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.NoNamespace.cs @@ -1712,13 +1712,6 @@ public bool? TotalHitsAsInteger set => Q("rest_total_hits_as_int", value); } - ///Indicate if the number of documents that match the query should be tracked - public bool? TrackTotalHits - { - get => Q("track_total_hits"); - set => Q("track_total_hits", value); - } - ///Specify whether aggregation and suggester names should be prefixed by their respective types in the response public bool? TypedKeys { diff --git a/src/Nest/Descriptors.NoNamespace.cs b/src/Nest/Descriptors.NoNamespace.cs index e746e4bf907..fa302f6708f 100644 --- a/src/Nest/Descriptors.NoNamespace.cs +++ b/src/Nest/Descriptors.NoNamespace.cs @@ -1300,8 +1300,6 @@ public SearchDescriptor Index() public SearchDescriptor SuggestText(string suggesttext) => Qs("suggest_text", suggesttext); ///Indicates whether hits.total should be rendered as an integer or an object in the rest search response public SearchDescriptor TotalHitsAsInteger(bool? totalhitsasinteger = true) => Qs("rest_total_hits_as_int", totalhitsasinteger); - ///Indicate if the number of documents that match the query should be tracked - public SearchDescriptor TrackTotalHits(bool? tracktotalhits = true) => Qs("track_total_hits", tracktotalhits); ///Specify whether aggregation and suggester names should be prefixed by their respective types in the response public SearchDescriptor TypedKeys(bool? typedkeys = true) => Qs("typed_keys", typedkeys); } diff --git a/src/Nest/Requests.NoNamespace.cs b/src/Nest/Requests.NoNamespace.cs index 10dfd363dc7..124b6cc4ee6 100644 --- a/src/Nest/Requests.NoNamespace.cs +++ b/src/Nest/Requests.NoNamespace.cs @@ -2809,13 +2809,6 @@ public bool? TotalHitsAsInteger set => Q("rest_total_hits_as_int", value); } - ///Indicate if the number of documents that match the query should be tracked - public bool? TrackTotalHits - { - get => Q("track_total_hits"); - set => Q("track_total_hits", value); - } - ///Specify whether aggregation and suggester names should be prefixed by their respective types in the response public bool? TypedKeys { diff --git a/src/Nest/Search/MultiSearch/MultiSearchJsonConverter.cs b/src/Nest/Search/MultiSearch/MultiSearchFormatter.cs similarity index 95% rename from src/Nest/Search/MultiSearch/MultiSearchJsonConverter.cs rename to src/Nest/Search/MultiSearch/MultiSearchFormatter.cs index 35c95621e30..de46ef816d0 100644 --- a/src/Nest/Search/MultiSearch/MultiSearchJsonConverter.cs +++ b/src/Nest/Search/MultiSearch/MultiSearchFormatter.cs @@ -4,7 +4,7 @@ namespace Nest { - internal class MultiSearchJsonConverter : IJsonFormatter + internal class MultiSearchFormatter : IJsonFormatter { private const byte Newline = (byte)'\n'; diff --git a/src/Nest/Search/MultiSearch/MultiSearchRequest.cs b/src/Nest/Search/MultiSearch/MultiSearchRequest.cs index 01c09f773d4..67a52df1f32 100644 --- a/src/Nest/Search/MultiSearch/MultiSearchRequest.cs +++ b/src/Nest/Search/MultiSearch/MultiSearchRequest.cs @@ -6,7 +6,7 @@ namespace Nest { [MapsApi("msearch.json")] - [JsonFormatter(typeof(MultiSearchJsonConverter))] + [JsonFormatter(typeof(MultiSearchFormatter))] public partial interface IMultiSearchRequest { IDictionary Operations { get; set; } diff --git a/src/Nest/Search/Search/SearchRequest.cs b/src/Nest/Search/Search/SearchRequest.cs index 210ce7d9cc1..b8e98d4f260 100644 --- a/src/Nest/Search/Search/SearchRequest.cs +++ b/src/Nest/Search/Search/SearchRequest.cs @@ -9,70 +9,162 @@ namespace Nest [ReadAs(typeof(SearchRequest))] public partial interface ISearchRequest : ITypedSearchRequest { + /// + /// Specifies the aggregations to perform + /// [DataMember(Name = "aggs")] AggregationDictionary Aggregations { get; set; } + /// + /// Allows to collapse search results based on field values. + /// The collapsing is done by selecting only the top sorted document per collapse key. + /// For instance the query below retrieves the best tweet for each user and sorts them by number of likes. + /// + /// NOTE: The collapsing is applied to the top hits only and does not affect aggregations. + /// You can only collapse to a depth of 2. + /// + /// [DataMember(Name = "collapse")] IFieldCollapse Collapse { get; set; } + /// + /// Enables explanation for each hit on how its score was computed + /// [DataMember(Name = "explain")] bool? Explain { get; set; } + /// + /// The starting from index of the hits to return. Defaults to 0. + /// [DataMember(Name = "from")] int? From { get; set; } + /// + /// Allow to highlight search results on one or more fields. The implementation uses the either lucene + /// fast-vector-highlighter or highlighter. + /// [DataMember(Name = "highlight")] IHighlight Highlight { get; set; } + /// + /// Allows to configure different boost level per index when searching across + /// more than one indices. This is very handy when hits coming from one index + /// matter more than hits coming from another index (think social graph where each user has an index). + /// [DataMember(Name = "indices_boost")] [JsonFormatter(typeof(IndicesBoostFormatter))] IDictionary IndicesBoost { get; set; } + /// + /// Allows to filter out documents based on a minimum score + /// [DataMember(Name = "min_score")] double? MinScore { get; set; } + /// + /// Specify a query to apply to the search hits at the very end of a search request, + /// after aggregations have already been calculated. Useful when both search hits and aggregations + /// will be returned in the response, and a filter should only be applied to the search hits. + /// [DataMember(Name = "post_filter")] QueryContainer PostFilter { get; set; } + /// + /// The Profile API provides detailed timing information about the execution of individual components in a query. + /// It gives the user insight into how queries are executed at a low level so that the user can understand + /// why certain queries are slow, and take steps to improve their slow queries. + /// [DataMember(Name = "profile")] bool? Profile { get; set; } + /// + /// Specify the search query to perform + /// [DataMember(Name = "query")] QueryContainer Query { get; set; } + /// + /// Specify one or more queries to use for rescoring + /// [DataMember(Name = "rescore")] IList Rescore { get; set; } + /// + /// Allows to return a script evaluation (based on different fields) for each hit + /// [DataMember(Name = "script_fields")] IScriptFields ScriptFields { get; set; } + /// + /// Sort values that can be used to start returning results "after" any document in the result list. + /// [DataMember(Name = "search_after")] IList SearchAfter { get; set; } + /// The number of hits to return. Defaults to 10. [DataMember(Name = "size")] int? Size { get; set; } + /// + /// For scroll queries that return a lot of documents it is possible to split the scroll in multiple slices which can be + /// consumed independently + /// [DataMember(Name = "slice")] ISlicedScroll Slice { get; set; } + /// + /// Specifies how to sort the search hits + /// [DataMember(Name = "sort")] IList Sort { get; set; } + /// + /// Specify how the _source field is returned for each search hit. + /// When true, _source retrieval is enabled (default) + /// When false, _source retrieval is disabled, and no _source will be returned for each hit + /// When is specified, fields to include/exclude can be controlled + /// [DataMember(Name = "_source")] Union Source { get; set; } + /// + /// The suggest feature suggests similar looking terms based on a provided text by using a suggester + /// [DataMember(Name = "suggest")] ISuggestContainer Suggest { get; set; } + /// + /// The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate + /// early. + /// If set, the response will have a boolean field terminated_early to indicate whether the query execution has actually + /// terminated_early. + /// [DataMember(Name = "terminate_after")] long? TerminateAfter { get; set; } + /// + /// A search timeout, bounding the search request to be executed within the + /// specified time value and bail with the hits accumulated up + /// to that point, when expired. Defaults to no timeout. + /// [DataMember(Name = "timeout")] string Timeout { get; set; } + /// + /// Make sure we keep calculating score even if we are sorting on a field. + /// [DataMember(Name = "track_scores")] bool? TrackScores { get; set; } + /// + /// Indicate if the number of documents that match the query should be tracked. + /// + [DataMember(Name = "track_total_hits")] + bool? TrackTotalHits { get; set; } + + /// + /// Return a version for each search hit + /// [DataMember(Name = "version")] bool? Version { get; set; } } @@ -85,33 +177,56 @@ public partial interface ISearchRequest : ISearchRequest { } [DataContract] public partial class SearchRequest { - public Fields StoredFields { get; set; } - public Fields DocValueFields { get; set; } + /// public AggregationDictionary Aggregations { get; set; } + /// public IFieldCollapse Collapse { get; set; } + /// + public Fields DocValueFields { get; set; } + /// public bool? Explain { get; set; } + /// public int? From { get; set; } - + /// public IHighlight Highlight { get; set; } - + /// [JsonFormatter(typeof(IndicesBoostFormatter))] public IDictionary IndicesBoost { get; set; } - + /// public double? MinScore { get; set; } + /// public QueryContainer PostFilter { get; set; } + /// public bool? Profile { get; set; } + /// public QueryContainer Query { get; set; } + /// public IList Rescore { get; set; } + /// public IScriptFields ScriptFields { get; set; } + /// public IList SearchAfter { get; set; } + /// public int? Size { get; set; } + /// public ISlicedScroll Slice { get; set; } + /// public IList Sort { get; set; } + /// public Union Source { get; set; } + /// + public Fields StoredFields { get; set; } + /// public ISuggestContainer Suggest { get; set; } + /// public long? TerminateAfter { get; set; } + /// public string Timeout { get; set; } + /// public bool? TrackScores { get; set; } + /// + public bool? TrackTotalHits { get; set; } + /// public bool? Version { get; set; } protected override HttpMethod HttpMethod => @@ -148,7 +263,6 @@ public partial class SearchDescriptor where TInferDocument : cla bool? ISearchRequest.Explain { get; set; } int? ISearchRequest.From { get; set; } IHighlight ISearchRequest.Highlight { get; set; } - IDictionary ISearchRequest.IndicesBoost { get; set; } double? ISearchRequest.MinScore { get; set; } QueryContainer ISearchRequest.PostFilter { get; set; } @@ -164,83 +278,61 @@ public partial class SearchDescriptor where TInferDocument : cla Fields ISearchRequest.StoredFields { get; set; } ISuggestContainer ISearchRequest.Suggest { get; set; } long? ISearchRequest.TerminateAfter { get; set; } - string ISearchRequest.Timeout { get; set; } bool? ISearchRequest.TrackScores { get; set; } + bool? ISearchRequest.TrackTotalHits { get; set; } bool? ISearchRequest.Version { get; set; } protected sealed override void RequestDefaults(SearchRequestParameters parameters) => TypedKeys(); - public SearchDescriptor Aggregations(Func, IAggregationContainer> aggregationsSelector) => + /// + public SearchDescriptor Aggregations( + Func, IAggregationContainer> aggregationsSelector + ) => Assign(aggregationsSelector(new AggregationContainerDescriptor())?.Aggregations, (a, v) => a.Aggregations = v); + /// public SearchDescriptor Aggregations(AggregationDictionary aggregations) => Assign(aggregations, (a, v) => a.Aggregations = v); + /// public SearchDescriptor Source(bool enabled = true) => Assign(enabled, (a, v) => a.Source = v); + /// public SearchDescriptor Source(Func, ISourceFilter> selector) => Assign(selector, (a, v) => a.Source = new Union(v?.Invoke(new SourceFilterDescriptor()))); - /// The number of hits to return. Defaults to 10. + /// public SearchDescriptor Size(int? size) => Assign(size, (a, v) => a.Size = v); - /// - /// The number of hits to return. Alias for . Defaults to 10. - /// + /// public SearchDescriptor Take(int? take) => Size(take); - /// - /// The starting from index of the hits to return. Defaults to 0. - /// + /// public SearchDescriptor From(int? from) => Assign(from, (a, v) => a.From = v); - /// - /// The starting from index of the hits to return. Alias for . Defaults to 0. - /// + /// public SearchDescriptor Skip(int? skip) => From(skip); - /// - /// A search timeout, bounding the search request to be executed within the - /// specified time value and bail with the hits accumulated up - /// to that point when expired. Defaults to no timeout. - /// + /// public SearchDescriptor Timeout(string timeout) => Assign(timeout, (a, v) => a.Timeout = v); - /// - /// Enables explanation for each hit on how its score was computed. - /// (Use .DocumentsWithMetadata on the return results) - /// + /// public SearchDescriptor Explain(bool? explain = true) => Assign(explain, (a, v) => a.Explain = v); - /// - /// Returns a version for each search hit. (Use .DocumentsWithMetadata on the return results) - /// + /// public SearchDescriptor Version(bool? version = true) => Assign(version, (a, v) => a.Version = v); - /// - /// Make sure we keep calculating score even if we are sorting on a field. - /// + /// public SearchDescriptor TrackScores(bool? trackscores = true) => Assign(trackscores, (a, v) => a.TrackScores = v); - /// - /// The Profile API provides detailed timing information about the execution of individual components in a query. - /// It gives the user insight into how queries are executed at a low level so that the user can understand - /// why certain queries are slow, and take steps to improve their slow queries. - /// + /// public SearchDescriptor Profile(bool? profile = true) => Assign(profile, (a, v) => a.Profile = v); - /// - /// Allows to filter out documents based on a minimum score: - /// + /// public SearchDescriptor MinScore(double? minScore) => Assign(minScore, (a, v) => a.MinScore = v); - /// - /// The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate - /// early. - /// If set, the response will have a boolean field terminated_early to indicate whether the query execution has actually - /// terminated_early. - /// + /// public SearchDescriptor TerminateAfter(long? terminateAfter) => Assign(terminateAfter, (a, v) => a.TerminateAfter = v); /// @@ -274,102 +366,76 @@ public SearchDescriptor Source(Func /// - public SearchDescriptor ExecuteOnPreferredNode(string node) => Preference(node.IsNullOrEmpty() ? null : $"_prefer_node:{node}"); + public SearchDescriptor ExecuteOnPreferredNode(string node) => + Preference(node.IsNullOrEmpty() ? null : $"_prefer_node:{node}"); - /// - /// Allows to configure different boost level per index when searching across - /// more than one indices. This is very handy when hits coming from one index - /// matter more than hits coming from another index (think social graph where each user has an index). - /// + /// public SearchDescriptor IndicesBoost(Func, FluentDictionary> boost) => Assign(boost, (a, v) => a.IndicesBoost = v?.Invoke(new FluentDictionary())); - /// - /// Allows to selectively load specific fields for each document - /// represented by a search hit. Defaults to load the internal _source field. - /// + /// public SearchDescriptor StoredFields(Func, IPromise> fields) => Assign(fields, (a, v) => a.StoredFields = v?.Invoke(new FieldsDescriptor())?.Value); + /// public SearchDescriptor StoredFields(Fields fields) => Assign(fields, (a, v) => a.StoredFields = v); + /// public SearchDescriptor ScriptFields(Func> selector) => Assign(selector, (a, v) => a.ScriptFields = v?.Invoke(new ScriptFieldsDescriptor())?.Value); + /// public SearchDescriptor DocValueFields(Func, IPromise> fields) => Assign(fields, (a, v) => a.DocValueFields = v?.Invoke(new FieldsDescriptor())?.Value); + /// public SearchDescriptor DocValueFields(Fields fields) => Assign(fields, (a, v) => a.DocValueFields = v); - /// - /// A comma-separated list of fields to return as the field data representation of a field for each hit - /// + /// public SearchDescriptor Sort(Func, IPromise>> selector) => Assign(selector, (a, v) => a.Sort = v?.Invoke(new SortDescriptor())?.Value); - /// - /// Sort values that can be used to start returning results "after" any document in the result list. - /// + /// public SearchDescriptor SearchAfter(IList searchAfter) => Assign(searchAfter, (a, v) => a.SearchAfter = v); - /// - /// Sort values that can be used to start returning results "after" any document in the result list. - /// + /// public SearchDescriptor SearchAfter(params object[] searchAfter) => Assign(searchAfter, (a, v) => a.SearchAfter = v); - /// - /// The suggest feature suggests similar looking terms based on a provided text by using a suggester - /// + /// public SearchDescriptor Suggest(Func, IPromise> selector) => Assign(selector, (a, v) => a.Suggest = v?.Invoke(new SuggestContainerDescriptor())?.Value); - /// - /// Describe the query to perform using a query descriptor lambda - /// + /// public SearchDescriptor Query(Func, QueryContainer> query) => Assign(query, (a, v) => a.Query = v?.Invoke(new QueryContainerDescriptor())); - /// - /// For scroll queries that return a lot of documents it is possible to split the scroll in multiple slices which can be - /// consumed independently - /// + /// public SearchDescriptor Slice(Func, ISlicedScroll> selector) => Assign(selector, (a, v) => a.Slice = v?.Invoke(new SlicedScrollDescriptor())); /// /// Shortcut to default to a match all query /// - public SearchDescriptor MatchAll(Func selector = null) => Query(q => q.MatchAll(selector)); + public SearchDescriptor MatchAll(Func selector = null) => + Query(q => q.MatchAll(selector)); - /// - /// Filter search using a filter descriptor lambda - /// + /// public SearchDescriptor PostFilter(Func, QueryContainer> filter) => Assign(filter, (a, v) => a.PostFilter = v?.Invoke(new QueryContainerDescriptor())); - /// - /// Allow to highlight search results on one or more fields. The implementation uses the either lucene - /// fast-vector-highlighter or highlighter. - /// + /// public SearchDescriptor Highlight(Func, IHighlight> highlightSelector) => Assign(highlightSelector, (a, v) => a.Highlight = v?.Invoke(new HighlightDescriptor())); - /// - /// Allows to collapse search results based on field values. - /// The collapsing is done by selecting only the top sorted document per collapse key. - /// For instance the query below retrieves the best tweet for each user and sorts them by number of likes. - /// - /// NOTE: The collapsing is applied to the top hits only and does not affect aggregations. - /// You can only collapse to a depth of 2. - /// - /// + /// public SearchDescriptor Collapse(Func, IFieldCollapse> collapseSelector) => Assign(collapseSelector, (a, v) => a.Collapse = v?.Invoke(new FieldCollapseDescriptor())); - /// - /// Allows you to specify one or more queries to use for rescoring - /// + /// public SearchDescriptor Rescore(Func, IPromise>> rescoreSelector) => Assign(rescoreSelector, (a, v) => a.Rescore = v?.Invoke(new RescoringDescriptor()).Value); + + /// + public SearchDescriptor TrackTotalHits(bool? trackTotalHits = true) => Assign(trackTotalHits, (a, v) => a.TrackTotalHits = v); } } diff --git a/src/Tests/Tests/Search/MultiSearch/MultiSearchApiTests.cs b/src/Tests/Tests/Search/MultiSearch/MultiSearchApiTests.cs index 1ccd6f1beb5..3192ac7fc7e 100644 --- a/src/Tests/Tests/Search/MultiSearch/MultiSearchApiTests.cs +++ b/src/Tests/Tests/Search/MultiSearch/MultiSearchApiTests.cs @@ -25,7 +25,7 @@ public MultiSearchApiTests(ReadOnlyCluster cluster, EndpointUsage usage) : base( protected override string UrlPath => "/project/_msearch"; protected override int ExpectStatusCode => 200; protected override HttpMethod HttpMethod => HttpMethod.POST; - + protected override LazyResponses ClientUsage() => Calls( (c, f) => c.MultiSearch(Index(), f), (c, f) => c.MultiSearchAsync(Index(), f), @@ -37,7 +37,7 @@ protected override LazyResponses ClientUsage() => Calls( protected override object ExpectJson => new object[] { new { }, - new { from = 0, size = 10, query = new { match_all = new { } } }, + new { from = 0, size = 10, query = new { match_all = new { } }, track_total_hits = true }, new { search_type = "dfs_query_then_fetch" }, new { }, new { index = "devs" }, @@ -60,7 +60,7 @@ protected override LazyResponses ClientUsage() => Calls( protected override MultiSearchDescriptor NewDescriptor() => new MultiSearchDescriptor(Index()); protected override Func Fluent => ms => ms - .Search("10projects", s => s.Query(q => q.MatchAll()).From(0).Size(10)) + .Search("10projects", s => s.Query(q => q.MatchAll()).From(0).Size(10).TrackTotalHits()) .Search("dfs_projects", s => s.SearchType(SearchType.DfsQueryThenFetch)) .Search("5developers", s => s.Query(q => q.MatchAll()).From(0).Size(5)) .Search("infer_type_name", s => s.Index("devs").From(0).Size(5).MatchAll()) @@ -90,7 +90,7 @@ protected override LazyResponses ClientUsage() => Calls( { Operations = new Dictionary { - { "10projects", new SearchRequest { From = 0, Size = 10, Query = new QueryContainer(new MatchAllQuery()) } }, + { "10projects", new SearchRequest { From = 0, Size = 10, Query = new QueryContainer(new MatchAllQuery()), TrackTotalHits = true } }, { "dfs_projects", new SearchRequest { SearchType = SearchType.DfsQueryThenFetch } }, { "5developers", new SearchRequest { From = 0, Size = 5, Query = new QueryContainer(new MatchAllQuery()) } }, { "infer_type_name", new SearchRequest("devs") { From = 0, Size = 5, Query = new QueryContainer(new MatchAllQuery()) } }, @@ -135,6 +135,7 @@ [I] public Task AssertResponse() => AssertOnAllResponses(r => var projects = r.GetResponse("10projects"); projects.ShouldBeValid(); projects.Documents.Should().HaveCount(10); + projects.HitsMetadata.Total.Relation.Should().Be(TotalHitsRelation.EqualTo); var projectsCount = r.GetResponse("count_project"); projectsCount.Should().BeNull(); From 77a133acfcf3a898e3970cac2b0540442db74f0d Mon Sep 17 00:00:00 2001 From: Stuart Cam Date: Tue, 9 Jul 2019 14:05:58 +1000 Subject: [PATCH 07/10] Include size of snapshot in snapshot metadata (#3911) (cherry picked from commit 0cba1353c4c0a5bba862e189f0da161c0abba9f8) --- .../SnapshotStatus/SnapshotStatusResponse.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Nest/Modules/SnapshotAndRestore/Snapshot/SnapshotStatus/SnapshotStatusResponse.cs b/src/Nest/Modules/SnapshotAndRestore/Snapshot/SnapshotStatus/SnapshotStatusResponse.cs index 45ad34018f1..2c168d40c60 100644 --- a/src/Nest/Modules/SnapshotAndRestore/Snapshot/SnapshotStatus/SnapshotStatusResponse.cs +++ b/src/Nest/Modules/SnapshotAndRestore/Snapshot/SnapshotStatus/SnapshotStatusResponse.cs @@ -85,10 +85,25 @@ public class SnapshotShardsStats public class SnapshotStats { + [DataMember(Name ="incremental")] + public FileCountSnapshotStats Incremental { get; internal set; } + + [DataMember(Name ="total")] + public FileCountSnapshotStats Total { get; internal set; } + [DataMember(Name ="start_time_in_millis")] public long StartTimeInMilliseconds { get; internal set; } [DataMember(Name ="time_in_millis")] public long TimeInMilliseconds { get; internal set; } } + + public class FileCountSnapshotStats + { + [DataMember(Name ="file_count")] + public int FileCount { get; internal set; } + + [DataMember(Name ="size_in_bytes")] + public long SizeInBytes { get; internal set; } + } } From 8abfe065f774204ca2308531bb5ef0b84fa5e95f Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Wed, 10 Jul 2019 09:57:39 +1000 Subject: [PATCH 08/10] Obsolete AllField and IndexField (#3918) This commit obsoletes the AllField and IndexField. Both should have been removed in 7.x but were missed. Remove the DataMemberAttribute and add IgnoreDataMemberAttribute so that any value is not sent in a request. Fixes #3905 (cherry picked from commit c4ed49d1122d879ffc030cac87b515f3de640fdd) --- .../GetFieldMapping/FieldMappingFormatter.cs | 4 + .../PutMapping/PutMappingRequest.cs | 7 ++ src/Nest/Mapping/Mappings.cs | 8 +- src/Nest/Mapping/MetaFields/All/AllField.cs | 7 +- .../Mapping/MetaFields/Index/IndexField.cs | 6 +- src/Nest/Mapping/TypeMapping.cs | 109 ++++++++++++++---- 6 files changed, 113 insertions(+), 28 deletions(-) diff --git a/src/Nest/Indices/MappingManagement/GetFieldMapping/FieldMappingFormatter.cs b/src/Nest/Indices/MappingManagement/GetFieldMapping/FieldMappingFormatter.cs index f6a6d4ae766..157684b1e97 100644 --- a/src/Nest/Indices/MappingManagement/GetFieldMapping/FieldMappingFormatter.cs +++ b/src/Nest/Indices/MappingManagement/GetFieldMapping/FieldMappingFormatter.cs @@ -37,8 +37,10 @@ public IReadOnlyDictionary Deserialize(ref JsonReader read switch (value) { case 0: +#pragma warning disable 618 mapping = formatterResolver.GetFormatter() .Deserialize(ref reader, formatterResolver); +#pragma warning restore 618 break; case 1: mapping = formatterResolver.GetFormatter() @@ -49,8 +51,10 @@ public IReadOnlyDictionary Deserialize(ref JsonReader read .Deserialize(ref reader, formatterResolver); break; case 3: +#pragma warning disable 618 mapping = formatterResolver.GetFormatter() .Deserialize(ref reader, formatterResolver); +#pragma warning restore 618 break; case 4: mapping = formatterResolver.GetFormatter() diff --git a/src/Nest/Indices/MappingManagement/PutMapping/PutMappingRequest.cs b/src/Nest/Indices/MappingManagement/PutMapping/PutMappingRequest.cs index 89c0356a9e0..8f245e6b4ea 100644 --- a/src/Nest/Indices/MappingManagement/PutMapping/PutMappingRequest.cs +++ b/src/Nest/Indices/MappingManagement/PutMapping/PutMappingRequest.cs @@ -18,6 +18,7 @@ public partial interface IPutMappingRequest where TDocument : class { public partial class PutMappingRequest { /// + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] public IAllField AllField { get; set; } /// @@ -36,6 +37,7 @@ public partial class PutMappingRequest public IFieldNamesField FieldNamesField { get; set; } /// + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public IIndexField IndexField { get; set; } /// @@ -63,12 +65,14 @@ public partial class PutMappingRequest where TDocument : class { } [DataContract] public partial class PutMappingDescriptor where TDocument : class { + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] IAllField ITypeMapping.AllField { get; set; } bool? ITypeMapping.DateDetection { get; set; } Union ITypeMapping.Dynamic { get; set; } IEnumerable ITypeMapping.DynamicDateFormats { get; set; } IDynamicTemplateContainer ITypeMapping.DynamicTemplates { get; set; } IFieldNamesField ITypeMapping.FieldNamesField { get; set; } + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] IIndexField ITypeMapping.IndexField { get; set; } IDictionary ITypeMapping.Meta { get; set; } bool? ITypeMapping.NumericDetection { get; set; } @@ -102,10 +106,12 @@ public PutMappingDescriptor AutoMap(IPropertyVisitor visitor = null, public PutMappingDescriptor Dynamic(bool? dynamic = true) => Assign(dynamic, (a, v) => a.Dynamic = v); /// + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] public PutMappingDescriptor AllField(Func allFieldSelector) => Assign(allFieldSelector, (a, v) => a.AllField = v?.Invoke(new AllFieldDescriptor())); /// + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public PutMappingDescriptor IndexField(Func indexFieldSelector) => Assign(indexFieldSelector, (a, v) => a.IndexField = v?.Invoke(new IndexFieldDescriptor())); @@ -118,6 +124,7 @@ public PutMappingDescriptor DisableSizeField(bool? disabled = true) = Assign(disabled, (a, v) => a.SizeField = new SizeField { Enabled = !v }); /// + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public PutMappingDescriptor DisableIndexField(bool? disabled = true) => Assign(disabled, (a, v) => a.IndexField = new IndexField { Enabled = !v }); diff --git a/src/Nest/Mapping/Mappings.cs b/src/Nest/Mapping/Mappings.cs index 920f42a156f..9a3ac504203 100644 --- a/src/Nest/Mapping/Mappings.cs +++ b/src/Nest/Mapping/Mappings.cs @@ -13,7 +13,7 @@ namespace Nest /// new features in the future. /// /// - [Obsolete("Mappings are no longer type dependant, please use TypeMapping directly")] + [Obsolete("Mappings are no longer type dependent, please use TypeMapping directly")] public class Mappings : ObsoleteMappingsBase, ITypeMapping, IEnumerable { private IEnumerable AsEnumerable => new[] { new TypeMapping() }; @@ -29,7 +29,8 @@ public class Mappings : ObsoleteMappingsBase, ITypeMapping, IEnumerable Wrapped.AllField; set => Wrapped.AllField = value; } [DataMember(Name = "date_detection")] bool? ITypeMapping.DateDetection { get => Wrapped.DateDetection; set => Wrapped.DateDetection = value; } @@ -41,7 +42,8 @@ public abstract class ObsoleteMappingsBase : ITypeMapping IDynamicTemplateContainer ITypeMapping.DynamicTemplates { get => Wrapped.DynamicTemplates; set => Wrapped.DynamicTemplates = value; } [DataMember(Name = "_field_names")] IFieldNamesField ITypeMapping.FieldNamesField { get => Wrapped.FieldNamesField; set => Wrapped.FieldNamesField = value; } - [DataMember(Name = "_index")] + [IgnoreDataMember] + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] IIndexField ITypeMapping.IndexField { get => Wrapped.IndexField; set => Wrapped.IndexField = value; } [DataMember(Name = "_meta")] IDictionary ITypeMapping.Meta { get => Wrapped.Meta; set => Wrapped.Meta = value; } diff --git a/src/Nest/Mapping/MetaFields/All/AllField.cs b/src/Nest/Mapping/MetaFields/All/AllField.cs index 7d2eaf0e9d3..58f33e6235e 100644 --- a/src/Nest/Mapping/MetaFields/All/AllField.cs +++ b/src/Nest/Mapping/MetaFields/All/AllField.cs @@ -1,7 +1,9 @@ -using System.Runtime.Serialization; +using System; +using System.Runtime.Serialization; namespace Nest { + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] [ReadAs(typeof(AllField))] public interface IAllField : IFieldMapping { @@ -36,6 +38,7 @@ public interface IAllField : IFieldMapping bool? StoreTermVectors { get; set; } } + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] public class AllField : IAllField { public string Analyzer { get; set; } @@ -50,7 +53,7 @@ public class AllField : IAllField public bool? StoreTermVectors { get; set; } } - //OBSOLETE + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] public class AllFieldDescriptor : DescriptorBase, IAllField { diff --git a/src/Nest/Mapping/MetaFields/Index/IndexField.cs b/src/Nest/Mapping/MetaFields/Index/IndexField.cs index 64f9504b699..e9b1bd82001 100644 --- a/src/Nest/Mapping/MetaFields/Index/IndexField.cs +++ b/src/Nest/Mapping/MetaFields/Index/IndexField.cs @@ -1,7 +1,9 @@ -using System.Runtime.Serialization; +using System; +using System.Runtime.Serialization; namespace Nest { + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] [ReadAs(typeof(IndexField))] public interface IIndexField : IFieldMapping { @@ -9,11 +11,13 @@ public interface IIndexField : IFieldMapping bool? Enabled { get; set; } } + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public class IndexField : IIndexField { public bool? Enabled { get; set; } } + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public class IndexFieldDescriptor : DescriptorBase, IIndexField { diff --git a/src/Nest/Mapping/TypeMapping.cs b/src/Nest/Mapping/TypeMapping.cs index 428504cac2c..b6770de7225 100644 --- a/src/Nest/Mapping/TypeMapping.cs +++ b/src/Nest/Mapping/TypeMapping.cs @@ -9,43 +9,98 @@ namespace Nest [ReadAs(typeof(TypeMapping))] public interface ITypeMapping { - [DataMember(Name = "_all")] + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] + [IgnoreDataMember] IAllField AllField { get; set; } + /// + /// If enabled (default), then new string fields are checked to see whether their contents match + /// any of the date patterns specified in . + /// If a match is found, a new date field is added with the corresponding format. + /// [DataMember(Name = "date_detection")] bool? DateDetection { get; set; } + /// + /// Whether new unseen fields will be added to the mapping. Default is true. + /// A value of false will ignore unknown fields and a value of + /// will result in an error if an unknown field is encountered in a document. + /// [DataMember(Name = "dynamic")] Union Dynamic { get; set; } + /// + /// Date formats used by + /// [DataMember(Name = "dynamic_date_formats")] IEnumerable DynamicDateFormats { get; set; } + /// + /// Dynamic templates allow you to define custom mappings that can be applied to dynamically added fields based on + /// - the datatype detected by Elasticsearch, with . + /// - the name of the field, with and or + /// . + /// - the full dotted path to the field, with and + /// . + /// The original field name {name} and the detected datatype {dynamic_type} template variables can be + /// used in the mapping specification as placeholders. + /// [DataMember(Name = "dynamic_templates")] IDynamicTemplateContainer DynamicTemplates { get; set; } + /// + /// Used to index the names of every field in a document that contains any value other than null. + /// This field was used by the exists query to find documents that either have or don’t have any non-null value for a particular field. + /// Now, it only indexes the names of fields that have doc_values and norms disabled. + /// Can be disabled. Disabling _field_names is often not necessary because it no longer carries the index overhead it once did. + /// If you have a lot of fields which have doc_values and norms disabled and you do not need to execute exists queries + /// using those fields you might want to disable + /// [DataMember(Name = "_field_names")] IFieldNamesField FieldNamesField { get; set; } - [DataMember(Name = "_index")] + + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] + [IgnoreDataMember] IIndexField IndexField { get; set; } + /// + /// Custom meta data to associate with a mapping. Not used by Elasticsearch, + /// but can be used to store application-specific metadata. + /// [DataMember(Name = "_meta")] [JsonFormatter(typeof(VerbatimDictionaryInterfaceKeysFormatter))] IDictionary Meta { get; set; } + /// + /// If enabled (not enabled by default), then new string fields are checked to see whether + /// they wholly contain a numeric value and if so, to map as a numeric field. + /// [DataMember(Name = "numeric_detection")] bool? NumericDetection { get; set; } + /// + /// Specifies the mapping properties + /// [DataMember(Name = "properties")] IProperties Properties { get; set; } + /// + /// Specifies configuration for the _routing parameter + /// [DataMember(Name = "_routing")] IRoutingField RoutingField { get; set; } + /// + /// If enabled, indexes the size in bytes of the original _source field. + /// Requires mapper-size plugin be installed + /// [DataMember(Name = "_size")] ISizeField SizeField { get; set; } + /// + /// Specifies configuration for the _source field + /// [DataMember(Name = "_source")] ISourceField SourceField { get; set; } } @@ -53,6 +108,7 @@ public interface ITypeMapping public class TypeMapping : ITypeMapping { /// + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] public IAllField AllField { get; set; } /// @@ -71,6 +127,7 @@ public class TypeMapping : ITypeMapping public IFieldNamesField FieldNamesField { get; set; } /// + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public IIndexField IndexField { get; set; } /// @@ -96,12 +153,14 @@ public class TypeMapping : ITypeMapping public class TypeMappingDescriptor : DescriptorBase, ITypeMapping>, ITypeMapping where T : class { + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] IAllField ITypeMapping.AllField { get; set; } bool? ITypeMapping.DateDetection { get; set; } Union ITypeMapping.Dynamic { get; set; } IEnumerable ITypeMapping.DynamicDateFormats { get; set; } IDynamicTemplateContainer ITypeMapping.DynamicTemplates { get; set; } IFieldNamesField ITypeMapping.FieldNamesField { get; set; } + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] IIndexField ITypeMapping.IndexField { get; set; } IDictionary ITypeMapping.Meta { get; set; } bool? ITypeMapping.NumericDetection { get; set; } @@ -112,7 +171,7 @@ public class TypeMappingDescriptor : DescriptorBase, /// /// Convenience method to map as much as it can based on attributes set on the - /// type. + /// type, as well as inferring mappings from the CLR property types. /// 
This method also automatically sets up mappings for known values types (int, long, double, datetime, etc)
/// 
Class types default to object and Enums to int
/// 
Later calls can override whatever is set is by this call.
@@ -122,7 +181,7 @@ public TypeMappingDescriptor AutoMap(IPropertyVisitor visitor = null, int max /// /// Convenience method to map as much as it can based on attributes set on the - /// type. + /// type, as well as inferring mappings from the CLR property types. /// This particular overload is useful for automapping any children /// 
This method also automatically sets up mappings for known values types (int, long, double, datetime, etc)
/// 
Class types default to object and Enums to int
@@ -136,7 +195,7 @@ public TypeMappingDescriptor AutoMap(Type documentType, IPropertyVisitor visi /// /// Convenience method to map as much as it can based on attributes set on the - /// type. + /// type, as well as inferring mappings from the CLR property types. /// This particular overload is useful for automapping any children /// 
This method also automatically sets up mappings for known values types (int, long, double, datetime, etc)
/// 
Class types default to object and Enums to int
@@ -146,70 +205,76 @@ public TypeMappingDescriptor AutoMap(IPropertyVisitor visitor = nu where TDocument : class => Assign(Self.Properties.AutoMap(visitor, maxRecursion), (a, v) => a.Properties = v); - /// + /// + /// Convenience method to map as much as it can based on attributes set on the + /// type, as well as inferring mappings from the CLR property types. + /// This overload determines how deep automapping should recurse on a complex CLR type. + /// public TypeMappingDescriptor AutoMap(int maxRecursion) => AutoMap(null, maxRecursion); - /// + /// public TypeMappingDescriptor Dynamic(Union dynamic) => Assign(dynamic, (a, v) => a.Dynamic = v); - /// + /// public TypeMappingDescriptor Dynamic(bool dynamic = true) => Assign(dynamic, (a, v) => a.Dynamic = v); - /// + [Obsolete("The _all field is no longer supported in Elasticsearch 7.x and will be removed in the next major release. The value will not be sent in a request. An _all like field can be achieved using copy_to")] public TypeMappingDescriptor AllField(Func allFieldSelector) => Assign(allFieldSelector, (a, v) => a.AllField = v?.Invoke(new AllFieldDescriptor())); - /// + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public TypeMappingDescriptor IndexField(Func indexFieldSelector) => Assign(indexFieldSelector, (a, v) => a.IndexField = v?.Invoke(new IndexFieldDescriptor())); - /// + /// public TypeMappingDescriptor SizeField(Func sizeFieldSelector) => Assign(sizeFieldSelector, (a, v) => a.SizeField = v?.Invoke(new SizeFieldDescriptor())); - /// + /// public TypeMappingDescriptor SourceField(Func sourceFieldSelector) => Assign(sourceFieldSelector, (a, v) => a.SourceField = v?.Invoke(new SourceFieldDescriptor())); - /// + /// public TypeMappingDescriptor DisableSizeField(bool? disabled = true) => Assign(new SizeField { Enabled = !disabled }, (a, v) => a.SizeField = v); - /// + [Obsolete("Configuration for the _index field is no longer supported in Elasticsearch 7.x and will be removed in the next major release.")] public TypeMappingDescriptor DisableIndexField(bool? disabled = true) => Assign(new IndexField { Enabled = !disabled }, (a, v) => a.IndexField = v); - /// + /// public TypeMappingDescriptor DynamicDateFormats(IEnumerable dateFormats) => Assign(dateFormats, (a, v) => a.DynamicDateFormats = v); - /// + /// public TypeMappingDescriptor DateDetection(bool? detect = true) => Assign(detect, (a, v) => a.DateDetection = v); - /// + /// public TypeMappingDescriptor NumericDetection(bool? detect = true) => Assign(detect, (a, v) => a.NumericDetection = v); - /// + /// public TypeMappingDescriptor RoutingField(Func, IRoutingField> routingFieldSelector) => Assign(routingFieldSelector, (a, v) => a.RoutingField = v?.Invoke(new RoutingFieldDescriptor())); - /// + /// public TypeMappingDescriptor FieldNamesField(Func, IFieldNamesField> fieldNamesFieldSelector) => Assign(fieldNamesFieldSelector.Invoke(new FieldNamesFieldDescriptor()), (a, v) => a.FieldNamesField = v); - /// + /// public TypeMappingDescriptor Meta(Func, FluentDictionary> metaSelector) => Assign(metaSelector(new FluentDictionary()), (a, v) => a.Meta = v); - /// + /// public TypeMappingDescriptor Meta(Dictionary metaDictionary) => Assign(metaDictionary, (a, v) => a.Meta = v); + /// public TypeMappingDescriptor Properties(Func, IPromise> propertiesSelector) => Assign(propertiesSelector, (a, v) => a.Properties = v?.Invoke(new PropertiesDescriptor(Self.Properties))?.Value); + /// public TypeMappingDescriptor Properties(Func, IPromise> propertiesSelector) where TDocument : class => Assign(propertiesSelector, (a, v) => a.Properties = v?.Invoke(new PropertiesDescriptor(Self.Properties))?.Value); - /// + /// public TypeMappingDescriptor DynamicTemplates( Func, IPromise> dynamicTemplatesSelector ) => From f3496750ae330ac672485be23668ba56a5678551 Mon Sep 17 00:00:00 2001 From: Stuart Cam Date: Wed, 10 Jul 2019 14:25:47 +1000 Subject: [PATCH 09/10] Implement Freeze and Unfreeze API --- .../Configuration/CodeConfiguration.cs | 2 - .../RequestParameters.Indices.cs | 100 ++++++++++++ .../ElasticLowLevelClient.Indices.cs | 20 +++ src/Nest/Descriptors.Indices.cs | 76 +++++++++ src/Nest/ElasticClient.Indices.cs | 48 ++++++ .../FreezeIndex/FreezeIndexRequest.cs | 9 + .../FreezeIndex/FreezeIndexResponse.cs | 10 ++ .../UnfreezeIndex/UnfreezeIndexRequest.cs | 9 + .../UnfreezeIndex/UnfreezeIndexResponse.cs | 10 ++ src/Nest/Requests.Indices.cs | 154 ++++++++++++++++++ .../_Generated/ApiUrlsLookup.generated.cs | 2 + 11 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexRequest.cs create mode 100644 src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexResponse.cs create mode 100644 src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexRequest.cs create mode 100644 src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexResponse.cs diff --git a/src/CodeGeneration/ApiGenerator/Configuration/CodeConfiguration.cs b/src/CodeGeneration/ApiGenerator/Configuration/CodeConfiguration.cs index 574f6327b02..e27c080e392 100644 --- a/src/CodeGeneration/ApiGenerator/Configuration/CodeConfiguration.cs +++ b/src/CodeGeneration/ApiGenerator/Configuration/CodeConfiguration.cs @@ -22,8 +22,6 @@ public static class CodeConfiguration "ml.set_upgrade_mode.json", "ml.find_file_structure.json", "monitoring.bulk.json", - "indices.freeze.json", - "indices.unfreeze.json", "ccr.follow_info.json", "ccr.forget_follower.json" diff --git a/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.Indices.cs b/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.Indices.cs index 2a2c012d962..8a14cee3b89 100644 --- a/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.Indices.cs +++ b/src/Elasticsearch.Net/Api/RequestParameters/RequestParameters.Indices.cs @@ -527,6 +527,56 @@ public bool? OnlyExpungeDeletes } } + ///Request options for Freeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + public class FreezeIndexRequestParameters : RequestParameters + { + public override HttpMethod DefaultHttpMethod => HttpMethod.POST; + /// + /// Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have + /// been specified) + /// + public bool? AllowNoIndices + { + get => Q("allow_no_indices"); + set => Q("allow_no_indices", value); + } + + ///Whether to expand wildcard expression to concrete indices that are open, closed or both. + public ExpandWildcards? ExpandWildcards + { + get => Q("expand_wildcards"); + set => Q("expand_wildcards", value); + } + + ///Whether specified concrete indices should be ignored when unavailable (missing or closed) + public bool? IgnoreUnavailable + { + get => Q("ignore_unavailable"); + set => Q("ignore_unavailable", value); + } + + ///Specify timeout for connection to master + public TimeSpan MasterTimeout + { + get => Q("master_timeout"); + set => Q("master_timeout", value); + } + + ///Explicit operation timeout + public TimeSpan Timeout + { + get => Q("timeout"); + set => Q("timeout", value); + } + + ///Sets the number of active shards to wait for before the operation returns. + public string WaitForActiveShards + { + get => Q("wait_for_active_shards"); + set => Q("wait_for_active_shards", value); + } + } + ///Request options for Get http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-index.html public class GetIndexRequestParameters : RequestParameters { @@ -1289,6 +1339,56 @@ public Level? Level } } + ///Request options for Unfreeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + public class UnfreezeIndexRequestParameters : RequestParameters + { + public override HttpMethod DefaultHttpMethod => HttpMethod.POST; + /// + /// Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have + /// been specified) + /// + public bool? AllowNoIndices + { + get => Q("allow_no_indices"); + set => Q("allow_no_indices", value); + } + + ///Whether to expand wildcard expression to concrete indices that are open, closed or both. + public ExpandWildcards? ExpandWildcards + { + get => Q("expand_wildcards"); + set => Q("expand_wildcards", value); + } + + ///Whether specified concrete indices should be ignored when unavailable (missing or closed) + public bool? IgnoreUnavailable + { + get => Q("ignore_unavailable"); + set => Q("ignore_unavailable", value); + } + + ///Specify timeout for connection to master + public TimeSpan MasterTimeout + { + get => Q("master_timeout"); + set => Q("master_timeout", value); + } + + ///Explicit operation timeout + public TimeSpan Timeout + { + get => Q("timeout"); + set => Q("timeout", value); + } + + ///Sets the number of active shards to wait for before the operation returns. + public string WaitForActiveShards + { + get => Q("wait_for_active_shards"); + set => Q("wait_for_active_shards", value); + } + } + ///Request options for BulkAlias http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html public class BulkAliasRequestParameters : RequestParameters { diff --git a/src/Elasticsearch.Net/ElasticLowLevelClient.Indices.cs b/src/Elasticsearch.Net/ElasticLowLevelClient.Indices.cs index a51c67b70fa..e8614d4f41e 100644 --- a/src/Elasticsearch.Net/ElasticLowLevelClient.Indices.cs +++ b/src/Elasticsearch.Net/ElasticLowLevelClient.Indices.cs @@ -245,6 +245,16 @@ public TResponse ForceMerge(string index, ForceMergeRequestParameters ///Request specific configuration such as querystring parameters & request specific connection settings. public Task ForceMergeAsync(string index, ForceMergeRequestParameters requestParameters = null, CancellationToken ctx = default) where TResponse : class, IElasticsearchResponse, new() => DoRequestAsync(POST, Url($"{index:index}/_forcemerge"), ctx, null, RequestParams(requestParameters)); + ///POST on /{index}/_freeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + ///The name of the index to freeze + ///Request specific configuration such as querystring parameters & request specific connection settings. + public TResponse Freeze(string index, FreezeIndexRequestParameters requestParameters = null) + where TResponse : class, IElasticsearchResponse, new() => DoRequest(POST, Url($"{index:index}/_freeze"), null, RequestParams(requestParameters)); + ///POST on /{index}/_freeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + ///The name of the index to freeze + ///Request specific configuration such as querystring parameters & request specific connection settings. + public Task FreezeAsync(string index, FreezeIndexRequestParameters requestParameters = null, CancellationToken ctx = default) + where TResponse : class, IElasticsearchResponse, new() => DoRequestAsync(POST, Url($"{index:index}/_freeze"), ctx, null, RequestParams(requestParameters)); ///GET on /{index} http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-index.html ///A comma-separated list of index names ///Request specific configuration such as querystring parameters & request specific connection settings. @@ -715,6 +725,16 @@ public TResponse Stats(string index, string metric, IndicesStatsReque ///Request specific configuration such as querystring parameters & request specific connection settings. public Task StatsAsync(string index, string metric, IndicesStatsRequestParameters requestParameters = null, CancellationToken ctx = default) where TResponse : class, IElasticsearchResponse, new() => DoRequestAsync(GET, Url($"{index:index}/_stats/{metric:metric}"), ctx, null, RequestParams(requestParameters)); + ///POST on /{index}/_unfreeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + ///The name of the index to unfreeze + ///Request specific configuration such as querystring parameters & request specific connection settings. + public TResponse Unfreeze(string index, UnfreezeIndexRequestParameters requestParameters = null) + where TResponse : class, IElasticsearchResponse, new() => DoRequest(POST, Url($"{index:index}/_unfreeze"), null, RequestParams(requestParameters)); + ///POST on /{index}/_unfreeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + ///The name of the index to unfreeze + ///Request specific configuration such as querystring parameters & request specific connection settings. + public Task UnfreezeAsync(string index, UnfreezeIndexRequestParameters requestParameters = null, CancellationToken ctx = default) + where TResponse : class, IElasticsearchResponse, new() => DoRequestAsync(POST, Url($"{index:index}/_unfreeze"), ctx, null, RequestParams(requestParameters)); ///POST on /_aliases http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html ///The definition of `actions` to perform ///Request specific configuration such as querystring parameters & request specific connection settings. diff --git a/src/Nest/Descriptors.Indices.cs b/src/Nest/Descriptors.Indices.cs index 5193880077a..bc32f4d3cb7 100644 --- a/src/Nest/Descriptors.Indices.cs +++ b/src/Nest/Descriptors.Indices.cs @@ -526,6 +526,44 @@ public ForceMergeDescriptor Index() public ForceMergeDescriptor OnlyExpungeDeletes(bool? onlyexpungedeletes = true) => Qs("only_expunge_deletes", onlyexpungedeletes); } + ///descriptor for Freeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + public partial class FreezeIndexDescriptor : RequestDescriptorBase, IFreezeIndexRequest + { + internal override ApiUrls ApiUrls => ApiUrlsLookups.IndicesFreeze; + ////{index}/_freeze + ///this parameter is required + public FreezeIndexDescriptor(IndexName index): base(r => r.Required("index", index)) + { + } + + ///Used for serialization purposes, making sure we have a parameterless constructor + [SerializationConstructor] + protected FreezeIndexDescriptor(): base() + { + } + + // values part of the url path + IndexName IFreezeIndexRequest.Index => Self.RouteValues.Get("index"); + ///The name of the index to freeze + public FreezeIndexDescriptor Index(IndexName index) => Assign(index, (a, v) => a.RouteValues.Required("index", v)); + ///a shortcut into calling Index(typeof(TOther)) + public FreezeIndexDescriptor Index() + where TOther : class => Assign(typeof(TOther), (a, v) => a.RouteValues.Required("index", (IndexName)v)); + // Request parameters + ///Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) + public FreezeIndexDescriptor AllowNoIndices(bool? allownoindices = true) => Qs("allow_no_indices", allownoindices); + ///Whether to expand wildcard expression to concrete indices that are open, closed or both. + public FreezeIndexDescriptor ExpandWildcards(ExpandWildcards? expandwildcards) => Qs("expand_wildcards", expandwildcards); + ///Whether specified concrete indices should be ignored when unavailable (missing or closed) + public FreezeIndexDescriptor IgnoreUnavailable(bool? ignoreunavailable = true) => Qs("ignore_unavailable", ignoreunavailable); + ///Specify timeout for connection to master + public FreezeIndexDescriptor MasterTimeout(Time mastertimeout) => Qs("master_timeout", mastertimeout); + ///Explicit operation timeout + public FreezeIndexDescriptor Timeout(Time timeout) => Qs("timeout", timeout); + ///Sets the number of active shards to wait for before the operation returns. + public FreezeIndexDescriptor WaitForActiveShards(string waitforactiveshards) => Qs("wait_for_active_shards", waitforactiveshards); + } + ///descriptor for Get http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-index.html public partial class GetIndexDescriptor : RequestDescriptorBase, IGetIndexRequest { @@ -1287,6 +1325,44 @@ public IndicesStatsDescriptor Fields(params Expression>[] fie public IndicesStatsDescriptor Level(Level? level) => Qs("level", level); } + ///descriptor for Unfreeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + public partial class UnfreezeIndexDescriptor : RequestDescriptorBase, IUnfreezeIndexRequest + { + internal override ApiUrls ApiUrls => ApiUrlsLookups.IndicesUnfreeze; + ////{index}/_unfreeze + ///this parameter is required + public UnfreezeIndexDescriptor(IndexName index): base(r => r.Required("index", index)) + { + } + + ///Used for serialization purposes, making sure we have a parameterless constructor + [SerializationConstructor] + protected UnfreezeIndexDescriptor(): base() + { + } + + // values part of the url path + IndexName IUnfreezeIndexRequest.Index => Self.RouteValues.Get("index"); + ///The name of the index to unfreeze + public UnfreezeIndexDescriptor Index(IndexName index) => Assign(index, (a, v) => a.RouteValues.Required("index", v)); + ///a shortcut into calling Index(typeof(TOther)) + public UnfreezeIndexDescriptor Index() + where TOther : class => Assign(typeof(TOther), (a, v) => a.RouteValues.Required("index", (IndexName)v)); + // Request parameters + ///Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified) + public UnfreezeIndexDescriptor AllowNoIndices(bool? allownoindices = true) => Qs("allow_no_indices", allownoindices); + ///Whether to expand wildcard expression to concrete indices that are open, closed or both. + public UnfreezeIndexDescriptor ExpandWildcards(ExpandWildcards? expandwildcards) => Qs("expand_wildcards", expandwildcards); + ///Whether specified concrete indices should be ignored when unavailable (missing or closed) + public UnfreezeIndexDescriptor IgnoreUnavailable(bool? ignoreunavailable = true) => Qs("ignore_unavailable", ignoreunavailable); + ///Specify timeout for connection to master + public UnfreezeIndexDescriptor MasterTimeout(Time mastertimeout) => Qs("master_timeout", mastertimeout); + ///Explicit operation timeout + public UnfreezeIndexDescriptor Timeout(Time timeout) => Qs("timeout", timeout); + ///Sets the number of active shards to wait for before the operation returns. + public UnfreezeIndexDescriptor WaitForActiveShards(string waitforactiveshards) => Qs("wait_for_active_shards", waitforactiveshards); + } + ///descriptor for BulkAlias http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html public partial class BulkAliasDescriptor : RequestDescriptorBase, IBulkAliasRequest { diff --git a/src/Nest/ElasticClient.Indices.cs b/src/Nest/ElasticClient.Indices.cs index be6d380430b..e89f2d2e788 100644 --- a/src/Nest/ElasticClient.Indices.cs +++ b/src/Nest/ElasticClient.Indices.cs @@ -373,6 +373,30 @@ internal IndicesNamespace(ElasticClient client): base(client) /// public Task ForceMergeAsync(IForceMergeRequest request, CancellationToken ct = default) => DoRequestAsync(request, request.RequestParameters, ct); /// + /// POST request to the indices.freeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public FreezeIndexResponse Freeze(IndexName index, Func selector = null) => Freeze(selector.InvokeOrDefault(new FreezeIndexDescriptor(index: index))); + /// + /// POST request to the indices.freeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public Task FreezeAsync(IndexName index, Func selector = null, CancellationToken ct = default) => FreezeAsync(selector.InvokeOrDefault(new FreezeIndexDescriptor(index: index)), ct); + /// + /// POST request to the indices.freeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public FreezeIndexResponse Freeze(IFreezeIndexRequest request) => DoRequest(request, request.RequestParameters); + /// + /// POST request to the indices.freeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public Task FreezeAsync(IFreezeIndexRequest request, CancellationToken ct = default) => DoRequestAsync(request, request.RequestParameters, ct); + /// /// GET request to the indices.get API, read more about this API online: /// /// http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-index.html @@ -835,6 +859,30 @@ public Task PutMappingAsync(Func public Task StatsAsync(IIndicesStatsRequest request, CancellationToken ct = default) => DoRequestAsync(request, request.RequestParameters, ct); /// + /// POST request to the indices.unfreeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public UnfreezeIndexResponse Unfreeze(IndexName index, Func selector = null) => Unfreeze(selector.InvokeOrDefault(new UnfreezeIndexDescriptor(index: index))); + /// + /// POST request to the indices.unfreeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public Task UnfreezeAsync(IndexName index, Func selector = null, CancellationToken ct = default) => UnfreezeAsync(selector.InvokeOrDefault(new UnfreezeIndexDescriptor(index: index)), ct); + /// + /// POST request to the indices.unfreeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public UnfreezeIndexResponse Unfreeze(IUnfreezeIndexRequest request) => DoRequest(request, request.RequestParameters); + /// + /// POST request to the indices.unfreeze API, read more about this API online: + /// + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + /// + public Task UnfreezeAsync(IUnfreezeIndexRequest request, CancellationToken ct = default) => DoRequestAsync(request, request.RequestParameters, ct); + /// /// POST request to the indices.update_aliases API, read more about this API online: /// /// http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html diff --git a/src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexRequest.cs b/src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexRequest.cs new file mode 100644 index 00000000000..e15e846dd36 --- /dev/null +++ b/src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexRequest.cs @@ -0,0 +1,9 @@ +namespace Nest +{ + [MapsApi("indices.freeze.json")] + public partial interface IFreezeIndexRequest { } + + public partial class FreezeIndexRequest { } + + public partial class FreezeIndexDescriptor { } +} diff --git a/src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexResponse.cs b/src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexResponse.cs new file mode 100644 index 00000000000..58951b1aa7c --- /dev/null +++ b/src/Nest/Indices/IndexManagement/FreezeIndex/FreezeIndexResponse.cs @@ -0,0 +1,10 @@ +using System.Runtime.Serialization; + +namespace Nest +{ + public class FreezeIndexResponse : AcknowledgedResponseBase + { + [DataMember(Name = "shards_acknowledged")] + public bool ShardsAcknowledged { get; internal set; } + } +} diff --git a/src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexRequest.cs b/src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexRequest.cs new file mode 100644 index 00000000000..987721dab99 --- /dev/null +++ b/src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexRequest.cs @@ -0,0 +1,9 @@ +namespace Nest +{ + [MapsApi("indices.unfreeze.json")] + public partial interface IUnfreezeIndexRequest { } + + public partial class UnfreezeIndexRequest { } + + public partial class UnfreezeIndexDescriptor { } +} diff --git a/src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexResponse.cs b/src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexResponse.cs new file mode 100644 index 00000000000..7970573ed28 --- /dev/null +++ b/src/Nest/Indices/IndexManagement/UnfreezeIndex/UnfreezeIndexResponse.cs @@ -0,0 +1,10 @@ +using System.Runtime.Serialization; + +namespace Nest +{ + public class UnfreezeIndexResponse : AcknowledgedResponseBase + { + [DataMember(Name = "shards_acknowledged")] + public bool ShardsAcknowledged { get; internal set; } + } +} diff --git a/src/Nest/Requests.Indices.cs b/src/Nest/Requests.Indices.cs index 6a57fc4eb85..4d187a99e2e 100644 --- a/src/Nest/Requests.Indices.cs +++ b/src/Nest/Requests.Indices.cs @@ -927,6 +927,83 @@ public bool? OnlyExpungeDeletes } } + [InterfaceDataContract] + public partial interface IFreezeIndexRequest : IRequest + { + [IgnoreDataMember] + IndexName Index + { + get; + } + } + + ///Request for Freeze https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html + public partial class FreezeIndexRequest : PlainRequestBase, IFreezeIndexRequest + { + protected IFreezeIndexRequest Self => this; + internal override ApiUrls ApiUrls => ApiUrlsLookups.IndicesFreeze; + ////{index}/_freeze + ///this parameter is required + public FreezeIndexRequest(IndexName index): base(r => r.Required("index", index)) + { + } + + ///Used for serialization purposes, making sure we have a parameterless constructor + [SerializationConstructor] + protected FreezeIndexRequest(): base() + { + } + + // values part of the url path + [IgnoreDataMember] + IndexName IFreezeIndexRequest.Index => Self.RouteValues.Get("index"); + // Request parameters + /// + /// Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have + /// been specified) + /// + public bool? AllowNoIndices + { + get => Q("allow_no_indices"); + set => Q("allow_no_indices", value); + } + + ///Whether to expand wildcard expression to concrete indices that are open, closed or both. + public ExpandWildcards? ExpandWildcards + { + get => Q("expand_wildcards"); + set => Q("expand_wildcards", value); + } + + ///Whether specified concrete indices should be ignored when unavailable (missing or closed) + public bool? IgnoreUnavailable + { + get => Q("ignore_unavailable"); + set => Q("ignore_unavailable", value); + } + + ///Specify timeout for connection to master + public Time MasterTimeout + { + get => Q