From 544d9d7ee14542d30b2c0074eb0fea56e17073cd Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Mon, 18 Mar 2024 10:59:56 -0400 Subject: [PATCH] Hack in an example of analyzer --- .../org/elasticsearch/TransportVersions.java | 1 + .../admin/indices/analyze/AnalyzeAction.java | 7 ++ .../xpack/application/rules/QueryRule.java | 5 +- .../application/rules/QueryRuleCriteria.java | 62 +++++++++++++---- .../rules/QueryRuleCriteriaType.java | 66 +++++++++++++++---- .../rules/QueryRulesAnalysisConfig.java | 36 ++++++++++ .../rules/QueryRulesAnalysisService.java | 56 ++++++++++++++++ .../rules/QueryRulesIndexService.java | 8 ++- .../application/rules/RuleQueryBuilder.java | 2 +- 9 files changed, 212 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisConfig.java create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisService.java diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 536a5db51e8a8..81dc435031001 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -149,6 +149,7 @@ static TransportVersion def(int id) { public static final TransportVersion AGGS_EXCLUDED_DELETED_DOCS = def(8_609_00_0); public static final TransportVersion ESQL_SERIALIZE_BIG_ARRAY = def(8_610_00_0); public static final TransportVersion AUTO_SHARDING_ROLLOVER_CONDITION = def(8_611_00_0); + public static final TransportVersion QUERY_RULES_CRITERIA_METADATA_PROPERTIES_ADDED = def(8_612_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java index d19aacb306414..61cd3b485a625 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java @@ -141,6 +141,13 @@ public Request addTokenFilter(String tokenFilter) { return this; } + public Request addTokenFilters(List tokenFilters) { + for (String tokenFilter : tokenFilters) { + this.tokenFilters.add(new NameOrDefinition(tokenFilter)); + } + return this; + } + public Request addTokenFilter(Map tokenFilter) { this.tokenFilters.add(new NameOrDefinition(tokenFilter)); return this; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java index b9093a2597d7d..9a7bd28533c13 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.application.rules; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -277,7 +278,7 @@ public String toString() { } @SuppressWarnings("unchecked") - public AppliedQueryRules applyRule(AppliedQueryRules appliedRules, Map matchCriteria) { + public AppliedQueryRules applyRule(Client client, AppliedQueryRules appliedRules, Map matchCriteria) { if (type != QueryRule.QueryRuleType.PINNED) { throw new UnsupportedOperationException("Only pinned query rules are supported"); } @@ -294,7 +295,7 @@ public AppliedQueryRules applyRule(AppliedQueryRules appliedRules, Map criteriaValues; + private final Map criteriaProperties; private static final Logger logger = LogManager.getLogger(QueryRuleCriteria.class); /** * - * @param criteriaType The {@link QueryRuleCriteriaType}, indicating how the criteria is matched - * @param criteriaMetadata The metadata for this identifier, indicating the criteria key of what is matched against. - * Required unless the CriteriaType is ALWAYS. - * @param criteriaValues The values to match against when evaluating {@link QueryRuleCriteria} against a {@link QueryRule} - * Required unless the CriteriaType is ALWAYS. + * @param criteriaType The {@link QueryRuleCriteriaType}, indicating how the criteria is matched + * @param criteriaMetadata The metadata for this identifier, indicating the criteria key of what is matched against. + * Required unless the CriteriaType is ALWAYS. + * @param criteriaValues The values to match against when evaluating {@link QueryRuleCriteria} against a {@link QueryRule} + * Required unless the CriteriaType is ALWAYS. + * @param criteriaProperties Additional configuration properties for this criteria, to override default criteria configuration. */ - public QueryRuleCriteria(QueryRuleCriteriaType criteriaType, @Nullable String criteriaMetadata, @Nullable List criteriaValues) { + public QueryRuleCriteria( + QueryRuleCriteriaType criteriaType, + @Nullable String criteriaMetadata, + @Nullable List criteriaValues, + Map criteriaProperties + ) { Objects.requireNonNull(criteriaType); @@ -69,6 +81,9 @@ public QueryRuleCriteria(QueryRuleCriteriaType criteriaType, @Nullable String cr this.criteriaValues = criteriaValues; this.criteriaType = criteriaType; + this.criteriaProperties = criteriaProperties == null ? Map.of() : criteriaProperties; + // TODO criteriaType.validateProperties(criteriaProperties); + } public QueryRuleCriteria(StreamInput in) throws IOException { @@ -80,6 +95,11 @@ public QueryRuleCriteria(StreamInput in) throws IOException { this.criteriaMetadata = in.readString(); this.criteriaValues = List.of(in.readGenericValue()); } + if (in.getTransportVersion().onOrAfter(CRITERIA_METADATA_PROPERTIES_TRANSPORT_VERSION)) { + this.criteriaProperties = in.readGenericMap(); + } else { + this.criteriaProperties = Map.of(); + } } @Override @@ -92,6 +112,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(criteriaMetadata); out.writeGenericValue(criteriaValues().get(0)); } + if (out.getTransportVersion().onOrAfter(CRITERIA_METADATA_PROPERTIES_TRANSPORT_VERSION)) { + out.writeGenericMap(criteriaProperties); + } } private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -102,18 +125,22 @@ public void writeTo(StreamOutput out) throws IOException { final String metadata = params.length >= 3 ? (String) params[1] : null; @SuppressWarnings("unchecked") final List values = params.length >= 3 ? (List) params[2] : null; - return new QueryRuleCriteria(type, metadata, values); + @SuppressWarnings("unchecked") + final Map properties = params.length >= 4 ? (Map) params[3] : null; + return new QueryRuleCriteria(type, metadata, values, properties); } ); public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField METADATA_FIELD = new ParseField("metadata"); public static final ParseField VALUES_FIELD = new ParseField("values"); + public static final ParseField PROPERTIES_FIELD = new ParseField("properties"); static { PARSER.declareString(constructorArg(), TYPE_FIELD); PARSER.declareStringOrNull(optionalConstructorArg(), METADATA_FIELD); PARSER.declareStringArray(optionalConstructorArg(), VALUES_FIELD); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.map(), PROPERTIES_FIELD); } /** @@ -153,6 +180,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (criteriaValues != null) { builder.array(VALUES_FIELD.getPreferredName(), criteriaValues.toArray()); } + if (criteriaProperties != null) { + builder.field(PROPERTIES_FIELD.getPreferredName(), criteriaProperties); + } } builder.endObject(); return builder; @@ -170,6 +200,10 @@ public List criteriaValues() { return criteriaValues; } + public Map criteriaProperties() { + return criteriaProperties; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -177,12 +211,13 @@ public boolean equals(Object o) { QueryRuleCriteria that = (QueryRuleCriteria) o; return criteriaType == that.criteriaType && Objects.equals(criteriaMetadata, that.criteriaMetadata) - && Objects.equals(criteriaValues, that.criteriaValues); + && Objects.equals(criteriaValues, that.criteriaValues) + && Objects.equals(criteriaProperties, that.criteriaProperties); } @Override public int hashCode() { - return Objects.hash(criteriaType, criteriaMetadata, criteriaValues); + return Objects.hash(criteriaType, criteriaMetadata, criteriaValues, criteriaProperties); } @Override @@ -190,11 +225,11 @@ public String toString() { return Strings.toString(this); } - public boolean isMatch(Object matchValue, QueryRuleCriteriaType matchType) { - return isMatch(matchValue, matchType, true); + public boolean isMatch(Client client, Object matchValue, QueryRuleCriteriaType matchType) { + return isMatch(client, matchValue, matchType, true); } - public boolean isMatch(Object matchValue, QueryRuleCriteriaType matchType, boolean throwOnInvalidInput) { + public boolean isMatch(Client client, Object matchValue, QueryRuleCriteriaType matchType, boolean throwOnInvalidInput) { if (matchType == ALWAYS) { return true; } @@ -204,7 +239,8 @@ public boolean isMatch(Object matchValue, QueryRuleCriteriaType matchType, boole if (isValid == false) { return false; } - boolean matchFound = matchType.isMatch(matchString, criteriaValue); + QueryRulesAnalysisService analysisService = new QueryRulesAnalysisService(client); + boolean matchFound = matchType.isMatch(analysisService, matchString, criteriaValue, criteriaProperties); if (matchFound) { return true; } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRuleCriteriaType.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRuleCriteriaType.java index 5606d47697d2e..212a154920bf4 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRuleCriteriaType.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRuleCriteriaType.java @@ -11,22 +11,51 @@ import java.util.List; import java.util.Locale; +import java.util.Map; /** * Defines the different types of query rule criteria and their rules for matching input against the criteria. */ public enum QueryRuleCriteriaType { + ALWAYS { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { return true; } }, EXACT { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { + throw new UnsupportedOperationException("[" + this + "] criteria type requires analysis service"); + } + + @Override + public boolean isMatch( + QueryRulesAnalysisService analysisService, + Object input, + Object criteriaValue, + Map criteriaProperties + ) { if (input instanceof String && criteriaValue instanceof String) { - return input.equals(criteriaValue); + + if (criteriaProperties.containsKey("analysis")) { + @SuppressWarnings("unchecked") + List> analysisChain = (List>) criteriaProperties.get("analysis"); + String analyzedInput = (String) input; + String analyzedCriteriaValue = (String) criteriaValue; + for (Map analysisConfig : analysisChain) { + String tokenizer = analysisConfig.containsKey("tokenizer") ? (String) analysisConfig.get("tokenizer") : "keyword"; + String filter = analysisConfig.containsKey("filter") ? (String) analysisConfig.get("filter") : "lowercase"; + QueryRulesAnalysisConfig config = new QueryRulesAnalysisConfig(null, tokenizer, List.of(filter)); + analyzedInput = analysisService.analyze(analyzedInput, config); + analyzedCriteriaValue = analysisService.analyze(analyzedCriteriaValue, config); + } + return analyzedInput.equals(analyzedCriteriaValue); + } else { + return input.equals(criteriaValue); + } + } else { return parseDouble(input) == parseDouble(criteriaValue); } @@ -34,7 +63,7 @@ public boolean isMatch(Object input, Object criteriaValue) { }, FUZZY { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { final LevenshteinDistance ld = new LevenshteinDistance(); if (input instanceof String && criteriaValue instanceof String) { return ld.getDistance((String) input, (String) criteriaValue) > 0.5f; @@ -44,43 +73,43 @@ public boolean isMatch(Object input, Object criteriaValue) { }, PREFIX { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { return ((String) input).startsWith((String) criteriaValue); } }, SUFFIX { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { return ((String) input).endsWith((String) criteriaValue); } }, CONTAINS { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { return ((String) input).contains((String) criteriaValue); } }, LT { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { return parseDouble(input) < parseDouble(criteriaValue); } }, LTE { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { return parseDouble(input) <= parseDouble(criteriaValue); } }, GT { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { return parseDouble(input) > parseDouble(criteriaValue); } }, GTE { @Override - public boolean isMatch(Object input, Object criteriaValue) { + public boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties) { validateInput(input); return parseDouble(input) >= parseDouble(criteriaValue); } @@ -94,11 +123,20 @@ public boolean validateInput(Object input, boolean throwOnInvalidInput) { return isValid; } - public boolean validateInput(Object input) { - return validateInput(input, true); + public void validateInput(Object input) { + validateInput(input, true); } - public abstract boolean isMatch(Object input, Object criteriaValue); + public abstract boolean isMatch(Object input, Object criteriaValue, Map criteriaProperties); + + public boolean isMatch( + QueryRulesAnalysisService analysisService, + Object input, + Object criteriaValue, + Map criteriaProperties + ) { + return isMatch(input, criteriaValue, criteriaProperties); + } public static QueryRuleCriteriaType type(String criteriaType) { for (QueryRuleCriteriaType type : values()) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisConfig.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisConfig.java new file mode 100644 index 0000000000000..2cfec219e6939 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisConfig.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules; + +import java.util.List; + +public class QueryRulesAnalysisConfig { + + private final String analyzer; + private final String tokenizer; + private final List filters; + + public QueryRulesAnalysisConfig(String analyzer, String tokenizer, List filters) { + this.analyzer = analyzer; + this.tokenizer = tokenizer; + this.filters = filters; + } + + public String analyzer() { + return analyzer; + } + + public String tokenizer() { + return tokenizer; + } + + public List filters() { + return filters; + } + +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisService.java new file mode 100644 index 0000000000000..491c582e24baa --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesAnalysisService.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.rules; + +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.List; + +import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; + +public class QueryRulesAnalysisService { + + private static final TimeValue TIMEOUT_MS = TimeValue.timeValueMillis(1000); + + private static final Logger logger = LogManager.getLogger(QueryRulesAnalysisService.class); + + private final Client clientWithOrigin; + + public QueryRulesAnalysisService(Client client) { + this.clientWithOrigin = new OriginSettingClient(client, ENT_SEARCH_ORIGIN); + } + + public String analyze(String text, QueryRulesAnalysisConfig analysisConfig) { + + logger.info("Analyzing original text [" + text + "]"); + + String analyzer = analysisConfig.analyzer(); + String tokenizer = analysisConfig.tokenizer(); + List filters = analysisConfig.filters(); + + AnalyzeAction.Request analyzeRequest = new AnalyzeAction.Request().analyzer(analyzer) + .tokenizer(tokenizer) + .addTokenFilters(filters) + .text(text); + AnalyzeAction.Response analyzeResponse = clientWithOrigin.execute(AnalyzeAction.INSTANCE, analyzeRequest).actionGet(TIMEOUT_MS); + List analyzeTokens = analyzeResponse.getTokens(); + StringBuilder sb = new StringBuilder(); + for (AnalyzeAction.AnalyzeToken analyzeToken : analyzeTokens) { + logger.info("Analyzed term: [" + analyzeToken.getTerm() + "]"); + sb.append(analyzeToken.getTerm()).append(" "); + + } + return sb.toString().trim(); + } + +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java index 1e98755cc7acf..278755d18ba0c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java @@ -143,6 +143,11 @@ private static XContentBuilder getIndexMappings() { builder.field("type", "object"); builder.field("enabled", false); builder.endObject(); + + builder.startObject(QueryRuleCriteria.PROPERTIES_FIELD.getPreferredName()); + builder.field("type", "object"); + builder.field("enabled", false); + builder.endObject(); } builder.endObject(); builder.endObject(); @@ -218,7 +223,8 @@ private static List parseCriteria(List> r new QueryRuleCriteria( QueryRuleCriteriaType.type((String) entry.get(QueryRuleCriteria.TYPE_FIELD.getPreferredName())), (String) entry.get(QueryRuleCriteria.METADATA_FIELD.getPreferredName()), - (List) entry.get(QueryRuleCriteria.VALUES_FIELD.getPreferredName()) + (List) entry.get(QueryRuleCriteria.VALUES_FIELD.getPreferredName()), + (Map) entry.get(QueryRuleCriteria.PROPERTIES_FIELD.getPreferredName()) ) ); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java index bc45b24027e0e..5208921a93426 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java @@ -213,7 +213,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) { QueryRuleset queryRuleset = QueryRuleset.fromXContentBytes(rulesetId, getResponse.getSourceAsBytesRef(), XContentType.JSON); for (QueryRule rule : queryRuleset.rules()) { - rule.applyRule(appliedRules, matchCriteria); + rule.applyRule(client, appliedRules, matchCriteria); } pinnedIdsSetOnce.set(appliedRules.pinnedIds().stream().distinct().toList()); pinnedDocsSetOnce.set(appliedRules.pinnedDocs().stream().distinct().toList());