fix(java) Use alias for name search sorting and fix missing mappings (#…

…8648)
datahub-project · Aug 23, 2023 · a4cb81c · a4cb81c
1 parent 52f1e32
commit a4cb81c
Show file tree

Hide file tree

Showing 36 changed files with 187 additions and 35 deletions.
diff --git a/datahub-web-react/src/app/search/context/constants.ts b/datahub-web-react/src/app/search/context/constants.ts
@@ -1,15 +1,23 @@
 import { SortOrder } from '../../../types.generated';
 
 export const RELEVANCE = 'relevance';
-export const NAME_FIELD = 'name';
+export const ENTITY_NAME_FIELD = '_entityName';
 export const LAST_OPERATION_TIME_FIELD = 'lastOperationTime';
 
 export const DEFAULT_SORT_OPTION = RELEVANCE;
 
 export const SORT_OPTIONS = {
     [RELEVANCE]: { label: 'Relevance', field: RELEVANCE, sortOrder: SortOrder.Descending },
-    [`${NAME_FIELD}_${SortOrder.Ascending}`]: { label: 'A to Z', field: NAME_FIELD, sortOrder: SortOrder.Ascending },
-    [`${NAME_FIELD}_${SortOrder.Descending}`]: { label: 'Z to A', field: NAME_FIELD, sortOrder: SortOrder.Descending },
+    [`${ENTITY_NAME_FIELD}_${SortOrder.Ascending}`]: {
+        label: 'A to Z',
+        field: ENTITY_NAME_FIELD,
+        sortOrder: SortOrder.Ascending,
+    },
+    [`${ENTITY_NAME_FIELD}_${SortOrder.Descending}`]: {
+        label: 'Z to A',
+        field: ENTITY_NAME_FIELD,
+        sortOrder: SortOrder.Descending,
+    },
     [`${LAST_OPERATION_TIME_FIELD}_${SortOrder.Descending}`]: {
         label: 'Last Modified in Platform',
         field: LAST_OPERATION_TIME_FIELD,

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java
@@ -155,7 +155,8 @@ private void extractSearchableAnnotation(final Object annotationObj, final DataS
             annotation.getBoostScore(),
             annotation.getHasValuesFieldName(),
             annotation.getNumValuesFieldName(),
-            annotation.getWeightsPerFieldValue());
+            annotation.getWeightsPerFieldValue(),
+            annotation.getFieldNameAliases());
       }
     }
     log.debug("Searchable annotation for field: {} : {}", schemaPathSpec, annotation);

diff --git a/...-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java b/...-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java
@@ -4,7 +4,10 @@
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.data.schema.DataSchema;
 import com.linkedin.metadata.models.ModelValidationException;
+
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -19,6 +22,7 @@
 @Value
 public class SearchableAnnotation {
 
+  public static final String FIELD_NAME_ALIASES = "fieldNameAliases";
   public static final String ANNOTATION_NAME = "Searchable";
   private static final Set<FieldType> DEFAULT_QUERY_FIELD_TYPES =
       ImmutableSet.of(FieldType.TEXT, FieldType.TEXT_PARTIAL, FieldType.WORD_GRAM, FieldType.URN, FieldType.URN_PARTIAL);
@@ -47,6 +51,8 @@ public class SearchableAnnotation {
   Optional<String> numValuesFieldName;
   // (Optional) Weights to apply to score for a given value
   Map<Object, Double> weightsPerFieldValue;
+  // (Optional) Aliases for this given field that can be used for sorting etc.
+  List<String> fieldNameAliases;
 
   public enum FieldType {
     KEYWORD,
@@ -94,6 +100,7 @@ public static SearchableAnnotation fromPegasusAnnotationObject(@Nonnull final Ob
     final Optional<String> numValuesFieldName = AnnotationUtils.getField(map, "numValuesFieldName", String.class);
     final Optional<Map> weightsPerFieldValueMap =
         AnnotationUtils.getField(map, "weightsPerFieldValue", Map.class).map(m -> (Map<Object, Double>) m);
+    final List<String> fieldNameAliases = getFieldNameAliases(map);
 
     final FieldType resolvedFieldType = getFieldType(fieldType, schemaDataType);
     return new SearchableAnnotation(
@@ -108,7 +115,8 @@ public static SearchableAnnotation fromPegasusAnnotationObject(@Nonnull final Ob
         boostScore.orElse(1.0),
         hasValuesFieldName,
         numValuesFieldName,
-        weightsPerFieldValueMap.orElse(ImmutableMap.of()));
+        weightsPerFieldValueMap.orElse(ImmutableMap.of()),
+        fieldNameAliases);
   }
 
   private static FieldType getFieldType(Optional<String> maybeFieldType, DataSchema.Type schemaDataType) {
@@ -156,4 +164,15 @@ private static String capitalizeFirstLetter(String str) {
       return str.substring(0, 1).toUpperCase() + str.substring(1);
     }
   }
+
+  private static List<String> getFieldNameAliases(Map map) {
+    final List<String> aliases = new ArrayList<>();
+    final Optional<List> fieldNameAliases = AnnotationUtils.getField(map, FIELD_NAME_ALIASES, List.class);
+    if (fieldNameAliases.isPresent()) {
+      for (Object alias : fieldNameAliases.get()) {
+        aliases.add((String) alias);
+      }
+    }
+    return aliases;
+  }
 }
diff --git a/...rc/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/...rc/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
@@ -6,6 +6,7 @@
 import com.linkedin.metadata.models.SearchableFieldSpec;
 import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -46,6 +47,10 @@ public static Map<String, String> getPartialNgramConfigWithOverrides(Map<String,
   public static final String WORD_GRAMS_LENGTH_3 = "wordGrams3";
   public static final String WORD_GRAMS_LENGTH_4 = "wordGrams4";
 
+  // Alias field mappings constants
+  public static final String ALIAS = "alias";
+  public static final String PATH = "path";
+
   private MappingsBuilder() {
   }
 
@@ -180,6 +185,7 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
     searchableFieldSpec.getSearchableAnnotation()
         .getNumValuesFieldName()
         .ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, LONG)));
+    mappings.putAll(getMappingsForFieldNameAliases(searchableFieldSpec));
 
     return mappings;
   }
@@ -189,4 +195,16 @@ private static Map<String, Object> getMappingsForSearchScoreField(
     return ImmutableMap.of(searchScoreFieldSpec.getSearchScoreAnnotation().getFieldName(),
         ImmutableMap.of(TYPE, DOUBLE));
   }
+
+  private static Map<String, Object> getMappingsForFieldNameAliases(@Nonnull final SearchableFieldSpec searchableFieldSpec) {
+    Map<String, Object> mappings = new HashMap<>();
+    List<String> fieldNameAliases = searchableFieldSpec.getSearchableAnnotation().getFieldNameAliases();
+    fieldNameAliases.forEach(alias -> {
+      Map<String, Object> aliasMappings = new HashMap<>();
+      aliasMappings.put(TYPE, ALIAS);
+      aliasMappings.put(PATH, searchableFieldSpec.getSearchableAnnotation().getFieldName());
+      mappings.put(alias, aliasMappings);
+    });
+    return mappings;
+  }
 }
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
@@ -45,6 +45,7 @@ public class ESUtils {
   public static final int MAX_RESULT_SIZE = 10000;
   public static final String OPAQUE_ID_HEADER = "X-Opaque-Id";
   public static final String HEADER_VALUE_DELIMITER = "|";
+  public static final String KEYWORD_TYPE = "keyword";
 
   // we use this to make sure we filter for editable & non-editable fields. Also expands out top-level properties
   // to field level properties
@@ -174,6 +175,8 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull final Criterion
    * If no sort criterion is provided then the default sorting criterion is chosen which is descending order of score
    * Furthermore to resolve conflicts, the results are further sorted by ascending order of urn
    * If the input sort criterion is urn itself, then no additional sort criterion is applied as there will be no conflicts.
+   * When sorting, set the unmappedType param to arbitrary "keyword" so we essentially ignore sorting where indices do not
+   * have the field we are sorting on.
    * </p>
    *
    * @param searchSourceBuilder {@link SearchSourceBuilder} that needs to be populated with sort order
@@ -187,7 +190,7 @@ public static void buildSortOrder(@Nonnull SearchSourceBuilder searchSourceBuild
       final SortOrder esSortOrder =
           (sortCriterion.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING) ? SortOrder.ASC
               : SortOrder.DESC;
-      searchSourceBuilder.sort(new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder));
+      searchSourceBuilder.sort(new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder).unmappedType(KEYWORD_TYPE));
     }
     if (sortCriterion == null || !sortCriterion.getField().equals(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD)) {
       searchSourceBuilder.sort(new FieldSortBuilder(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD).order(SortOrder.ASC));

diff --git a/...est/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java b/...est/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java
@@ -16,7 +16,7 @@ public void testMappingsBuilder() {
     Map<String, Object> result = MappingsBuilder.getMappings(TestEntitySpecBuilder.getSpec());
     assertEquals(result.size(), 1);
     Map<String, Object> properties = (Map<String, Object>) result.get("properties");
-    assertEquals(properties.size(), 18);
+    assertEquals(properties.size(), 19);
     assertEquals(properties.get("urn"), ImmutableMap.of("type", "keyword",
             "fields",
             ImmutableMap.of("delimited",
@@ -66,6 +66,11 @@ public void testMappingsBuilder() {
     assertTrue(textFieldSubfields.containsKey("delimited"));
     assertTrue(textFieldSubfields.containsKey("keyword"));
 
+    // TEXT with addToFilters aliased under "_entityName"
+    Map<String, Object> textFieldAlias = (Map<String, Object>) properties.get("_entityName");
+    assertEquals(textFieldAlias.get("type"), "alias");
+    assertEquals(textFieldAlias.get("path"), "textFieldOverride");
+
     // TEXT_PARTIAL
     Map<String, Object> textArrayField = (Map<String, Object>) properties.get("textArrayField");
     assertEquals(textArrayField.get("type"), "keyword");

diff --git a/...com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java b/...com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilderTest.java
@@ -31,7 +31,8 @@ public void testGetDefaultAggregationsHasFields() {
         1.0,
         Optional.of("hasTest"),
         Optional.empty(),
-        Collections.emptyMap()
+        Collections.emptyMap(),
+        Collections.emptyList()
     );
 
     SearchConfiguration config = new SearchConfiguration();
@@ -60,7 +61,8 @@ public void testGetDefaultAggregationsFields() {
         1.0,
         Optional.empty(),
         Optional.empty(),
-        Collections.emptyMap()
+        Collections.emptyMap(),
+        Collections.emptyList()
     );
 
     SearchConfiguration config = new SearchConfiguration();
@@ -89,7 +91,8 @@ public void testGetSpecificAggregationsHasFields() {
         1.0,
         Optional.of("hasTest1"),
         Optional.empty(),
-        Collections.emptyMap()
+        Collections.emptyMap(),
+        Collections.emptyList()
     );
 
     SearchableAnnotation annotation2 = new SearchableAnnotation(
@@ -104,7 +107,8 @@ public void testGetSpecificAggregationsHasFields() {
         1.0,
         Optional.empty(),
         Optional.empty(),
-        Collections.emptyMap()
+        Collections.emptyMap(),
+        Collections.emptyList()
     );
 
     SearchConfiguration config = new SearchConfiguration();

diff --git a/metadata-models/src/main/pegasus/com/linkedin/chart/ChartInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/chart/ChartInfo.pdl
@@ -21,7 +21,8 @@ record ChartInfo includes CustomProperties, ExternalReference {
    */
   @Searchable = {
     "fieldType": "WORD_GRAM",
-    "enableAutocomplete": true
+    "enableAutocomplete": true,
+    "fieldNameAliases": [ "_entityName" ]
   }
   title: string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl
@@ -17,7 +17,8 @@ record ContainerProperties includes CustomProperties, ExternalReference {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl
@@ -24,7 +24,8 @@ record DashboardInfo includes CustomProperties, ExternalReference {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   title: string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/datajob/DataFlowInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/datajob/DataFlowInfo.pdl
@@ -19,7 +19,8 @@ record DataFlowInfo includes CustomProperties, ExternalReference {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/datajob/DataJobInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/datajob/DataJobInfo.pdl
@@ -20,7 +20,8 @@ record DataJobInfo includes CustomProperties, ExternalReference {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataplatform/DataPlatformInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataplatform/DataPlatformInfo.pdl
@@ -17,7 +17,8 @@ record DataPlatformInfo {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": false,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string
 

diff --git a/...els/src/main/pegasus/com/linkedin/dataplatforminstance/DataPlatformInstanceProperties.pdl b/...els/src/main/pegasus/com/linkedin/dataplatforminstance/DataPlatformInstanceProperties.pdl
@@ -18,7 +18,8 @@ record DataPlatformInstanceProperties includes CustomProperties, ExternalReferen
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: optional string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataproduct/DataProductProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataproduct/DataProductProperties.pdl
@@ -15,7 +15,8 @@ record DataProductProperties includes CustomProperties, ExternalReference  {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: optional string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProperties.pdl
@@ -19,7 +19,8 @@ record DatasetProperties includes CustomProperties, ExternalReference {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: optional string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/domain/DomainProperties.pdl
@@ -16,7 +16,8 @@ record DomainProperties {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryNodeInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryNodeInfo.pdl
@@ -37,7 +37,8 @@ record GlossaryNodeInfo {
       "fieldName": "displayName",
       "fieldType": "WORD_GRAM",
       "enableAutocomplete": true,
-      "boostScore": 10.0
+      "boostScore": 10.0,
+      "fieldNameAliases": [ "_entityName" ]
     }
     name: optional string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryTermInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryTermInfo.pdl
@@ -25,7 +25,8 @@ record GlossaryTermInfo includes CustomProperties {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: optional string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpGroupInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpGroupInfo.pdl
@@ -21,7 +21,8 @@ record CorpGroupInfo {
     "fieldType": "TEXT_PARTIAL"
     "queryByDefault": true,
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   displayName: optional string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserInfo.pdl
@@ -29,7 +29,8 @@ record CorpUserInfo includes CustomProperties {
     "fieldType": "WORD_GRAM",
     "queryByDefault": true,
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   displayName: optional string
 

diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/MLFeatureKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/MLFeatureKey.pdl
@@ -22,7 +22,8 @@ record MLFeatureKey {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 8.0
+    "boostScore": 8.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string
 }
diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/MLFeatureTableKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/MLFeatureTableKey.pdl
@@ -24,7 +24,8 @@ record MLFeatureTableKey {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 8.0
+    "boostScore": 8.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string
 }
diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/MLModelDeploymentKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/MLModelDeploymentKey.pdl
@@ -21,7 +21,8 @@ record MLModelDeploymentKey {
   @Searchable = {
     "fieldType": "WORD_GRAM",
     "enableAutocomplete": true,
-    "boostScore": 10.0
+    "boostScore": 10.0,
+    "fieldNameAliases": [ "_entityName" ]
   }
   name: string