Refactor tableSource for Column and Argument (#2075)

* Populating namespaces in metadatastore * Added support for versioned default namespaces * Added test for static type namespaces * Changed Include.type to Include.name. Added metadata to Include * Fixed build * Removed NamespaceMeta annotation * Round one - doesn't work * Fixed broken build * Fix build * Elide model config builds. Changed tableSource from a string to an object with more metadata * Fleshed out metadata models for new table source definition * Fixed build after rebase Co-authored-by: Aaron Klish <[email protected]>
yahoo · May 10, 2021 · e722d3e · e722d3e
1 parent 1bb35be
commit e722d3e
Show file tree

Hide file tree

Showing 29 changed files with 375 additions and 70 deletions.
diff --git a/...tastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java b/...tastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java
@@ -17,6 +17,7 @@
 import com.yahoo.elide.datastores.aggregation.metadata.models.Metric;
 import com.yahoo.elide.datastores.aggregation.metadata.models.Namespace;
 import com.yahoo.elide.datastores.aggregation.metadata.models.Table;
+import com.yahoo.elide.datastores.aggregation.metadata.models.TableSource;
 import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimension;
 import com.yahoo.elide.datastores.aggregation.query.DimensionProjection;
 import com.yahoo.elide.datastores.aggregation.query.MetricProjection;
@@ -164,6 +165,28 @@ protected void populateMetaData(MetaDataStore metaDataStore) {
         metaDataStore.getModelsToBind().stream()
                 .map(model -> constructTable(metaDataStore.getNamespace(model), model, metadataDictionary))
                 .forEach(metaDataStore::addTable);
+
+        //Populate table sources.
+        metaDataStore.getTables().forEach(table -> {
+            table.getColumns().forEach(column -> {
+
+                //Populate column sources.
+                column.setTableSource(TableSource.fromDefinition(
+                        column.getTableSourceDefinition(),
+                        table.getVersion(),
+                        metaDataStore
+                ));
+
+                //Populate column argument sources.
+                column.getArguments().forEach(argument -> {
+                    argument.setTableSource(TableSource.fromDefinition(
+                            argument.getTableSourceDefinition(),
+                            table.getVersion(),
+                            metaDataStore
+                    ));
+                });
+            });
+        });
     }
 
     /**

diff --git a/...n/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/ArgumentDefinition.java b/...n/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/ArgumentDefinition.java
@@ -11,11 +11,10 @@
  * The definition of Argument.
  */
 public @interface ArgumentDefinition {
-
     String name() default "";
     String description() default "";
     ValueType type() default ValueType.TEXT;
-    String tableSource() default "";
+    TableSource tableSource() default @TableSource(table = "", column = "");
     String [] values() default {};
     String defaultValue() default "";
 }
diff --git a/...gregation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/ColumnMeta.java b/...gregation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/ColumnMeta.java
@@ -18,11 +18,10 @@
 @Target({ElementType.FIELD, ElementType.METHOD})
 @Retention(RetentionPolicy.RUNTIME)
 public @interface ColumnMeta {
-
     String friendlyName() default "";
     String description() default "";
     String category() default "";
-    String tableSource() default "";
+    TableSource tableSource() default @TableSource(table = "", column = "");
     String [] tags() default {};
     String [] values() default {};
 

diff --git a/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/TableSource.java b/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/annotation/TableSource.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright 2019, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+package com.yahoo.elide.datastores.aggregation.annotation;
+
+import static com.yahoo.elide.datastores.aggregation.dynamic.NamespacePackage.DEFAULT;
+
+/**
+ * The definition of TableSource.
+ */
+public @interface TableSource {
+    String table();
+    String namespace() default DEFAULT;
+    String column();
+    String [] suggestionColumns() default {};
+}
diff --git a/...gation/src/main/java/com/yahoo/elide/datastores/aggregation/dynamic/NamespacePackage.java b/...gation/src/main/java/com/yahoo/elide/datastores/aggregation/dynamic/NamespacePackage.java
@@ -21,8 +21,8 @@
  */
 public class NamespacePackage implements Package {
 
-    public static String EMPTY = "";
-    public static String DEFAULT = "default";
+    public static final String EMPTY = "";
+    public static final String DEFAULT = "default";
     public static NamespacePackage DEFAULT_NAMESPACE =
             new NamespacePackage(EMPTY, "Default Namespace", DEFAULT, NO_VERSION);
 

diff --git a/...e-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dynamic/TableType.java b/...e-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dynamic/TableType.java
@@ -9,6 +9,7 @@
 import static com.yahoo.elide.core.type.ClassType.BOOLEAN_TYPE;
 import static com.yahoo.elide.core.type.ClassType.LONG_TYPE;
 import static com.yahoo.elide.core.type.ClassType.STRING_TYPE;
+import static com.yahoo.elide.datastores.aggregation.dynamic.NamespacePackage.DEFAULT;
 import static com.yahoo.elide.datastores.aggregation.dynamic.NamespacePackage.DEFAULT_NAMESPACE;
 import static com.yahoo.elide.datastores.aggregation.timegrains.Time.TIME_TYPE;
 import static com.yahoo.elide.modelconfig.model.Type.TIME;
@@ -26,6 +27,7 @@
 import com.yahoo.elide.datastores.aggregation.annotation.JoinType;
 import com.yahoo.elide.datastores.aggregation.annotation.MetricFormula;
 import com.yahoo.elide.datastores.aggregation.annotation.TableMeta;
+import com.yahoo.elide.datastores.aggregation.annotation.TableSource;
 import com.yahoo.elide.datastores.aggregation.annotation.Temporal;
 import com.yahoo.elide.datastores.aggregation.annotation.TimeGrainDefinition;
 import com.yahoo.elide.datastores.aggregation.metadata.enums.TimeGrain;
@@ -45,6 +47,7 @@
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Array;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -399,8 +402,8 @@ public ValueType type() {
                 }
 
                 @Override
-                public String tableSource() {
-                    return argument.getTableSource();
+                public TableSource tableSource() {
+                    return buildTableSource(argument.getTableSource());
                 }
 
                 @Override
@@ -450,8 +453,8 @@ public String category() {
             }
 
             @Override
-            public String tableSource() {
-                return "";
+            public TableSource tableSource() {
+                return buildTableSource(null);
             }
 
             @Override
@@ -519,6 +522,40 @@ public String expression() {
         return annotations;
     }
 
+    private static TableSource buildTableSource(com.yahoo.elide.modelconfig.model.TableSource source) {
+        if (source == null) {
+            return buildTableSource(
+                    new com.yahoo.elide.modelconfig.model.TableSource("", DEFAULT, "", new HashSet<>()));
+        }
+        return new TableSource() {
+
+            @Override
+            public Class<? extends Annotation> annotationType() {
+                return TableSource.class;
+            }
+
+            @Override
+            public String table() {
+                return source.getTable();
+            }
+
+            @Override
+            public String namespace() {
+                return source.getNamespace();
+            }
+
+            @Override
+            public String column() {
+                return source.getColumn();
+            }
+
+            @Override
+            public String[] suggestionColumns() {
+                return source.getSuggestionColumns().toArray(new String[0]);
+            }
+        };
+    }
+
     private static Map<Class<? extends Annotation>, Annotation> buildAnnotations(Dimension dimension) {
         Map<Class<? extends Annotation>, Annotation> annotations = new HashMap<>();
 
@@ -548,8 +585,8 @@ public String category() {
             }
 
             @Override
-            public String tableSource() {
-                return dimension.getTableSource() == null ? "" : dimension.getTableSource();
+            public TableSource tableSource() {
+                return buildTableSource(dimension.getTableSource());
             }
 
             @Override

diff --git a/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java b/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java
@@ -305,6 +305,14 @@ public Table getTable(String name, String version) {
                 .orElse(null);
     }
 
+    /**
+     * Returns the complete set of tables.
+     * @return a set of tables.
+     */
+    public Set<Table> getTables() {
+        return new HashSet<>(tables.values());
+    }
+
     /**
      * Get a {@link Column} from a table.
      *

diff --git a/.../src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/ValueSourceType.java b/.../src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/ValueSourceType.java
@@ -5,6 +5,7 @@
  */
 package com.yahoo.elide.datastores.aggregation.metadata.enums;
 
+import com.yahoo.elide.datastores.aggregation.annotation.TableSource;
 import org.apache.commons.collections4.CollectionUtils;
 
 import java.util.Set;
@@ -17,11 +18,11 @@ public enum ValueSourceType {
     TABLE,
     NONE;
 
-    public static ValueSourceType getValueSourceType(Set<String> values, String tableSource) {
+    public static ValueSourceType getValueSourceType(Set<String> values, TableSource tableSource) {
         if (CollectionUtils.isNotEmpty(values)) {
             return ValueSourceType.ENUM;
         }
-        if (tableSource != null) {
+        if (tableSource != null && ! tableSource.table().isEmpty()) {
             return ValueSourceType.TABLE;
         }
         return ValueSourceType.NONE;

diff --git a/...gation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Argument.java b/...gation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Argument.java
@@ -5,6 +5,7 @@
  */
 package com.yahoo.elide.datastores.aggregation.metadata.models;
 
+import com.yahoo.elide.annotation.Exclude;
 import com.yahoo.elide.annotation.Include;
 import com.yahoo.elide.datastores.aggregation.annotation.ArgumentDefinition;
 import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueSourceType;
@@ -17,6 +18,7 @@
 import java.util.HashSet;
 import java.util.Set;
 import javax.persistence.Id;
+import javax.persistence.OneToOne;
 
 /**
  * Arguments that can be provided into a table/column.
@@ -39,7 +41,11 @@ public class Argument {
 
     private Set<String> values;
 
-    private String tableSource;
+    @OneToOne
+    private TableSource tableSource;
+
+    @Exclude
+    private com.yahoo.elide.datastores.aggregation.annotation.TableSource tableSourceDefinition;
 
     private Object defaultValue;
 
@@ -49,8 +55,8 @@ public Argument(String idPrefix, ArgumentDefinition argument) {
         this.description = argument.description();
         this.type = argument.type();
         this.values = new HashSet<>(Arrays.asList(argument.values()));
-        this.tableSource = argument.tableSource();
+        this.tableSourceDefinition = argument.tableSource();
         this.defaultValue = argument.defaultValue();
-        this.valueSourceType = ValueSourceType.getValueSourceType(this.values, this.tableSource);
+        this.valueSourceType = ValueSourceType.getValueSourceType(this.values, this.tableSourceDefinition);
     }
 }
diff --git a/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java b/...regation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java
@@ -7,6 +7,7 @@
 
 import static com.yahoo.elide.datastores.aggregation.metadata.enums.ColumnType.FIELD;
 import static com.yahoo.elide.datastores.aggregation.metadata.enums.ColumnType.FORMULA;
+import com.yahoo.elide.annotation.Exclude;
 import com.yahoo.elide.annotation.Include;
 import com.yahoo.elide.annotation.ToOne;
 import com.yahoo.elide.core.dictionary.EntityDictionary;
@@ -20,9 +21,9 @@
 import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueSourceType;
 import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType;
 import com.yahoo.elide.datastores.aggregation.query.ColumnProjection;
-import org.apache.commons.lang3.StringUtils;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.Setter;
 import lombok.ToString;
 
 import java.util.Arrays;
@@ -32,6 +33,7 @@
 import java.util.stream.Collectors;
 import javax.persistence.Id;
 import javax.persistence.OneToMany;
+import javax.persistence.OneToOne;
 
 /**
  * Column is the super class of a field in a table, it can be either dimension or metric.
@@ -70,7 +72,12 @@ public abstract class Column implements Versioned {
 
     private final Set<String> values;
 
-    private final String tableSource;
+    @OneToOne
+    @Setter
+    private TableSource tableSource = null;
+
+    @Exclude
+    private com.yahoo.elide.datastores.aggregation.annotation.TableSource tableSourceDefinition;
 
     @OneToMany
     @ToString.Exclude
@@ -95,17 +102,16 @@ protected Column(Table table, String fieldName, EntityDictionary dictionary) {
             this.category = meta.category();
             this.values = new HashSet<>(Arrays.asList(meta.values()));
             this.tags = new HashSet<>(Arrays.asList(meta.tags()));
-            this.tableSource = StringUtils.isBlank(meta.tableSource()) ? null : meta.tableSource();
-            this.valueSourceType = ValueSourceType.getValueSourceType(this.values,
-                    this.tableSource);
+            this.tableSourceDefinition = meta.tableSource();
+            this.valueSourceType = ValueSourceType.getValueSourceType(this.values, this.tableSourceDefinition);
             this.cardinality = meta.size();
         } else {
             this.friendlyName = name;
             this.description = null;
             this.category = null;
             this.values = null;
             this.tags = new HashSet<>();
-            this.tableSource = null;
+            this.tableSourceDefinition = null;
             this.valueSourceType = ValueSourceType.NONE;
             this.cardinality = CardinalitySize.UNKNOWN;
         }