From a86a6e3200227ec1c1d39b5bd773828a0a484300 Mon Sep 17 00:00:00 2001 From: hchen04 Date: Thu, 17 Oct 2019 11:22:18 -0500 Subject: [PATCH 01/10] Manager transacton manually --- .../aggregation/queryengines/sql/SQLQueryEngine.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 7d8d5aa38e..7169da6ce2 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -45,6 +45,7 @@ import java.util.stream.Collectors; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; +import javax.persistence.EntityTransaction; import javax.persistence.Table; /** @@ -84,8 +85,16 @@ public Schema getSchema(Class entityClass) { @Override public Iterable executeQuery(Query query) { EntityManager entityManager = null; + EntityTransaction transaction = null; try { entityManager = emf.createEntityManager(); + + // manually begin the transaction + transaction = entityManager.getTransaction(); + if (!transaction.isActive()) { + transaction.begin(); + } + SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); //Make sure we actually manage this schema. @@ -129,6 +138,9 @@ public Iterable executeQuery(Query query) { return new SQLEntityHydrator(results, query, dictionary, entityManager).hydrate(); } finally { + if (transaction != null && transaction.isActive()) { + transaction.commit(); + } if (entityManager != null) { entityManager.close(); } From cd05415ed2096f6a14f46d9a13731d0a14b38391 Mon Sep 17 00:00:00 2001 From: hchen04 Date: Thu, 17 Oct 2019 11:30:23 -0500 Subject: [PATCH 02/10] Add readonly --- .../aggregation/queryengines/sql/SQLQueryEngine.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 7169da6ce2..8b0968169e 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -31,6 +31,8 @@ import com.google.common.base.Preconditions; +import org.hibernate.jpa.QueryHints; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -115,7 +117,8 @@ public Iterable executeQuery(Query query) { SQLQuery paginationSQL = toPageTotalSQL(sql); javax.persistence.Query pageTotalQuery = - entityManager.createNativeQuery(paginationSQL.toString()); + entityManager.createNativeQuery(paginationSQL.toString()) + .setHint(QueryHints.HINT_READONLY, true); //Supply the query parameters to the query supplyFilterQueryParameters(query, pageTotalQuery); @@ -134,7 +137,9 @@ public Iterable executeQuery(Query query) { supplyFilterQueryParameters(query, jpaQuery); //Run the primary query and log the time spent. - List results = new TimedFunction<>(() -> jpaQuery.getResultList(), "Running Query: " + sql).get(); + List results = new TimedFunction<>( + () -> jpaQuery.setHint(QueryHints.HINT_READONLY, true).getResultList(), + "Running Query: " + sql).get(); return new SQLEntityHydrator(results, query, dictionary, entityManager).hydrate(); } finally { From b3d3e6c2f330773c2dff7ece6a0676bac41aca4b Mon Sep 17 00:00:00 2001 From: hchen04 Date: Fri, 18 Oct 2019 16:39:10 -0500 Subject: [PATCH 03/10] some rework --- .../AggregationDataStoreHelper.java | 90 ++++++++++--------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java index d81c0fcec9..f115146540 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java @@ -93,51 +93,57 @@ private void splitFilters() { } //TODO - Add tests in the next PR. + /** + * Gets time dimensions based on relationships and attributes from {@link EntityProjection}. + * + * @return projections for time dimension columns + * @throws InvalidOperationException Thrown if a requested time grain is not supported. + */ private Set resolveTimeDimensions() { - Set timeDims = new LinkedHashSet<>(); - //Only attributes can be time dimensions - entityProjection.getAttributes().stream().forEach((attribute -> { - TimeDimensionColumn timeDim = schema.getTimeDimension(attribute.getName()); - if (timeDim == null) { - return; - } - - Argument timeGrainArgument = attribute.getArguments().stream() - .filter(attr -> attr.getName().equals("grain")) - .findAny() - .orElse(null); - - TimeGrainDefinition requestedGrainDefinition; - if (timeGrainArgument == null) { - - //The first grain is the default. - requestedGrainDefinition = timeDim.getSupportedGrains().iterator().next(); - } else { - String requestedGrainName = timeGrainArgument.getValue().toString(); - - TimeGrain requestedGrain; - try { - requestedGrain = TimeGrain.valueOf(requestedGrainName); - } catch (IllegalArgumentException e) { - throw new InvalidOperationException(String.format("Invalid grain %s", requestedGrainName)); - } - - requestedGrainDefinition = timeDim.getSupportedGrains().stream() - .filter(supportedGrainDef -> supportedGrainDef.grain().equals(requestedGrain)) - .findAny() - .orElseThrow(() -> new InvalidOperationException( - String.format("Requested grain %s, not supported on %s", - requestedGrainName, attribute.getName()))); - } - - timeDims.add(timeDim.toProjectedDimension(requestedGrainDefinition)); - })); - - return timeDims; + return entityProjection.getAttributes().stream() + .filter(attribute -> schema.getDimension(attribute.getName()) instanceof TimeDimensionColumn) + .map(attribute -> { + TimeDimensionColumn timeDim = schema.getTimeDimension(attribute.getName()); + + Argument timeGrainArgument = attribute.getArguments().stream() + .filter(attr -> attr.getName().equals("grain")) + .findAny() + .orElse(null); + + TimeGrainDefinition requestedGrainDefinition; + if (timeGrainArgument == null) { + + //The first grain is the default. + requestedGrainDefinition = timeDim.getSupportedGrains().stream() + .findFirst() + .orElseThrow(() -> new InvalidOperationException( + String.format("Requested default grain, no grain defined on %s", + attribute.getName()))); + } else { + String requestedGrainName = timeGrainArgument.getValue().toString(); + + TimeGrain requestedGrain; + try { + requestedGrain = TimeGrain.valueOf(requestedGrainName); + } catch (IllegalArgumentException e) { + throw new InvalidOperationException(String.format("Invalid grain %s", requestedGrainName)); + } + + requestedGrainDefinition = timeDim.getSupportedGrains().stream() + .filter(supportedGrainDef -> supportedGrainDef.grain().equals(requestedGrain)) + .findAny() + .orElseThrow(() -> new InvalidOperationException( + String.format("Requested grain %s, not supported on %s", + requestedGrainName, attribute.getName()))); + } + + return timeDim.toProjectedDimension(requestedGrainDefinition); + }) + .collect(Collectors.toCollection(LinkedHashSet::new)); } /** - * Gets dimensions based on relationships and attributes from {@link EntityProjection}. + * Gets dimensions except time dimensions based on relationships and attributes from {@link EntityProjection}. */ private Set resolveNonTimeDimensions() { @@ -145,9 +151,9 @@ private Set resolveNonTimeDimensions() { allColumns.addAll(getRelationships()); return allColumns.stream() + .filter(columnName -> !(schema.getDimension(columnName) instanceof TimeDimensionColumn)) .map(columnName -> schema.getDimension(columnName)) .filter(Objects::nonNull) - .filter(column -> ! (column instanceof TimeDimensionColumn)) .map(DimensionColumn::toProjectedDimension) .collect(Collectors.toCollection(LinkedHashSet::new)); } From ad7629384ae1ac3a24c6e0cdf7ea1aff6be59504 Mon Sep 17 00:00:00 2001 From: hchen04 Date: Fri, 18 Oct 2019 16:49:14 -0500 Subject: [PATCH 04/10] use getTimeDimension() --- .../datastores/aggregation/AggregationDataStoreHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java index f115146540..14d9d33352 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java @@ -101,7 +101,7 @@ private void splitFilters() { */ private Set resolveTimeDimensions() { return entityProjection.getAttributes().stream() - .filter(attribute -> schema.getDimension(attribute.getName()) instanceof TimeDimensionColumn) + .filter(attribute -> schema.getTimeDimension(attribute.getName()) != null) .map(attribute -> { TimeDimensionColumn timeDim = schema.getTimeDimension(attribute.getName()); @@ -151,7 +151,7 @@ private Set resolveNonTimeDimensions() { allColumns.addAll(getRelationships()); return allColumns.stream() - .filter(columnName -> !(schema.getDimension(columnName) instanceof TimeDimensionColumn)) + .filter(columnName -> schema.getTimeDimension(columnName) == null) .map(columnName -> schema.getDimension(columnName)) .filter(Objects::nonNull) .map(DimensionColumn::toProjectedDimension) From 6b0a9ab6cf6d9b53be81cf97a41d78923e343d71 Mon Sep 17 00:00:00 2001 From: hchen04 Date: Fri, 18 Oct 2019 16:54:48 -0500 Subject: [PATCH 05/10] change exception --- .../datastores/aggregation/AggregationDataStoreHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java index 14d9d33352..fd1b27bc73 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java @@ -116,7 +116,7 @@ private Set resolveTimeDimensions() { //The first grain is the default. requestedGrainDefinition = timeDim.getSupportedGrains().stream() .findFirst() - .orElseThrow(() -> new InvalidOperationException( + .orElseThrow(() -> new IllegalStateException( String.format("Requested default grain, no grain defined on %s", attribute.getName()))); } else { From 5d3a0b736a8df8258686a1ff8cf5127b02dce5b2 Mon Sep 17 00:00:00 2001 From: hchen04 Date: Fri, 8 Nov 2019 11:04:33 -0600 Subject: [PATCH 06/10] Metadatastore models --- .../aggregation/metadata/MetaDataStore.java | 147 ++++++++++++++++++ .../metadata/enums/Aggregation.java | 15 ++ .../aggregation/metadata/enums/Format.java | 13 ++ .../aggregation/metadata/enums/Tag.java | 13 ++ .../aggregation/metadata/enums/ValueType.java | 19 +++ .../metadata/metric/AggregatedField.java | 38 +++++ .../metadata/metric/BasicMetricFunction.java | 64 ++++++++ .../metadata/metric/MetricComputation.java | 40 +++++ .../metric/MetricFunctionInvocation.java | 22 +++ .../metadata/metric/functions/Max.java | 17 ++ .../metadata/metric/functions/Min.java | 17 ++ .../metadata/metric/functions/Sum.java | 17 ++ .../metadata/models/AnalyticView.java | 50 ++++++ .../aggregation/metadata/models/Column.java | 81 ++++++++++ .../aggregation/metadata/models/DataType.java | 60 +++++++ .../metadata/models/Dimension.java | 27 ++++ .../metadata/models/FunctionArgument.java | 41 +++++ .../aggregation/metadata/models/Metric.java | 51 ++++++ .../metadata/models/MetricFunction.java | 98 ++++++++++++ .../metadata/models/RelationshipType.java | 30 ++++ .../aggregation/metadata/models/Table.java | 129 +++++++++++++++ .../metadata/models/TimeDimension.java | 43 +++++ .../metadata/models/TimeDimensionGrain.java | 37 +++++ 23 files changed, 1069 insertions(+) create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Aggregation.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Format.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Tag.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/ValueType.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Max.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Min.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Sum.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/AnalyticView.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/DataType.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/FunctionArgument.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java new file mode 100644 index 0000000000..7e32718cff --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java @@ -0,0 +1,147 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata; + +import static com.yahoo.elide.datastores.aggregation.AggregationDictionary.isAnalyticView; + +import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.core.datastore.inmemory.HashMapDataStore; +import com.yahoo.elide.core.exceptions.DuplicateMappingException; +import com.yahoo.elide.datastores.aggregation.AggregationDataStore; +import com.yahoo.elide.datastores.aggregation.AggregationDictionary; +import com.yahoo.elide.datastores.aggregation.metadata.models.AnalyticView; +import com.yahoo.elide.datastores.aggregation.metadata.models.Column; +import com.yahoo.elide.datastores.aggregation.metadata.models.DataType; +import com.yahoo.elide.datastores.aggregation.metadata.models.FunctionArgument; +import com.yahoo.elide.datastores.aggregation.metadata.models.Metric; +import com.yahoo.elide.datastores.aggregation.metadata.models.MetricFunction; +import com.yahoo.elide.datastores.aggregation.metadata.models.Table; +import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimension; +import com.yahoo.elide.datastores.aggregation.metadata.models.TimeDimensionGrain; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * MetaDataStore is a in-memory data store that manage data models for an {@link AggregationDataStore}. + */ +public class MetaDataStore extends HashMapDataStore { + private static final Package MODEL_PACKAGE = + Package.getPackage("com.yahoo.elide.datastores.aggregation.metadata.models"); + + public MetaDataStore() { + super(MODEL_PACKAGE); + } + + public void populateEntityDictionary(EntityDictionary dictionary) { + super.populateEntityDictionary(dictionary); + + if (dictionary instanceof AggregationDictionary) { + loadMetaData((AggregationDictionary) dictionary); + } + } + + /** + * Load meta data of models from an populated entity dictionary. + * + * @param dictionary entity dictionary used by an aggregation data store. + */ + private void loadMetaData(AggregationDictionary dictionary) { + Set> classes = dictionary.getBindings(); + + classes.stream() + .filter(cls -> !MODEL_PACKAGE.equals(cls.getPackage())) + .forEach(cls -> addTable( + isAnalyticView(cls) + ? new AnalyticView(cls, dictionary) + : new Table(cls, dictionary))); + } + + /** + * Add a table metadata object. + * + * @param table table metadata + */ + private void addTable(Table table) { + addMetaData(table); + table.getColumns().forEach(this::addColumn); + } + + /** + * Add a column metadata object. + * + * @param column column metadata + */ + private void addColumn(Column column) { + addMetaData(column); + addDataType(column.getDataType()); + + if (column instanceof TimeDimension) { + ((TimeDimension) column).getSupportedGrains().forEach(this::addTimeDimensionGrain); + } else if (column instanceof Metric) { + addMetricFunction(((Metric) column).getMetricFunction()); + } + } + + /** + * Add a metric function metadata object. + * + * @param metricFunction metric function metadata + */ + private void addMetricFunction(MetricFunction metricFunction) { + addMetaData(metricFunction); + metricFunction.getArguments().forEach(this::addFunctionArgument); + } + + /** + * Add a datatype metadata object. + * + * @param dataType datatype metadata + */ + private void addDataType(DataType dataType) { + addMetaData(dataType); + } + + /** + * Add a function argument metadata object. + * + * @param functionArgument function argument metadata + */ + private void addFunctionArgument(FunctionArgument functionArgument) { + addMetaData(functionArgument); + } + + /** + * Add a time dimension grain metadata object. + * + * @param timeDimensionGrain time dimension grain metadata + */ + private void addTimeDimensionGrain(TimeDimensionGrain timeDimensionGrain) { + addMetaData(timeDimensionGrain); + } + + /** + * Add a meta data object into this data store, check for duplication. + * + * @param object a meta data object + */ + private void addMetaData(Object object) { + Class cls = this.getDictionary().lookupEntityClass(object.getClass()); + String id = getDictionary().getId(object); + + if (dataStore.get(cls).containsKey(id)) { + if (!dataStore.get(cls).get(id).equals(object)) { + throw new DuplicateMappingException("Duplicated " + cls.getSimpleName() + " metadata " + id); + } + } else { + dataStore.get(cls).put(id, object); + } + } + + public Set getMetaData(Class cls) { + return dataStore.get(cls).values().stream().map(cls::cast).collect(Collectors.toSet()); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Aggregation.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Aggregation.java new file mode 100644 index 0000000000..2bfa910b0d --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Aggregation.java @@ -0,0 +1,15 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.enums; + +/** + * Aggregation functions + */ +public enum Aggregation { + SUM, + MIN, + MAX; +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Format.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Format.java new file mode 100644 index 0000000000..24e9b4f90d --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Format.java @@ -0,0 +1,13 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.datastores.aggregation.metadata.enums; + +/** + * Format of a value field, e.g. decimal for numbers + */ +public enum Format { +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Tag.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Tag.java new file mode 100644 index 0000000000..e01ddcc84b --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/Tag.java @@ -0,0 +1,13 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.enums; + +/** + * Tag attached to fields + */ +public enum Tag { + DISPLAY +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/ValueType.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/ValueType.java new file mode 100644 index 0000000000..699fd3ce54 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/enums/ValueType.java @@ -0,0 +1,19 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.enums; + +/** + * Actual value type of a data type + */ +public enum ValueType { + DATE, + NUMBER, + TEXT, + COORDINATE, + BOOLEAN, + RELATIONSHIP, + ID +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java new file mode 100644 index 0000000000..e465fcfe31 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.metric; + +import com.yahoo.elide.datastores.aggregation.metadata.models.Metric; + +import lombok.Getter; + +/** + * A field in a physical/logical table that is actually aggregated by a metric function. + * It can be either a metric field from a physical table or a field alias in a subquery. + */ +public class AggregatedField { + @Getter + private boolean isMetricField; + + @Getter + private Metric metric; + + private String alias; + + public final String getFieldName() { + return isMetricField ? metric.getName() : alias; + } + + public AggregatedField(String alias) { + this.isMetricField = false; + this.alias = alias; + } + + public AggregatedField(Metric metric) { + this.isMetricField = true; + this.metric = metric; + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java new file mode 100644 index 0000000000..ae41ee3843 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.metric; + +import com.yahoo.elide.datastores.aggregation.metadata.models.FunctionArgument; +import com.yahoo.elide.datastores.aggregation.metadata.models.MetricFunction; +import com.yahoo.elide.request.Argument; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** + * Basic implementation for Metric function. + */ +@EqualsAndHashCode(callSuper = true) +@Data +@AllArgsConstructor +public abstract class BasicMetricFunction extends MetricFunction { + private String name; + + private String longName; + + private String description; + + private Set arguments; + + protected BasicMetricFunction(String name, String longName, String description) { + this(name, longName, description, Collections.emptySet()); + } + + @Override + public MetricFunctionInvocation invoke(Map arguments, AggregatedField field, String alias) { + final MetricFunction function = this; + return new MetricFunctionInvocation() { + @Override + public Map getArguments() { + return arguments; + } + + @Override + public MetricFunction getFunction() { + return function; + } + + @Override + public AggregatedField getAggregatedField() { + return field; + } + + @Override + public String getAlias() { + return alias; + } + }; + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java new file mode 100644 index 0000000000..8dccae3343 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.metric; + +import com.yahoo.elide.datastores.aggregation.metadata.models.MetricFunction; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * Special Metric function that represents computation of other metrics. It contains a argumentNameMap to convert + * arguments assigned to this metric to actual arguments for sub metric functions. e.g. + * 
+ *     @Metric(function = funWithArgs(a1, a2))
+ *     private int metricA;
+ *
+ *     @Metric(function = funWithArgs(b1, b2))
+ *     private int metricB;
+ *
+ *     @MetricComputation(expression = metricA(p1, p2) / metricB(p1, p3))
+ *     private float ratio();
+ *
+ * Would map 'p1' to 'a1' and 'b1', 'p2' to 'a2' and 'p3' to 'b2'. + */ +public abstract class MetricComputation extends MetricFunction { + private final Map> argumentNameMap; + + @Override + public Function> getArgumentNameMapper() { + return argumentNameMap::get; + } + + public MetricComputation(Map> argumentNameMap) { + this.argumentNameMap = argumentNameMap; + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java new file mode 100644 index 0000000000..0f052fa039 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.metric; + +import com.yahoo.elide.datastores.aggregation.metadata.models.MetricFunction; +import com.yahoo.elide.request.Argument; + +import java.util.Map; + +/** + * An invoked metric function instance applied on an aggregated field with provided arguments to project the result + * as the alias. + */ +public interface MetricFunctionInvocation { + Map getArguments(); + MetricFunction getFunction(); + AggregatedField getAggregatedField(); + String getAlias(); +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Max.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Max.java new file mode 100644 index 0000000000..2a4417c00c --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Max.java @@ -0,0 +1,17 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.metric.functions; + +import com.yahoo.elide.datastores.aggregation.metadata.metric.BasicMetricFunction; + +/** + * Canned MAX metric function. + */ +public class Max extends BasicMetricFunction { + public Max() { + super("Max", "Max", "Calculate max of a metric column"); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Min.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Min.java new file mode 100644 index 0000000000..2ee91faafe --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Min.java @@ -0,0 +1,17 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.metric.functions; + +import com.yahoo.elide.datastores.aggregation.metadata.metric.BasicMetricFunction; + +/** + * Canned MIN metric function. + */ +public class Min extends BasicMetricFunction { + public Min() { + super("Min", "Min", "Calculate min of a metric column"); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Sum.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Sum.java new file mode 100644 index 0000000000..57d599b26d --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/functions/Sum.java @@ -0,0 +1,17 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.metric.functions; + +import com.yahoo.elide.datastores.aggregation.metadata.metric.BasicMetricFunction; + +/** + * Canned SUM metric function. + */ +public class Sum extends BasicMetricFunction { + public Sum() { + super("Sum", "Sum", "Calculate sum of a metric column"); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/AnalyticView.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/AnalyticView.java new file mode 100644 index 0000000000..44668c6c68 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/AnalyticView.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.AggregationDictionary; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +import java.util.Set; +import java.util.stream.Collectors; +import javax.persistence.Entity; +import javax.persistence.OneToMany; + +/** + * AnalyticViews are logical tables that support aggregation functionality, but don't support join or relationship. + */ +@EqualsAndHashCode(callSuper = true) +@Include(rootLevel = true, type = "analyticView") +@Entity +@Data +public class AnalyticView extends Table { + + @OneToMany + @ToString.Exclude + private Set metrics; + + @OneToMany + @ToString.Exclude + private Set dimensions; + + public AnalyticView(Class cls, AggregationDictionary dictionary) { + super(cls, dictionary); + + metrics = getColumns().stream() + .filter(col -> col instanceof Metric) + .map(Metric.class::cast) + .collect(Collectors.toSet()); + + dimensions = getColumns().stream() + .filter(col -> !(col instanceof Metric)) + .map(Dimension.class::cast) + .collect(Collectors.toSet()); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java new file mode 100644 index 0000000000..403f1bfcfc --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.datastores.aggregation.AggregationDictionary; +import com.yahoo.elide.datastores.aggregation.annotation.Meta; +import com.yahoo.elide.datastores.aggregation.metadata.enums.Tag; +import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType; + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import javax.persistence.Id; +import javax.persistence.ManyToOne; + +/** + * Column is the super class of a field in a table, it can be either dimension or metric. + */ +@Data +@ToString +public abstract class Column { + @Id + private String id; + + private String name; + + private String longName; + + private String tableName; + + private String description; + + private String category; + + @ManyToOne + private DataType dataType; + + @ToString.Exclude + private Set columnTags; + + protected Column(Class tableClass, String fieldName, AggregationDictionary dictionary) { + this.tableName = dictionary.getJsonAliasFor(tableClass); + this.id = tableName + "." + fieldName; + this.name = fieldName; + this.columnTags = new HashSet<>(); + + Meta meta = dictionary.getAttributeOrRelationAnnotation(tableClass, Meta.class, fieldName); + if (meta != null) { + this.longName = meta.longName(); + this.description = meta.description(); + } + + if (dictionary.isRelation(tableClass, fieldName)) { + Class relationshipClass = dictionary.getParameterizedType(tableClass, fieldName); + String relationshipName = dictionary.getJsonAliasFor(relationshipClass); + this.dataType = new RelationshipType( + this.id + "." + relationshipClass.getSimpleName().toLowerCase(Locale.ENGLISH), relationshipName); + } else { + Class fieldClass = dictionary.getType(tableClass, fieldName); + + if (dictionary.getIdFieldName(tableClass).equals(fieldName)) { + this.dataType = new DataType(tableName + "." + fieldName, ValueType.ID); + } else if (Date.class.isAssignableFrom(fieldClass)) { + this.dataType = new DataType(fieldClass.getSimpleName().toLowerCase(Locale.ENGLISH), ValueType.DATE); + } else { + DataType dataType = DataType.getScalarType(fieldClass); + if (dataType == null) { + throw new IllegalArgumentException("Unknown data type for " + this.id); + } + this.dataType = dataType; + } + } + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/DataType.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/DataType.java new file mode 100644 index 0000000000..67d35d30f8 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/DataType.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import static com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType.BOOLEAN; +import static com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType.NUMBER; +import static com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType.TEXT; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.ToString; + +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.Map; +import javax.persistence.Entity; +import javax.persistence.Id; + +/** + * Data type of a column + */ +@Include(rootLevel = true, type = "dataType") +@Entity +@Data +@AllArgsConstructor +@ToString +public class DataType { + private static final Map, DataType> SCALAR_TYPES = new HashMap, DataType>() {{ + put(short.class, new DataType("p_short", NUMBER)); + put(Short.class, new DataType("short", NUMBER)); + put(int.class, new DataType("p_int", NUMBER)); + put(Integer.class, new DataType("int", NUMBER)); + put(long.class, new DataType("p_bigint", NUMBER)); + put(Long.class, new DataType("bigint", NUMBER)); + put(BigDecimal.class, new DataType("bigDecimal", NUMBER)); + put(float.class, new DataType("p_float", NUMBER)); + put(Float.class, new DataType("float", NUMBER)); + put(double.class, new DataType("p_double", NUMBER)); + put(Double.class, new DataType("double", NUMBER)); + put(boolean.class, new DataType("p_boolean", BOOLEAN)); + put(Boolean.class, new DataType("boolean", BOOLEAN)); + put(char.class, new DataType("p_char", TEXT)); + put(String.class, new DataType("string", TEXT)); + }}; + + @Id + private String name; + + private ValueType valueType; + + public static DataType getScalarType(Class valueClass) { + return SCALAR_TYPES.get(valueClass); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java new file mode 100644 index 0000000000..52c6302838 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.AggregationDictionary; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.persistence.Entity; + +/** + * Regular field in tables, can be grouped by if the table is an AnalyticView + */ +@EqualsAndHashCode(callSuper = true) +@Include(rootLevel = true, type = "dimension") +@Entity +@Data +public class Dimension extends Column { + public Dimension(Class tableClass, String fieldName, AggregationDictionary dictionary) { + super(tableClass, fieldName, dictionary); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/FunctionArgument.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/FunctionArgument.java new file mode 100644 index 0000000000..524443d232 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/FunctionArgument.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; + +import lombok.Data; +import lombok.ToString; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.ManyToOne; + +/** + * Arguments that can be provided into a metric function. + */ +@Include(type = "functionArgument") +@Entity +@Data +@ToString +public class FunctionArgument { + @Id + private String id; + + private String name; + + private String description; + + @ManyToOne + private DataType dataType; + + public FunctionArgument(String functionName, FunctionArgument argument) { + this.id = functionName + "." + argument.getName(); + this.name = argument.getName(); + this.description = argument.getDescription(); + this.dataType = argument.getDataType(); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java new file mode 100644 index 0000000000..be49409042 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.AggregationDictionary; +import com.yahoo.elide.datastores.aggregation.metadata.enums.Format; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +import javax.persistence.Entity; +import javax.persistence.ManyToOne; + +/** + * Special column for AnalyticView which supports aggregation. + */ +@EqualsAndHashCode(callSuper = true) +@Include(rootLevel = true, type = "metric") +@Entity +@Data +public class Metric extends Column { + private Format defaultFormat; + + @ManyToOne + @ToString.Exclude + private MetricFunction metricFunction; + + public Metric(Class tableClass, String fieldName, AggregationDictionary dictionary) { + super(tableClass, fieldName, dictionary); + + com.yahoo.elide.datastores.aggregation.annotation.Metric metric = dictionary.getAttributeOrRelationAnnotation( + tableClass, + com.yahoo.elide.datastores.aggregation.annotation.Metric.class, + fieldName); + + if (metric == null) { + throw new IllegalArgumentException(getId() + " is not a metric field"); + } else { + try { + this.metricFunction = metric.function().newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalArgumentException("Can't initialize function for metric " + getId()); + } + } + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java new file mode 100644 index 0000000000..eb7efdc054 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java @@ -0,0 +1,98 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.core.exceptions.InvalidPredicateException; +import com.yahoo.elide.datastores.aggregation.metadata.metric.AggregatedField; +import com.yahoo.elide.datastores.aggregation.metadata.metric.MetricFunctionInvocation; +import com.yahoo.elide.request.Argument; + +import lombok.Data; +import lombok.ToString; + +import java.util.AbstractMap; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.OneToMany; +import javax.persistence.Transient; + +/** + * Functions used to compute metrics. + */ +@Include(rootLevel = true, type = "metricFunction") +@Entity +@Data +@ToString +public abstract class MetricFunction { + @Id + public abstract String getName(); + + public abstract String getLongName(); + + public abstract String getDescription(); + + @OneToMany + public abstract Set getArguments(); + + private Set getArgumentNames() { + return getArguments().stream().map(FunctionArgument::getName).collect(Collectors.toSet()); + } + + /** + * Metric function can have an argument name converter to convert an argument alias to actually argument name. + * + * @return a string-to-string mapping function + */ + @Transient + protected Function> getArgumentNameMapper() { + return Collections::singletonList; + } + + @Transient + protected abstract MetricFunctionInvocation invoke(Map arguments, + AggregatedField field, + String alias); + + /** + * Invoke this metric function with arguments, an aggregated field and projection alias. + * + * @param arguments arguments provided in the request + * @param field field to apply this function + * @param alias result alias + * @return an invoked metric function + */ + @Transient + public final MetricFunctionInvocation invoke(Set arguments, AggregatedField field, String alias) { + Set requiredArguments = getArgumentNames(); + + Map providedArguments = arguments.stream() + .filter(arg -> requiredArguments.contains(arg.getName())) + .collect(Collectors.toMap(Argument::getName, Function.identity())); + + // map arguments to their actual name + Map resolvedArguments = requiredArguments.stream() + .map(argName -> { + if (!providedArguments.containsKey(argName)) { + throw new InvalidPredicateException("Argument Not Found " + getName() + "." + argName + "."); + } + Argument arg = providedArguments.get(argName); + List actualNames = getArgumentNameMapper().apply(argName); + return actualNames.stream().map(name -> new AbstractMap.SimpleEntry<>(name, arg)); + }) + .reduce(Stream.empty(), Stream::concat) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return invoke(resolvedArguments, field, alias); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java new file mode 100644 index 0000000000..990f7a1405 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.persistence.Entity; + +/** + * Special data type that represents a relationship between tables. + */ +@Entity +@Include(rootLevel = true, type = "relationshipType") +@Data +@EqualsAndHashCode(callSuper = true) +public class RelationshipType extends DataType { + private String tableName; + + public RelationshipType(String name, String tableName) { + super(name, ValueType.RELATIONSHIP); + this.tableName = tableName; + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java new file mode 100644 index 0000000000..30ff2e81a4 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java @@ -0,0 +1,129 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.AggregationDictionary; +import com.yahoo.elide.datastores.aggregation.annotation.Cardinality; +import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; +import com.yahoo.elide.datastores.aggregation.annotation.Meta; +import com.yahoo.elide.datastores.aggregation.annotation.Temporal; + +import lombok.Data; +import lombok.ToString; + +import java.util.Set; +import java.util.stream.Collectors; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.OneToMany; +import javax.persistence.Transient; + +/** + * Super class of all logical or physical tables + */ +@Include(rootLevel = true, type = "table") +@Entity +@Data +@ToString +public class Table { + @Transient + private Class cls; + + @Id + private String name; + + private String longName; + + private String description; + + private String category; + + private CardinalitySize cardinality; + + @OneToMany + @ToString.Exclude + private Set columns; + + public Table(Class cls, AggregationDictionary dictionary) { + if (!dictionary.getBindings().contains(cls)) { + throw new IllegalArgumentException( + String.format("Table class {%s} is not defined in dictionary.", cls)); + } + + this.cls = cls; + this.name = dictionary.getJsonAliasFor(cls); + this.columns = resolveColumns(cls, dictionary); + + Meta meta = cls.getAnnotation(Meta.class); + if (meta != null) { + this.longName = meta.longName(); + this.description = meta.description(); + } + + Cardinality cardinality = dictionary.getAnnotation(cls, Cardinality.class); + if (cardinality != null) { + this.cardinality = cardinality.size(); + } + } + + @Transient + private static Set resolveColumns(Class cls, AggregationDictionary dictionary) { + Set fields = dictionary.getAllFields(cls).stream() + .map(field -> { + if (dictionary.isMetricField(cls, field)) { + return new Metric(cls, field, dictionary); + } else if (dictionary.attributeOrRelationAnnotationExists(cls, field, Temporal.class)) { + return new TimeDimension(cls, field, dictionary); + } else { + return new Dimension(cls, field, dictionary); + } + }) + .collect(Collectors.toSet()); + + // add id field + fields.add(new Dimension(cls, dictionary.getIdFieldName(cls), dictionary)); + + return fields; + } + + @Transient + private T getColumn(Class cls, String fieldName) { + return columns.stream() + .filter(col -> cls.isAssignableFrom(col.getClass()) && (col.getName().equals(fieldName))) + .map(cls::cast) + .findFirst() + .orElse(null); + } + + @Transient + public Metric getMetric(String fieldName) { + return getColumn(Metric.class, fieldName); + } + + @Transient + public Dimension getDimension(String fieldName) { + return getColumn(Dimension.class, fieldName); + } + + @Transient + public TimeDimension getTimeDimension(String fieldName) { + return getColumn(TimeDimension.class, fieldName); + } + + @Transient + public Set getColumns(Class cls) { + return columns.stream() + .filter(col -> cls.isAssignableFrom(col.getClass())) + .map(cls::cast) + .collect(Collectors.toSet()); + } + + @Transient + public boolean isMetric(String fieldName) { + return getColumns(Metric.class).stream().anyMatch(metric -> metric.getName().equals(fieldName)); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java new file mode 100644 index 0000000000..594cf56f9e --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.AggregationDictionary; +import com.yahoo.elide.datastores.aggregation.annotation.Temporal; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.Arrays; +import java.util.Set; +import java.util.TimeZone; +import java.util.stream.Collectors; +import javax.persistence.Entity; + +/** + * TimeDimension is a dimension that represents time value. + * This type of dimension can be used to support more specific aggregation logic e.g. DAILY/MONTHLY aggregation + */ +@EqualsAndHashCode(callSuper = true) +@Include(rootLevel = true, type = "timeDimension") +@Entity +@Data +public class TimeDimension extends Dimension { + Set supportedGrains; + + private TimeZone timezone; + + public TimeDimension(Class tableClass, String fieldName, AggregationDictionary dictionary) { + super(tableClass, fieldName, dictionary); + + Temporal temporal = dictionary.getAttributeOrRelationAnnotation(tableClass, Temporal.class, fieldName); + + this.supportedGrains = Arrays.stream(temporal.grains()) + .map(grain -> new TimeDimensionGrain(getId(), grain)) + .collect(Collectors.toSet()); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java new file mode 100644 index 0000000000..1a07f5dfc4 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.metadata.models; + +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.datastores.aggregation.annotation.TimeGrainDefinition; +import com.yahoo.elide.datastores.aggregation.time.TimeGrain; + +import lombok.Data; + +import java.util.Locale; +import javax.persistence.Entity; +import javax.persistence.Id; + +/** + * Defines how to extract a time dimension for a specific grain from a table. + */ +@Include(rootLevel = true, type = "timeDimensionGrain") +@Entity +@Data +public class TimeDimensionGrain { + @Id + private String id; + + private TimeGrain grain; + + private String expression; + + public TimeDimensionGrain(String fieldName, TimeGrainDefinition definition) { + this.id = fieldName + "." + definition.grain().name().toLowerCase(Locale.ENGLISH); + this.grain = definition.grain(); + this.expression = definition.expression(); + } +} From 043520a8650e0863dd0d040ce59868b1eb2e4965 Mon Sep 17 00:00:00 2001 From: hchen04 Date: Fri, 8 Nov 2019 16:02:49 -0600 Subject: [PATCH 07/10] Address comments --- .../metadata/metric/MetricComputation.java | 40 ----------------- .../metadata/models/MetricFunction.java | 27 ++++------- .../aggregation/metadata/models/Table.java | 45 ++++++++++++------- .../metadata/models/TimeDimensionGrain.java | 2 +- 4 files changed, 40 insertions(+), 74 deletions(-) delete mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java deleted file mode 100644 index 8dccae3343..0000000000 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricComputation.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019, Yahoo Inc. - * Licensed under the Apache License, Version 2.0 - * See LICENSE file in project root for terms. - */ -package com.yahoo.elide.datastores.aggregation.metadata.metric; - -import com.yahoo.elide.datastores.aggregation.metadata.models.MetricFunction; - -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -/** - * Special Metric function that represents computation of other metrics. It contains a argumentNameMap to convert - * arguments assigned to this metric to actual arguments for sub metric functions. e.g. - * 
- *     @Metric(function = funWithArgs(a1, a2))
- *     private int metricA;
- *
- *     @Metric(function = funWithArgs(b1, b2))
- *     private int metricB;
- *
- *     @MetricComputation(expression = metricA(p1, p2) / metricB(p1, p3))
- *     private float ratio();
- *
- * Would map 'p1' to 'a1' and 'b1', 'p2' to 'a2' and 'p3' to 'b2'. - */ -public abstract class MetricComputation extends MetricFunction { - private final Map> argumentNameMap; - - @Override - public Function> getArgumentNameMapper() { - return argumentNameMap::get; - } - - public MetricComputation(Map> argumentNameMap) { - this.argumentNameMap = argumentNameMap; - } -} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java index eb7efdc054..8c34823840 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java @@ -25,7 +25,6 @@ import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.OneToMany; -import javax.persistence.Transient; /** * Functions used to compute metrics. @@ -54,12 +53,10 @@ private Set getArgumentNames() { * * @return a string-to-string mapping function */ - @Transient protected Function> getArgumentNameMapper() { return Collections::singletonList; } - @Transient protected abstract MetricFunctionInvocation invoke(Map arguments, AggregatedField field, String alias); @@ -72,26 +69,20 @@ protected abstract MetricFunctionInvocation invoke(Map argumen * @param alias result alias * @return an invoked metric function */ - @Transient public final MetricFunctionInvocation invoke(Set arguments, AggregatedField field, String alias) { Set requiredArguments = getArgumentNames(); + Set providedArguments = arguments.stream() + .map(Argument::getName) + .collect(Collectors.toSet()); - Map providedArguments = arguments.stream() - .filter(arg -> requiredArguments.contains(arg.getName())) - .collect(Collectors.toMap(Argument::getName, Function.identity())); + if (!requiredArguments.equals(providedArguments)) { + throw new InvalidPredicateException( + "Provided arguments doesn't match requirement for function " + getName() + "."); + } // map arguments to their actual name - Map resolvedArguments = requiredArguments.stream() - .map(argName -> { - if (!providedArguments.containsKey(argName)) { - throw new InvalidPredicateException("Argument Not Found " + getName() + "." + argName + "."); - } - Argument arg = providedArguments.get(argName); - List actualNames = getArgumentNameMapper().apply(argName); - return actualNames.stream().map(name -> new AbstractMap.SimpleEntry<>(name, arg)); - }) - .reduce(Stream.empty(), Stream::concat) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map resolvedArguments = arguments.stream() + .collect(Collectors.toMap(Argument::getName, Function.identity())); return invoke(resolvedArguments, field, alias); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java index 30ff2e81a4..e4f2d16f4d 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Table.java @@ -70,7 +70,13 @@ public Table(Class cls, AggregationDictionary dictionary) { } } - @Transient + /** + * Add all columns of this table. + * + * @param cls table class + * @param dictionary dictionary contains the table class + * @return all resolved column metadata + */ private static Set resolveColumns(Class cls, AggregationDictionary dictionary) { Set fields = dictionary.getAllFields(cls).stream() .map(field -> { @@ -90,7 +96,28 @@ private static Set resolveColumns(Class cls, AggregationDictionary di return fields; } - @Transient + /** + * Get all columns of a specific class, can be {@link Metric}, {@link TimeDimension} or {@link Dimension} + * + * @param cls metadata class + * @param metadata class + * @return columns as requested type if found + */ + public Set getColumns(Class cls) { + return columns.stream() + .filter(col -> cls.isAssignableFrom(col.getClass())) + .map(cls::cast) + .collect(Collectors.toSet()); + } + + /** + * Get a field column as a specific class, can be {@link Metric}, {@link TimeDimension} or {@link Dimension} + * + * @param cls metadata class + * @param fieldName logical column name + * @param metadata class + * @return column as requested type if found + */ private T getColumn(Class cls, String fieldName) { return columns.stream() .filter(col -> cls.isAssignableFrom(col.getClass()) && (col.getName().equals(fieldName))) @@ -99,31 +126,19 @@ private T getColumn(Class cls, String fieldName) { .orElse(null); } - @Transient public Metric getMetric(String fieldName) { return getColumn(Metric.class, fieldName); } - @Transient public Dimension getDimension(String fieldName) { return getColumn(Dimension.class, fieldName); } - @Transient public TimeDimension getTimeDimension(String fieldName) { return getColumn(TimeDimension.class, fieldName); } - @Transient - public Set getColumns(Class cls) { - return columns.stream() - .filter(col -> cls.isAssignableFrom(col.getClass())) - .map(cls::cast) - .collect(Collectors.toSet()); - } - - @Transient public boolean isMetric(String fieldName) { - return getColumns(Metric.class).stream().anyMatch(metric -> metric.getName().equals(fieldName)); + return getMetric(fieldName) != null; } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java index 1a07f5dfc4..d3c2922ea1 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimensionGrain.java @@ -18,7 +18,7 @@ /** * Defines how to extract a time dimension for a specific grain from a table. */ -@Include(rootLevel = true, type = "timeDimensionGrain") +@Include(type = "timeDimensionGrain") @Entity @Data public class TimeDimensionGrain { From 413112668bdf09945111e8352a4df06e96300a5b Mon Sep 17 00:00:00 2001 From: hchen04 Date: Tue, 12 Nov 2019 12:52:32 -0600 Subject: [PATCH 08/10] address comments --- .../metadata/metric/AggregatedField.java | 38 ------------------- .../metadata/metric/BasicMetricFunction.java | 12 +++++- .../metric/MetricFunctionInvocation.java | 9 ++++- .../aggregation/metadata/models/Column.java | 2 +- .../metadata/models/MetricFunction.java | 13 ------- .../metadata/models/RelationshipType.java | 10 +---- 6 files changed, 19 insertions(+), 65 deletions(-) delete mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java deleted file mode 100644 index e465fcfe31..0000000000 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/AggregatedField.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019, Yahoo Inc. - * Licensed under the Apache License, Version 2.0 - * See LICENSE file in project root for terms. - */ -package com.yahoo.elide.datastores.aggregation.metadata.metric; - -import com.yahoo.elide.datastores.aggregation.metadata.models.Metric; - -import lombok.Getter; - -/** - * A field in a physical/logical table that is actually aggregated by a metric function. - * It can be either a metric field from a physical table or a field alias in a subquery. - */ -public class AggregatedField { - @Getter - private boolean isMetricField; - - @Getter - private Metric metric; - - private String alias; - - public final String getFieldName() { - return isMetricField ? metric.getName() : alias; - } - - public AggregatedField(String alias) { - this.isMetricField = false; - this.alias = alias; - } - - public AggregatedField(Metric metric) { - this.isMetricField = true; - this.metric = metric; - } -} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java index ae41ee3843..50984b39da 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java @@ -13,9 +13,12 @@ import lombok.Data; import lombok.EqualsAndHashCode; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Basic implementation for Metric function. @@ -41,8 +44,13 @@ public MetricFunctionInvocation invoke(Map arguments, Aggregat final MetricFunction function = this; return new MetricFunctionInvocation() { @Override - public Map getArguments() { - return arguments; + public List getArguments() { + return new ArrayList<>(arguments.values()); + } + + @Override + public Argument getArgument(String argumentName) { + return arguments.get(argumentName); } @Override diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java index 0f052fa039..0ee112b3a9 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/MetricFunctionInvocation.java @@ -8,15 +8,20 @@ import com.yahoo.elide.datastores.aggregation.metadata.models.MetricFunction; import com.yahoo.elide.request.Argument; -import java.util.Map; +import java.util.List; /** * An invoked metric function instance applied on an aggregated field with provided arguments to project the result * as the alias. */ public interface MetricFunctionInvocation { - Map getArguments(); + List getArguments(); + + Argument getArgument(String argumentName); + MetricFunction getFunction(); + AggregatedField getAggregatedField(); + String getAlias(); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java index 403f1bfcfc..7c82e7f87b 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java @@ -61,7 +61,7 @@ protected Column(Class tableClass, String fieldName, AggregationDictionary di Class relationshipClass = dictionary.getParameterizedType(tableClass, fieldName); String relationshipName = dictionary.getJsonAliasFor(relationshipClass); this.dataType = new RelationshipType( - this.id + "." + relationshipClass.getSimpleName().toLowerCase(Locale.ENGLISH), relationshipName); + this.id + "[" + relationshipClass.getSimpleName().toLowerCase(Locale.ENGLISH) + "]"); } else { Class fieldClass = dictionary.getType(tableClass, fieldName); diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java index 8c34823840..ddf3c82ebe 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java @@ -14,14 +14,10 @@ import lombok.Data; import lombok.ToString; -import java.util.AbstractMap; -import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.OneToMany; @@ -48,15 +44,6 @@ private Set getArgumentNames() { return getArguments().stream().map(FunctionArgument::getName).collect(Collectors.toSet()); } - /** - * Metric function can have an argument name converter to convert an argument alias to actually argument name. - * - * @return a string-to-string mapping function - */ - protected Function> getArgumentNameMapper() { - return Collections::singletonList; - } - protected abstract MetricFunctionInvocation invoke(Map arguments, AggregatedField field, String alias); diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java index 990f7a1405..c0e8f49587 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java @@ -5,26 +5,18 @@ */ package com.yahoo.elide.datastores.aggregation.metadata.models; -import com.yahoo.elide.annotation.Include; import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType; import lombok.Data; import lombok.EqualsAndHashCode; -import javax.persistence.Entity; - /** * Special data type that represents a relationship between tables. */ -@Entity -@Include(rootLevel = true, type = "relationshipType") @Data @EqualsAndHashCode(callSuper = true) public class RelationshipType extends DataType { - private String tableName; - - public RelationshipType(String name, String tableName) { + public RelationshipType(String name) { super(name, ValueType.RELATIONSHIP); - this.tableName = tableName; } } From e5abcc6394d9e118cfe8bc6c2000fa02a8b3ee1f Mon Sep 17 00:00:00 2001 From: hchen04 Date: Tue, 12 Nov 2019 12:53:52 -0600 Subject: [PATCH 09/10] move root --- .../elide/datastores/aggregation/metadata/models/Dimension.java | 2 +- .../elide/datastores/aggregation/metadata/models/Metric.java | 2 +- .../datastores/aggregation/metadata/models/TimeDimension.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java index 52c6302838..3e4121ead0 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Dimension.java @@ -17,7 +17,7 @@ * Regular field in tables, can be grouped by if the table is an AnalyticView */ @EqualsAndHashCode(callSuper = true) -@Include(rootLevel = true, type = "dimension") +@Include(type = "dimension") @Entity @Data public class Dimension extends Column { diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java index be49409042..3c83bad5e6 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Metric.java @@ -20,7 +20,7 @@ * Special column for AnalyticView which supports aggregation. */ @EqualsAndHashCode(callSuper = true) -@Include(rootLevel = true, type = "metric") +@Include(type = "metric") @Entity @Data public class Metric extends Column { diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java index 594cf56f9e..92543dd6f5 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java @@ -23,7 +23,7 @@ * This type of dimension can be used to support more specific aggregation logic e.g. DAILY/MONTHLY aggregation */ @EqualsAndHashCode(callSuper = true) -@Include(rootLevel = true, type = "timeDimension") +@Include(type = "timeDimension") @Entity @Data public class TimeDimension extends Dimension { From 9d2f42f47176113d9bec147533f84395c5974a53 Mon Sep 17 00:00:00 2001 From: hchen04 Date: Thu, 14 Nov 2019 14:06:32 -0600 Subject: [PATCH 10/10] fix style check --- .../datastores/aggregation/metadata/MetaDataStore.java | 6 +++--- .../aggregation/metadata/metric/BasicMetricFunction.java | 1 - .../datastores/aggregation/metadata/models/Column.java | 4 ++++ .../aggregation/metadata/models/MetricFunction.java | 2 +- .../aggregation/metadata/models/RelationshipType.java | 2 ++ .../aggregation/metadata/models/TimeDimension.java | 2 ++ 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java index 7e32718cff..aa8ae6d186 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java @@ -29,11 +29,11 @@ * MetaDataStore is a in-memory data store that manage data models for an {@link AggregationDataStore}. */ public class MetaDataStore extends HashMapDataStore { - private static final Package MODEL_PACKAGE = + public static final Package META_DATA_PACKAGE = Package.getPackage("com.yahoo.elide.datastores.aggregation.metadata.models"); public MetaDataStore() { - super(MODEL_PACKAGE); + super(META_DATA_PACKAGE); } public void populateEntityDictionary(EntityDictionary dictionary) { @@ -53,7 +53,7 @@ private void loadMetaData(AggregationDictionary dictionary) { Set> classes = dictionary.getBindings(); classes.stream() - .filter(cls -> !MODEL_PACKAGE.equals(cls.getPackage())) + .filter(cls -> !META_DATA_PACKAGE.equals(cls.getPackage())) .forEach(cls -> addTable( isAnalyticView(cls) ? new AnalyticView(cls, dictionary) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java index 50984b39da..1b0de2b2ac 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/metric/BasicMetricFunction.java @@ -18,7 +18,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; /** * Basic implementation for Metric function. diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java index 7c82e7f87b..41084bb887 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/Column.java @@ -5,6 +5,7 @@ */ package com.yahoo.elide.datastores.aggregation.metadata.models; +import com.yahoo.elide.annotation.Include; import com.yahoo.elide.datastores.aggregation.AggregationDictionary; import com.yahoo.elide.datastores.aggregation.annotation.Meta; import com.yahoo.elide.datastores.aggregation.metadata.enums.Tag; @@ -17,12 +18,15 @@ import java.util.HashSet; import java.util.Locale; import java.util.Set; +import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.ManyToOne; /** * Column is the super class of a field in a table, it can be either dimension or metric. */ +@Entity +@Include(type = "column") @Data @ToString public abstract class Column { diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java index ddf3c82ebe..2ef6755306 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/MetricFunction.java @@ -25,7 +25,7 @@ /** * Functions used to compute metrics. */ -@Include(rootLevel = true, type = "metricFunction") +@Include(type = "metricFunction") @Entity @Data @ToString diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java index c0e8f49587..227f3d5d0e 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/RelationshipType.java @@ -5,6 +5,7 @@ */ package com.yahoo.elide.datastores.aggregation.metadata.models; +import com.yahoo.elide.annotation.Exclude; import com.yahoo.elide.datastores.aggregation.metadata.enums.ValueType; import lombok.Data; @@ -14,6 +15,7 @@ * Special data type that represents a relationship between tables. */ @Data +@Exclude @EqualsAndHashCode(callSuper = true) public class RelationshipType extends DataType { public RelationshipType(String name) { diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java index 92543dd6f5..ff1cc67902 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/models/TimeDimension.java @@ -17,6 +17,7 @@ import java.util.TimeZone; import java.util.stream.Collectors; import javax.persistence.Entity; +import javax.persistence.ManyToMany; /** * TimeDimension is a dimension that represents time value. @@ -27,6 +28,7 @@ @Entity @Data public class TimeDimension extends Dimension { + @ManyToMany Set supportedGrains; private TimeZone timezone;