Expression reference ast and parser (#1994)

* Adding expression reference AST and visitor classes

* Added unit tests

* Small refactoring

* More refactoring.  Adding more useful fields to reference classes

Co-authored-by: Aaron Klish <[email protected]>

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/expression/ExpressionParser.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright 2021, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.datastores.aggregation.queryengines.sql.expression;
+
+import com.yahoo.elide.core.Path;
+import com.yahoo.elide.core.dictionary.EntityDictionary;
+import com.yahoo.elide.core.type.Type;
+import com.yahoo.elide.datastores.aggregation.core.JoinPath;
+import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore;
+import com.yahoo.elide.datastores.aggregation.metadata.enums.ColumnType;
+import com.yahoo.elide.datastores.aggregation.query.ColumnProjection;
+import com.yahoo.elide.datastores.aggregation.query.Queryable;
+
+import com.google.common.base.Preconditions;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Parses a column or join expression and returns the reference abstract syntax tree for each discovered
+ * reference.
+ */
+public class ExpressionParser {
+
+    private static final Pattern REFERENCE_PARENTHESES = Pattern.compile("\\{\\{(.+?)}}");
+
+    private MetaDataStore metaDataStore;
+    private EntityDictionary dictionary;
+
+    public ExpressionParser(MetaDataStore store) {
+        this.dictionary = store.getMetadataDictionary();
+        this.metaDataStore = store;
+    }
+
+    /**
+     * Parses the column or join expression and returns the list of discovered references.
+     * @param source The source table where the column or join expression lives.
+     * @param expression The expression to parse.
+     * @return A list of discovered references.
+     */
+    public List<Reference> parse(Queryable source, String expression) {
+        List<String> referenceNames = resolveFormulaReferences(expression);
+
+        List<Reference> results = new ArrayList<>();
+
+        for (String referenceName : referenceNames) {
+            if (referenceName.startsWith("$$")) {
+                continue;
+            }
+            if (referenceName.startsWith("$")) {
+                results.add(PhysicalReference
+                        .builder()
+                        .source(source)
+                        .name(referenceName.substring(1))
+                        .build());
+            } else if (referenceName.contains(".")) {
+                results.add(buildJoin(source, referenceName));
+            } else {
+                Reference reference = buildReferenceFromField(source, referenceName);
+                results.add(LogicalReference
+                        .builder()
+                        .source(source)
+                        .column(source.getColumnProjection(referenceName))
+                        .reference(reference)
+                        .build());
+            }
+        }
+
+        return results;
+    }
+
+    private Reference buildReferenceFromField(Queryable source, String fieldName) {
+        ColumnProjection column = source.getColumnProjection(fieldName);
+
+        Preconditions.checkNotNull(column);
+
+        if (column.getColumnType() == ColumnType.FIELD) {
+            return PhysicalReference
+                    .builder()
+                    .source(source)
+                    .name(column.getName())
+                    .build();
+        } else {
+            return LogicalReference
+                    .builder()
+                    .source(source)
+                    .column(column)
+                    .references(parse(source, column.getExpression()))
+                    .build();
+        }
+    }
+
+    private JoinReference buildJoin(Queryable source, String referenceName) {
+        Queryable root = source.getRoot();
+        Type<?> tableClass = dictionary.getEntityClass(root.getName(), root.getVersion());
+
+        JoinPath joinPath = new JoinPath(tableClass, metaDataStore, referenceName);
+
+        Path.PathElement lastElement = joinPath.lastElement().get();
+        Queryable joinSource = metaDataStore.getTable(lastElement.getType());
+        String fieldName = lastElement.getFieldName();
+
+        Reference reference;
+        if (fieldName.startsWith("$")) {
+            reference = PhysicalReference
+                    .builder()
+                    .source(joinSource)
+                    .name(fieldName.substring(1))
+                    .build();
+        } else {
+            reference = buildReferenceFromField(joinSource, fieldName);
+        }
+
+        return JoinReference
+                .builder()
+                .path(joinPath)
+                .source(source)
+                .reference(reference)
+                .build();
+    }
+
+    /**
+     * Use regex to get all references from a formula expression.
+     *
+     * @param expression expression
+     * @return references appear in the expression.
+     */
+    private static List<String> resolveFormulaReferences(String expression) {
+        Matcher matcher = REFERENCE_PARENTHESES.matcher(expression);
+        List<String> references = new ArrayList<>();
+
+        while (matcher.find()) {
+            references.add(matcher.group(1));
+        }
+
+        return references;
+    }
+}

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/expression/JoinReference.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.datastores.aggregation.queryengines.sql.expression;
+
+import com.yahoo.elide.datastores.aggregation.core.JoinPath;
+import com.yahoo.elide.datastores.aggregation.query.Queryable;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+
+/**
+ * A reference to a column in another table ("author.books.title").
+ */
+@Value
+@Builder
+public class JoinReference implements Reference {
+
+    @NonNull
+    private Reference reference;
+
+    @NonNull
+    private Queryable source;
+
+    @NonNull
+    private JoinPath path;
+
+    @Override
+    public <T> T accept(ReferenceVisitor<T> visitor) {
+        return visitor.visitJoinReference(this);
+    }
+}

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/expression/LogicalReference.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2021, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.datastores.aggregation.queryengines.sql.expression;
+
+import com.yahoo.elide.datastores.aggregation.query.ColumnProjection;
+import com.yahoo.elide.datastores.aggregation.query.Queryable;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Singular;
+import lombok.Value;
+
+import java.util.List;
+
+/**
+ * A reference to a logical column in the same table.
+ */
+@Value
+@Builder
+public class LogicalReference implements Reference {
+
+    @Singular
+    private List<Reference> references;
+
+    @NonNull
+    private Queryable source;
+
+    @NonNull
+    private ColumnProjection column;
+
+    @Override
+    public <T> T accept(ReferenceVisitor<T> visitor) {
+        return visitor.visitLogicalReference(this);
+    }
+}

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/expression/PhysicalReference.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.datastores.aggregation.queryengines.sql.expression;
+
+import com.yahoo.elide.datastores.aggregation.query.Queryable;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+
+/**
+ * A reference to a column in the database: "$revenue"
+ */
+@Value
+@Builder
+public class PhysicalReference implements Reference {
+    @NonNull
+    private Queryable source;
+
+    @NonNull
+    private String name;
+
+    @Override
+    public <T> T accept(ReferenceVisitor<T> visitor) {
+        return visitor.visitPhysicalReference(this);
+    }
+}

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/expression/Reference.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2021, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.datastores.aggregation.queryengines.sql.expression;
+
+/**
+ * A reference to a column in a column or join expression.
+ */
+public interface Reference {
+
+    /**
+     * Accepts a visitor that walks the reference AST.
+     * @param visitor The visitor
+     * @param <T> The return type of the visitor
+     * @return a T
+     */
+    <T> T accept(ReferenceVisitor<T> visitor);
+}

elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/expression/ReferenceVisitor.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+
+package com.yahoo.elide.datastores.aggregation.queryengines.sql.expression;
+
+/**
+ * Walks a reference abstract syntax tree and builds a T.
+ * @param <T> The type the visitor constructs/returns.
+ */
+public interface ReferenceVisitor<T> {
+
+    /**
+     * Visits a physical reference.
+     * @param reference The physical reference
+     * @return a type T.
+     */
+    T visitPhysicalReference(PhysicalReference reference);
+
+    /**
+     * Visits a logical reference.
+     * @param reference The logical reference
+     * @return a type T.
+     */
+    T visitLogicalReference(LogicalReference reference);
+
+    /**
+     * Visits a join reference.
+     * @param reference The join reference
+     * @return a type T.
+     */
+    T visitJoinReference(JoinReference reference);
+}