#4475 Add mask() function and deprecate countPrevious()

stroom-app/src/test/java/stroom/analytics/TestTableBuilderAnalytics.java

@@ -77,7 +77,6 @@ void testWindowCount() {
                 where UserId = user5
                 window EventTime by 1y advance 1y
                 group by UserId
-                // having count > countPrevious
                 select UserId""";
         basicTest(query, 9, 3);
     }

stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/LmdbDataStore.java

@@ -32,7 +32,6 @@
 import stroom.query.api.v2.TimeFilter;
 import stroom.query.common.v2.SearchProgressLog.SearchPhase;
 import stroom.query.language.functions.ChildData;
-import stroom.query.language.functions.CountPrevious;
 import stroom.query.language.functions.ExpressionContext;
 import stroom.query.language.functions.FieldIndex;
 import stroom.query.language.functions.Generator;
@@ -270,6 +269,7 @@ private void addInternal(final Val[] values,
         final LmdbKV[] rows = new LmdbKV[groupIndicesByDepth.length];
         for (int depth = 0; depth < groupIndicesByDepth.length; depth++) {
             final StoredValues storedValues = valueReferenceIndex.createStoredValues();
+            storedValues.setIteration(iteration);
             final boolean[] valueIndices = valueIndicesByDepth[depth];
 
             for (int columnIndex = 0; columnIndex < compiledColumnArray.length; columnIndex++) {
@@ -279,17 +279,7 @@ private void addInternal(final Val[] values,
                 // If we need a value at this level then set the raw values.
                 if (valueIndices[columnIndex] ||
                         columnIndex == keyFactoryConfig.getTimeColumnIndex()) {
-                    if (iteration != -1) {
-                        if (generator instanceof CountPrevious.Gen gen) {
-                            if (gen.getIteration() == iteration) {
-                                generator.set(values, storedValues);
-                            }
-                        } else {
-                            generator.set(values, storedValues);
-                        }
-                    } else {
-                        generator.set(values, storedValues);
-                    }
+                    generator.set(values, storedValues);
                 }
             }
 

stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/WindowSupport.java

@@ -98,13 +98,13 @@ public WindowSupport(final TableSettings tableSettings) {
                         final int index = columns.indexOf(column);
                         columns.set(index, column
                                 .copy()
-                                .expression("countPrevious(" + i + ")")
+                                .expression("mask(" + i + ", count())")
                                 .build());
                     } else {
                         columns.add(Column.builder()
                                 .id(fieldId)
                                 .name(fieldId)
-                                .expression("countPrevious(" + i + ")")
+                                .expression("mask(" + i + ", count())")
                                 .visible(true)
                                 .build());
                     }

stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/Functions.java

@@ -115,7 +115,8 @@ public Functions(final Provider<UiConfig> uiConfigProvider) {
                                         convertSignature(functionDef, functionSignature, countsByCategoryPath);
                                 addSignature(localMap, row);
                             });
-                        } catch (Exception e) {
+                        } catch (final Exception e) {
+                            LOGGER.error(e.getMessage(), e);
                             throw new RuntimeException("Error converting FunctionDef " + functionDef.name(), e);
                         }
                     }

stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/CountPrevious.java

@@ -16,49 +16,68 @@
 
 package stroom.query.language.functions;
 
-import stroom.query.language.functions.ref.CountIterationReference;
 import stroom.query.language.functions.ref.StoredValues;
 import stroom.query.language.functions.ref.ValueReferenceIndex;
 import stroom.query.language.token.Param;
 
 import java.text.ParseException;
 import java.util.function.Supplier;
 
+@Deprecated
 @SuppressWarnings("unused") //Used by FunctionFactory
 @FunctionDef(
         name = CountPrevious.NAME,
         commonCategory = FunctionCategory.AGGREGATE,
-        commonDescription = "Counts the number of records that are passed through it for a previous time period. " +
-                "Doesn't take any notice of the values of any fields.",
+        commonReturnType = Val.class,
+        commonDescription = "DEPRECATED alias for mask(), here for backward compatibility.",
         signatures = @FunctionSignature(
-                returnType = ValLong.class,
-                returnDescription = "Number of records",
-                args = {}))
+                returnDescription = "Computed values for masked functions",
+                args = {
+                        @FunctionArg(
+                                name = "iteration",
+                                description = "The window iteration to mask",
+                                argType = ValInteger.class),
+                        @FunctionArg(
+                                name = "function",
+                                description = "Inner function to apply mask to",
+                                argType = Val.class)
+                }))
 public class CountPrevious extends AbstractFunction implements AggregateFunction {
 
     static final String NAME = "countPrevious";
     private int iteration;
-    private CountIterationReference valueReference;
+    private Function function;
 
     public CountPrevious(final String name) {
-        super(name, 1, 1);
+        super(name, 1, 2);
     }
 
     @Override
     public void setParams(final Param[] params) throws ParseException {
         super.setParams(params);
         iteration = Integer.parseInt(params[0].toString());
+
+        if (params.length == 2) {
+            final Param param = params[1];
+            if (param instanceof Function) {
+                function = (Function) param;
+            } else {
+                function = new StaticValueFunction((Val) param);
+            }
+        } else {
+            function = new Count(Count.NAME);
+        }
     }
 
     @Override
     public void addValueReferences(final ValueReferenceIndex valueReferenceIndex) {
-        valueReference = valueReferenceIndex.addCountIteration(name + " - " + iteration, iteration);
-        super.addValueReferences(valueReferenceIndex);
+        function.addValueReferences(valueReferenceIndex);
     }
 
     @Override
     public Generator createGenerator() {
-        return new Gen(iteration, valueReference);
+        final Generator childGenerator = function.createGenerator();
+        return new Gen(iteration, childGenerator);
     }
 
     @Override
@@ -74,29 +93,29 @@ public boolean hasAggregate() {
     public static final class Gen extends AbstractNoChildGenerator {
 
         private final int iteration;
-        private final CountIterationReference valueReference;
+        private final Generator childGenerator;
 
-        public Gen(final int iteration, final CountIterationReference valueReference) {
+        public Gen(final int iteration, final Generator childGenerator) {
             this.iteration = iteration;
-            this.valueReference = valueReference;
+            this.childGenerator = childGenerator;
         }
 
         @Override
         public void set(final Val[] values, final StoredValues storedValues) {
-            valueReference.increment(storedValues);
+            // Filter on iteration.
+            if (storedValues.getIteration() == iteration) {
+                childGenerator.set(values, storedValues);
+            }
         }
 
         @Override
         public Val eval(final StoredValues storedValues, final Supplier<ChildData> childDataSupplier) {
-            return ValLong.create(valueReference.get(storedValues));
+            return childGenerator.eval(storedValues, childDataSupplier);
         }
 
         @Override
         public void merge(final StoredValues existingValues, final StoredValues newValues) {
-            final long existingValue = valueReference.get(existingValues);
-            final long newValue = valueReference.get(newValues);
-            valueReference.set(existingValues, existingValue + newValue);
-            super.merge(existingValues, newValues);
+            childGenerator.merge(existingValues, newValues);
         }
 
         public int getIteration() {

stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Mask.java

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2017 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package stroom.query.language.functions;
+
+import stroom.query.language.functions.ref.StoredValues;
+import stroom.query.language.functions.ref.ValueReferenceIndex;
+import stroom.query.language.token.Param;
+
+import java.text.ParseException;
+import java.util.function.Supplier;
+
+@SuppressWarnings("unused") //Used by FunctionFactory
+@FunctionDef(
+        name = Mask.NAME,
+        commonCategory = FunctionCategory.AGGREGATE,
+        commonReturnType = Val.class,
+        commonDescription = "Provides a masking effect for nested functions for specific window iterations",
+        signatures = @FunctionSignature(
+                returnDescription = "Computed values for masked functions",
+                args = {
+                        @FunctionArg(
+                                name = "iteration",
+                                description = "The window iteration to mask",
+                                argType = ValInteger.class),
+                        @FunctionArg(
+                                name = "function",
+                                description = "Inner function to apply mask to",
+                                argType = Val.class)
+                }))
+public class Mask extends AbstractFunction implements AggregateFunction {
+
+    static final String NAME = "mask";
+    private int iteration;
+    private Function function;
+
+    public Mask(final String name) {
+        super(name, 1, 2);
+    }
+
+    @Override
+    public void setParams(final Param[] params) throws ParseException {
+        super.setParams(params);
+        iteration = Integer.parseInt(params[0].toString());
+
+        if (params.length == 2) {
+            final Param param = params[1];
+            if (param instanceof Function) {
+                function = (Function) param;
+            } else {
+                function = new StaticValueFunction((Val) param);
+            }
+        } else {
+            function = new Count(Count.NAME);
+        }
+    }
+
+    @Override
+    public void addValueReferences(final ValueReferenceIndex valueReferenceIndex) {
+        function.addValueReferences(valueReferenceIndex);
+    }
+
+    @Override
+    public Generator createGenerator() {
+        final Generator childGenerator = function.createGenerator();
+        return new Gen(iteration, childGenerator);
+    }
+
+    @Override
+    public boolean isAggregate() {
+        return true;
+    }
+
+    @Override
+    public boolean hasAggregate() {
+        return isAggregate();
+    }
+
+    public static final class Gen extends AbstractNoChildGenerator {
+
+        private final int iteration;
+        private final Generator childGenerator;
+
+        public Gen(final int iteration, final Generator childGenerator) {
+            this.iteration = iteration;
+            this.childGenerator = childGenerator;
+        }
+
+        @Override
+        public void set(final Val[] values, final StoredValues storedValues) {
+            // Filter on iteration.
+            if (storedValues.getIteration() == iteration) {
+                childGenerator.set(values, storedValues);
+            }
+        }
+
+        @Override
+        public Val eval(final StoredValues storedValues, final Supplier<ChildData> childDataSupplier) {
+            return childGenerator.eval(storedValues, childDataSupplier);
+        }
+
+        @Override
+        public void merge(final StoredValues existingValues, final StoredValues newValues) {
+            childGenerator.merge(existingValues, newValues);
+        }
+
+        public int getIteration() {
+            return iteration;
+        }
+    }
+}

stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ref/StoredValues.java

@@ -5,6 +5,7 @@
 public class StoredValues {
 
     private final Object[] values;
+    private int iteration;
 
     public StoredValues(final Object[] values) {
         this.values = values;
@@ -18,6 +19,14 @@ public void set(final int index, final Object val) {
         this.values[index] = val;
     }
 
+    public int getIteration() {
+        return iteration;
+    }
+
+    public void setIteration(final int iteration) {
+        this.iteration = iteration;
+    }
+
     @Override
     public boolean equals(final Object o) {
         if (this == o) {

stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ref/ValueReferenceIndex.java

@@ -16,10 +16,6 @@ public CountReference addCount(final String name) {
         return add(new CountReference(list.size(), name));
     }
 
-    public CountIterationReference addCountIteration(final String name, final int iteration) {
-        return add(new CountIterationReference(list.size(), name, iteration));
-    }
-
     public DoubleListReference addDoubleList(final String name) {
         return add(new DoubleListReference(list.size(), name));
     }

stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/TestValueReferenceIndex.java

@@ -1,6 +1,5 @@
 package stroom.query.language.functions;
 
-import stroom.query.language.functions.ref.CountIterationReference;
 import stroom.query.language.functions.ref.CountReference;
 import stroom.query.language.functions.ref.DoubleListReference;
 import stroom.query.language.functions.ref.FieldValReference;
@@ -35,19 +34,6 @@ void testCountReference() {
         testWriteRead(valueReferenceIndex, storedValues);
     }
 
-    @Test
-    void testCountIterationReference() {
-        final ValueReferenceIndex valueReferenceIndex = new ValueReferenceIndex();
-        final CountIterationReference r1 = valueReferenceIndex.addCountIteration("test1", 1);
-        final CountIterationReference r2 = valueReferenceIndex.addCountIteration("test2", 2);
-        final CountIterationReference r3 = valueReferenceIndex.addCountIteration("test3", 3);
-        final StoredValues storedValues = valueReferenceIndex.createStoredValues();
-        r1.increment(storedValues);
-        r2.add(storedValues, 3);
-        r3.increment(storedValues);
-        testWriteRead(valueReferenceIndex, storedValues);
-    }
-
     @Test
     void testDoubleListReference() {
         final ValueReferenceIndex valueReferenceIndex = new ValueReferenceIndex();