Analytics API: Add Options aggregation and disaggregation[2.42-DHIS2-18368_3]

dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.analytics;
+
+import java.util.List;
+import lombok.Builder;
+import lombok.Getter;
+
+@Builder
+@Getter
+public class OptionSetSelection {
+  private String optionSetUid;
+  private List<String> options;
+  private OptionSetSelectionMode optionSetSelectionMode;
+}

dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.analytics;
+
+import java.util.Map;
+import lombok.Builder;
+import lombok.Getter;
+
+@Builder
+@Getter
+public class OptionSetSelectionCriteria {
+  private Map<String, OptionSetSelection> optionSetSelections;
+}

dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2004-2024, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.analytics;
+
+import java.util.Arrays;
+import java.util.List;
+
+/** The selection modes for items with option sets */
+public enum OptionSetSelectionMode {
+  // All options in an option set are chosen and aggregated into a single column.
+  // This selection is relative, so any new options added to the option set are included.
+  AGGREGATED,
+  // All options in an option set are chosen and displayed as data items.
+  // This selection is relative, so any new options added to the option set are included.
+  DISAGGREGATED;
+
+  public static List<String> getOptionSetSelectionModes() {
+    return Arrays.stream(OptionSetSelectionMode.values()).map(Enum::toString).toList();
+  }
+}

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java

@@ -57,6 +57,7 @@
 import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Triple;
+import org.hisp.dhis.analytics.OptionSetSelectionMode;
 import org.hisp.dhis.common.comparator.ObjectStringValueComparator;
 import org.hisp.dhis.dataelement.DataElementOperand;
 import org.hisp.dhis.eventvisualization.Attribute;
@@ -81,12 +82,16 @@ public class DimensionalObjectUtils {
 
   public static final String COL_SEP = " ";
 
+  public static final String OPTION_SET_SELECTION_MODE_SEP = "-";
+
   /**
    * Matching data element operand, program data element, program attribute, data set reporting rate
    * metric.
    */
+  // Luqe6ps5KZ9.uTLkjHWtSL8.R0jROOT3zni-AGGREGATED
   private static final Pattern COMPOSITE_DIM_OBJECT_PATTERN =
-      Pattern.compile("(?<id1>\\w+)\\.(?<id2>\\w+|\\*)(\\.(?<id3>\\w+|\\*))?");
+      Pattern.compile(
+          "(?<id1>\\w+)\\.(?<id2>\\w+|\\*)(\\.(?<id3>\\w+|\\*))?(\\[(?<list>[^\\]]*?)\\])?(-(?<suffix>AGGREGATED|DISAGGREGATED)?)?");
 
   private static final Set<QueryOperator> IGNORED_OPERATORS =
       Set.of(QueryOperator.LIKE, QueryOperator.IN, QueryOperator.SW, QueryOperator.EW);
@@ -359,6 +364,20 @@ public static String getDimensionFromParam(String param) {
     return param.split(DIMENSION_NAME_SEP).length > 0 ? param.split(DIMENSION_NAME_SEP)[0] : param;
   }
 
+  /**
+   * Retrieves the param name from the given string. Returns the part of the string after the
+   * dimension name separator, or the whole string if the separator is not present.
+   *
+   * @param param the parameter.
+   */
+  public static String getParamFromDimension(String param) {
+    if (param == null) {
+      return null;
+    }
+
+    return param.split(DIMENSION_NAME_SEP).length > 1 ? param.split(DIMENSION_NAME_SEP)[1] : param;
+  }
+
   /**
    * Retrieves the dimension options from the given string. Looks for the part succeeding the
    * dimension name separator, if exists, splits the string part on the option separator and returns
@@ -521,9 +540,28 @@ public static boolean isCompositeDimensionalObject(String expression) {
    * @param compositeItem the composite dimension object identifier.
    * @return the first identifier, or null if not a valid composite identifier or no match.
    */
-  public static String getFirstIdentifer(String compositeItem) {
+  public static String getFirstIdentifier(String compositeItem) {
+    if (compositeItem == null) {
+      return null;
+    }
+
+    Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem);
+    return matcher.matches() ? matcher.group("id1") : null;
+  }
+
+  /**
+   * Returns the second identifier in a composite dimension object identifier.
+   *
+   * @param compositeItem the composite dimension object identifier.
+   * @return the second identifier, or null if not a valid composite identifier or no match.
+   */
+  public static String getSecondIdentifier(String compositeItem) {
+    if (compositeItem == null) {
+      return null;
+    }
+
     Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem);
-    return matcher.matches() ? matcher.group(1) : null;
+    return matcher.matches() ? matcher.group("id2") : null;
   }
 
   /**
@@ -532,9 +570,34 @@ public static String getFirstIdentifer(String compositeItem) {
    * @param compositeItem the composite dimension object identifier.
    * @return the second identifier, or null if not a valid composite identifier or no match.
    */
-  public static String getSecondIdentifer(String compositeItem) {
+  public static String getThirdIdentifier(String compositeItem) {
+    if (compositeItem == null) {
+      return null;
+    }
+
+    Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem);
+    return matcher.matches() ? matcher.group("id3") : null;
+  }
+
+  public static OptionSetSelectionMode getOptionSetSelectionMode(String compositeItem) {
+    if (compositeItem == null) {
+      return null;
+    }
+
+    Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem);
+    if (matcher.matches()) {
+      String suffix = matcher.group("suffix");
+      return suffix != null
+          ? OptionSetSelectionMode.valueOf(suffix)
+          : OptionSetSelectionMode.AGGREGATED;
+    }
+
+    return OptionSetSelectionMode.AGGREGATED;
+  }
+
+  public static String getOptions(String compositeItem) {
     Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem);
-    return matcher.matches() ? matcher.group(2) : null;
+    return matcher.matches() ? matcher.group("list") : null;
   }
 
   /**

dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java

@@ -72,6 +72,7 @@ public enum ErrorCode {
   E1126("Category combo {0} cannot combine more than {1} categories, but had: {2}"),
   E1127("Category {0} cannot have more than {1} options, but had: {2} "),
   E1128("Category combo {0} cannot have more than {1} combinations, but requires: {2}"),
+  E1129("Option set selection mode must match valid mode: `{0}`"),
 
   /* Org unit merge */
   E1500("At least two source orgs unit must be specified"),

dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java

@@ -177,15 +177,15 @@ void testGetDataElementOperandIdSchemeCodeMap() {
   @Test
   void testGetFirstSecondIdentifier() {
     assertEquals(
-        "A123456789A", DimensionalObjectUtils.getFirstIdentifer("A123456789A.P123456789A"));
-    assertNull(DimensionalObjectUtils.getFirstIdentifer("A123456789A"));
+        "A123456789A", DimensionalObjectUtils.getFirstIdentifier("A123456789A.P123456789A"));
+    assertNull(DimensionalObjectUtils.getFirstIdentifier("A123456789A"));
   }
 
   @Test
   void testGetSecondIdentifier() {
     assertEquals(
-        "P123456789A", DimensionalObjectUtils.getSecondIdentifer("A123456789A.P123456789A"));
-    assertNull(DimensionalObjectUtils.getSecondIdentifer("A123456789A"));
+        "P123456789A", DimensionalObjectUtils.getSecondIdentifier("A123456789A.P123456789A"));
+    assertNull(DimensionalObjectUtils.getSecondIdentifier("A123456789A"));
   }
 
   @Test

dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java

@@ -87,6 +87,7 @@
 import org.hisp.dhis.common.DhisApiVersion;
 import org.hisp.dhis.common.DimensionItemKeywords;
 import org.hisp.dhis.common.DimensionItemObjectValue;
+import org.hisp.dhis.common.DimensionItemType;
 import org.hisp.dhis.common.DimensionType;
 import org.hisp.dhis.common.DimensionalItemObject;
 import org.hisp.dhis.common.DimensionalObject;
@@ -234,6 +235,9 @@ public class DataQueryParams {
   /** The aggregation type. */
   protected AnalyticsAggregationType aggregationType;
 
+  /** The option set selection criteria. */
+  protected OptionSetSelectionCriteria optionSetSelectionCriteria;
+
   /** The measure criteria, which is measure filters and corresponding values. */
   protected Map<MeasureFilter, Double> measureCriteria = new HashMap<>();
 
@@ -498,6 +502,7 @@ public <T extends DataQueryParams> T copyTo(T params) {
     params.dimensions = DimensionalObjectUtils.getCopies(this.dimensions);
     params.filters = DimensionalObjectUtils.getCopies(this.filters);
     params.aggregationType = this.aggregationType != null ? this.aggregationType.instance() : null;
+    params.optionSetSelectionCriteria = this.optionSetSelectionCriteria;
     params.measureCriteria = new HashMap<>(this.measureCriteria);
     params.preAggregateMeasureCriteria = new HashMap<>(this.preAggregateMeasureCriteria);
     params.skipMeta = this.skipMeta;
@@ -591,6 +596,7 @@ protected QueryKey getQueryKey() {
         (k, v) -> key.add("preAggregateMeasureCriteria", (String.valueOf(k) + v)));
 
     return key.add("aggregationType", aggregationType)
+        .add("optionSetSelectionCriteria", optionSetSelectionCriteria)
         .add("skipMeta", skipMeta)
         .add("skipData", skipData)
         .add("skipHeaders", skipHeaders)
@@ -746,6 +752,11 @@ public boolean hasOrganisationUnitGroupSets() {
     return !getDimensionsAndFilters(ORGANISATION_UNIT_GROUP_SET).isEmpty();
   }
 
+  /** Indicates whether option set selection criteria are present as dimension. */
+  public boolean hasOptionSetSelectionCriteria() {
+    return optionSetSelectionCriteria != null;
+  }
+
   /**
    * Returns the period type of the first period specified as filter, or null if there is no period
    * filter.
@@ -861,6 +872,17 @@ public boolean isOutputFormat(OutputFormat format) {
     return this.outputFormat != null && this.outputFormat == format;
   }
 
+  public boolean hasOptionSetInDimensionItems() {
+    return dimensions.stream()
+        .anyMatch(
+            d ->
+                d.getItems().stream()
+                    .anyMatch(
+                        it ->
+                            it.getDimensionItemType() == DimensionItemType.DATA_ELEMENT
+                                && ((DataElement) it).getOptionSet() != null));
+  }
+
   /**
    * Creates a mapping between the data periods, based on the data period type for this query, and
    * the aggregation periods for this query.
@@ -1952,6 +1974,10 @@ public AnalyticsAggregationType getAggregationType() {
     return aggregationType;
   }
 
+  public OptionSetSelectionCriteria getOptionSetSelectionCriteria() {
+    return optionSetSelectionCriteria;
+  }
+
   public Map<MeasureFilter, Double> getMeasureCriteria() {
     return measureCriteria;
   }
@@ -2796,6 +2822,12 @@ public Builder withAggregationType(AnalyticsAggregationType aggregationType) {
       return this;
     }
 
+    public Builder withOptionSetSelectionCriteria(
+        OptionSetSelectionCriteria optionSetSelectionCriteria) {
+      this.params.optionSetSelectionCriteria = optionSetSelectionCriteria;
+      return this;
+    }
+
     public Builder withSkipMeta(boolean skipMeta) {
       this.params.skipMeta = skipMeta;
       return this;