From 5aa25823903bb460d737b19717b202ac2a8ecfbd Mon Sep 17 00:00:00 2001 From: d-bernat Date: Wed, 18 Dec 2024 13:17:48 +0100 Subject: [PATCH 01/16] cherrypicking, resolve conflicts, part1 --- .../dhis/analytics/OptionSetSelection.java | 40 ++++++++++ .../analytics/OptionSetSelectionCriteria.java | 74 +++++++++++++++++++ .../OptionSetSelectionCriteriaV2.java | 38 ++++++++++ .../analytics/OptionSetSelectionMode.java | 45 +++++++++++ .../dhis/common/DimensionalObjectUtils.java | 68 ++++++++++++++++- .../org/hisp/dhis/feedback/ErrorCode.java | 1 + .../common/DimensionalObjectUtilsTest.java | 8 +- .../hisp/dhis/analytics/DataQueryParams.java | 32 ++++++++ .../data/DefaultDataQueryService.java | 72 +++++++++++++++++- .../dhis/analytics/util/AnalyticsUtils.java | 4 +- 10 files changed, 371 insertions(+), 11 deletions(-) create mode 100644 dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java create mode 100644 dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java create mode 100644 dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java create mode 100644 dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java new file mode 100644 index 000000000000..d18baeb326ef --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.List; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class OptionSetSelection { + private String optionSetUid; + private List options; + private OptionSetSelectionMode optionSetSelectionMode; +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java new file mode 100644 index 000000000000..e11c2b3ee8c7 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +public class OptionSetSelectionCriteria { + // @Getter private Map optionSetSelectionModes; + // @Getter private List options; + // + // OptionSetSelectionCriteria( + // Map optionSetSelectionModes, List options) { + // this.optionSetSelectionModes = optionSetSelectionModes; + // this.options = options; + // } + // + // public static OptionSetSelectionCriteriaBuilder builder() { + // return new OptionSetSelectionCriteriaBuilder(); + // } + // + // public static class OptionSetSelectionCriteriaBuilder { + // private Map optionSetSelectionModes; + // private List options; + // + // OptionSetSelectionCriteriaBuilder() {} + // + // public OptionSetSelectionCriteriaBuilder optionSetSelectionModes( + // Map optionSetSelectionModes) { + // this.optionSetSelectionModes = Collections.unmodifiableMap(optionSetSelectionModes); + // return this; + // } + // + // public OptionSetSelectionCriteriaBuilder options(List options) { + // this.options = Collections.unmodifiableList(options); + // return this; + // } + // + // public OptionSetSelectionCriteria build() { + // return new OptionSetSelectionCriteria(this.optionSetSelectionModes, this.options); + // } + // + // public String toString() { + // return + // "OptionSetSelectionCriteria.OptionSetSelectionCriteriaBuilder(optionSetSelectionModes=" + // + this.optionSetSelectionModes + // + ", options=" + // + this.options + // + ")"; + // } + // } +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java new file mode 100644 index 000000000000..350ea38cb4e0 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.Map; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class OptionSetSelectionCriteriaV2 { + private Map optionSetSelections; +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java new file mode 100644 index 000000000000..34c42a441821 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.Arrays; +import java.util.List; + +/** The selection modes for items with option sets */ +public enum OptionSetSelectionMode { + // All options in an option set are chosen and aggregated into a single column. + // This selection is relative, so any new options added to the option set are included. + AGGREGATED, + // All options in an option set are chosen and displayed as data items. + // This selection is relative, so any new options added to the option set are included. + DISAGGREGATED; + + public static List getOptionSetSelectionModes() { + return Arrays.stream(OptionSetSelectionMode.values()).map(Enum::toString).toList(); + } +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index a7786ad46813..e2e9e060e705 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -57,6 +57,7 @@ import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.common.comparator.ObjectStringValueComparator; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.eventvisualization.Attribute; @@ -81,12 +82,16 @@ public class DimensionalObjectUtils { public static final String COL_SEP = " "; + public static final String OPTION_SET_SELECTION_MODE_SEP = "-"; + /** * Matching data element operand, program data element, program attribute, data set reporting rate * metric. */ + // Luqe6ps5KZ9.uTLkjHWtSL8.R0jROOT3zni-AGGREGATED private static final Pattern COMPOSITE_DIM_OBJECT_PATTERN = - Pattern.compile("(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?"); + Pattern.compile( + "(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?(\\[(?[^\\]]*?)\\])?(-(?AGGREGATED|DISAGGREGATED)?)?"); private static final Set IGNORED_OPERATORS = Set.of(QueryOperator.LIKE, QueryOperator.IN, QueryOperator.SW, QueryOperator.EW); @@ -359,6 +364,19 @@ public static String getDimensionFromParam(String param) { return param.split(DIMENSION_NAME_SEP).length > 0 ? param.split(DIMENSION_NAME_SEP)[0] : param; } + /** + * Retrieves the param name from the given string. Returns the part of the string after the + * dimension name separator, or the whole string if the separator is not present. + * + * @param param the parameter. + */ + public static String getParamFromDimension(String param) { + if (param == null) { + return null; + } + + return param.split(DIMENSION_NAME_SEP).length > 1 ? param.split(DIMENSION_NAME_SEP)[1] : param; + } /** * Retrieves the dimension options from the given string. Looks for the part succeeding the * dimension name separator, if exists, splits the string part on the option separator and returns @@ -521,7 +539,7 @@ public static boolean isCompositeDimensionalObject(String expression) { * @param compositeItem the composite dimension object identifier. * @return the first identifier, or null if not a valid composite identifier or no match. */ - public static String getFirstIdentifer(String compositeItem) { + public static String getFirstIdentifier(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); return matcher.matches() ? matcher.group(1) : null; } @@ -532,9 +550,51 @@ public static String getFirstIdentifer(String compositeItem) { * @param compositeItem the composite dimension object identifier. * @return the second identifier, or null if not a valid composite identifier or no match. */ - public static String getSecondIdentifer(String compositeItem) { + public static String getSecondIdentifier(String compositeItem) { + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + return matcher.matches() ? extractOptionSetSelectionModeIdentifier(matcher.group(2)) : null; + } + + public static String getOptionSetSelectionModeIdentifier(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? matcher.group(2) : null; + return matcher.matches() && matcher.groupCount() >= 4 + ? extractOptionSetSelectionModeIdentifier(matcher.group(4)) + : null; + } + + public static String getOptions(String compositeItem) { + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + return matcher.matches() && matcher.groupCount() >= 6 ? matcher.group(6) : null; + } + + public static OptionSetSelectionMode getOptionSetSelectionMode(String compositeItem) { + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + return matcher.matches() ? getOptionSetSelectionMode(matcher) : null; + } + + private static OptionSetSelectionMode getOptionSetSelectionMode(Matcher matcher) { + List modes = OptionSetSelectionMode.getOptionSetSelectionModes(); + for (int i = 0; i <= matcher.groupCount(); i++) { + if (modes.contains(matcher.group(i))) { + return OptionSetSelectionMode.valueOf(matcher.group(i)); + } + } + + return null; + } + + private static String extractOptionSetSelectionModeIdentifier(String identifier) { + if (identifier == null) { + return null; + } + + List modes = OptionSetSelectionMode.getOptionSetSelectionModes(); + + if (modes.stream().anyMatch(identifier::contains)) { + return identifier.split(OPTION_SET_SELECTION_MODE_SEP)[0]; + } + + return identifier; } /** diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index e21ab1118fc3..0fd3dcaec55b 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -72,6 +72,7 @@ public enum ErrorCode { E1126("Category combo {0} cannot combine more than {1} categories, but had: {2}"), E1127("Category {0} cannot have more than {1} options, but had: {2} "), E1128("Category combo {0} cannot have more than {1} combinations, but requires: {2}"), + E1129("Option set selection mode must match valid mode: `{0}`"), /* Org unit merge */ E1500("At least two source orgs unit must be specified"), diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java index d68df61a942f..c4a124a31b32 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java @@ -177,15 +177,15 @@ void testGetDataElementOperandIdSchemeCodeMap() { @Test void testGetFirstSecondIdentifier() { assertEquals( - "A123456789A", DimensionalObjectUtils.getFirstIdentifer("A123456789A.P123456789A")); - assertNull(DimensionalObjectUtils.getFirstIdentifer("A123456789A")); + "A123456789A", DimensionalObjectUtils.getFirstIdentifier("A123456789A.P123456789A")); + assertNull(DimensionalObjectUtils.getFirstIdentifier("A123456789A")); } @Test void testGetSecondIdentifier() { assertEquals( - "P123456789A", DimensionalObjectUtils.getSecondIdentifer("A123456789A.P123456789A")); - assertNull(DimensionalObjectUtils.getSecondIdentifer("A123456789A")); + "P123456789A", DimensionalObjectUtils.getSecondIdentifier("A123456789A.P123456789A")); + assertNull(DimensionalObjectUtils.getSecondIdentifier("A123456789A")); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java index c350db389e18..49eef0f15d04 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java @@ -87,6 +87,7 @@ import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.common.DimensionItemKeywords; import org.hisp.dhis.common.DimensionItemObjectValue; +import org.hisp.dhis.common.DimensionItemType; import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; @@ -234,6 +235,9 @@ public class DataQueryParams { /** The aggregation type. */ protected AnalyticsAggregationType aggregationType; + /** The option set selection criteria. */ + protected OptionSetSelectionCriteriaV2 optionSetSelectionCriteria; + /** The measure criteria, which is measure filters and corresponding values. */ protected Map measureCriteria = new HashMap<>(); @@ -498,6 +502,7 @@ public T copyTo(T params) { params.dimensions = DimensionalObjectUtils.getCopies(this.dimensions); params.filters = DimensionalObjectUtils.getCopies(this.filters); params.aggregationType = this.aggregationType != null ? this.aggregationType.instance() : null; + params.optionSetSelectionCriteria = this.optionSetSelectionCriteria; params.measureCriteria = new HashMap<>(this.measureCriteria); params.preAggregateMeasureCriteria = new HashMap<>(this.preAggregateMeasureCriteria); params.skipMeta = this.skipMeta; @@ -591,6 +596,7 @@ protected QueryKey getQueryKey() { (k, v) -> key.add("preAggregateMeasureCriteria", (String.valueOf(k) + v))); return key.add("aggregationType", aggregationType) + .add("optionSetSelectionCriteria", optionSetSelectionCriteria) .add("skipMeta", skipMeta) .add("skipData", skipData) .add("skipHeaders", skipHeaders) @@ -746,6 +752,11 @@ public boolean hasOrganisationUnitGroupSets() { return !getDimensionsAndFilters(ORGANISATION_UNIT_GROUP_SET).isEmpty(); } + /** Indicates whether option set selection criteria are present as dimension. */ + public boolean hasOptionSetSelectionCriteria() { + return optionSetSelectionCriteria != null; + } + /** * Returns the period type of the first period specified as filter, or null if there is no period * filter. @@ -861,6 +872,17 @@ public boolean isOutputFormat(OutputFormat format) { return this.outputFormat != null && this.outputFormat == format; } + public boolean hasOptionSetInDimensionItems() { + return dimensions.stream() + .anyMatch( + d -> + d.getItems().stream() + .anyMatch( + it -> + it.getDimensionItemType() == DimensionItemType.DATA_ELEMENT + && ((DataElement) it).getOptionSet() != null)); + } + /** * Creates a mapping between the data periods, based on the data period type for this query, and * the aggregation periods for this query. @@ -1952,6 +1974,10 @@ public AnalyticsAggregationType getAggregationType() { return aggregationType; } + public OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria() { + return optionSetSelectionCriteria; + } + public Map getMeasureCriteria() { return measureCriteria; } @@ -2796,6 +2822,12 @@ public Builder withAggregationType(AnalyticsAggregationType aggregationType) { return this; } + public Builder withOptionSetSelectionCriteria( + OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { + this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; + return this; + } + public Builder withSkipMeta(boolean skipMeta) { this.params.skipMeta = skipMeta; return this; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index 6ed9ff5c5838..8feee854fccc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -59,15 +59,22 @@ import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; + import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsSecurityManager; import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataQueryService; +import org.hisp.dhis.analytics.OptionSetSelection; +import org.hisp.dhis.analytics.OptionSetSelectionCriteriaV2; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.common.AnalyticalObject; @@ -75,12 +82,15 @@ import org.hisp.dhis.common.DataQueryRequest; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; +import org.hisp.dhis.common.DimensionalObjectUtils; import org.hisp.dhis.common.DisplayProperty; import org.hisp.dhis.common.EventDataQueryRequest; import org.hisp.dhis.common.IdScheme; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.feedback.ErrorMessage; +import org.hisp.dhis.option.Option; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.setting.UserSettings; import org.hisp.dhis.user.User; @@ -171,6 +181,66 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { .build(); } + private OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria(Set dimensions) { + OptionSetSelectionCriteriaV2.OptionSetSelectionCriteriaV2Builder builder = + OptionSetSelectionCriteriaV2.builder(); + Map optionSetSelections = new HashMap<>(); + for (String dimension : dimensions) { + String param = DimensionalObjectUtils.getParamFromDimension(dimension); + + if (!hasOptionSet(param)) { + continue; + } + + OptionSetSelectionMode mode = DimensionalObjectUtils.getOptionSetSelectionMode(param); + if (mode == null) { + mode = OptionSetSelectionMode.AGGREGATED; + } + + String key = DimensionalObjectUtils.getOptionSetSelectionModeIdentifier(param); + if (key == null) { + key = + DimensionalObjectUtils.getFirstIdentifier(param) + + "." + + DimensionalObjectUtils.getSecondIdentifier(param); + } else { + key = DimensionalObjectUtils.getSecondIdentifier(param) + "." + key; + } + + OptionSetSelection.OptionSetSelectionBuilder optionSetSelectionBuilder = + OptionSetSelection.builder().optionSetSelectionMode(mode).optionSetUid(key); + String options = DimensionalObjectUtils.getOptions(param); + + if (options != null && !options.isEmpty()) { + List optionList = + Stream.of(options.split("#")) + .map( + uid -> + Objects.requireNonNull(this.idObjectManager.get(Option.class, uid)) + .getCode()) + .toList(); + optionSetSelectionBuilder.options(optionList); + } + + optionSetSelections.put(key, optionSetSelectionBuilder.build()); + } + + if (optionSetSelections.isEmpty()) { + return null; + } + + return builder.optionSetSelections(optionSetSelections).build(); + } + + private boolean hasOptionSet(String param) { + String uid = DimensionalObjectUtils.getOptionSetSelectionModeIdentifier(param); + if (uid == null) { + uid = DimensionalObjectUtils.getSecondIdentifier(param); + } + + return uid != null && idObjectManager.exists(OptionSet.class, uid); + } + @Override @Transactional(readOnly = true) public DataQueryParams getFromAnalyticalObject(AnalyticalObject object) { @@ -290,7 +360,7 @@ public List getUserOrgUnits(DataQueryParams params, String use getItemsFromParam(userOrgUnit).stream() .map(ou -> idObjectManager.get(OrganisationUnit.class, ou)) .filter(Objects::nonNull) - .collect(toList())); + .toList()); } else if (currentUser != null && params != null && params.getUserOrgUnitType() != null) { switch (params.getUserOrgUnitType()) { case DATA_CAPTURE: diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index b61b7a82bbaa..acbbae870898 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -644,8 +644,8 @@ public static void handleGridForDataValueSet(DataQueryParams params, Grid grid) coc = dataItem.getAggregateExportCategoryOptionCombo(); aoc = dataItem.getAggregateExportAttributeOptionCombo(); } else if (DataElementOperand.class.isAssignableFrom(item.getClass())) { - row.set(dxInx, DimensionalObjectUtils.getFirstIdentifer(dx)); - coc = DimensionalObjectUtils.getSecondIdentifer(dx); + row.set(dxInx, DimensionalObjectUtils.getFirstIdentifier(dx)); + coc = DimensionalObjectUtils.getSecondIdentifier(dx); } cocCol.add(coc); From 8bbb20f9342154be7e0df51bb50ad83c1e3e1879 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Wed, 18 Dec 2024 13:45:56 +0100 Subject: [PATCH 02/16] cherrypicking, resolve conflicts, part2 --- .../analytics/data/JdbcAnalyticsManager.java | 78 +++++++++++++++++-- .../analytics/data/handler/DataHandler.java | 4 + .../analytics/event/EventQueryParams.java | 8 ++ .../analytics/event/EventQueryPlanner.java | 8 ++ .../AbstractJdbcEventAnalyticsManager.java | 29 ++++++- .../event/data/DefaultEventQueryPlanner.java | 7 ++ .../event/data/EventQueryService.java | 20 ++++- .../event/data/JdbcEventAnalyticsManager.java | 27 +++++++ .../data/AnalyticsServiceBaseTest.java | 4 + .../data/handler/DataHandlerTest.java | 2 +- 10 files changed, 177 insertions(+), 10 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index 89935206684f..b617d93e3026 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -58,6 +58,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; @@ -71,6 +72,7 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.analytics.MeasureFilter; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.QueryPlanner; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.table.model.Partitions; @@ -85,6 +87,7 @@ import org.hisp.dhis.commons.util.DebugUtils; import org.hisp.dhis.commons.util.SqlHelper; import org.hisp.dhis.commons.util.TextUtils; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -342,6 +345,7 @@ private String getSelectClause(DataQueryParams params) { String sql = "select " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + ", "; sql += getValueClause(params); + sql += getAggregatedOptionValueClause(params); return sql; } @@ -355,7 +359,7 @@ private String getSelectClause(DataQueryParams params) { protected String getValueClause(DataQueryParams params) { String sql = ""; - if (params.isAggregation()) { + if (hasAggregation(params)) { sql += getAggregateValueColumn(params); } else { sql += params.getValueColumn(); @@ -364,6 +368,38 @@ protected String getValueClause(DataQueryParams params) { return sql + " as value "; } + private boolean hasAggregation(DataQueryParams params) { + // analytics query is an item of sequential queries with one data element only. + if (params.getDataElements().size() != 1) { + return params.isAggregation(); + } + + Optional optionSetSelectionMode = + params.getDataElements().stream() + .map( + de -> + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) + .getOptionSetSelectionMode()) + .findFirst(); + OptionSetSelectionMode mode = optionSetSelectionMode.orElse(OptionSetSelectionMode.AGGREGATED); + + return params.isAggregation() && mode == OptionSetSelectionMode.AGGREGATED; + } + + protected String getAggregatedOptionValueClause(DataQueryParams params) { + String sql = ""; + + if (params.hasOptionSetInDimensionItems() && hasAggregation(params)) { + sql += ", count(" + params.getValueColumn() + ") as valuecount "; + return sql; + } + + return sql; + } + /** * Returns an aggregate clause for the numeric value column. * @@ -463,12 +499,38 @@ protected String getWhereClause(DataQueryParams params, AnalyticsTableType table getWhereClauseDimensions(params, sqlHelper, sql); getWhereClauseFilters(params, sqlHelper, sql); + getWhereClauseOptions(params, sqlHelper, sql); getWhereClauseDataApproval(params, sqlHelper, sql); getWhereClauseRestrictions(params, sqlHelper, sql, tableType); return sql.toString(); } + /** Add where clause for option set selection. */ + private void getWhereClauseOptions( + DataQueryParams params, SqlHelper sqlHelper, StringBuilder sql) { + if (!params.hasOptionSetSelectionCriteria()) { + return; + } + + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + List options = value.getOptions(); + if (options != null && !options.isEmpty()) { + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(quote("optionvalueuid")) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); + } + /** Add where clause dimensions. */ private void getWhereClauseDimensions( DataQueryParams params, SqlHelper sqlHelper, StringBuilder sql) { @@ -640,13 +702,11 @@ private void getWhereClauseRestrictions( * @return a SQL group by clause. */ protected String getGroupByClause(DataQueryParams params) { - String sql = ""; - - if (params.isAggregation()) { - sql = "group by " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + " "; + if (hasAggregation(params)) { + return "group by " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + " "; } - return sql; + return ""; } /** @@ -961,7 +1021,11 @@ private Map getKeyValueMap(DataQueryParams params, String sql, i } else // NUMERIC { Double value = rowSet.getDouble(VALUE_ID); - map.put(key.toString(), value); + if (params.hasOptionSetInDimensionItems()) { + map.put(key + DIMENSION_SEP + value, rowSet.getString("valuecount")); + } else { + map.put(key.toString(), value); + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java index 998fcb80d55c..d9e8735b44a5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java @@ -130,6 +130,7 @@ import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.event.data.EventAggregateService; +import org.hisp.dhis.analytics.event.data.EventQueryService; import org.hisp.dhis.analytics.resolver.ExpressionResolver; import org.hisp.dhis.analytics.resolver.ExpressionResolvers; import org.hisp.dhis.analytics.util.PeriodOffsetUtils; @@ -173,6 +174,8 @@ public class DataHandler { private final EventAggregateService eventAggregatedService; + private final EventQueryService eventQueryService; + private final RawAnalyticsManager rawAnalyticsManager; private final ExpressionResolvers resolvers; @@ -426,6 +429,7 @@ public void addProgramDataElementAttributeIndicatorValues(DataQueryParams params EventQueryParams eventQueryParams = new EventQueryParams.Builder(fromDataQueryParams(dataSourceParams)) .withSkipMeta(true) + .withOptionSetSelectionCriteria(params.getOptionSetSelectionCriteria()) .build(); Grid eventGrid = eventAggregatedService.getAggregatedData(eventQueryParams); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 4d6f0683f691..32173f9c3e83 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -59,6 +59,7 @@ import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.EventOutputType; +import org.hisp.dhis.analytics.OptionSetSelectionCriteriaV2; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.QueryKey; import org.hisp.dhis.analytics.QueryParamsBuilder; @@ -305,6 +306,7 @@ protected EventQueryParams instance() { params.rowContext = this.rowContext; params.multipleQueries = this.multipleQueries; params.userOrganisationUnitsCriteria = this.userOrganisationUnitsCriteria; + params.optionSetSelectionCriteria = this.optionSetSelectionCriteria; return params; } @@ -1360,5 +1362,11 @@ public Builder withMultipleQueries(boolean multipleQueries) { this.params.multipleQueries = multipleQueries; return this; } + + public Builder withOptionSetSelectionCriteria( + OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { + this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; + return this; + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java index 6f5ccba6f53c..7b0f2795c9d0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java @@ -41,6 +41,14 @@ public interface EventQueryPlanner { */ List planAggregateQuery(EventQueryParams params); + /** + * Plans the given parameters and returns a list of parameters. + * + * @param params the event query parameters. + * @return a list of {@link EventQueryParams}. + */ + List planQuery(EventQueryParams params); + /** * Plans the given parameters and returns a list of parameters. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 55a58883bd63..d5399ca41084 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -46,6 +46,7 @@ import static org.hisp.dhis.analytics.QueryKey.NV; import static org.hisp.dhis.analytics.SortOrder.ASC; import static org.hisp.dhis.analytics.SortOrder.DESC; +import static org.hisp.dhis.analytics.data.QueryPlannerUtils.getAggregationType; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getHeaderColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getOrgUnitLevelColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getPeriodColumns; @@ -89,6 +90,7 @@ import org.apache.commons.lang3.time.DateUtils; import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.analytics.EventOutputType; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; @@ -111,6 +113,7 @@ import org.hisp.dhis.commons.collection.ListUtils; import org.hisp.dhis.commons.util.SqlHelper; import org.hisp.dhis.commons.util.TextUtils; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.option.Option; @@ -591,6 +594,11 @@ public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLi private String getGroupByClause(EventQueryParams params) { String sql = ""; + AggregationType aggregationType = getAggregationType(params); + if (aggregationType == NONE) { + return sql; + } + if (params.isAggregation()) { List selectColumnNames = getGroupByColumnNames(params, true); @@ -697,7 +705,7 @@ protected String getAggregateClause(EventQueryParams params) { EventOutputType outputType = params.getOutputType(); - AggregationType aggregationType = params.getAggregationTypeFallback().getAggregationType(); + AggregationType aggregationType = getAggregationType(params); String function = (aggregationType == NONE || aggregationType == CUSTOM) ? "" : aggregationType.getValue(); @@ -747,6 +755,25 @@ protected String getAggregateClause(EventQueryParams params) { } } + private AggregationType getAggregationType(EventQueryParams params) { + + if (params.getValue() instanceof DataElement + && ((DataElement) params.getValue()).hasOptionSet() + && params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get( + params.getValue().getUid() + + "." + + ((DataElement) params.getValue()).getOptionSet().getUid()) + .getOptionSetSelectionMode() + != OptionSetSelectionMode.AGGREGATED) { + return NONE; + } + + return params.getAggregationTypeFallback().getAggregationType(); + } + /** * Creates a coordinate base column "selector" for the given item name. The item is expected to be * of type Coordinate. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java index 2ee4a0fb3d9a..bde443d696ee 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java @@ -90,6 +90,13 @@ public List planAggregateQuery(EventQueryParams params) { return withTableNameAndPartitions(queries); } + @Override + public List planQuery(EventQueryParams params) { + List queries = Lists.newArrayList(params); + + return withTableNameAndPartitions(queries); + } + @Override public EventQueryParams planEventQuery(EventQueryParams params) { return withTableNameAndPartitions(params); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java index 1bef2394a06a..a8cde4e236b9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java @@ -70,6 +70,7 @@ import static org.hisp.dhis.feedback.ErrorCode.E7218; import java.util.List; +import java.util.Optional; import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsSecurityManager; @@ -83,6 +84,7 @@ import org.hisp.dhis.common.DimensionItemKeywords.Keyword; import org.hisp.dhis.common.Grid; import org.hisp.dhis.common.GridHeader; +import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.util.Timer; @@ -133,6 +135,9 @@ public Grid getEvents(EventQueryParams params) { // Set periods. params = new EventQueryParams.Builder(params).withStartEndDatesForPeriods().build(); + // Set program if null. + params = getEventQueryParamsWithProgram(params); + // Populate headers. Grid grid = createGridWithHeaders(params); addCommonHeaders(grid, params, List.of()); @@ -155,6 +160,19 @@ public Grid getEvents(EventQueryParams params) { return grid; } + private static EventQueryParams getEventQueryParamsWithProgram(EventQueryParams params) { + if (!params.hasProgram()) { + Optional itemWithProgram = + params.getItems().stream().filter(QueryItem::hasProgram).findFirst(); + if (itemWithProgram.isPresent()) { + EventQueryParams.Builder builder = + new EventQueryParams.Builder(params).withProgram(itemWithProgram.get().getProgram()); + params = builder.build(); + } + } + return params; + } + /** * Returns a list of event clusters matching the given query. * @@ -270,7 +288,7 @@ private Grid createGridWithHeaders(EventQueryParams params) { false, true)); - if (params.getProgram().isRegistration()) { + if (params.getProgram() != null && params.getProgram().isRegistration()) { grid.addHeader( new GridHeader( ENROLLMENT_DATE.getItem(), diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 667a139a2209..a26f81635f22 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -50,6 +50,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; @@ -59,6 +60,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.math3.util.Precision; import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.Rectangle; import org.hisp.dhis.analytics.TimeField; @@ -515,6 +517,8 @@ protected String getWhereClause(EventQueryParams params) { sql += getQueryItemsAndFiltersWhereClause(params, hlp); + sql += getWhereClauseOptions(params, hlp); + // --------------------------------------------------------------------- // Filter expression // --------------------------------------------------------------------- @@ -624,6 +628,29 @@ protected String getWhereClause(EventQueryParams params) { return sql; } + private String getWhereClauseOptions(DataQueryParams params, SqlHelper sqlHelper) { + StringBuilder sql = new StringBuilder(); + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + List uids = Arrays.stream(key.split("\\.")).toList(); + List options = value.getOptions(); + if (!uids.isEmpty() && options != null && !options.isEmpty()) { + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(quote(uids.get(0))) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); + + return sql.toString(); + } + /** Generates a sub query which provides a filter by organisation descendant level. */ private String getOrgUnitDescendantsClause( OrgUnitField orgUnitField, List dimensionOrFilterItems) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java index 576a9fc9a32c..bc35b3e3f59e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java @@ -50,6 +50,7 @@ import org.hisp.dhis.analytics.data.handler.MetadataHandler; import org.hisp.dhis.analytics.data.handler.SchemeIdResponseMapper; import org.hisp.dhis.analytics.event.data.EventAggregateService; +import org.hisp.dhis.analytics.event.data.EventQueryService; import org.hisp.dhis.analytics.resolver.ExpressionResolvers; import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.external.conf.DhisConfigurationProvider; @@ -87,6 +88,8 @@ abstract class AnalyticsServiceBaseTest { @Mock protected EventAggregateService eventAggregatedService; + @Mock protected EventQueryService eventQueryService; + @Mock private DataQueryService dataQueryService; @Mock private SchemeIdResponseMapper schemeIdResponseMapper; @@ -115,6 +118,7 @@ public void baseSetUp() { DataHandler dataHandler = new DataHandler( eventAggregatedService, + eventQueryService, rawAnalyticsManager, resolvers, expressionService, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java index 79766d139804..8892449e4550 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java @@ -180,6 +180,6 @@ void testOperandDataQueryParamsWithAttrOptionComboInFilter() { } private DataHandler withNullDependencies() { - return new DataHandler(null, null, null, null, null, null, null, null, null); + return new DataHandler(null, null, null, null, null, null, null, null, null, null); } } From 3a85626f7c2daaa707fa4aaacf799b3e97e75ce3 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Fri, 20 Dec 2024 10:20:58 +0100 Subject: [PATCH 03/16] cherrypicking, resolve conflicts, part3 --- .../table/JdbcAnalyticsTableManager.java | 15 ++- .../dhis/analytics/util/AnalyticsUtils.java | 25 ++++ .../DefaultResourceTableService.java | 4 +- .../table/DataElementOptionResourceTable.java | 118 ++++++++++++++++++ .../dimension/DefaultDimensionService.java | 12 ++ 5 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 31d587e51392..3068a1222d4f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -125,6 +125,18 @@ public class JdbcAnalyticsTableManager extends AbstractJdbcTableManager { .selectExpression("acs.categoryoptioncombouid as ao") .indexColumns(List.of("dx", "ao")) .build(), + AnalyticsTableColumn.builder() + .name("optionsetuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionsetuid as optionsetuid") + .build(), + AnalyticsTableColumn.builder() + .name("optionvalueuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionvalueuid as optionvalueuid") + .build(), AnalyticsTableColumn.builder() .name("pestartdate") .dataType(DATE) @@ -366,7 +378,8 @@ private void populateTable( inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ inner join analytics_rs_categoryoptioncomboname aon on dv.attributeoptioncomboid=aon.categoryoptioncomboid \ - inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid\s""", + inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid \ + left outer join analytics_rs_dataelementoption deo on dv.dataelementid = deo.dataelementid and dv.value = deo.optionvaluecode \s""", Map.of( "approvalSelectExpression", approvalSelectExpression, "valueExpression", valueExpression, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index acbbae870898..d61542330763 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -105,11 +105,13 @@ import org.hisp.dhis.feedback.ErrorMessage; import org.hisp.dhis.hibernate.HibernateProxyUtils; import org.hisp.dhis.indicator.Indicator; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.period.FinancialPeriodType; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramDataElementDimensionItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramStage; import org.hisp.dhis.system.grid.ListGrid; @@ -825,6 +827,29 @@ public static Map getDimensionMetadataItemMap( coc.getDisplayProperty(params.getDisplayProperty()), includeMetadataDetails ? coc : null)); } + + OptionSet optionSet = dataElement.getOptionSet(); + if (optionSet != null) { + map.put( + dataElement.getUid() + "." + optionSet.getUid(), + includeMetadataDetails + ? new MetadataItem( + optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) + : new MetadataItem(optionSet.getName())); + } + } + if (DimensionItemType.PROGRAM_DATA_ELEMENT == item.getDimensionItemType() + && item instanceof ProgramDataElementDimensionItem programDataElement) { + + OptionSet optionSet = programDataElement.getOptionSet(); + if (optionSet != null) { + map.put( + programDataElement.getDataElement().getUid() + "." + optionSet.getUid(), + includeMetadataDetails + ? new MetadataItem( + optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) + : new MetadataItem(optionSet.getName())); + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java index 04ba319c9e30..d8b7bbc82ea3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java @@ -61,6 +61,7 @@ import org.hisp.dhis.resourcetable.table.DataApprovalMinLevelResourceTable; import org.hisp.dhis.resourcetable.table.DataApprovalRemapLevelResourceTable; import org.hisp.dhis.resourcetable.table.DataElementGroupSetResourceTable; +import org.hisp.dhis.resourcetable.table.DataElementOptionResourceTable; import org.hisp.dhis.resourcetable.table.DataElementResourceTable; import org.hisp.dhis.resourcetable.table.DataSetOrganisationUnitCategoryResourceTable; import org.hisp.dhis.resourcetable.table.DataSetResourceTable; @@ -162,7 +163,8 @@ private final List getResourceTables() { new DataElementResourceTable(logged, idObjectManager.getAllNoAcl(DataElement.class)), new DatePeriodResourceTable(logged, getAndValidateAvailableDataYears()), new PeriodResourceTable(logged, periodService.getAllPeriods()), - new CategoryOptionComboResourceTable(logged)); + new CategoryOptionComboResourceTable(logged), + new DataElementOptionResourceTable(logged)); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java new file mode 100644 index 000000000000..66d7ccdb5fd4 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.resourcetable.table; + +import static org.hisp.dhis.commons.util.TextUtils.replace; +import static org.hisp.dhis.db.model.Table.toStaging; +import static org.hisp.dhis.system.util.SqlUtils.appendRandom; + +import java.util.List; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Index; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.hisp.dhis.resourcetable.ResourceTable; +import org.hisp.dhis.resourcetable.ResourceTableType; + +@RequiredArgsConstructor +public class DataElementOptionResourceTable implements ResourceTable { + public static final String TABLE_NAME = "analytics_rs_dataelementoption"; + + private final Logged logged; + + @Override + public Table getTable() { + return new Table(toStaging(TABLE_NAME), getColumns(), getPrimaryKey(), logged); + } + + @Override + public Table getMainTable() { + return new Table(TABLE_NAME, getColumns(), getPrimaryKey(), logged); + } + + private List getColumns() { + return List.of( + new Column("dataelementid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("optionsetid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("optionvalueid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("dataelementuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionsetuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionvalueuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionvaluecode", DataType.VARCHAR_255, Nullable.NOT_NULL)); + } + + private List getPrimaryKey() { + return List.of("dataelementid", "optionsetid", "optionvalueid"); + } + + @Override + public List getIndexes() { + return List.of( + Index.builder() + .name(appendRandom("in_optionsetoptionvalue")) + .tableName(toStaging(TABLE_NAME)) + .columns(List.of("dataelementuid", "optionsetuid", "optionvalueuid")) + .build(), + Index.builder() + .name(appendRandom("in_dataelementoptioncode")) + .tableName(toStaging(TABLE_NAME)) + .columns(List.of("dataelementuid", "optionvaluecode")) + .build()); + } + + @Override + public ResourceTableType getTableType() { + return ResourceTableType.DATA_ELEMENT_CATEGORY_OPTION_COMBO; + } + + @Override + public Optional getPopulateTempTableStatement() { + String sql = + replace( + """ + insert into ${tableName} \ + (dataelementid, optionsetid, optionvalueid, dataelementuid, optionsetuid, optionvalueuid, optionvaluecode) \ + select de.dataelementid, os.optionsetid as optionsetid, ov.optionvalueid as optionvalueid, \ + de.uid as dataelementuid, os.uid as optionsetuid, ov.uid as optionvalueuid, ov.code as optionvaluecode from optionvalue ov \ + inner join optionset os on ov.optionsetid = os.optionsetid \ + inner join dataelement de on os.optionsetid = de.optionsetid;""", + "tableName", + toStaging(TABLE_NAME)); + + return Optional.of(sql); + } + + @Override + public Optional> getPopulateTempTableContent() { + return Optional.empty(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java index 8d37336eda60..a3093eeef652 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java @@ -352,6 +352,13 @@ public DimensionalItemObject getDataDimensionalItemObject( String id1 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 1); String id2 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 2); + String optionSetSelectionMode = splitSafe(dimensionItem, "-", 1); + if (optionSetSelectionMode != null && id2 != null) { + id2 = splitSafe(id2, "-", 0); + } else if (optionSetSelectionMode != null && id1 != null) { + id1 = splitSafe(id1, "-", 0); + } + DataElementOperand operand; ReportingRate reportingRate; ProgramDataElementDimensionItem programDataElement; @@ -373,6 +380,11 @@ public DimensionalItemObject getDataDimensionalItemObject( != null) { return programAttribute; } + + if (!idScheme.is(IdentifiableProperty.UID) || CodeGenerator.isValidUid(id0)) { + return idObjectManager.get(DataDimensionItem.DATA_DIM_CLASSES, idScheme, id0); + } + } else if (!idScheme.is(IdentifiableProperty.UID) || CodeGenerator.isValidUid(dimensionItem)) { return idObjectManager.get(DataDimensionItem.DATA_DIM_CLASSES, idScheme, dimensionItem); } From 123bc051f86a8e263f8ca05fd80c69e95cb4555d Mon Sep 17 00:00:00 2001 From: d-bernat Date: Fri, 20 Dec 2024 10:56:59 +0100 Subject: [PATCH 04/16] cherrypicking, resolve conflicts, part3.1 --- .../table/JdbcEventAnalyticsTableManager.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 2a07e0b69b89..6ca4e17b8af0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -39,6 +39,7 @@ import static org.hisp.dhis.db.model.DataType.INTEGER; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; +import static org.hisp.dhis.system.util.SqlUtils.singleQuote; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; @@ -51,6 +52,7 @@ import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -467,6 +469,12 @@ private List getDataElementColumns(Program program) { .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); + columns.addAll( + program.getAnalyticsDataElements().stream() + .filter(DataElement::hasOptionSet) + .map(this::getColumnFromDataElementOptionSet) + .flatMap(Collection::stream) + .toList()); columns.addAll( program.getAnalyticsDataElementsWithLegendSet().stream() .map(de -> getColumnForDataElement(de, true)) @@ -475,6 +483,92 @@ private List getDataElementColumns(Program program) { return columns; } + private List getColumnFromDataElementOptionSet(DataElement dataElement) { + List columns = new ArrayList<>(); + + if (!dataElement.hasOptionSet()) { + return columns; + } + + String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); + String columnName = "eventdatavalues #>> '{" + dataElement.getUid() + ", value}'"; + String select = getSelectClause(dataElement.getValueType(), columnName); + String sql = selectOptionValueCodeForInsert(dataElement, select, dataClause); + + columns.add( + AnalyticsTableColumn.builder() + .name(dataElement.getUid() + ".optionvalueuid") + .columnType(AnalyticsColumnType.DYNAMIC) + .dataType(DataType.VARCHAR_255) + .selectExpression(sql) + .skipIndex(Skip.INCLUDE) + .build()); + + return columns; + } + + private String selectOptionValueCodeForInsert( + DataElement dataElement, String fromType, String dataClause) { + String innerSql = + replaceQualify( + """ + (select ${fromType} from ${event} \ + where eventid=ev.eventid ${dataClause})${closingParentheses}""", + Map.of( + "fromType", + fromType, + "dataClause", + dataClause, + "closingParentheses", + getClosingParentheses(fromType), + "dataElementUid", + quote(dataElement.getUid()))); + + return replaceQualify( + """ + (select optionvalueuid \ + from analytics_rs_dataelementoption \ + where dataelementuid = ${dataElementUid} \ + and optionvaluecode = ${selectForInsert}::varchar) as ${alias}""", + Map.of( + "dataElementUid", + singleQuote(dataElement.getUid()), + "selectForInsert", + innerSql, + "alias", + quote(dataElement.getUid() + ".optionvalueuid"))); + } + + /** + * Returns a string containing closing parenthesis. The number of parenthesis is based on the + * number of missing closing parenthesis in the argument string. + * + *

Example: + * + *

{@code} input: "((( ))" -> output: ")" {@code} + * + * @param str a string. + * @return a String containing 0 or more "closing" parenthesis + */ + private static String getClosingParentheses(String str) { + if (StringUtils.isEmpty(str)) { + return EMPTY; + } + + int open = 0; + + for (int i = 0; i < str.length(); i++) { + if (str.charAt(i) == '(') { + open++; + } else if ((str.charAt(i) == ')') && open >= 1) { + open--; + } + } + + return StringUtils.repeat(")", open); + } +} + /** * Returns a column for the given data element. If the value type of the data element is {@link * ValueType#ORGANISATION_UNIT}, an extra column will be included. From 10f0c8db0cfc59141beba04b306cdc6c82d815a1 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Fri, 20 Dec 2024 12:55:15 +0100 Subject: [PATCH 05/16] Revert "cherrypicking, resolve conflicts, part3.1" This reverts commit 123bc051f86a8e263f8ca05fd80c69e95cb4555d. --- .../table/JdbcEventAnalyticsTableManager.java | 94 ------------------- 1 file changed, 94 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6ca4e17b8af0..2a07e0b69b89 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -39,7 +39,6 @@ import static org.hisp.dhis.db.model.DataType.INTEGER; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; -import static org.hisp.dhis.system.util.SqlUtils.singleQuote; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; @@ -52,7 +51,6 @@ import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -469,12 +467,6 @@ private List getDataElementColumns(Program program) { .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); - columns.addAll( - program.getAnalyticsDataElements().stream() - .filter(DataElement::hasOptionSet) - .map(this::getColumnFromDataElementOptionSet) - .flatMap(Collection::stream) - .toList()); columns.addAll( program.getAnalyticsDataElementsWithLegendSet().stream() .map(de -> getColumnForDataElement(de, true)) @@ -483,92 +475,6 @@ private List getDataElementColumns(Program program) { return columns; } - private List getColumnFromDataElementOptionSet(DataElement dataElement) { - List columns = new ArrayList<>(); - - if (!dataElement.hasOptionSet()) { - return columns; - } - - String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); - String columnName = "eventdatavalues #>> '{" + dataElement.getUid() + ", value}'"; - String select = getSelectClause(dataElement.getValueType(), columnName); - String sql = selectOptionValueCodeForInsert(dataElement, select, dataClause); - - columns.add( - AnalyticsTableColumn.builder() - .name(dataElement.getUid() + ".optionvalueuid") - .columnType(AnalyticsColumnType.DYNAMIC) - .dataType(DataType.VARCHAR_255) - .selectExpression(sql) - .skipIndex(Skip.INCLUDE) - .build()); - - return columns; - } - - private String selectOptionValueCodeForInsert( - DataElement dataElement, String fromType, String dataClause) { - String innerSql = - replaceQualify( - """ - (select ${fromType} from ${event} \ - where eventid=ev.eventid ${dataClause})${closingParentheses}""", - Map.of( - "fromType", - fromType, - "dataClause", - dataClause, - "closingParentheses", - getClosingParentheses(fromType), - "dataElementUid", - quote(dataElement.getUid()))); - - return replaceQualify( - """ - (select optionvalueuid \ - from analytics_rs_dataelementoption \ - where dataelementuid = ${dataElementUid} \ - and optionvaluecode = ${selectForInsert}::varchar) as ${alias}""", - Map.of( - "dataElementUid", - singleQuote(dataElement.getUid()), - "selectForInsert", - innerSql, - "alias", - quote(dataElement.getUid() + ".optionvalueuid"))); - } - - /** - * Returns a string containing closing parenthesis. The number of parenthesis is based on the - * number of missing closing parenthesis in the argument string. - * - *

Example: - * - *

{@code} input: "((( ))" -> output: ")" {@code} - * - * @param str a string. - * @return a String containing 0 or more "closing" parenthesis - */ - private static String getClosingParentheses(String str) { - if (StringUtils.isEmpty(str)) { - return EMPTY; - } - - int open = 0; - - for (int i = 0; i < str.length(); i++) { - if (str.charAt(i) == '(') { - open++; - } else if ((str.charAt(i) == ')') && open >= 1) { - open--; - } - } - - return StringUtils.repeat(")", open); - } -} - /** * Returns a column for the given data element. If the value type of the data element is {@link * ValueType#ORGANISATION_UNIT}, an extra column will be included. From b2a5cdcf8f991670a2c25c6ad64e750d91cd1ba8 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Fri, 20 Dec 2024 17:13:35 +0100 Subject: [PATCH 06/16] cherrypicking, resolve conflicts, part3.2 --- .../dhis/common/DimensionalObjectUtils.java | 49 +--- .../data/DefaultDataQueryService.java | 32 +-- .../analytics/data/JdbcAnalyticsManager.java | 1 + .../table/JdbcEventAnalyticsTableManager.java | 269 +++++++++++++++++- 4 files changed, 298 insertions(+), 53 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index e2e9e060e705..43ac1f37f3a9 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -541,7 +541,7 @@ public static boolean isCompositeDimensionalObject(String expression) { */ public static String getFirstIdentifier(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? matcher.group(1) : null; + return matcher.matches() ? matcher.group("id1") : null; } /** @@ -552,49 +552,28 @@ public static String getFirstIdentifier(String compositeItem) { */ public static String getSecondIdentifier(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? extractOptionSetSelectionModeIdentifier(matcher.group(2)) : null; + return matcher.matches() ? matcher.group("id2") : null; } - public static String getOptionSetSelectionModeIdentifier(String compositeItem) { - Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() && matcher.groupCount() >= 4 - ? extractOptionSetSelectionModeIdentifier(matcher.group(4)) - : null; - } - - public static String getOptions(String compositeItem) { + /** + * Returns the second identifier in a composite dimension object identifier. + * + * @param compositeItem the composite dimension object identifier. + * @return the second identifier, or null if not a valid composite identifier or no match. + */ + public static String getThirdIdentifier(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() && matcher.groupCount() >= 6 ? matcher.group(6) : null; + return matcher.matches() ? matcher.group("id3") : null; } public static OptionSetSelectionMode getOptionSetSelectionMode(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? getOptionSetSelectionMode(matcher) : null; - } - - private static OptionSetSelectionMode getOptionSetSelectionMode(Matcher matcher) { - List modes = OptionSetSelectionMode.getOptionSetSelectionModes(); - for (int i = 0; i <= matcher.groupCount(); i++) { - if (modes.contains(matcher.group(i))) { - return OptionSetSelectionMode.valueOf(matcher.group(i)); - } - } - - return null; - } - - private static String extractOptionSetSelectionModeIdentifier(String identifier) { - if (identifier == null) { - return null; - } - - List modes = OptionSetSelectionMode.getOptionSetSelectionModes(); - - if (modes.stream().anyMatch(identifier::contains)) { - return identifier.split(OPTION_SET_SELECTION_MODE_SEP)[0]; + if(matcher.matches()){ + String suffix = matcher.group("suffix"); + return suffix != null ? OptionSetSelectionMode.valueOf(suffix): OptionSetSelectionMode.AGGREGATED; } - return identifier; + return OptionSetSelectionMode.AGGREGATED; } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index 8feee854fccc..e431a3ab36e5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -124,6 +124,7 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { if (isNotEmpty(request.getDimension())) { params.addDimensions(getDimensionalObjects(request)); + params.withOptionSetSelectionCriteria(getOptionSetSelectionCriteria(request.getDimension())); } if (isNotEmpty(request.getFilter())) { @@ -193,11 +194,8 @@ private OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria(Set d } OptionSetSelectionMode mode = DimensionalObjectUtils.getOptionSetSelectionMode(param); - if (mode == null) { - mode = OptionSetSelectionMode.AGGREGATED; - } - String key = DimensionalObjectUtils.getOptionSetSelectionModeIdentifier(param); + String key = DimensionalObjectUtils.getThirdIdentifier(param); if (key == null) { key = DimensionalObjectUtils.getFirstIdentifier(param) @@ -209,18 +207,18 @@ private OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria(Set d OptionSetSelection.OptionSetSelectionBuilder optionSetSelectionBuilder = OptionSetSelection.builder().optionSetSelectionMode(mode).optionSetUid(key); - String options = DimensionalObjectUtils.getOptions(param); - - if (options != null && !options.isEmpty()) { - List optionList = - Stream.of(options.split("#")) - .map( - uid -> - Objects.requireNonNull(this.idObjectManager.get(Option.class, uid)) - .getCode()) - .toList(); - optionSetSelectionBuilder.options(optionList); - } +// String options = DimensionalObjectUtils.getOptions(param); +// +// if (options != null && !options.isEmpty()) { +// List optionList = +// Stream.of(options.split("#")) +// .map( +// uid -> +// Objects.requireNonNull(this.idObjectManager.get(Option.class, uid)) +// .getCode()) +// .toList(); +// optionSetSelectionBuilder.options(optionList); +// } optionSetSelections.put(key, optionSetSelectionBuilder.build()); } @@ -233,7 +231,7 @@ private OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria(Set d } private boolean hasOptionSet(String param) { - String uid = DimensionalObjectUtils.getOptionSetSelectionModeIdentifier(param); + String uid = DimensionalObjectUtils.getThirdIdentifier(param); if (uid == null) { uid = DimensionalObjectUtils.getSecondIdentifier(param); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index b617d93e3026..558125a5a07a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -376,6 +376,7 @@ private boolean hasAggregation(DataQueryParams params) { Optional optionSetSelectionMode = params.getDataElements().stream() + .filter(de -> params.getOptionSetSelectionCriteria() != null) .map( de -> params diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 2a07e0b69b89..a63db132b50a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -27,8 +27,12 @@ */ package org.hisp.dhis.analytics.table; +import static java.lang.String.join; import static java.util.stream.Collectors.toList; import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.hisp.dhis.analytics.AggregationType.AVERAGE; +import static org.hisp.dhis.analytics.AggregationType.SUM; + import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.commons.util.TextUtils.emptyIfTrue; @@ -39,6 +43,7 @@ import static org.hisp.dhis.db.model.DataType.INTEGER; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; +import static org.hisp.dhis.system.util.SqlUtils.singleQuote; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; @@ -50,10 +55,17 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; +import org.hisp.dhis.analytics.DataQueryParams; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTable; @@ -65,6 +77,7 @@ import org.hisp.dhis.calendar.Calendar; import org.hisp.dhis.category.Category; import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; @@ -362,7 +375,7 @@ and ev.status in (${exportableEventStatues}) \ "programId", String.valueOf(program.getId()), "firstDataYear", String.valueOf(firstDataYear), "latestDataYear", String.valueOf(latestDataYear), - "exportableEventStatues", String.join(",", EXPORTABLE_EVENT_STATUSES))); + "exportableEventStatues", join(",", EXPORTABLE_EVENT_STATUSES))); populateTableInternal(partition, fromClause); } @@ -467,6 +480,13 @@ private List getDataElementColumns(Program program) { .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); + columns.addAll( + program.getAnalyticsDataElements().stream() + .filter(DataElement::hasOptionSet) + .map(this::getColumnFromDataElementOptionSet) + .flatMap(Collection::stream) + .toList()); + columns.addAll( program.getAnalyticsDataElementsWithLegendSet().stream() .map(de -> getColumnForDataElement(de, true)) @@ -788,4 +808,251 @@ private final String getNumericClause(String value) { private List getYearsForPartitionTable(List dataYears) { return ListUtils.mutableCopy(!dataYears.isEmpty() ? dataYears : List.of(Year.now().getValue())); } + + private List getColumnFromDataElementOptionSet(DataElement dataElement) { + List columns = new ArrayList<>(); + + if (!dataElement.hasOptionSet()) { + return columns; + } + + String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); + String columnName = "eventdatavalues #>> '{" + dataElement.getUid() + ", value}'"; + String select = getSelectClause(dataElement.getValueType(), columnName); + String sql = selectOptionValueCodeForInsert(dataElement, select, dataClause); + + columns.add( + AnalyticsTableColumn.builder() + .name(dataElement.getUid() + ".optionvalueuid") + .dataType(DataType.VARCHAR_255) + .selectExpression(sql) + .skipIndex(Skip.INCLUDE) + .build()); + + return columns; + } + + private String getDataClause(String uid, ValueType valueType) { + if (valueType.isNumeric() || valueType.isDate()) { + String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; + + return replace( + " and eventdatavalues #>> '{${uid},value}' ~* '${regex}'", + Map.of("uid", uid, "regex", regex)); + } + + return ""; + } + + private String selectOptionValueCodeForInsert( + DataElement dataElement, String fromType, String dataClause) { + String innerSql = + replaceQualify( + """ + (select ${fromType} from ${event} \ + where eventid=ev.eventid ${dataClause})${closingParentheses}""", + Map.of( + "fromType", + fromType, + "dataClause", + dataClause, + "closingParentheses", + getClosingParentheses(fromType), + "dataElementUid", + quote(dataElement.getUid()))); + + return replaceQualify( + """ + (select optionvalueuid \ + from analytics_rs_dataelementoption \ + where dataelementuid = ${dataElementUid} \ + and optionvaluecode = ${selectForInsert}::varchar) as ${alias}""", + Map.of( + "dataElementUid", + singleQuote(dataElement.getUid()), + "selectForInsert", + innerSql, + "alias", + quote(dataElement.getUid() + ".optionvalueuid"))); + } + + private String getClosingParentheses(String str) { + if (StringUtils.isEmpty(str)) { + return EMPTY; + } + + int open = 0; + + for (int i = 0; i < str.length(); i++) { + if (str.charAt(i) == '(') { + open++; + } else if ((str.charAt(i) == ')') && open >= 1) { + open--; + } + } + + return StringUtils.repeat(")", open); + } + + /** + * Generates the select clause of the query SQL. + * + * @param params the {@link DataQueryParams}. + * @return a SQL select clause. + */ + private String getSelectClause(DataQueryParams params) { + String sql = "select " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + ", "; + + sql += getValueClause(params); + + sql += getAggregatedOptionValueClause(params); + + return sql; + } + + /** + * Generates the value clause of the query SQL. + * + * @param params the {@link DataQueryParams}. + * @return a SQL value clause. + */ + protected String getValueClause(DataQueryParams params) { + String sql = ""; + + if (hasAggregation(params)) { + sql += getAggregateValueColumn(params); + } else { + sql += params.getValueColumn(); + } + + return sql + " as value "; + } + + private boolean hasAggregation(DataQueryParams params) { + // analytics query is an item of sequential queries with one data element only. + if (params.getDataElements().size() != 1) { + return params.isAggregation(); + } + + Optional optionSetSelectionMode = + params.getDataElements().stream() + .map( + de -> + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) + .getOptionSetSelectionMode()) + .findFirst(); + OptionSetSelectionMode mode = optionSetSelectionMode.orElse(OptionSetSelectionMode.AGGREGATED); + + return params.isAggregation() && mode == OptionSetSelectionMode.AGGREGATED; + } + + /** + * Returns an aggregate clause for the numeric value column. + * + * @param params the {@link DataQueryParams}. + * @return a SQL numeric value column. + */ + private String getAggregateValueColumn(DataQueryParams params) { + String sql; + + AnalyticsAggregationType aggType = params.getAggregationType(); + + String valueColumn = params.getValueColumn(); + + if (aggType.isAggregationType(SUM) + && aggType.isPeriodAggregationType(AVERAGE) + && aggType.isNumericDataType()) { + sql = "sum(daysxvalue) / " + params.getDaysForAvgSumIntAggregation(); + } else if (aggType.isAggregationType(AVERAGE) && aggType.isNumericDataType()) { + sql = "avg(" + valueColumn + ")"; + } else if (aggType.isAggregationType(AVERAGE) && aggType.isBooleanDataType()) { + sql = "sum(daysxvalue) / sum(daysno) * 100"; + } else // SUM and no value + { + sql = "sum(" + valueColumn + ")"; + } + + return sql; + } + + private String getAggregatedOptionValueClause(DataQueryParams params) { + String sql = ""; + + if (params.hasOptionSetInDimensionItems() && hasAggregation(params)) { + sql += ", count(" + params.getValueColumn() + ") as valuecount "; + return sql; + } + + return sql; + } + + /** + * Generates a comma-delimited string with the dimension names of the given dimensions where each + * dimension name is quoted. Dimensions which are considered fixed will be excluded. + * + * @param dimensions the collection of {@link DimensionalObject}. + * @return a comma-delimited string of quoted dimension names. + */ + private String getCommaDelimitedQuotedDimensionColumns( + Collection dimensions) { + return join(",", getQuotedDimensionColumns(dimensions)); + } + + /** + * Generates a list of the dimension names of the given dimensions where each dimension name is + * quoted. Dimensions which are considered fixed will be excluded. + * + * @param dimensions the collection of {@link DimensionalObject}. + * @return a list of quoted dimension names. + */ + protected List getQuotedDimensionColumns(Collection dimensions) { + return dimensions.stream() + .filter(d -> !d.isFixed()) + .map(DimensionalObject::getDimensionName) + .map(this::quoteAlias) + .collect(Collectors.toList()); + } + + /** + * @param relation the relation to quote. + * @return an "ax" aliased and double quoted relation. + */ + private String quoteAlias(String relation) { + return sqlBuilder.quoteAx(relation); + } + + /** + * Returns the select clause, potentially with a cast statement, based on the given value type. + * + * @param valueType the value type to represent as database column type. + */ + private String getSelectClause(ValueType valueType, String columnName) { + String doubleType = sqlBuilder.dataTypeDouble(); + if (valueType.isDecimal()) { + return "cast(" + columnName + " as " + doubleType + ")"; + } else if (valueType.isInteger()) { + return "cast(" + columnName + " as bigint)"; + } else if (valueType.isBoolean()) { + return "case when " + + columnName + + " = 'true' then 1 when " + + columnName + + " = 'false' then 0 else null end"; + } else if (valueType.isDate()) { + return "cast(" + columnName + " as timestamp)"; + } else if (valueType.isGeo() && isSpatialSupport()) { + return "ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (" + + columnName + + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; + } else if (valueType.isOrganisationUnit()) { + return replaceQualify( + "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}", + Map.of("columnName", columnName)); + } else { + return columnName; + } + } } From 9641e6b20853ab89a105010e1ec43ef7e2723c90 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 09:57:56 +0100 Subject: [PATCH 07/16] cherrypicking, resolve conflicts, part3.2 --- .../dhis/common/DimensionalObjectUtils.java | 14 ++- .../hisp/dhis/analytics/DataQueryParams.java | 16 +-- .../data/DefaultDataQueryService.java | 35 +++--- .../analytics/data/JdbcAnalyticsManager.java | 62 +++++----- .../analytics/event/EventQueryParams.java | 2 +- .../AbstractJdbcEventAnalyticsManager.java | 16 +-- .../event/data/EventQueryService.java | 4 +- .../event/data/JdbcEventAnalyticsManager.java | 32 ++--- .../table/JdbcAnalyticsTableManager.java | 24 ++-- .../table/JdbcEventAnalyticsTableManager.java | 117 +++++++++--------- .../dhis/analytics/util/AnalyticsUtils.java | 22 ++-- .../DefaultResourceTableService.java | 4 +- 12 files changed, 177 insertions(+), 171 deletions(-) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index 43ac1f37f3a9..a1e06797fddb 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -91,7 +91,7 @@ public class DimensionalObjectUtils { // Luqe6ps5KZ9.uTLkjHWtSL8.R0jROOT3zni-AGGREGATED private static final Pattern COMPOSITE_DIM_OBJECT_PATTERN = Pattern.compile( - "(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?(\\[(?[^\\]]*?)\\])?(-(?AGGREGATED|DISAGGREGATED)?)?"); + "(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?(\\[(?[^\\]]*?)\\])?(-(?AGGREGATED|DISAGGREGATED)?)?"); private static final Set IGNORED_OPERATORS = Set.of(QueryOperator.LIKE, QueryOperator.IN, QueryOperator.SW, QueryOperator.EW); @@ -377,6 +377,7 @@ public static String getParamFromDimension(String param) { return param.split(DIMENSION_NAME_SEP).length > 1 ? param.split(DIMENSION_NAME_SEP)[1] : param; } + /** * Retrieves the dimension options from the given string. Looks for the part succeeding the * dimension name separator, if exists, splits the string part on the option separator and returns @@ -568,14 +569,21 @@ public static String getThirdIdentifier(String compositeItem) { public static OptionSetSelectionMode getOptionSetSelectionMode(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - if(matcher.matches()){ + if (matcher.matches()) { String suffix = matcher.group("suffix"); - return suffix != null ? OptionSetSelectionMode.valueOf(suffix): OptionSetSelectionMode.AGGREGATED; + return suffix != null + ? OptionSetSelectionMode.valueOf(suffix) + : OptionSetSelectionMode.AGGREGATED; } return OptionSetSelectionMode.AGGREGATED; } + public static String getOptions(String compositeItem) { + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + return matcher.matches() ? matcher.group("list") : null; + } + /** * Indicates whether the given identifier is a wildcard. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java index 49eef0f15d04..d97fe24a5e1d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java @@ -874,13 +874,13 @@ public boolean isOutputFormat(OutputFormat format) { public boolean hasOptionSetInDimensionItems() { return dimensions.stream() - .anyMatch( - d -> - d.getItems().stream() - .anyMatch( - it -> - it.getDimensionItemType() == DimensionItemType.DATA_ELEMENT - && ((DataElement) it).getOptionSet() != null)); + .anyMatch( + d -> + d.getItems().stream() + .anyMatch( + it -> + it.getDimensionItemType() == DimensionItemType.DATA_ELEMENT + && ((DataElement) it).getOptionSet() != null)); } /** @@ -2823,7 +2823,7 @@ public Builder withAggregationType(AnalyticsAggregationType aggregationType) { } public Builder withOptionSetSelectionCriteria( - OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { + OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; return this; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index e431a3ab36e5..559eaf01b061 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -67,7 +67,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Stream; - import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsSecurityManager; import org.hisp.dhis.analytics.DataQueryParams; @@ -184,7 +183,7 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { private OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria(Set dimensions) { OptionSetSelectionCriteriaV2.OptionSetSelectionCriteriaV2Builder builder = - OptionSetSelectionCriteriaV2.builder(); + OptionSetSelectionCriteriaV2.builder(); Map optionSetSelections = new HashMap<>(); for (String dimension : dimensions) { String param = DimensionalObjectUtils.getParamFromDimension(dimension); @@ -198,27 +197,27 @@ private OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria(Set d String key = DimensionalObjectUtils.getThirdIdentifier(param); if (key == null) { key = - DimensionalObjectUtils.getFirstIdentifier(param) - + "." - + DimensionalObjectUtils.getSecondIdentifier(param); + DimensionalObjectUtils.getFirstIdentifier(param) + + "." + + DimensionalObjectUtils.getSecondIdentifier(param); } else { key = DimensionalObjectUtils.getSecondIdentifier(param) + "." + key; } OptionSetSelection.OptionSetSelectionBuilder optionSetSelectionBuilder = - OptionSetSelection.builder().optionSetSelectionMode(mode).optionSetUid(key); -// String options = DimensionalObjectUtils.getOptions(param); -// -// if (options != null && !options.isEmpty()) { -// List optionList = -// Stream.of(options.split("#")) -// .map( -// uid -> -// Objects.requireNonNull(this.idObjectManager.get(Option.class, uid)) -// .getCode()) -// .toList(); -// optionSetSelectionBuilder.options(optionList); -// } + OptionSetSelection.builder().optionSetSelectionMode(mode).optionSetUid(key); + String options = DimensionalObjectUtils.getOptions(param); + + if (options != null && !options.isEmpty()) { + List optionList = + Stream.of(options.split("#")) + .map( + uid -> + Objects.requireNonNull(this.idObjectManager.get(Option.class, uid)) + .getUid()) + .toList(); + optionSetSelectionBuilder.options(optionList); + } optionSetSelections.put(key, optionSetSelectionBuilder.build()); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index 558125a5a07a..b984df305eab 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -375,16 +375,16 @@ private boolean hasAggregation(DataQueryParams params) { } Optional optionSetSelectionMode = - params.getDataElements().stream() - .filter(de -> params.getOptionSetSelectionCriteria() != null) - .map( - de -> - params - .getOptionSetSelectionCriteria() - .getOptionSetSelections() - .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) - .getOptionSetSelectionMode()) - .findFirst(); + params.getDataElements().stream() + .filter(de -> params.getOptionSetSelectionCriteria() != null) + .map( + de -> + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) + .getOptionSetSelectionMode()) + .findFirst(); OptionSetSelectionMode mode = optionSetSelectionMode.orElse(OptionSetSelectionMode.AGGREGATED); return params.isAggregation() && mode == OptionSetSelectionMode.AGGREGATED; @@ -509,27 +509,27 @@ protected String getWhereClause(DataQueryParams params, AnalyticsTableType table /** Add where clause for option set selection. */ private void getWhereClauseOptions( - DataQueryParams params, SqlHelper sqlHelper, StringBuilder sql) { + DataQueryParams params, SqlHelper sqlHelper, StringBuilder sql) { if (!params.hasOptionSetSelectionCriteria()) { return; } params - .getOptionSetSelectionCriteria() - .getOptionSetSelections() - .forEach( - (key, value) -> { - List options = value.getOptions(); - if (options != null && !options.isEmpty()) { - sql.append(" ") - .append(sqlHelper.whereAnd()) - .append(" ") - .append(quote("optionvalueuid")) - .append(" in ('") - .append(String.join("','", options)) - .append("') "); - } - }); + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + List options = value.getOptions(); + if (options != null && !options.isEmpty()) { + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(quote("optionvalueuid")) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); } /** Add where clause dimensions. */ @@ -1018,15 +1018,17 @@ private Map getKeyValueMap(DataQueryParams params, String sql, i if (params.isDataType(TEXT)) { String value = rowSet.getString(VALUE_ID); - map.put(key.toString(), value); - } else // NUMERIC - { - Double value = rowSet.getDouble(VALUE_ID); + if (params.hasOptionSetInDimensionItems()) { map.put(key + DIMENSION_SEP + value, rowSet.getString("valuecount")); } else { map.put(key.toString(), value); } + } else // NUMERIC + { + Double value = rowSet.getDouble(VALUE_ID); + + map.put(key.toString() + counter, value); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 32173f9c3e83..d3a557fe994f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -1364,7 +1364,7 @@ public Builder withMultipleQueries(boolean multipleQueries) { } public Builder withOptionSetSelectionCriteria( - OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { + OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; return this; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index d5399ca41084..472346d40e64 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -758,15 +758,15 @@ protected String getAggregateClause(EventQueryParams params) { private AggregationType getAggregationType(EventQueryParams params) { if (params.getValue() instanceof DataElement - && ((DataElement) params.getValue()).hasOptionSet() - && params - .getOptionSetSelectionCriteria() - .getOptionSetSelections() - .get( + && ((DataElement) params.getValue()).hasOptionSet() + && params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get( params.getValue().getUid() - + "." - + ((DataElement) params.getValue()).getOptionSet().getUid()) - .getOptionSetSelectionMode() + + "." + + ((DataElement) params.getValue()).getOptionSet().getUid()) + .getOptionSetSelectionMode() != OptionSetSelectionMode.AGGREGATED) { return NONE; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java index a8cde4e236b9..4cedc1602854 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java @@ -163,10 +163,10 @@ public Grid getEvents(EventQueryParams params) { private static EventQueryParams getEventQueryParamsWithProgram(EventQueryParams params) { if (!params.hasProgram()) { Optional itemWithProgram = - params.getItems().stream().filter(QueryItem::hasProgram).findFirst(); + params.getItems().stream().filter(QueryItem::hasProgram).findFirst(); if (itemWithProgram.isPresent()) { EventQueryParams.Builder builder = - new EventQueryParams.Builder(params).withProgram(itemWithProgram.get().getProgram()); + new EventQueryParams.Builder(params).withProgram(itemWithProgram.get().getProgram()); params = builder.build(); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index a26f81635f22..1b8080f1663b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -631,22 +631,22 @@ protected String getWhereClause(EventQueryParams params) { private String getWhereClauseOptions(DataQueryParams params, SqlHelper sqlHelper) { StringBuilder sql = new StringBuilder(); params - .getOptionSetSelectionCriteria() - .getOptionSetSelections() - .forEach( - (key, value) -> { - List uids = Arrays.stream(key.split("\\.")).toList(); - List options = value.getOptions(); - if (!uids.isEmpty() && options != null && !options.isEmpty()) { - sql.append(" ") - .append(sqlHelper.whereAnd()) - .append(" ") - .append(quote(uids.get(0))) - .append(" in ('") - .append(String.join("','", options)) - .append("') "); - } - }); + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + List uids = Arrays.stream(key.split("\\.")).toList(); + List options = value.getOptions(); + if (!uids.isEmpty() && options != null && !options.isEmpty()) { + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(quote(uids.get(0) + ".optionvalueuid")) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); return sql.toString(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 3068a1222d4f..7f79311cb32a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -125,18 +125,18 @@ public class JdbcAnalyticsTableManager extends AbstractJdbcTableManager { .selectExpression("acs.categoryoptioncombouid as ao") .indexColumns(List.of("dx", "ao")) .build(), - AnalyticsTableColumn.builder() - .name("optionsetuid") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("deo.optionsetuid as optionsetuid") - .build(), - AnalyticsTableColumn.builder() - .name("optionvalueuid") - .dataType(CHARACTER_11) - .nullable(NULL) - .selectExpression("deo.optionvalueuid as optionvalueuid") - .build(), + AnalyticsTableColumn.builder() + .name("optionsetuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionsetuid as optionsetuid") + .build(), + AnalyticsTableColumn.builder() + .name("optionvalueuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionvalueuid as optionvalueuid") + .build(), AnalyticsTableColumn.builder() .name("pestartdate") .dataType(DATE) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index a63db132b50a..6a2c9529895e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -32,7 +32,6 @@ import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.AggregationType.AVERAGE; import static org.hisp.dhis.analytics.AggregationType.SUM; - import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.commons.util.TextUtils.emptyIfTrue; @@ -57,7 +56,6 @@ import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.AnalyticsAggregationType; @@ -481,11 +479,11 @@ private List getDataElementColumns(Program program) { .flatMap(Collection::stream) .toList()); columns.addAll( - program.getAnalyticsDataElements().stream() - .filter(DataElement::hasOptionSet) - .map(this::getColumnFromDataElementOptionSet) - .flatMap(Collection::stream) - .toList()); + program.getAnalyticsDataElements().stream() + .filter(DataElement::hasOptionSet) + .map(this::getColumnFromDataElementOptionSet) + .flatMap(Collection::stream) + .toList()); columns.addAll( program.getAnalyticsDataElementsWithLegendSet().stream() @@ -822,12 +820,12 @@ private List getColumnFromDataElementOptionSet(DataElement String sql = selectOptionValueCodeForInsert(dataElement, select, dataClause); columns.add( - AnalyticsTableColumn.builder() - .name(dataElement.getUid() + ".optionvalueuid") - .dataType(DataType.VARCHAR_255) - .selectExpression(sql) - .skipIndex(Skip.INCLUDE) - .build()); + AnalyticsTableColumn.builder() + .name(dataElement.getUid() + ".optionvalueuid") + .dataType(DataType.VARCHAR_255) + .selectExpression(sql) + .skipIndex(Skip.INCLUDE) + .build()); return columns; } @@ -837,43 +835,43 @@ private String getDataClause(String uid, ValueType valueType) { String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; return replace( - " and eventdatavalues #>> '{${uid},value}' ~* '${regex}'", - Map.of("uid", uid, "regex", regex)); + " and eventdatavalues #>> '{${uid},value}' ~* '${regex}'", + Map.of("uid", uid, "regex", regex)); } return ""; } private String selectOptionValueCodeForInsert( - DataElement dataElement, String fromType, String dataClause) { + DataElement dataElement, String fromType, String dataClause) { String innerSql = - replaceQualify( - """ + replaceQualify( + """ (select ${fromType} from ${event} \ where eventid=ev.eventid ${dataClause})${closingParentheses}""", - Map.of( - "fromType", - fromType, - "dataClause", - dataClause, - "closingParentheses", - getClosingParentheses(fromType), - "dataElementUid", - quote(dataElement.getUid()))); + Map.of( + "fromType", + fromType, + "dataClause", + dataClause, + "closingParentheses", + getClosingParentheses(fromType), + "dataElementUid", + quote(dataElement.getUid()))); return replaceQualify( - """ + """ (select optionvalueuid \ from analytics_rs_dataelementoption \ where dataelementuid = ${dataElementUid} \ and optionvaluecode = ${selectForInsert}::varchar) as ${alias}""", - Map.of( - "dataElementUid", - singleQuote(dataElement.getUid()), - "selectForInsert", - innerSql, - "alias", - quote(dataElement.getUid() + ".optionvalueuid"))); + Map.of( + "dataElementUid", + singleQuote(dataElement.getUid()), + "selectForInsert", + innerSql, + "alias", + quote(dataElement.getUid() + ".optionvalueuid"))); } private String getClosingParentheses(String str) { @@ -935,15 +933,15 @@ private boolean hasAggregation(DataQueryParams params) { } Optional optionSetSelectionMode = - params.getDataElements().stream() - .map( - de -> - params - .getOptionSetSelectionCriteria() - .getOptionSetSelections() - .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) - .getOptionSetSelectionMode()) - .findFirst(); + params.getDataElements().stream() + .map( + de -> + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) + .getOptionSetSelectionMode()) + .findFirst(); OptionSetSelectionMode mode = optionSetSelectionMode.orElse(OptionSetSelectionMode.AGGREGATED); return params.isAggregation() && mode == OptionSetSelectionMode.AGGREGATED; @@ -963,8 +961,8 @@ private String getAggregateValueColumn(DataQueryParams params) { String valueColumn = params.getValueColumn(); if (aggType.isAggregationType(SUM) - && aggType.isPeriodAggregationType(AVERAGE) - && aggType.isNumericDataType()) { + && aggType.isPeriodAggregationType(AVERAGE) + && aggType.isNumericDataType()) { sql = "sum(daysxvalue) / " + params.getDaysForAvgSumIntAggregation(); } else if (aggType.isAggregationType(AVERAGE) && aggType.isNumericDataType()) { sql = "avg(" + valueColumn + ")"; @@ -996,8 +994,7 @@ private String getAggregatedOptionValueClause(DataQueryParams params) { * @param dimensions the collection of {@link DimensionalObject}. * @return a comma-delimited string of quoted dimension names. */ - private String getCommaDelimitedQuotedDimensionColumns( - Collection dimensions) { + private String getCommaDelimitedQuotedDimensionColumns(Collection dimensions) { return join(",", getQuotedDimensionColumns(dimensions)); } @@ -1010,10 +1007,10 @@ private String getCommaDelimitedQuotedDimensionColumns( */ protected List getQuotedDimensionColumns(Collection dimensions) { return dimensions.stream() - .filter(d -> !d.isFixed()) - .map(DimensionalObject::getDimensionName) - .map(this::quoteAlias) - .collect(Collectors.toList()); + .filter(d -> !d.isFixed()) + .map(DimensionalObject::getDimensionName) + .map(this::quoteAlias) + .collect(Collectors.toList()); } /** @@ -1037,20 +1034,20 @@ private String getSelectClause(ValueType valueType, String columnName) { return "cast(" + columnName + " as bigint)"; } else if (valueType.isBoolean()) { return "case when " - + columnName - + " = 'true' then 1 when " - + columnName - + " = 'false' then 0 else null end"; + + columnName + + " = 'true' then 1 when " + + columnName + + " = 'false' then 0 else null end"; } else if (valueType.isDate()) { return "cast(" + columnName + " as timestamp)"; } else if (valueType.isGeo() && isSpatialSupport()) { return "ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (" - + columnName - + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; + + columnName + + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; } else if (valueType.isOrganisationUnit()) { return replaceQualify( - "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}", - Map.of("columnName", columnName)); + "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}", + Map.of("columnName", columnName)); } else { return columnName; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index d61542330763..5ab3e9de5dbb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -831,24 +831,24 @@ public static Map getDimensionMetadataItemMap( OptionSet optionSet = dataElement.getOptionSet(); if (optionSet != null) { map.put( - dataElement.getUid() + "." + optionSet.getUid(), - includeMetadataDetails - ? new MetadataItem( - optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) - : new MetadataItem(optionSet.getName())); + dataElement.getUid() + "." + optionSet.getUid(), + includeMetadataDetails + ? new MetadataItem( + optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) + : new MetadataItem(optionSet.getName())); } } if (DimensionItemType.PROGRAM_DATA_ELEMENT == item.getDimensionItemType() - && item instanceof ProgramDataElementDimensionItem programDataElement) { + && item instanceof ProgramDataElementDimensionItem programDataElement) { OptionSet optionSet = programDataElement.getOptionSet(); if (optionSet != null) { map.put( - programDataElement.getDataElement().getUid() + "." + optionSet.getUid(), - includeMetadataDetails - ? new MetadataItem( - optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) - : new MetadataItem(optionSet.getName())); + programDataElement.getDataElement().getUid() + "." + optionSet.getUid(), + includeMetadataDetails + ? new MetadataItem( + optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) + : new MetadataItem(optionSet.getName())); } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java index d8b7bbc82ea3..d032ba88003f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java @@ -163,8 +163,8 @@ private final List getResourceTables() { new DataElementResourceTable(logged, idObjectManager.getAllNoAcl(DataElement.class)), new DatePeriodResourceTable(logged, getAndValidateAvailableDataYears()), new PeriodResourceTable(logged, periodService.getAllPeriods()), - new CategoryOptionComboResourceTable(logged), - new DataElementOptionResourceTable(logged)); + new CategoryOptionComboResourceTable(logged), + new DataElementOptionResourceTable(logged)); } /** From 36520de35d2b7481171388423997d1ec246d7c80 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 09:58:01 +0100 Subject: [PATCH 08/16] cherrypicking, resolve conflicts, part3.2 --- .../java/org/hisp/dhis/analytics/data/handler/DataHandler.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java index d9e8735b44a5..ec9ca0eb36f2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java @@ -130,7 +130,6 @@ import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.event.data.EventAggregateService; -import org.hisp.dhis.analytics.event.data.EventQueryService; import org.hisp.dhis.analytics.resolver.ExpressionResolver; import org.hisp.dhis.analytics.resolver.ExpressionResolvers; import org.hisp.dhis.analytics.util.PeriodOffsetUtils; @@ -174,8 +173,6 @@ public class DataHandler { private final EventAggregateService eventAggregatedService; - private final EventQueryService eventQueryService; - private final RawAnalyticsManager rawAnalyticsManager; private final ExpressionResolvers resolvers; From 30ab37670fa8f60516f5c2f7ebba38b17f488828 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 10:03:45 +0100 Subject: [PATCH 09/16] code cleaning --- .../analytics/OptionSetSelectionCriteria.java | 50 +++---------------- .../OptionSetSelectionCriteriaV2.java | 38 -------------- .../hisp/dhis/analytics/DataQueryParams.java | 6 +-- .../data/DefaultDataQueryService.java | 8 +-- .../analytics/event/EventQueryParams.java | 4 +- .../data/AnalyticsServiceBaseTest.java | 4 -- .../data/handler/DataHandlerTest.java | 2 +- 7 files changed, 17 insertions(+), 95 deletions(-) delete mode 100644 dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java index e11c2b3ee8c7..5ec7324dd900 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java @@ -27,48 +27,12 @@ */ package org.hisp.dhis.analytics; +import java.util.Map; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter public class OptionSetSelectionCriteria { - // @Getter private Map optionSetSelectionModes; - // @Getter private List options; - // - // OptionSetSelectionCriteria( - // Map optionSetSelectionModes, List options) { - // this.optionSetSelectionModes = optionSetSelectionModes; - // this.options = options; - // } - // - // public static OptionSetSelectionCriteriaBuilder builder() { - // return new OptionSetSelectionCriteriaBuilder(); - // } - // - // public static class OptionSetSelectionCriteriaBuilder { - // private Map optionSetSelectionModes; - // private List options; - // - // OptionSetSelectionCriteriaBuilder() {} - // - // public OptionSetSelectionCriteriaBuilder optionSetSelectionModes( - // Map optionSetSelectionModes) { - // this.optionSetSelectionModes = Collections.unmodifiableMap(optionSetSelectionModes); - // return this; - // } - // - // public OptionSetSelectionCriteriaBuilder options(List options) { - // this.options = Collections.unmodifiableList(options); - // return this; - // } - // - // public OptionSetSelectionCriteria build() { - // return new OptionSetSelectionCriteria(this.optionSetSelectionModes, this.options); - // } - // - // public String toString() { - // return - // "OptionSetSelectionCriteria.OptionSetSelectionCriteriaBuilder(optionSetSelectionModes=" - // + this.optionSetSelectionModes - // + ", options=" - // + this.options - // + ")"; - // } - // } + private Map optionSetSelections; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java deleted file mode 100644 index 350ea38cb4e0..000000000000 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteriaV2.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2004-2024, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.analytics; - -import java.util.Map; -import lombok.Builder; -import lombok.Getter; - -@Builder -@Getter -public class OptionSetSelectionCriteriaV2 { - private Map optionSetSelections; -} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java index d97fe24a5e1d..09d6e1eea4b9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java @@ -236,7 +236,7 @@ public class DataQueryParams { protected AnalyticsAggregationType aggregationType; /** The option set selection criteria. */ - protected OptionSetSelectionCriteriaV2 optionSetSelectionCriteria; + protected OptionSetSelectionCriteria optionSetSelectionCriteria; /** The measure criteria, which is measure filters and corresponding values. */ protected Map measureCriteria = new HashMap<>(); @@ -1974,7 +1974,7 @@ public AnalyticsAggregationType getAggregationType() { return aggregationType; } - public OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria() { + public OptionSetSelectionCriteria getOptionSetSelectionCriteria() { return optionSetSelectionCriteria; } @@ -2823,7 +2823,7 @@ public Builder withAggregationType(AnalyticsAggregationType aggregationType) { } public Builder withOptionSetSelectionCriteria( - OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { + OptionSetSelectionCriteria optionSetSelectionCriteria) { this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; return this; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index 559eaf01b061..d16e9210f20d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -72,7 +72,7 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataQueryService; import org.hisp.dhis.analytics.OptionSetSelection; -import org.hisp.dhis.analytics.OptionSetSelectionCriteriaV2; +import org.hisp.dhis.analytics.OptionSetSelectionCriteria; import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.category.CategoryOptionCombo; @@ -181,9 +181,9 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { .build(); } - private OptionSetSelectionCriteriaV2 getOptionSetSelectionCriteria(Set dimensions) { - OptionSetSelectionCriteriaV2.OptionSetSelectionCriteriaV2Builder builder = - OptionSetSelectionCriteriaV2.builder(); + private OptionSetSelectionCriteria getOptionSetSelectionCriteria(Set dimensions) { + OptionSetSelectionCriteria.OptionSetSelectionCriteriaBuilder builder = + OptionSetSelectionCriteria.builder(); Map optionSetSelections = new HashMap<>(); for (String dimension : dimensions) { String param = DimensionalObjectUtils.getParamFromDimension(dimension); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index d3a557fe994f..d50be11d4a0d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -59,7 +59,7 @@ import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.EventOutputType; -import org.hisp.dhis.analytics.OptionSetSelectionCriteriaV2; +import org.hisp.dhis.analytics.OptionSetSelectionCriteria; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.QueryKey; import org.hisp.dhis.analytics.QueryParamsBuilder; @@ -1364,7 +1364,7 @@ public Builder withMultipleQueries(boolean multipleQueries) { } public Builder withOptionSetSelectionCriteria( - OptionSetSelectionCriteriaV2 optionSetSelectionCriteria) { + OptionSetSelectionCriteria optionSetSelectionCriteria) { this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; return this; } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java index bc35b3e3f59e..576a9fc9a32c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceBaseTest.java @@ -50,7 +50,6 @@ import org.hisp.dhis.analytics.data.handler.MetadataHandler; import org.hisp.dhis.analytics.data.handler.SchemeIdResponseMapper; import org.hisp.dhis.analytics.event.data.EventAggregateService; -import org.hisp.dhis.analytics.event.data.EventQueryService; import org.hisp.dhis.analytics.resolver.ExpressionResolvers; import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.external.conf.DhisConfigurationProvider; @@ -88,8 +87,6 @@ abstract class AnalyticsServiceBaseTest { @Mock protected EventAggregateService eventAggregatedService; - @Mock protected EventQueryService eventQueryService; - @Mock private DataQueryService dataQueryService; @Mock private SchemeIdResponseMapper schemeIdResponseMapper; @@ -118,7 +115,6 @@ public void baseSetUp() { DataHandler dataHandler = new DataHandler( eventAggregatedService, - eventQueryService, rawAnalyticsManager, resolvers, expressionService, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java index 8892449e4550..79766d139804 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/handler/DataHandlerTest.java @@ -180,6 +180,6 @@ void testOperandDataQueryParamsWithAttrOptionComboInFilter() { } private DataHandler withNullDependencies() { - return new DataHandler(null, null, null, null, null, null, null, null, null, null); + return new DataHandler(null, null, null, null, null, null, null, null, null); } } From 5fd534348072bc7d1bab4869c79af743eb13125f Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 10:08:05 +0100 Subject: [PATCH 10/16] code cleaning --- .../org/hisp/dhis/analytics/data/DefaultDataQueryService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index 0ea846626b32..715ced1c037c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -65,8 +65,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Stream; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsSecurityManager; import org.hisp.dhis.analytics.DataQueryParams; From 4c48f68ecb17129d65098708f77a6d91e3d20297 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 10:22:35 +0100 Subject: [PATCH 11/16] QA issues --- .../AbstractJdbcEventAnalyticsManager.java | 8 ++++---- .../table/JdbcEventAnalyticsTableManager.java | 19 +------------------ 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 472346d40e64..ae927a11be01 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -757,15 +757,15 @@ protected String getAggregateClause(EventQueryParams params) { private AggregationType getAggregationType(EventQueryParams params) { - if (params.getValue() instanceof DataElement - && ((DataElement) params.getValue()).hasOptionSet() + if (params.getValue() instanceof DataElement dataElement + && dataElement.hasOptionSet() && params .getOptionSetSelectionCriteria() .getOptionSetSelections() .get( - params.getValue().getUid() + dataElement.getUid() + "." - + ((DataElement) params.getValue()).getOptionSet().getUid()) + + dataElement.getOptionSet().getUid()) .getOptionSetSelectionMode() != OptionSetSelectionMode.AGGREGATED) { return NONE; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 6a2c9529895e..4e00bc30f54b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -55,7 +55,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.AnalyticsAggregationType; @@ -892,22 +891,6 @@ private String getClosingParentheses(String str) { return StringUtils.repeat(")", open); } - /** - * Generates the select clause of the query SQL. - * - * @param params the {@link DataQueryParams}. - * @return a SQL select clause. - */ - private String getSelectClause(DataQueryParams params) { - String sql = "select " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + ", "; - - sql += getValueClause(params); - - sql += getAggregatedOptionValueClause(params); - - return sql; - } - /** * Generates the value clause of the query SQL. * @@ -1010,7 +993,7 @@ protected List getQuotedDimensionColumns(Collection d .filter(d -> !d.isFixed()) .map(DimensionalObject::getDimensionName) .map(this::quoteAlias) - .collect(Collectors.toList()); + .toList(); } /** From 55ed40dfbd8c34318363a4d2a8425933b93b62db Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 10:22:46 +0100 Subject: [PATCH 12/16] QA issues --- .../event/data/AbstractJdbcEventAnalyticsManager.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index ae927a11be01..e2360ed7fbad 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -762,10 +762,7 @@ private AggregationType getAggregationType(EventQueryParams params) { && params .getOptionSetSelectionCriteria() .getOptionSetSelections() - .get( - dataElement.getUid() - + "." - + dataElement.getOptionSet().getUid()) + .get(dataElement.getUid() + "." + dataElement.getOptionSet().getUid()) .getOptionSetSelectionMode() != OptionSetSelectionMode.AGGREGATED) { return NONE; From f5c2da517653a3f732f4cb95ab670bb4676f13b0 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 10:38:35 +0100 Subject: [PATCH 13/16] QA issues --- .../event/data/JdbcEventAnalyticsManager.java | 5 + .../table/JdbcEventAnalyticsTableManager.java | 120 ------------------ 2 files changed, 5 insertions(+), 120 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 1b8080f1663b..2d95b71e02dc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -629,6 +629,11 @@ protected String getWhereClause(EventQueryParams params) { } private String getWhereClauseOptions(DataQueryParams params, SqlHelper sqlHelper) { + if (!params.hasOptionSetSelectionCriteria() + || params.getOptionSetSelectionCriteria().getOptionSetSelections() == null) { + return ""; + } + StringBuilder sql = new StringBuilder(); params .getOptionSetSelectionCriteria() diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4e00bc30f54b..a84d344c01ad 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -30,8 +30,6 @@ import static java.lang.String.join; import static java.util.stream.Collectors.toList; import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.hisp.dhis.analytics.AggregationType.AVERAGE; -import static org.hisp.dhis.analytics.AggregationType.SUM; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; import static org.hisp.dhis.commons.util.TextUtils.emptyIfTrue; @@ -54,15 +52,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; -import org.hisp.dhis.analytics.DataQueryParams; -import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsDimensionType; import org.hisp.dhis.analytics.table.model.AnalyticsTable; @@ -74,7 +68,6 @@ import org.hisp.dhis.calendar.Calendar; import org.hisp.dhis.category.Category; import org.hisp.dhis.category.CategoryService; -import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.collection.ListUtils; @@ -891,119 +884,6 @@ private String getClosingParentheses(String str) { return StringUtils.repeat(")", open); } - /** - * Generates the value clause of the query SQL. - * - * @param params the {@link DataQueryParams}. - * @return a SQL value clause. - */ - protected String getValueClause(DataQueryParams params) { - String sql = ""; - - if (hasAggregation(params)) { - sql += getAggregateValueColumn(params); - } else { - sql += params.getValueColumn(); - } - - return sql + " as value "; - } - - private boolean hasAggregation(DataQueryParams params) { - // analytics query is an item of sequential queries with one data element only. - if (params.getDataElements().size() != 1) { - return params.isAggregation(); - } - - Optional optionSetSelectionMode = - params.getDataElements().stream() - .map( - de -> - params - .getOptionSetSelectionCriteria() - .getOptionSetSelections() - .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) - .getOptionSetSelectionMode()) - .findFirst(); - OptionSetSelectionMode mode = optionSetSelectionMode.orElse(OptionSetSelectionMode.AGGREGATED); - - return params.isAggregation() && mode == OptionSetSelectionMode.AGGREGATED; - } - - /** - * Returns an aggregate clause for the numeric value column. - * - * @param params the {@link DataQueryParams}. - * @return a SQL numeric value column. - */ - private String getAggregateValueColumn(DataQueryParams params) { - String sql; - - AnalyticsAggregationType aggType = params.getAggregationType(); - - String valueColumn = params.getValueColumn(); - - if (aggType.isAggregationType(SUM) - && aggType.isPeriodAggregationType(AVERAGE) - && aggType.isNumericDataType()) { - sql = "sum(daysxvalue) / " + params.getDaysForAvgSumIntAggregation(); - } else if (aggType.isAggregationType(AVERAGE) && aggType.isNumericDataType()) { - sql = "avg(" + valueColumn + ")"; - } else if (aggType.isAggregationType(AVERAGE) && aggType.isBooleanDataType()) { - sql = "sum(daysxvalue) / sum(daysno) * 100"; - } else // SUM and no value - { - sql = "sum(" + valueColumn + ")"; - } - - return sql; - } - - private String getAggregatedOptionValueClause(DataQueryParams params) { - String sql = ""; - - if (params.hasOptionSetInDimensionItems() && hasAggregation(params)) { - sql += ", count(" + params.getValueColumn() + ") as valuecount "; - return sql; - } - - return sql; - } - - /** - * Generates a comma-delimited string with the dimension names of the given dimensions where each - * dimension name is quoted. Dimensions which are considered fixed will be excluded. - * - * @param dimensions the collection of {@link DimensionalObject}. - * @return a comma-delimited string of quoted dimension names. - */ - private String getCommaDelimitedQuotedDimensionColumns(Collection dimensions) { - return join(",", getQuotedDimensionColumns(dimensions)); - } - - /** - * Generates a list of the dimension names of the given dimensions where each dimension name is - * quoted. Dimensions which are considered fixed will be excluded. - * - * @param dimensions the collection of {@link DimensionalObject}. - * @return a list of quoted dimension names. - */ - protected List getQuotedDimensionColumns(Collection dimensions) { - return dimensions.stream() - .filter(d -> !d.isFixed()) - .map(DimensionalObject::getDimensionName) - .map(this::quoteAlias) - .toList(); - } - - /** - * @param relation the relation to quote. - * @return an "ax" aliased and double quoted relation. - */ - private String quoteAlias(String relation) { - return sqlBuilder.quoteAx(relation); - } - /** * Returns the select clause, potentially with a cast statement, based on the given value type. * From 5bcf1189c1387729da8a721c1e7179cc1731eb7f Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 10:57:31 +0100 Subject: [PATCH 14/16] bugfix unit test --- .../hisp/dhis/common/DimensionalObjectUtils.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index a1e06797fddb..a03a24b8ebab 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -541,6 +541,10 @@ public static boolean isCompositeDimensionalObject(String expression) { * @return the first identifier, or null if not a valid composite identifier or no match. */ public static String getFirstIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); return matcher.matches() ? matcher.group("id1") : null; } @@ -552,6 +556,10 @@ public static String getFirstIdentifier(String compositeItem) { * @return the second identifier, or null if not a valid composite identifier or no match. */ public static String getSecondIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); return matcher.matches() ? matcher.group("id2") : null; } @@ -563,11 +571,19 @@ public static String getSecondIdentifier(String compositeItem) { * @return the second identifier, or null if not a valid composite identifier or no match. */ public static String getThirdIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); return matcher.matches() ? matcher.group("id3") : null; } public static OptionSetSelectionMode getOptionSetSelectionMode(String compositeItem) { + if (compositeItem == null) { + return null; + } + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); if (matcher.matches()) { String suffix = matcher.group("suffix"); From 092ab3e570149a9c4c4f021659c1aef56fbab265 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 11:39:36 +0100 Subject: [PATCH 15/16] bugfix unit test --- .../org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index b984df305eab..7889544c140e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -1027,8 +1027,11 @@ private Map getKeyValueMap(DataQueryParams params, String sql, i } else // NUMERIC { Double value = rowSet.getDouble(VALUE_ID); - - map.put(key.toString() + counter, value); + if (params.hasOptionSetInDimensionItems()) { + map.put(key.toString() + counter, value); + } else { + map.put(key.toString(), value); + } } } From ad4b4a86e64e0e83a1dff928793ac8834debd8b1 Mon Sep 17 00:00:00 2001 From: d-bernat Date: Mon, 23 Dec 2024 13:38:00 +0100 Subject: [PATCH 16/16] bugfix unit test --- .../dimension/DataDimensionExtractor.java | 21 +++++++++++++++++++ .../dimension/DefaultDimensionService.java | 15 +++++++------ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java index b7dea4affd99..6e1a229fd76d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java @@ -52,6 +52,7 @@ import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.indicator.Indicator; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramDataElementDimensionItem; import org.hisp.dhis.program.ProgramIndicator; @@ -230,6 +231,26 @@ public ReportingRate getReportingRate(IdScheme idScheme, String dataSetId, Strin return new ReportingRate(dataSet, ReportingRateMetric.valueOf(metric)); } + /** + * Returns a {@link DataElement}. + * + * @param idScheme the identifier scheme. + * @param dataElementId the data element identifier. + * @param optionSetId the option set identifier. + */ + @Transactional(readOnly = true) + public DataElement getOptionSetDataElementDimensionItem( + IdScheme idScheme, String dataElementId, String optionSetId) { + DataElement dataElement = idObjectManager.getObject(DataElement.class, idScheme, dataElementId); + OptionSet optionSet = idObjectManager.getObject(OptionSet.class, idScheme, optionSetId); + + if (dataElement == null || optionSet == null) { + return null; + } + + return dataElement; + } + /** * Returns a {@link ProgramTrackedEntityAttributeDimensionItem}. * diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java index a3093eeef652..3b1f43c75e61 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java @@ -40,7 +40,6 @@ import static org.hisp.dhis.common.DimensionType.PROGRAM_ATTRIBUTE; import static org.hisp.dhis.common.DimensionType.PROGRAM_DATA_ELEMENT; import static org.hisp.dhis.common.DimensionType.PROGRAM_INDICATOR; -import static org.hisp.dhis.common.DimensionalObjectUtils.COMPOSITE_DIM_OBJECT_ESCAPED_SEP; import static org.hisp.dhis.common.IdScheme.UID; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.commons.util.TextUtils.splitSafe; @@ -348,9 +347,10 @@ public DimensionalItemObject getDataDimensionalItemObject(String dimensionItem) public DimensionalItemObject getDataDimensionalItemObject( IdScheme idScheme, String dimensionItem) { if (DimensionalObjectUtils.isCompositeDimensionalObject(dimensionItem)) { - String id0 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 0); - String id1 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 1); - String id2 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 2); + + String id0 = DimensionalObjectUtils.getFirstIdentifier(dimensionItem); + String id1 = DimensionalObjectUtils.getSecondIdentifier(dimensionItem); + String id2 = DimensionalObjectUtils.getThirdIdentifier(dimensionItem); String optionSetSelectionMode = splitSafe(dimensionItem, "-", 1); if (optionSetSelectionMode != null && id2 != null) { @@ -359,6 +359,7 @@ public DimensionalItemObject getDataDimensionalItemObject( id1 = splitSafe(id1, "-", 0); } + DataElement dataElementWithOptionSet; DataElementOperand operand; ReportingRate reportingRate; ProgramDataElementDimensionItem programDataElement; @@ -381,8 +382,10 @@ public DimensionalItemObject getDataDimensionalItemObject( return programAttribute; } - if (!idScheme.is(IdentifiableProperty.UID) || CodeGenerator.isValidUid(id0)) { - return idObjectManager.get(DataDimensionItem.DATA_DIM_CLASSES, idScheme, id0); + if ((dataElementWithOptionSet = + dataDimensionExtractor.getOptionSetDataElementDimensionItem(idScheme, id0, id1)) + != null) { + return dataElementWithOptionSet; } } else if (!idScheme.is(IdentifiableProperty.UID) || CodeGenerator.isValidUid(dimensionItem)) {