diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java new file mode 100644 index 000000000000..d18baeb326ef --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelection.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.List; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class OptionSetSelection { + private String optionSetUid; + private List options; + private OptionSetSelectionMode optionSetSelectionMode; +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java new file mode 100644 index 000000000000..5ec7324dd900 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionCriteria.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.Map; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class OptionSetSelectionCriteria { + private Map optionSetSelections; +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java new file mode 100644 index 000000000000..34c42a441821 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/analytics/OptionSetSelectionMode.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics; + +import java.util.Arrays; +import java.util.List; + +/** The selection modes for items with option sets */ +public enum OptionSetSelectionMode { + // All options in an option set are chosen and aggregated into a single column. + // This selection is relative, so any new options added to the option set are included. + AGGREGATED, + // All options in an option set are chosen and displayed as data items. + // This selection is relative, so any new options added to the option set are included. + DISAGGREGATED; + + public static List getOptionSetSelectionModes() { + return Arrays.stream(OptionSetSelectionMode.values()).map(Enum::toString).toList(); + } +} diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java index a7786ad46813..a03a24b8ebab 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/DimensionalObjectUtils.java @@ -57,6 +57,7 @@ import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.common.comparator.ObjectStringValueComparator; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.eventvisualization.Attribute; @@ -81,12 +82,16 @@ public class DimensionalObjectUtils { public static final String COL_SEP = " "; + public static final String OPTION_SET_SELECTION_MODE_SEP = "-"; + /** * Matching data element operand, program data element, program attribute, data set reporting rate * metric. */ + // Luqe6ps5KZ9.uTLkjHWtSL8.R0jROOT3zni-AGGREGATED private static final Pattern COMPOSITE_DIM_OBJECT_PATTERN = - Pattern.compile("(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?"); + Pattern.compile( + "(?\\w+)\\.(?\\w+|\\*)(\\.(?\\w+|\\*))?(\\[(?[^\\]]*?)\\])?(-(?AGGREGATED|DISAGGREGATED)?)?"); private static final Set IGNORED_OPERATORS = Set.of(QueryOperator.LIKE, QueryOperator.IN, QueryOperator.SW, QueryOperator.EW); @@ -359,6 +364,20 @@ public static String getDimensionFromParam(String param) { return param.split(DIMENSION_NAME_SEP).length > 0 ? param.split(DIMENSION_NAME_SEP)[0] : param; } + /** + * Retrieves the param name from the given string. Returns the part of the string after the + * dimension name separator, or the whole string if the separator is not present. + * + * @param param the parameter. + */ + public static String getParamFromDimension(String param) { + if (param == null) { + return null; + } + + return param.split(DIMENSION_NAME_SEP).length > 1 ? param.split(DIMENSION_NAME_SEP)[1] : param; + } + /** * Retrieves the dimension options from the given string. Looks for the part succeeding the * dimension name separator, if exists, splits the string part on the option separator and returns @@ -521,9 +540,28 @@ public static boolean isCompositeDimensionalObject(String expression) { * @param compositeItem the composite dimension object identifier. * @return the first identifier, or null if not a valid composite identifier or no match. */ - public static String getFirstIdentifer(String compositeItem) { + public static String getFirstIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + return matcher.matches() ? matcher.group("id1") : null; + } + + /** + * Returns the second identifier in a composite dimension object identifier. + * + * @param compositeItem the composite dimension object identifier. + * @return the second identifier, or null if not a valid composite identifier or no match. + */ + public static String getSecondIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? matcher.group(1) : null; + return matcher.matches() ? matcher.group("id2") : null; } /** @@ -532,9 +570,34 @@ public static String getFirstIdentifer(String compositeItem) { * @param compositeItem the composite dimension object identifier. * @return the second identifier, or null if not a valid composite identifier or no match. */ - public static String getSecondIdentifer(String compositeItem) { + public static String getThirdIdentifier(String compositeItem) { + if (compositeItem == null) { + return null; + } + + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + return matcher.matches() ? matcher.group("id3") : null; + } + + public static OptionSetSelectionMode getOptionSetSelectionMode(String compositeItem) { + if (compositeItem == null) { + return null; + } + + Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); + if (matcher.matches()) { + String suffix = matcher.group("suffix"); + return suffix != null + ? OptionSetSelectionMode.valueOf(suffix) + : OptionSetSelectionMode.AGGREGATED; + } + + return OptionSetSelectionMode.AGGREGATED; + } + + public static String getOptions(String compositeItem) { Matcher matcher = COMPOSITE_DIM_OBJECT_PATTERN.matcher(compositeItem); - return matcher.matches() ? matcher.group(2) : null; + return matcher.matches() ? matcher.group("list") : null; } /** diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index e21ab1118fc3..0fd3dcaec55b 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -72,6 +72,7 @@ public enum ErrorCode { E1126("Category combo {0} cannot combine more than {1} categories, but had: {2}"), E1127("Category {0} cannot have more than {1} options, but had: {2} "), E1128("Category combo {0} cannot have more than {1} combinations, but requires: {2}"), + E1129("Option set selection mode must match valid mode: `{0}`"), /* Org unit merge */ E1500("At least two source orgs unit must be specified"), diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java index d68df61a942f..c4a124a31b32 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/common/DimensionalObjectUtilsTest.java @@ -177,15 +177,15 @@ void testGetDataElementOperandIdSchemeCodeMap() { @Test void testGetFirstSecondIdentifier() { assertEquals( - "A123456789A", DimensionalObjectUtils.getFirstIdentifer("A123456789A.P123456789A")); - assertNull(DimensionalObjectUtils.getFirstIdentifer("A123456789A")); + "A123456789A", DimensionalObjectUtils.getFirstIdentifier("A123456789A.P123456789A")); + assertNull(DimensionalObjectUtils.getFirstIdentifier("A123456789A")); } @Test void testGetSecondIdentifier() { assertEquals( - "P123456789A", DimensionalObjectUtils.getSecondIdentifer("A123456789A.P123456789A")); - assertNull(DimensionalObjectUtils.getSecondIdentifer("A123456789A")); + "P123456789A", DimensionalObjectUtils.getSecondIdentifier("A123456789A.P123456789A")); + assertNull(DimensionalObjectUtils.getSecondIdentifier("A123456789A")); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java index c350db389e18..09d6e1eea4b9 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java @@ -87,6 +87,7 @@ import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.common.DimensionItemKeywords; import org.hisp.dhis.common.DimensionItemObjectValue; +import org.hisp.dhis.common.DimensionItemType; import org.hisp.dhis.common.DimensionType; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; @@ -234,6 +235,9 @@ public class DataQueryParams { /** The aggregation type. */ protected AnalyticsAggregationType aggregationType; + /** The option set selection criteria. */ + protected OptionSetSelectionCriteria optionSetSelectionCriteria; + /** The measure criteria, which is measure filters and corresponding values. */ protected Map measureCriteria = new HashMap<>(); @@ -498,6 +502,7 @@ public T copyTo(T params) { params.dimensions = DimensionalObjectUtils.getCopies(this.dimensions); params.filters = DimensionalObjectUtils.getCopies(this.filters); params.aggregationType = this.aggregationType != null ? this.aggregationType.instance() : null; + params.optionSetSelectionCriteria = this.optionSetSelectionCriteria; params.measureCriteria = new HashMap<>(this.measureCriteria); params.preAggregateMeasureCriteria = new HashMap<>(this.preAggregateMeasureCriteria); params.skipMeta = this.skipMeta; @@ -591,6 +596,7 @@ protected QueryKey getQueryKey() { (k, v) -> key.add("preAggregateMeasureCriteria", (String.valueOf(k) + v))); return key.add("aggregationType", aggregationType) + .add("optionSetSelectionCriteria", optionSetSelectionCriteria) .add("skipMeta", skipMeta) .add("skipData", skipData) .add("skipHeaders", skipHeaders) @@ -746,6 +752,11 @@ public boolean hasOrganisationUnitGroupSets() { return !getDimensionsAndFilters(ORGANISATION_UNIT_GROUP_SET).isEmpty(); } + /** Indicates whether option set selection criteria are present as dimension. */ + public boolean hasOptionSetSelectionCriteria() { + return optionSetSelectionCriteria != null; + } + /** * Returns the period type of the first period specified as filter, or null if there is no period * filter. @@ -861,6 +872,17 @@ public boolean isOutputFormat(OutputFormat format) { return this.outputFormat != null && this.outputFormat == format; } + public boolean hasOptionSetInDimensionItems() { + return dimensions.stream() + .anyMatch( + d -> + d.getItems().stream() + .anyMatch( + it -> + it.getDimensionItemType() == DimensionItemType.DATA_ELEMENT + && ((DataElement) it).getOptionSet() != null)); + } + /** * Creates a mapping between the data periods, based on the data period type for this query, and * the aggregation periods for this query. @@ -1952,6 +1974,10 @@ public AnalyticsAggregationType getAggregationType() { return aggregationType; } + public OptionSetSelectionCriteria getOptionSetSelectionCriteria() { + return optionSetSelectionCriteria; + } + public Map getMeasureCriteria() { return measureCriteria; } @@ -2796,6 +2822,12 @@ public Builder withAggregationType(AnalyticsAggregationType aggregationType) { return this; } + public Builder withOptionSetSelectionCriteria( + OptionSetSelectionCriteria optionSetSelectionCriteria) { + this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; + return this; + } + public Builder withSkipMeta(boolean skipMeta) { this.params.skipMeta = skipMeta; return this; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java index b5511f5ebad6..715ced1c037c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultDataQueryService.java @@ -58,16 +58,22 @@ import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsSecurityManager; import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataQueryService; +import org.hisp.dhis.analytics.OptionSetSelection; +import org.hisp.dhis.analytics.OptionSetSelectionCriteria; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.common.AnalyticalObject; @@ -75,12 +81,15 @@ import org.hisp.dhis.common.DataQueryRequest; import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.common.DimensionalObject; +import org.hisp.dhis.common.DimensionalObjectUtils; import org.hisp.dhis.common.DisplayProperty; import org.hisp.dhis.common.EventDataQueryRequest; import org.hisp.dhis.common.IdScheme; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.feedback.ErrorMessage; +import org.hisp.dhis.option.Option; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.setting.UserSettings; import org.hisp.dhis.user.User; @@ -114,6 +123,7 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { if (isNotEmpty(request.getDimension())) { params.addDimensions(getDimensionalObjects(request)); + params.withOptionSetSelectionCriteria(getOptionSetSelectionCriteria(request.getDimension())); } if (isNotEmpty(request.getFilter())) { @@ -171,6 +181,63 @@ public DataQueryParams getFromRequest(DataQueryRequest request) { .build(); } + private OptionSetSelectionCriteria getOptionSetSelectionCriteria(Set dimensions) { + OptionSetSelectionCriteria.OptionSetSelectionCriteriaBuilder builder = + OptionSetSelectionCriteria.builder(); + Map optionSetSelections = new HashMap<>(); + for (String dimension : dimensions) { + String param = DimensionalObjectUtils.getParamFromDimension(dimension); + + if (!hasOptionSet(param)) { + continue; + } + + OptionSetSelectionMode mode = DimensionalObjectUtils.getOptionSetSelectionMode(param); + + String key = DimensionalObjectUtils.getThirdIdentifier(param); + if (key == null) { + key = + DimensionalObjectUtils.getFirstIdentifier(param) + + "." + + DimensionalObjectUtils.getSecondIdentifier(param); + } else { + key = DimensionalObjectUtils.getSecondIdentifier(param) + "." + key; + } + + OptionSetSelection.OptionSetSelectionBuilder optionSetSelectionBuilder = + OptionSetSelection.builder().optionSetSelectionMode(mode).optionSetUid(key); + String options = DimensionalObjectUtils.getOptions(param); + + if (options != null && !options.isEmpty()) { + List optionList = + Stream.of(options.split("#")) + .map( + uid -> + Objects.requireNonNull(this.idObjectManager.get(Option.class, uid)) + .getUid()) + .toList(); + optionSetSelectionBuilder.options(optionList); + } + + optionSetSelections.put(key, optionSetSelectionBuilder.build()); + } + + if (optionSetSelections.isEmpty()) { + return null; + } + + return builder.optionSetSelections(optionSetSelections).build(); + } + + private boolean hasOptionSet(String param) { + String uid = DimensionalObjectUtils.getThirdIdentifier(param); + if (uid == null) { + uid = DimensionalObjectUtils.getSecondIdentifier(param); + } + + return uid != null && idObjectManager.exists(OptionSet.class, uid); + } + @Override @Transactional(readOnly = true) public DataQueryParams getFromAnalyticalObject(AnalyticalObject object) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index 89935206684f..7889544c140e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -58,6 +58,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; @@ -71,6 +72,7 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.DataType; import org.hisp.dhis.analytics.MeasureFilter; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.QueryPlanner; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.table.model.Partitions; @@ -85,6 +87,7 @@ import org.hisp.dhis.commons.util.DebugUtils; import org.hisp.dhis.commons.util.SqlHelper; import org.hisp.dhis.commons.util.TextUtils; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -342,6 +345,7 @@ private String getSelectClause(DataQueryParams params) { String sql = "select " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + ", "; sql += getValueClause(params); + sql += getAggregatedOptionValueClause(params); return sql; } @@ -355,7 +359,7 @@ private String getSelectClause(DataQueryParams params) { protected String getValueClause(DataQueryParams params) { String sql = ""; - if (params.isAggregation()) { + if (hasAggregation(params)) { sql += getAggregateValueColumn(params); } else { sql += params.getValueColumn(); @@ -364,6 +368,39 @@ protected String getValueClause(DataQueryParams params) { return sql + " as value "; } + private boolean hasAggregation(DataQueryParams params) { + // analytics query is an item of sequential queries with one data element only. + if (params.getDataElements().size() != 1) { + return params.isAggregation(); + } + + Optional optionSetSelectionMode = + params.getDataElements().stream() + .filter(de -> params.getOptionSetSelectionCriteria() != null) + .map( + de -> + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get(de.getUid() + "." + ((DataElement) de).getOptionSet().getUid()) + .getOptionSetSelectionMode()) + .findFirst(); + OptionSetSelectionMode mode = optionSetSelectionMode.orElse(OptionSetSelectionMode.AGGREGATED); + + return params.isAggregation() && mode == OptionSetSelectionMode.AGGREGATED; + } + + protected String getAggregatedOptionValueClause(DataQueryParams params) { + String sql = ""; + + if (params.hasOptionSetInDimensionItems() && hasAggregation(params)) { + sql += ", count(" + params.getValueColumn() + ") as valuecount "; + return sql; + } + + return sql; + } + /** * Returns an aggregate clause for the numeric value column. * @@ -463,12 +500,38 @@ protected String getWhereClause(DataQueryParams params, AnalyticsTableType table getWhereClauseDimensions(params, sqlHelper, sql); getWhereClauseFilters(params, sqlHelper, sql); + getWhereClauseOptions(params, sqlHelper, sql); getWhereClauseDataApproval(params, sqlHelper, sql); getWhereClauseRestrictions(params, sqlHelper, sql, tableType); return sql.toString(); } + /** Add where clause for option set selection. */ + private void getWhereClauseOptions( + DataQueryParams params, SqlHelper sqlHelper, StringBuilder sql) { + if (!params.hasOptionSetSelectionCriteria()) { + return; + } + + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + List options = value.getOptions(); + if (options != null && !options.isEmpty()) { + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(quote("optionvalueuid")) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); + } + /** Add where clause dimensions. */ private void getWhereClauseDimensions( DataQueryParams params, SqlHelper sqlHelper, StringBuilder sql) { @@ -640,13 +703,11 @@ private void getWhereClauseRestrictions( * @return a SQL group by clause. */ protected String getGroupByClause(DataQueryParams params) { - String sql = ""; - - if (params.isAggregation()) { - sql = "group by " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + " "; + if (hasAggregation(params)) { + return "group by " + getCommaDelimitedQuotedDimensionColumns(params.getDimensions()) + " "; } - return sql; + return ""; } /** @@ -957,11 +1018,20 @@ private Map getKeyValueMap(DataQueryParams params, String sql, i if (params.isDataType(TEXT)) { String value = rowSet.getString(VALUE_ID); - map.put(key.toString(), value); + + if (params.hasOptionSetInDimensionItems()) { + map.put(key + DIMENSION_SEP + value, rowSet.getString("valuecount")); + } else { + map.put(key.toString(), value); + } } else // NUMERIC { Double value = rowSet.getDouble(VALUE_ID); - map.put(key.toString(), value); + if (params.hasOptionSetInDimensionItems()) { + map.put(key.toString() + counter, value); + } else { + map.put(key.toString(), value); + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java index 998fcb80d55c..ec9ca0eb36f2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java @@ -426,6 +426,7 @@ public void addProgramDataElementAttributeIndicatorValues(DataQueryParams params EventQueryParams eventQueryParams = new EventQueryParams.Builder(fromDataQueryParams(dataSourceParams)) .withSkipMeta(true) + .withOptionSetSelectionCriteria(params.getOptionSetSelectionCriteria()) .build(); Grid eventGrid = eventAggregatedService.getAggregatedData(eventQueryParams); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java index 4d6f0683f691..d50be11d4a0d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java @@ -59,6 +59,7 @@ import org.hisp.dhis.analytics.AnalyticsAggregationType; import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.EventOutputType; +import org.hisp.dhis.analytics.OptionSetSelectionCriteria; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.QueryKey; import org.hisp.dhis.analytics.QueryParamsBuilder; @@ -305,6 +306,7 @@ protected EventQueryParams instance() { params.rowContext = this.rowContext; params.multipleQueries = this.multipleQueries; params.userOrganisationUnitsCriteria = this.userOrganisationUnitsCriteria; + params.optionSetSelectionCriteria = this.optionSetSelectionCriteria; return params; } @@ -1360,5 +1362,11 @@ public Builder withMultipleQueries(boolean multipleQueries) { this.params.multipleQueries = multipleQueries; return this; } + + public Builder withOptionSetSelectionCriteria( + OptionSetSelectionCriteria optionSetSelectionCriteria) { + this.params.optionSetSelectionCriteria = optionSetSelectionCriteria; + return this; + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java index 6f5ccba6f53c..7b0f2795c9d0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java @@ -41,6 +41,14 @@ public interface EventQueryPlanner { */ List planAggregateQuery(EventQueryParams params); + /** + * Plans the given parameters and returns a list of parameters. + * + * @param params the event query parameters. + * @return a list of {@link EventQueryParams}. + */ + List planQuery(EventQueryParams params); + /** * Plans the given parameters and returns a list of parameters. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java index 55a58883bd63..e2360ed7fbad 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java @@ -46,6 +46,7 @@ import static org.hisp.dhis.analytics.QueryKey.NV; import static org.hisp.dhis.analytics.SortOrder.ASC; import static org.hisp.dhis.analytics.SortOrder.DESC; +import static org.hisp.dhis.analytics.data.QueryPlannerUtils.getAggregationType; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getHeaderColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getOrgUnitLevelColumns; import static org.hisp.dhis.analytics.event.data.EnrollmentQueryHelper.getPeriodColumns; @@ -89,6 +90,7 @@ import org.apache.commons.lang3.time.DateUtils; import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.analytics.EventOutputType; +import org.hisp.dhis.analytics.OptionSetSelectionMode; import org.hisp.dhis.analytics.SortOrder; import org.hisp.dhis.analytics.analyze.ExecutionPlanStore; import org.hisp.dhis.analytics.common.ProgramIndicatorSubqueryBuilder; @@ -111,6 +113,7 @@ import org.hisp.dhis.commons.collection.ListUtils; import org.hisp.dhis.commons.util.SqlHelper; import org.hisp.dhis.commons.util.TextUtils; +import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.option.Option; @@ -591,6 +594,11 @@ public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLi private String getGroupByClause(EventQueryParams params) { String sql = ""; + AggregationType aggregationType = getAggregationType(params); + if (aggregationType == NONE) { + return sql; + } + if (params.isAggregation()) { List selectColumnNames = getGroupByColumnNames(params, true); @@ -697,7 +705,7 @@ protected String getAggregateClause(EventQueryParams params) { EventOutputType outputType = params.getOutputType(); - AggregationType aggregationType = params.getAggregationTypeFallback().getAggregationType(); + AggregationType aggregationType = getAggregationType(params); String function = (aggregationType == NONE || aggregationType == CUSTOM) ? "" : aggregationType.getValue(); @@ -747,6 +755,22 @@ protected String getAggregateClause(EventQueryParams params) { } } + private AggregationType getAggregationType(EventQueryParams params) { + + if (params.getValue() instanceof DataElement dataElement + && dataElement.hasOptionSet() + && params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .get(dataElement.getUid() + "." + dataElement.getOptionSet().getUid()) + .getOptionSetSelectionMode() + != OptionSetSelectionMode.AGGREGATED) { + return NONE; + } + + return params.getAggregationTypeFallback().getAggregationType(); + } + /** * Creates a coordinate base column "selector" for the given item name. The item is expected to be * of type Coordinate. diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java index 2ee4a0fb3d9a..bde443d696ee 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java @@ -90,6 +90,13 @@ public List planAggregateQuery(EventQueryParams params) { return withTableNameAndPartitions(queries); } + @Override + public List planQuery(EventQueryParams params) { + List queries = Lists.newArrayList(params); + + return withTableNameAndPartitions(queries); + } + @Override public EventQueryParams planEventQuery(EventQueryParams params) { return withTableNameAndPartitions(params); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java index 1bef2394a06a..4cedc1602854 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/EventQueryService.java @@ -70,6 +70,7 @@ import static org.hisp.dhis.feedback.ErrorCode.E7218; import java.util.List; +import java.util.Optional; import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import org.hisp.dhis.analytics.AnalyticsSecurityManager; @@ -83,6 +84,7 @@ import org.hisp.dhis.common.DimensionItemKeywords.Keyword; import org.hisp.dhis.common.Grid; import org.hisp.dhis.common.GridHeader; +import org.hisp.dhis.common.QueryItem; import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.util.Timer; @@ -133,6 +135,9 @@ public Grid getEvents(EventQueryParams params) { // Set periods. params = new EventQueryParams.Builder(params).withStartEndDatesForPeriods().build(); + // Set program if null. + params = getEventQueryParamsWithProgram(params); + // Populate headers. Grid grid = createGridWithHeaders(params); addCommonHeaders(grid, params, List.of()); @@ -155,6 +160,19 @@ public Grid getEvents(EventQueryParams params) { return grid; } + private static EventQueryParams getEventQueryParamsWithProgram(EventQueryParams params) { + if (!params.hasProgram()) { + Optional itemWithProgram = + params.getItems().stream().filter(QueryItem::hasProgram).findFirst(); + if (itemWithProgram.isPresent()) { + EventQueryParams.Builder builder = + new EventQueryParams.Builder(params).withProgram(itemWithProgram.get().getProgram()); + params = builder.build(); + } + } + return params; + } + /** * Returns a list of event clusters matching the given query. * @@ -270,7 +288,7 @@ private Grid createGridWithHeaders(EventQueryParams params) { false, true)); - if (params.getProgram().isRegistration()) { + if (params.getProgram() != null && params.getProgram().isRegistration()) { grid.addHeader( new GridHeader( ENROLLMENT_DATE.getItem(), diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java index 667a139a2209..2d95b71e02dc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/JdbcEventAnalyticsManager.java @@ -50,6 +50,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; @@ -59,6 +60,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.math3.util.Precision; import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.OrgUnitField; import org.hisp.dhis.analytics.Rectangle; import org.hisp.dhis.analytics.TimeField; @@ -515,6 +517,8 @@ protected String getWhereClause(EventQueryParams params) { sql += getQueryItemsAndFiltersWhereClause(params, hlp); + sql += getWhereClauseOptions(params, hlp); + // --------------------------------------------------------------------- // Filter expression // --------------------------------------------------------------------- @@ -624,6 +628,34 @@ protected String getWhereClause(EventQueryParams params) { return sql; } + private String getWhereClauseOptions(DataQueryParams params, SqlHelper sqlHelper) { + if (!params.hasOptionSetSelectionCriteria() + || params.getOptionSetSelectionCriteria().getOptionSetSelections() == null) { + return ""; + } + + StringBuilder sql = new StringBuilder(); + params + .getOptionSetSelectionCriteria() + .getOptionSetSelections() + .forEach( + (key, value) -> { + List uids = Arrays.stream(key.split("\\.")).toList(); + List options = value.getOptions(); + if (!uids.isEmpty() && options != null && !options.isEmpty()) { + sql.append(" ") + .append(sqlHelper.whereAnd()) + .append(" ") + .append(quote(uids.get(0) + ".optionvalueuid")) + .append(" in ('") + .append(String.join("','", options)) + .append("') "); + } + }); + + return sql.toString(); + } + /** Generates a sub query which provides a filter by organisation descendant level. */ private String getOrgUnitDescendantsClause( OrgUnitField orgUnitField, List dimensionOrFilterItems) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 31d587e51392..7f79311cb32a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -125,6 +125,18 @@ public class JdbcAnalyticsTableManager extends AbstractJdbcTableManager { .selectExpression("acs.categoryoptioncombouid as ao") .indexColumns(List.of("dx", "ao")) .build(), + AnalyticsTableColumn.builder() + .name("optionsetuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionsetuid as optionsetuid") + .build(), + AnalyticsTableColumn.builder() + .name("optionvalueuid") + .dataType(CHARACTER_11) + .nullable(NULL) + .selectExpression("deo.optionvalueuid as optionvalueuid") + .build(), AnalyticsTableColumn.builder() .name("pestartdate") .dataType(DATE) @@ -366,7 +378,8 @@ private void populateTable( inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ inner join analytics_rs_categoryoptioncomboname aon on dv.attributeoptioncomboid=aon.categoryoptioncomboid \ - inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid\s""", + inner join analytics_rs_categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid \ + left outer join analytics_rs_dataelementoption deo on dv.dataelementid = deo.dataelementid and dv.value = deo.optionvaluecode \s""", Map.of( "approvalSelectExpression", approvalSelectExpression, "valueExpression", valueExpression, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 2a07e0b69b89..a84d344c01ad 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -27,6 +27,7 @@ */ package org.hisp.dhis.analytics.table; +import static java.lang.String.join; import static java.util.stream.Collectors.toList; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; @@ -39,6 +40,7 @@ import static org.hisp.dhis.db.model.DataType.INTEGER; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.system.util.MathUtils.NUMERIC_LENIENT_REGEXP; +import static org.hisp.dhis.system.util.SqlUtils.singleQuote; import static org.hisp.dhis.util.DateUtils.toLongDate; import static org.hisp.dhis.util.DateUtils.toMediumDate; @@ -51,6 +53,7 @@ import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; @@ -362,7 +365,7 @@ and ev.status in (${exportableEventStatues}) \ "programId", String.valueOf(program.getId()), "firstDataYear", String.valueOf(firstDataYear), "latestDataYear", String.valueOf(latestDataYear), - "exportableEventStatues", String.join(",", EXPORTABLE_EVENT_STATUSES))); + "exportableEventStatues", join(",", EXPORTABLE_EVENT_STATUSES))); populateTableInternal(partition, fromClause); } @@ -467,6 +470,13 @@ private List getDataElementColumns(Program program) { .map(de -> getColumnForDataElement(de, false)) .flatMap(Collection::stream) .toList()); + columns.addAll( + program.getAnalyticsDataElements().stream() + .filter(DataElement::hasOptionSet) + .map(this::getColumnFromDataElementOptionSet) + .flatMap(Collection::stream) + .toList()); + columns.addAll( program.getAnalyticsDataElementsWithLegendSet().stream() .map(de -> getColumnForDataElement(de, true)) @@ -788,4 +798,121 @@ private final String getNumericClause(String value) { private List getYearsForPartitionTable(List dataYears) { return ListUtils.mutableCopy(!dataYears.isEmpty() ? dataYears : List.of(Year.now().getValue())); } + + private List getColumnFromDataElementOptionSet(DataElement dataElement) { + List columns = new ArrayList<>(); + + if (!dataElement.hasOptionSet()) { + return columns; + } + + String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); + String columnName = "eventdatavalues #>> '{" + dataElement.getUid() + ", value}'"; + String select = getSelectClause(dataElement.getValueType(), columnName); + String sql = selectOptionValueCodeForInsert(dataElement, select, dataClause); + + columns.add( + AnalyticsTableColumn.builder() + .name(dataElement.getUid() + ".optionvalueuid") + .dataType(DataType.VARCHAR_255) + .selectExpression(sql) + .skipIndex(Skip.INCLUDE) + .build()); + + return columns; + } + + private String getDataClause(String uid, ValueType valueType) { + if (valueType.isNumeric() || valueType.isDate()) { + String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; + + return replace( + " and eventdatavalues #>> '{${uid},value}' ~* '${regex}'", + Map.of("uid", uid, "regex", regex)); + } + + return ""; + } + + private String selectOptionValueCodeForInsert( + DataElement dataElement, String fromType, String dataClause) { + String innerSql = + replaceQualify( + """ + (select ${fromType} from ${event} \ + where eventid=ev.eventid ${dataClause})${closingParentheses}""", + Map.of( + "fromType", + fromType, + "dataClause", + dataClause, + "closingParentheses", + getClosingParentheses(fromType), + "dataElementUid", + quote(dataElement.getUid()))); + + return replaceQualify( + """ + (select optionvalueuid \ + from analytics_rs_dataelementoption \ + where dataelementuid = ${dataElementUid} \ + and optionvaluecode = ${selectForInsert}::varchar) as ${alias}""", + Map.of( + "dataElementUid", + singleQuote(dataElement.getUid()), + "selectForInsert", + innerSql, + "alias", + quote(dataElement.getUid() + ".optionvalueuid"))); + } + + private String getClosingParentheses(String str) { + if (StringUtils.isEmpty(str)) { + return EMPTY; + } + + int open = 0; + + for (int i = 0; i < str.length(); i++) { + if (str.charAt(i) == '(') { + open++; + } else if ((str.charAt(i) == ')') && open >= 1) { + open--; + } + } + + return StringUtils.repeat(")", open); + } + + /** + * Returns the select clause, potentially with a cast statement, based on the given value type. + * + * @param valueType the value type to represent as database column type. + */ + private String getSelectClause(ValueType valueType, String columnName) { + String doubleType = sqlBuilder.dataTypeDouble(); + if (valueType.isDecimal()) { + return "cast(" + columnName + " as " + doubleType + ")"; + } else if (valueType.isInteger()) { + return "cast(" + columnName + " as bigint)"; + } else if (valueType.isBoolean()) { + return "case when " + + columnName + + " = 'true' then 1 when " + + columnName + + " = 'false' then 0 else null end"; + } else if (valueType.isDate()) { + return "cast(" + columnName + " as timestamp)"; + } else if (valueType.isGeo() && isSpatialSupport()) { + return "ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (" + + columnName + + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; + } else if (valueType.isOrganisationUnit()) { + return replaceQualify( + "ou.uid from ${organisationunit} ou where ou.uid = (select ${columnName}", + Map.of("columnName", columnName)); + } else { + return columnName; + } + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index b61b7a82bbaa..5ab3e9de5dbb 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -105,11 +105,13 @@ import org.hisp.dhis.feedback.ErrorMessage; import org.hisp.dhis.hibernate.HibernateProxyUtils; import org.hisp.dhis.indicator.Indicator; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.period.FinancialPeriodType; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramDataElementDimensionItem; import org.hisp.dhis.program.ProgramIndicator; import org.hisp.dhis.program.ProgramStage; import org.hisp.dhis.system.grid.ListGrid; @@ -644,8 +646,8 @@ public static void handleGridForDataValueSet(DataQueryParams params, Grid grid) coc = dataItem.getAggregateExportCategoryOptionCombo(); aoc = dataItem.getAggregateExportAttributeOptionCombo(); } else if (DataElementOperand.class.isAssignableFrom(item.getClass())) { - row.set(dxInx, DimensionalObjectUtils.getFirstIdentifer(dx)); - coc = DimensionalObjectUtils.getSecondIdentifer(dx); + row.set(dxInx, DimensionalObjectUtils.getFirstIdentifier(dx)); + coc = DimensionalObjectUtils.getSecondIdentifier(dx); } cocCol.add(coc); @@ -825,6 +827,29 @@ public static Map getDimensionMetadataItemMap( coc.getDisplayProperty(params.getDisplayProperty()), includeMetadataDetails ? coc : null)); } + + OptionSet optionSet = dataElement.getOptionSet(); + if (optionSet != null) { + map.put( + dataElement.getUid() + "." + optionSet.getUid(), + includeMetadataDetails + ? new MetadataItem( + optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) + : new MetadataItem(optionSet.getName())); + } + } + if (DimensionItemType.PROGRAM_DATA_ELEMENT == item.getDimensionItemType() + && item instanceof ProgramDataElementDimensionItem programDataElement) { + + OptionSet optionSet = programDataElement.getOptionSet(); + if (optionSet != null) { + map.put( + programDataElement.getDataElement().getUid() + "." + optionSet.getUid(), + includeMetadataDetails + ? new MetadataItem( + optionSet.getName(), optionSet, new HashSet<>(optionSet.getOptions())) + : new MetadataItem(optionSet.getName())); + } } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java index 04ba319c9e30..d032ba88003f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java @@ -61,6 +61,7 @@ import org.hisp.dhis.resourcetable.table.DataApprovalMinLevelResourceTable; import org.hisp.dhis.resourcetable.table.DataApprovalRemapLevelResourceTable; import org.hisp.dhis.resourcetable.table.DataElementGroupSetResourceTable; +import org.hisp.dhis.resourcetable.table.DataElementOptionResourceTable; import org.hisp.dhis.resourcetable.table.DataElementResourceTable; import org.hisp.dhis.resourcetable.table.DataSetOrganisationUnitCategoryResourceTable; import org.hisp.dhis.resourcetable.table.DataSetResourceTable; @@ -162,7 +163,8 @@ private final List getResourceTables() { new DataElementResourceTable(logged, idObjectManager.getAllNoAcl(DataElement.class)), new DatePeriodResourceTable(logged, getAndValidateAvailableDataYears()), new PeriodResourceTable(logged, periodService.getAllPeriods()), - new CategoryOptionComboResourceTable(logged)); + new CategoryOptionComboResourceTable(logged), + new DataElementOptionResourceTable(logged)); } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java new file mode 100644 index 000000000000..66d7ccdb5fd4 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/resourcetable/table/DataElementOptionResourceTable.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2004-2024, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.resourcetable.table; + +import static org.hisp.dhis.commons.util.TextUtils.replace; +import static org.hisp.dhis.db.model.Table.toStaging; +import static org.hisp.dhis.system.util.SqlUtils.appendRandom; + +import java.util.List; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.db.model.Column; +import org.hisp.dhis.db.model.DataType; +import org.hisp.dhis.db.model.Index; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; +import org.hisp.dhis.db.model.constraint.Nullable; +import org.hisp.dhis.resourcetable.ResourceTable; +import org.hisp.dhis.resourcetable.ResourceTableType; + +@RequiredArgsConstructor +public class DataElementOptionResourceTable implements ResourceTable { + public static final String TABLE_NAME = "analytics_rs_dataelementoption"; + + private final Logged logged; + + @Override + public Table getTable() { + return new Table(toStaging(TABLE_NAME), getColumns(), getPrimaryKey(), logged); + } + + @Override + public Table getMainTable() { + return new Table(TABLE_NAME, getColumns(), getPrimaryKey(), logged); + } + + private List getColumns() { + return List.of( + new Column("dataelementid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("optionsetid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("optionvalueid", DataType.BIGINT, Nullable.NOT_NULL), + new Column("dataelementuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionsetuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionvalueuid", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("optionvaluecode", DataType.VARCHAR_255, Nullable.NOT_NULL)); + } + + private List getPrimaryKey() { + return List.of("dataelementid", "optionsetid", "optionvalueid"); + } + + @Override + public List getIndexes() { + return List.of( + Index.builder() + .name(appendRandom("in_optionsetoptionvalue")) + .tableName(toStaging(TABLE_NAME)) + .columns(List.of("dataelementuid", "optionsetuid", "optionvalueuid")) + .build(), + Index.builder() + .name(appendRandom("in_dataelementoptioncode")) + .tableName(toStaging(TABLE_NAME)) + .columns(List.of("dataelementuid", "optionvaluecode")) + .build()); + } + + @Override + public ResourceTableType getTableType() { + return ResourceTableType.DATA_ELEMENT_CATEGORY_OPTION_COMBO; + } + + @Override + public Optional getPopulateTempTableStatement() { + String sql = + replace( + """ + insert into ${tableName} \ + (dataelementid, optionsetid, optionvalueid, dataelementuid, optionsetuid, optionvalueuid, optionvaluecode) \ + select de.dataelementid, os.optionsetid as optionsetid, ov.optionvalueid as optionvalueid, \ + de.uid as dataelementuid, os.uid as optionsetuid, ov.uid as optionvalueuid, ov.code as optionvaluecode from optionvalue ov \ + inner join optionset os on ov.optionsetid = os.optionsetid \ + inner join dataelement de on os.optionsetid = de.optionsetid;""", + "tableName", + toStaging(TABLE_NAME)); + + return Optional.of(sql); + } + + @Override + public Optional> getPopulateTempTableContent() { + return Optional.empty(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java index b7dea4affd99..6e1a229fd76d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DataDimensionExtractor.java @@ -52,6 +52,7 @@ import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.indicator.Indicator; +import org.hisp.dhis.option.OptionSet; import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramDataElementDimensionItem; import org.hisp.dhis.program.ProgramIndicator; @@ -230,6 +231,26 @@ public ReportingRate getReportingRate(IdScheme idScheme, String dataSetId, Strin return new ReportingRate(dataSet, ReportingRateMetric.valueOf(metric)); } + /** + * Returns a {@link DataElement}. + * + * @param idScheme the identifier scheme. + * @param dataElementId the data element identifier. + * @param optionSetId the option set identifier. + */ + @Transactional(readOnly = true) + public DataElement getOptionSetDataElementDimensionItem( + IdScheme idScheme, String dataElementId, String optionSetId) { + DataElement dataElement = idObjectManager.getObject(DataElement.class, idScheme, dataElementId); + OptionSet optionSet = idObjectManager.getObject(OptionSet.class, idScheme, optionSetId); + + if (dataElement == null || optionSet == null) { + return null; + } + + return dataElement; + } + /** * Returns a {@link ProgramTrackedEntityAttributeDimensionItem}. * diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java index 8d37336eda60..3b1f43c75e61 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dimension/DefaultDimensionService.java @@ -40,7 +40,6 @@ import static org.hisp.dhis.common.DimensionType.PROGRAM_ATTRIBUTE; import static org.hisp.dhis.common.DimensionType.PROGRAM_DATA_ELEMENT; import static org.hisp.dhis.common.DimensionType.PROGRAM_INDICATOR; -import static org.hisp.dhis.common.DimensionalObjectUtils.COMPOSITE_DIM_OBJECT_ESCAPED_SEP; import static org.hisp.dhis.common.IdScheme.UID; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.commons.util.TextUtils.splitSafe; @@ -348,10 +347,19 @@ public DimensionalItemObject getDataDimensionalItemObject(String dimensionItem) public DimensionalItemObject getDataDimensionalItemObject( IdScheme idScheme, String dimensionItem) { if (DimensionalObjectUtils.isCompositeDimensionalObject(dimensionItem)) { - String id0 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 0); - String id1 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 1); - String id2 = splitSafe(dimensionItem, COMPOSITE_DIM_OBJECT_ESCAPED_SEP, 2); + String id0 = DimensionalObjectUtils.getFirstIdentifier(dimensionItem); + String id1 = DimensionalObjectUtils.getSecondIdentifier(dimensionItem); + String id2 = DimensionalObjectUtils.getThirdIdentifier(dimensionItem); + + String optionSetSelectionMode = splitSafe(dimensionItem, "-", 1); + if (optionSetSelectionMode != null && id2 != null) { + id2 = splitSafe(id2, "-", 0); + } else if (optionSetSelectionMode != null && id1 != null) { + id1 = splitSafe(id1, "-", 0); + } + + DataElement dataElementWithOptionSet; DataElementOperand operand; ReportingRate reportingRate; ProgramDataElementDimensionItem programDataElement; @@ -373,6 +381,13 @@ public DimensionalItemObject getDataDimensionalItemObject( != null) { return programAttribute; } + + if ((dataElementWithOptionSet = + dataDimensionExtractor.getOptionSetDataElementDimensionItem(idScheme, id0, id1)) + != null) { + return dataElementWithOptionSet; + } + } else if (!idScheme.is(IdentifiableProperty.UID) || CodeGenerator.isValidUid(dimensionItem)) { return idObjectManager.get(DataDimensionItem.DATA_DIM_CLASSES, idScheme, dimensionItem); }