Merge remote-tracking branch 'origin/DHIS2-15787_2.39' into DHIS2-157…

…87_2.39
dhis2 · Nov 1, 2023 · d09760f · d09760f
2 parents 9c9dc21 + 370a456
commit d09760f
Show file tree

Hide file tree

Showing 44 changed files with 830 additions and 251 deletions.
diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java
@@ -382,11 +382,14 @@ public enum ErrorCode {
   E7129("Program is specified but does not exist: `{0}`"),
   E7130("Program stage is specified but does not exist: `{0}`"),
   E7131("Query failed, likely because the query timed out"),
-  E7132("An indicator expression caused division by zero operation"),
+  E7132("Expression violation. Maybe an indicator caused division by zero?"),
   E7133("Query cannot be executed, possibly because of invalid types or invalid operation"),
   E7134("Cannot retrieve total value for data elements with skip total category combination"),
   E7135("Date time is not parsable: `{0}`"),
   E7143("Organisation unit or organisation unit level is not valid"),
+  E7144(
+      "Query failed because a referenced table does not exist. Please ensure analytics job was run"),
+  E7145("Query failed because of a syntax error"),
 
   /* Event analytics */
   E7200("At least one organisation unit must be specified"),

diff --git a/.../dhis-api/src/main/java/org/hisp/dhis/trackedentity/TrackedEntityInstanceQueryParams.java b/.../dhis-api/src/main/java/org/hisp/dhis/trackedentity/TrackedEntityInstanceQueryParams.java
@@ -44,7 +44,6 @@
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.time.DateUtils;
 import org.hisp.dhis.common.AssignedUserSelectionMode;
 import org.hisp.dhis.common.OrganisationUnitSelectionMode;
 import org.hisp.dhis.common.QueryFilter;
@@ -758,9 +757,7 @@ public TrackedEntityInstanceQueryParams setProgramEnrollmentStartDate(
   }
 
   public Date getProgramEnrollmentEndDate() {
-    return programEnrollmentEndDate != null
-        ? DateUtils.addDays(programEnrollmentEndDate, 1)
-        : programEnrollmentEndDate;
+    return programEnrollmentEndDate;
   }
 
   public TrackedEntityInstanceQueryParams setProgramEnrollmentEndDate(
@@ -780,9 +777,7 @@ public TrackedEntityInstanceQueryParams setProgramIncidentStartDate(
   }
 
   public Date getProgramIncidentEndDate() {
-    return programIncidentEndDate != null
-        ? DateUtils.addDays(programIncidentEndDate, 1)
-        : programIncidentEndDate;
+    return programIncidentEndDate;
   }
 
   public TrackedEntityInstanceQueryParams setProgramIncidentEndDate(Date programIncidentEndDate) {

diff --git a/...administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java b/...administration/src/main/java/org/hisp/dhis/resourcetable/DefaultResourceTableService.java
@@ -30,6 +30,8 @@
 import static java.time.temporal.ChronoUnit.YEARS;
 import static java.util.Comparator.reverseOrder;
 import static java.util.stream.Collectors.toList;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.SYSTEM_DEFINED;
 import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_ITEM;
 
 import com.google.common.collect.Lists;
@@ -183,7 +185,9 @@ public void generateDataElementTable() {
 
   @Override
   public void generateDatePeriodTable() {
-    List<Integer> availableYears = periodDataProvider.getAvailableYears();
+    List<Integer> availableYears =
+        periodDataProvider.getAvailableYears(
+            analyticsExportSettings.getMaxPeriodYearsOffset() == null ? SYSTEM_DEFINED : DATABASE);
     checkYearsOffset(availableYears);
 
     resourceTableStore.generateResourceTable(
@@ -202,30 +206,33 @@ public void generateDatePeriodTable() {
    * @param yearsToCheck the list of years to be checked.
    */
   private void checkYearsOffset(List<Integer> yearsToCheck) {
-    int maxYearsOffset = analyticsExportSettings.getMaxPeriodYearsOffset();
-    int minRangeAllowed = Year.now().minus(maxYearsOffset, YEARS).getValue();
-    int maxRangeAllowed = Year.now().plus(maxYearsOffset, YEARS).getValue();
-
-    boolean yearsOutOfRange =
-        yearsToCheck.stream().anyMatch(year -> year < minRangeAllowed || year > maxRangeAllowed);
-
-    if (yearsOutOfRange) {
-      String errorMessage = "Your database contains years out of the allowed offset.";
-      errorMessage +=
-          "\n Range of years allowed (based on your system settings and existing data): "
-              + yearsToCheck.stream()
-                  .filter(year -> year >= minRangeAllowed && year <= maxRangeAllowed)
-                  .collect(toList())
-              + ".";
-      errorMessage +=
-          "\n Years out of range found: "
-              + yearsToCheck.stream()
-                  .filter(year -> year < minRangeAllowed || year > maxRangeAllowed)
-                  .collect(toList())
-              + ".";
-
-      log.warn(errorMessage);
-      throw new RuntimeException(errorMessage);
+    Integer maxYearsOffset = analyticsExportSettings.getMaxPeriodYearsOffset();
+
+    if (maxYearsOffset != null) {
+      int minRangeAllowed = Year.now().minus(maxYearsOffset, YEARS).getValue();
+      int maxRangeAllowed = Year.now().plus(maxYearsOffset, YEARS).getValue();
+
+      boolean yearsOutOfRange =
+          yearsToCheck.stream().anyMatch(year -> year < minRangeAllowed || year > maxRangeAllowed);
+
+      if (yearsOutOfRange) {
+        String errorMessage = "Your database contains years out of the allowed offset.";
+        errorMessage +=
+            "\n Range of years allowed (based on your system settings and existing data): "
+                + yearsToCheck.stream()
+                    .filter(year -> year >= minRangeAllowed && year <= maxRangeAllowed)
+                    .collect(toList())
+                + ".";
+        errorMessage +=
+            "\n Years out of range found: "
+                + yearsToCheck.stream()
+                    .filter(year -> year < minRangeAllowed || year > maxRangeAllowed)
+                    .collect(toList())
+                + ".";
+
+        log.warn(errorMessage);
+        throw new RuntimeException(errorMessage);
+      }
     }
   }
 

diff --git a/...nistration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java b/...nistration/src/test/java/org/hisp/dhis/resourcetable/DefaultResourceTableServiceTest.java
@@ -28,6 +28,7 @@
 package org.hisp.dhis.resourcetable;
 
 import static java.time.temporal.ChronoUnit.YEARS;
+import static org.hisp.dhis.period.PeriodDataProvider.DataSource.DATABASE;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -63,7 +64,7 @@ void generateDatePeriodTableWhenYearIsOutOfRange() {
     int defaultOffset = 22;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(defaultOffset);
 
     // Then
@@ -82,7 +83,7 @@ void generateDatePeriodTableWhenOffsetIsZeroWithPreviousYears() {
     int zeroOffset = 0;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(zeroOffset);
 
     // Then
@@ -101,7 +102,7 @@ void generateDatePeriodTableWhenOffsetIsZeroWithCurrentYear() {
     int zeroOffset = 0;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(zeroOffset);
     doNothing().when(resourceTableStore).generateResourceTable(any());
 
@@ -120,7 +121,7 @@ void generateDatePeriodTableWhenYearsAreInExpectedRange() {
     int defaultOffset = 2;
 
     // When
-    when(periodDataProvider.getAvailableYears()).thenReturn(yearsToCheck);
+    when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(yearsToCheck);
     when(analyticsExportSettings.getMaxPeriodYearsOffset()).thenReturn(defaultOffset);
     doNothing().when(resourceTableStore).generateResourceTable(any());
 

diff --git a/...is-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/...is-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java
@@ -44,11 +44,14 @@
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote;
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAlias;
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAliasCommaSeparate;
+import static org.hisp.dhis.analytics.util.AnalyticsUtils.ERR_MSG_SILENT_FALLBACK;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx;
+import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling;
 import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP;
 import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids;
 import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString;
 import static org.hisp.dhis.commons.util.TextUtils.removeLastOr;
+import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.relationDoesNotExist;
 import static org.hisp.dhis.util.DateUtils.getMediumDateString;
 
 import com.google.common.collect.ImmutableMap;
@@ -177,25 +180,35 @@ public Future<Map<String, Object>> getAggregatedDataValues(
 
       log.debug(sql);
 
+      final String finalSqlValue = sql;
+      final DataQueryParams immutableParams = DataQueryParams.newBuilder(params).build();
+
       if (params.analyzeOnly()) {
-        executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql);
+        withExceptionHandling(
+            () ->
+                executionPlanStore.addExecutionPlan(
+                    immutableParams.getExplainOrderId(), finalSqlValue));
         return new AsyncResult<>(Maps.newHashMap());
       }
 
       Map<String, Object> map;
 
       try {
-        map = getKeyValueMap(params, sql, maxLimit);
+        map =
+            withExceptionHandling(() -> getKeyValueMap(immutableParams, finalSqlValue, maxLimit))
+                .orElse(Map.of());
       } catch (BadSqlGrammarException ex) {
-        log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex);
+        if (relationDoesNotExist(ex.getSQLException())) {
+          throw ex;
+        }
+        log.warn(ERR_MSG_SILENT_FALLBACK, ex);
         return new AsyncResult<>(Maps.newHashMap());
       }
 
       replaceDataPeriodsWithAggregationPeriods(map, params, dataPeriodAggregationPeriodMap);
 
       return new AsyncResult<>(map);
     } catch (DataAccessResourceFailureException ex) {
-      log.warn(ErrorCode.E7131.getMessage(), ex);
       throw new QueryRuntimeException(ErrorCode.E7131);
     } catch (RuntimeException ex) {
       log.error(DebugUtils.getStackTrace(ex));

diff --git a/...his-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java b/...his-service-analytics/src/main/java/org/hisp/dhis/analytics/data/handler/DataHandler.java
@@ -63,6 +63,7 @@
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.getRoundedValueObject;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.hasPeriod;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.isPeriodInPeriods;
+import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling;
 import static org.hisp.dhis.analytics.util.PeriodOffsetUtils.buildYearToDateRows;
 import static org.hisp.dhis.analytics.util.PeriodOffsetUtils.getPeriodOffsetRow;
 import static org.hisp.dhis.analytics.util.PeriodOffsetUtils.isYearToDate;
@@ -481,7 +482,8 @@ public void addRawData(DataQueryParams params, Grid grid) {
 
       params = queryPlanner.withTableNameAndPartitions(params, plannerParams);
 
-      rawAnalyticsManager.getRawDataValues(params, grid);
+      final DataQueryParams immutableParams = DataQueryParams.newBuilder(params).build();
+      withExceptionHandling(() -> rawAnalyticsManager.getRawDataValues(immutableParams, grid));
     }
   }
 

diff --git a/...ice-analytics/src/main/java/org/hisp/dhis/analytics/event/EnrollmentAnalyticsManager.java b/...ice-analytics/src/main/java/org/hisp/dhis/analytics/event/EnrollmentAnalyticsManager.java
@@ -54,7 +54,7 @@ public interface EnrollmentAnalyticsManager {
   void getEnrollments(EventQueryParams params, Grid grid, int maxLimit);
 
   /**
-   * Retreives count of enrollments based on params.
+   * Retrieves count of enrollments based on params.
    *
    * @param params the qyery to count enrollments for,
    * @return number of enrollments macting the parameter criteria.

diff --git a/.../dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java b/.../dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryParams.java
@@ -243,6 +243,8 @@ public class EventQueryParams extends DataQueryParams {
 
   @Getter protected EndpointAction endpointAction;
 
+  @Getter protected boolean multipleQueries = false;
+
   // -------------------------------------------------------------------------
   // Constructors
   // -------------------------------------------------------------------------
@@ -312,6 +314,7 @@ protected EventQueryParams instance() {
     params.endpointItem = this.endpointItem;
     params.endpointAction = this.endpointAction;
     params.rowContext = this.rowContext;
+    params.multipleQueries = this.multipleQueries;
     return params;
   }
 
@@ -1381,5 +1384,10 @@ public Builder withRowContext(boolean rowContext) {
       this.params.rowContext = rowContext;
       return this;
     }
+
+    public Builder withMultipleQueries(boolean multipleQueries) {
+      this.params.multipleQueries = multipleQueries;
+      return this;
+    }
   }
 }
diff --git a/...s/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java b/...s/src/main/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManager.java
@@ -53,13 +53,15 @@
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote;
 import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAlias;
 import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx;
+import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling;
 import static org.hisp.dhis.common.DimensionItemType.DATA_ELEMENT;
 import static org.hisp.dhis.common.DimensionItemType.PROGRAM_INDICATOR;
 import static org.hisp.dhis.common.DimensionalObjectUtils.COMPOSITE_DIM_OBJECT_PLAIN_SEP;
 import static org.hisp.dhis.common.QueryOperator.IN;
 import static org.hisp.dhis.common.RequestTypeAware.EndpointItem.ENROLLMENT;
 import static org.hisp.dhis.commons.util.TextUtils.getCommaDelimitedString;
 import static org.hisp.dhis.system.util.MathUtils.getRounded;
+import static org.springframework.transaction.annotation.Propagation.REQUIRES_NEW;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -101,7 +103,6 @@
 import org.hisp.dhis.common.InQueryFilter;
 import org.hisp.dhis.common.QueryFilter;
 import org.hisp.dhis.common.QueryItem;
-import org.hisp.dhis.common.QueryRuntimeException;
 import org.hisp.dhis.common.Reference;
 import org.hisp.dhis.common.RepeatableStageParams;
 import org.hisp.dhis.common.ValueType;
@@ -116,10 +117,9 @@
 import org.hisp.dhis.program.ProgramIndicatorService;
 import org.hisp.dhis.system.util.MathUtils;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.dao.DataAccessResourceFailureException;
-import org.springframework.jdbc.BadSqlGrammarException;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.jdbc.support.rowset.SqlRowSet;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.Assert;
 
 /**
@@ -492,6 +492,7 @@ protected Optional<String> getAlias(QueryItem queryItem) {
         .map(RepeatableStageParams::getDimension);
   }
 
+  @Transactional(readOnly = true, propagation = REQUIRES_NEW)
   public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLimit) {
     String aggregateClause = getAggregateClause(params);
 
@@ -535,17 +536,14 @@ public Grid getAggregatedEventData(EventQueryParams params, Grid grid, int maxLi
     // Grid
     // ---------------------------------------------------------------------
 
-    try {
-      if (params.analyzeOnly()) {
-        executionPlanStore.addExecutionPlan(params.getExplainOrderId(), sql);
-      } else {
-        getAggregatedEventData(grid, params, sql);
-      }
-    } catch (BadSqlGrammarException ex) {
-      log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex);
-    } catch (DataAccessResourceFailureException ex) {
-      log.warn(ErrorCode.E7131.getMessage(), ex);
-      throw new QueryRuntimeException(ErrorCode.E7131);
+    final String finalSqlValue = sql;
+
+    if (params.analyzeOnly()) {
+      withExceptionHandling(
+          () -> executionPlanStore.addExecutionPlan(params.getExplainOrderId(), finalSqlValue));
+    } else {
+      withExceptionHandling(
+          () -> getAggregatedEventData(grid, params, finalSqlValue), params.isMultipleQueries());
     }
 
     return grid;
@@ -903,23 +901,6 @@ protected String getEventsOrEnrollmentsSql(EventQueryParams params, int maxLimit
     return sql;
   }
 
-  /**
-   * Wraps the provided interface around a common exception handling strategy.
-   *
-   * @param runnable a {@link Runnable} containing the code block to execute and wrap around the
-   *     exception handling.
-   */
-  protected void withExceptionHandling(Runnable runnable) {
-    try {
-      runnable.run();
-    } catch (BadSqlGrammarException ex) {
-      log.info(AnalyticsUtils.ERR_MSG_TABLE_NOT_EXISTING, ex);
-    } catch (DataAccessResourceFailureException ex) {
-      log.warn(ErrorCode.E7131.getMessage(), ex);
-      throw new QueryRuntimeException(ErrorCode.E7131);
-    }
-  }
-
   /**
    * Adds a value from the given row set to the grid.
    *

diff --git a/...lytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java b/...lytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java
@@ -772,11 +772,13 @@ protected long addEventData(Grid grid, EventQueryParams params) {
 
     long count = 0;
 
+    EventQueryParams immutableParams = new EventQueryParams.Builder(params).build();
+
     if (params.getPartitions().hasAny() || params.isSkipPartitioning()) {
-      eventAnalyticsManager.getEvents(params, grid, queryValidator.getMaxLimit());
+      eventAnalyticsManager.getEvents(immutableParams, grid, queryValidator.getMaxLimit());
 
       if (params.isPaging() && params.isTotalPages()) {
-        count = eventAnalyticsManager.getEventCount(params);
+        count = eventAnalyticsManager.getEventCount(immutableParams);
       }
 
       timer.getTime("Got events " + grid.getHeight());