FINERACT-1996: Reporting should support query that includes json path…

… for DB field in Postgres
ruchiD · Oct 31, 2023 · 03e71a2 · 03e71a2
1 parent 24cb624
commit 03e71a2
Show file tree

Hide file tree

Showing 13 changed files with 167 additions and 175 deletions.
diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/DateUtils.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/DateUtils.java
@@ -193,6 +193,8 @@ public static int compare(OffsetDateTime first, OffsetDateTime second, ChronoUni
         if (second == null) {
             return 1;
         }
+        first = first.withOffsetSameInstant(ZoneOffset.UTC);
+        second = second.withOffsetSameInstant(ZoneOffset.UTC);
         return truncate == null ? first.compareTo(second) : first.truncatedTo(truncate).compareTo(second.truncatedTo(truncate));
     }
 

diff --git a/...rg/apache/fineract/infrastructure/core/service/database/DatabaseSpecificSQLGenerator.java b/...rg/apache/fineract/infrastructure/core/service/database/DatabaseSpecificSQLGenerator.java
@@ -23,8 +23,10 @@
 import jakarta.validation.constraints.NotNull;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import org.apache.fineract.infrastructure.core.service.DateUtils;
+import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Sort;
@@ -251,18 +253,33 @@ public String buildOrderBy(List<Sort.Order> orders, String alias, boolean embedd
                 .collect(Collectors.joining(", "));
     }
 
-    public String buildInsert(@NotNull String definition, Collection<String> fields) {
+    public String buildInsert(@NotNull String definition, List<String> fields, Map<String, ResultsetColumnHeaderData> headers) {
         if (fields == null || fields.isEmpty()) {
             return "";
         }
         return "INSERT INTO " + escape(definition) + '(' + fields.stream().map(this::escape).collect(Collectors.joining(", "))
-                + ") VALUES (?" + ", ?".repeat(fields.size() - 1) + ')';
+                + ") VALUES (" + fields.stream().map(e -> decoratePlaceHolder(headers, e, "?")).collect(Collectors.joining(", ")) + ')';
     }
 
-    public String buildUpdate(@NotNull String definition, Collection<String> fields) {
+    public String buildUpdate(@NotNull String definition, List<String> fields, Map<String, ResultsetColumnHeaderData> headers) {
         if (fields == null || fields.isEmpty()) {
             return "";
         }
-        return "UPDATE " + escape(definition) + " SET " + fields.stream().map(e -> escape(e) + " = ?").collect(Collectors.joining(", "));
+        return "UPDATE " + escape(definition) + " SET "
+                + fields.stream().map(e -> escape(e) + " = " + decoratePlaceHolder(headers, e, "?")).collect(Collectors.joining(", "));
+    }
+
+    private String decoratePlaceHolder(Map<String, ResultsetColumnHeaderData> headers, String field, String placeHolder) {
+        DatabaseType dialect = getDialect();
+        if (dialect.isPostgres()) {
+            ResultsetColumnHeaderData header = headers.get(field);
+            if (header != null) {
+                JdbcJavaType columnType = header.getColumnType();
+                if (columnType.isJsonType()) {
+                    return placeHolder + "::" + columnType.getJdbcName(dialect);
+                }
+            }
+        }
+        return placeHolder;
     }
 }
diff --git a/.../src/main/java/org/apache/fineract/infrastructure/core/service/database/JdbcJavaType.java b/.../src/main/java/org/apache/fineract/infrastructure/core/service/database/JdbcJavaType.java
@@ -74,6 +74,7 @@ public Object toJdbcValueImpl(@NotNull DatabaseType dialect, Object value) {
     TINYTEXT(JavaType.STRING, new DialectType(JDBCType.VARCHAR, "TINYTEXT"), new DialectType(JDBCType.VARCHAR, "TEXT")), //
     MEDIUMTEXT(JavaType.STRING, new DialectType(JDBCType.VARCHAR, "MEDIUMTEXT"), new DialectType(JDBCType.VARCHAR, "TEXT")), //
     LONGTEXT(JavaType.STRING, new DialectType(JDBCType.VARCHAR, "LONGTEXT"), new DialectType(JDBCType.VARCHAR, "TEXT")), //
+    JSON(JavaType.STRING, new DialectType(JDBCType.VARCHAR, "JSON"), new DialectType(JDBCType.VARCHAR, "JSON")), //
     DATE(JavaType.LOCAL_DATE, new DialectType(JDBCType.DATE), new DialectType(JDBCType.DATE)), //
     // precision for TIME, TIMESTAMP (postgres) and INTERVAL specifies the number of fractional digits retained in the
     // seconds field, but by default, there is no explicit bound on precision
@@ -126,7 +127,7 @@ public JavaType getJavaType() {
         return javaType;
     }
 
-    public static JdbcJavaType getByTypeName(@NotNull DatabaseType dialect, String name) {
+    public static JdbcJavaType getByTypeName(@NotNull DatabaseType dialect, String name, boolean check) {
         if (name == null) {
             return null;
         }
@@ -145,6 +146,10 @@ public static JdbcJavaType getByTypeName(@NotNull DatabaseType dialect, String n
                 }
             }
         }
+        if (check) {
+            throw new PlatformServiceUnavailableException("error.msg.database.type.not.supported",
+                    "Data type '" + name + "' is not supported ");
+        }
         return null;
     }
 
@@ -179,7 +184,11 @@ public boolean isVarcharType() {
     }
 
     public boolean isTextType() {
-        return this == TEXT || this == TINYTEXT || this == MEDIUMTEXT || this == LONGTEXT;
+        return this == TEXT || this == TINYTEXT || this == MEDIUMTEXT || this == LONGTEXT || this == JSON;
+    }
+
+    public boolean isJsonType() {
+        return this == JSON;
     }
 
     public boolean isSerialType() {
@@ -262,7 +271,7 @@ public String formatSql(@NotNull DatabaseType dialect, Integer precision, Intege
 
     public Object toJdbcValue(@NotNull DatabaseType dialect, Object value, boolean check) {
         if (value != null && check && !javaType.getObjectType().matchType(value.getClass(), false)) {
-            throw new PlatformServiceUnavailableException("error.msg.database.type.not.allowed",
+            throw new PlatformServiceUnavailableException("error.msg.database.type.not.valid",
                     "Data type of parameter " + value + " does not match " + this);
         }
         return toJdbcValueImpl(dialect, value);

diff --git a/...n/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java b/...n/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java
@@ -75,7 +75,7 @@ private ResultsetColumnHeaderData(final String columnName, String columnType, fi
         this.isColumnIndexed = columnIsIndexed;
 
         // Refer org.drizzle.jdbc.internal.mysql.MySQLType.java
-        this.columnType = JdbcJavaType.getByTypeName(dialect, adjustColumnType(columnType));
+        this.columnType = JdbcJavaType.getByTypeName(dialect, adjustColumnType(columnType), true);
 
         this.columnDisplayType = calcDisplayType();
     }

diff --git a/...e/fineract/infrastructure/core/serialization/DatatableCommandFromApiJsonDeserializer.java b/...e/fineract/infrastructure/core/serialization/DatatableCommandFromApiJsonDeserializer.java
@@ -23,6 +23,7 @@
 import static org.apache.fineract.infrastructure.core.service.database.JdbcJavaType.DATETIME;
 import static org.apache.fineract.infrastructure.core.service.database.JdbcJavaType.DECIMAL;
 import static org.apache.fineract.infrastructure.core.service.database.JdbcJavaType.INTEGER;
+import static org.apache.fineract.infrastructure.core.service.database.JdbcJavaType.JSON;
 import static org.apache.fineract.infrastructure.core.service.database.JdbcJavaType.TEXT;
 import static org.apache.fineract.infrastructure.core.service.database.JdbcJavaType.TIMESTAMP;
 import static org.apache.fineract.infrastructure.core.service.database.JdbcJavaType.VARCHAR;
@@ -40,6 +41,7 @@
 import static org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant.API_FIELD_TYPE_DATETIME;
 import static org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant.API_FIELD_TYPE_DECIMAL;
 import static org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant.API_FIELD_TYPE_DROPDOWN;
+import static org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant.API_FIELD_TYPE_JSON;
 import static org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant.API_FIELD_TYPE_NUMBER;
 import static org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant.API_FIELD_TYPE_STRING;
 import static org.apache.fineract.infrastructure.dataqueries.api.DataTableApiConstant.API_FIELD_TYPE_TEXT;
@@ -97,7 +99,8 @@ public class DatatableCommandFromApiJsonDeserializer {
             API_FIELD_MANDATORY, API_FIELD_AFTER, API_FIELD_CODE, API_FIELD_NEWCODE, API_FIELD_UNIQUE, API_FIELD_INDEXED);
     private static final Set<String> SUPPORTED_PARAMETERS_FOR_DROP_COLUMNS = Set.of(API_FIELD_NAME);
     private static final Object[] SUPPORTED_COLUMN_TYPES = { API_FIELD_TYPE_STRING, API_FIELD_TYPE_NUMBER, API_FIELD_TYPE_BOOLEAN,
-            API_FIELD_TYPE_DECIMAL, API_FIELD_TYPE_DATE, API_FIELD_TYPE_DATETIME, API_FIELD_TYPE_TEXT, API_FIELD_TYPE_DROPDOWN };
+            API_FIELD_TYPE_DECIMAL, API_FIELD_TYPE_DATE, API_FIELD_TYPE_DATETIME, API_FIELD_TYPE_TEXT, API_FIELD_TYPE_JSON,
+            API_FIELD_TYPE_DROPDOWN };
 
     private final FromJsonHelper fromApiJsonHelper;
     private final DatabaseTypeResolver databaseTypeResolver;
@@ -344,6 +347,8 @@ private static JdbcJavaType mapApiTypeToJdbcType(@NotNull String apiType, boolea
                 return TIMESTAMP;
             case API_FIELD_TYPE_TEXT:
                 return TEXT;
+            case API_FIELD_TYPE_JSON:
+                return JSON;
             default: {
                 if (fail) {
                     throw new PlatformDataIntegrityException("error.msg.datatable.column.type.invalid",

diff --git a/...rc/main/java/org/apache/fineract/infrastructure/dataqueries/api/DataTableApiConstant.java b/...rc/main/java/org/apache/fineract/infrastructure/dataqueries/api/DataTableApiConstant.java
@@ -85,5 +85,6 @@ private DataTableApiConstant() {
     public static final String API_FIELD_TYPE_DATETIME = "datetime";
     public static final String API_FIELD_TYPE_TIMESTAMP = "timestamp";
     public static final String API_FIELD_TYPE_TEXT = "text";
+    public static final String API_FIELD_TYPE_JSON = "json";
     public static final String API_FIELD_TYPE_DROPDOWN = "dropdown";
 }
diff --git a/...n/java/org/apache/fineract/infrastructure/dataqueries/service/GenericDataServiceImpl.java b/...n/java/org/apache/fineract/infrastructure/dataqueries/service/GenericDataServiceImpl.java
@@ -117,7 +117,7 @@ public List<ResultsetColumnHeaderData> fillResultsetColumnHeaders(final String t
             // primary keys and unique constrained columns are automatically indexed
             final boolean columnIsIndexed = columnIsPrimaryKey || columnIsUnique
                     || isExplicitlyIndexed(tableName, columnName, indexDefinitions);
-            JdbcJavaType jdbcType = JdbcJavaType.getByTypeName(dialect, columnType);
+            JdbcJavaType jdbcType = JdbcJavaType.getByTypeName(dialect, columnType, false);
 
             List<ResultsetColumnValueData> columnValues = new ArrayList<>();
             String codeName = null;

diff --git a/...g/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java b/...g/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java
@@ -1323,7 +1323,6 @@ private CommandProcessingResult createNewDatatableEntry(final String dataTableNa
             params.add(SearchUtil.parseJdbcColumnValue(columnHeader, dataParams.get(key), dateFormat, dateTimeFormat, locale, false,
                     sqlGenerator));
         }
-        final String sql = sqlGenerator.buildInsert(dataTableName, insertColumns);
         if (addScore) {
             List<Object> scoreIds = params.stream().filter(e -> e != null && !String.valueOf(e).isBlank()).toList();
             int scoreValue;
@@ -1339,6 +1338,8 @@ private CommandProcessingResult createNewDatatableEntry(final String dataTableNa
             insertColumns.add("score");
             params.add(scoreValue);
         }
+
+        final String sql = sqlGenerator.buildInsert(dataTableName, insertColumns, headersByName);
         try {
             int updated = jdbcTemplate.update(sql, params.toArray(Object[]::new));
             if (updated != 1) {
@@ -1446,8 +1447,8 @@ private CommandProcessingResult updateDatatableEntry(final String dataTableName,
         Locale locale = localeString == null ? null : JsonParserHelper.localeFromString(localeString);
 
         DatabaseType dialect = sqlGenerator.getDialect();
-        List<String> updateColumns = new ArrayList<>(List.of(UPDATEDAT_FIELD_NAME));
-        List<Object> params = new ArrayList<>(List.of(DateUtils.getAuditLocalDateTime()));
+        ArrayList<String> updateColumns = new ArrayList<>(List.of(UPDATEDAT_FIELD_NAME));
+        ArrayList<Object> params = new ArrayList<>(List.of(DateUtils.getAuditLocalDateTime()));
         final HashMap<String, Object> changes = new HashMap<>();
         for (String key : dataParams.keySet()) {
             if (isTechnicalParam(key)) {
@@ -1474,7 +1475,8 @@ private CommandProcessingResult updateDatatableEntry(final String dataTableName,
         if (!updateColumns.isEmpty()) {
             ResultsetColumnHeaderData pkColumn = SearchUtil.getFiltered(columnHeaders, ResultsetColumnHeaderData::getIsColumnPrimaryKey);
             params.add(primaryKey);
-            final String sql = sqlGenerator.buildUpdate(dataTableName, updateColumns) + " WHERE " + pkColumn.getColumnName() + " = ?";
+            final String sql = sqlGenerator.buildUpdate(dataTableName, updateColumns, headersByName) + " WHERE " + pkColumn.getColumnName()
+                    + " = ?";
             int updated = jdbcTemplate.update(sql, params.toArray(Object[]::new));
             if (updated != 1) {
                 throw new PlatformDataIntegrityException("error.msg.invalid.update", "Expected one updated row.");

diff --git a/...sts/src/test/java/org/apache/fineract/integrationtests/ClientAuditingIntegrationTest.java b/...sts/src/test/java/org/apache/fineract/integrationtests/ClientAuditingIntegrationTest.java
@@ -23,16 +23,18 @@
 import static org.apache.fineract.infrastructure.core.domain.AuditableFieldsConstants.LAST_MODIFIED_BY;
 import static org.apache.fineract.infrastructure.core.domain.AuditableFieldsConstants.LAST_MODIFIED_DATE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import io.restassured.builder.RequestSpecBuilder;
 import io.restassured.builder.ResponseSpecBuilder;
 import io.restassured.http.ContentType;
 import io.restassured.specification.RequestSpecification;
 import io.restassured.specification.ResponseSpecification;
 import java.time.OffsetDateTime;
-import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
 import java.util.Map;
+import org.apache.fineract.infrastructure.core.service.DateUtils;
 import org.apache.fineract.integrationtests.common.ClientHelper;
 import org.apache.fineract.integrationtests.common.Utils;
 import org.apache.fineract.integrationtests.common.organisation.StaffHelper;
@@ -65,12 +67,7 @@ public void checkAuditDates() throws InterruptedException {
         String username = Utils.uniqueRandomStringGenerator("user", 8);
         final Integer userId = (Integer) UserHelper.createUser(this.requestSpec, this.responseSpec, 1, staffId, username, "password",
                 "resourceId");
-        OffsetDateTime now = OffsetDateTime.now(ZoneId.of("Asia/Kolkata"));
-        // Testing in minutes precision, but still need to take care around the end of the actual minute
-        if (now.getSecond() > 56) {
-            Thread.sleep(5000);
-            now = OffsetDateTime.now(ZoneId.of("Asia/Kolkata"));
-        }
+        OffsetDateTime now = Utils.getAuditDateTimeToCompare();
         LOG.info("-------------------------Creating Client---------------------------");
 
         final Integer clientID = ClientHelper.createClientPending(requestSpec, responseSpec);
@@ -79,55 +76,36 @@ public void checkAuditDates() throws InterruptedException {
 
         OffsetDateTime createdDate = OffsetDateTime.parse((String) auditFieldsResponse.get(CREATED_DATE),
                 DateTimeFormatter.ISO_OFFSET_DATE_TIME);
-
         OffsetDateTime lastModifiedDate = OffsetDateTime.parse((String) auditFieldsResponse.get(LAST_MODIFIED_DATE),
                 DateTimeFormatter.ISO_OFFSET_DATE_TIME);
 
         LOG.info("-------------------------Check Audit dates---------------------------");
         assertEquals(1, auditFieldsResponse.get(CREATED_BY));
         assertEquals(1, auditFieldsResponse.get(LAST_MODIFIED_BY));
-        assertEquals(now.getYear(), createdDate.getYear());
-        assertEquals(now.getMonth(), createdDate.getMonth());
-        assertEquals(now.getDayOfMonth(), createdDate.getDayOfMonth());
-        assertEquals(now.getHour(), createdDate.getHour());
-        assertEquals(now.getMinute(), createdDate.getMinute());
-
-        assertEquals(now.getYear(), lastModifiedDate.getYear());
-        assertEquals(now.getMonth(), lastModifiedDate.getMonth());
-        assertEquals(now.getDayOfMonth(), lastModifiedDate.getDayOfMonth());
-        assertEquals(now.getHour(), lastModifiedDate.getHour());
-        assertEquals(now.getMinute(), lastModifiedDate.getMinute());
+        assertTrue(DateUtils.isEqual(now, createdDate, ChronoUnit.MINUTES));
+        assertTrue(DateUtils.isEqual(now, lastModifiedDate, ChronoUnit.MINUTES));
 
         LOG.info("-------------------------Modify Client with System user---------------------------");
         this.requestSpec = new RequestSpecBuilder().setContentType(ContentType.JSON).build();
         this.requestSpec.header("Authorization",
                 "Basic " + Utils.loginIntoServerAndGetBase64EncodedAuthenticationKey(username, "password"));
-
         this.clientHelper = new ClientHelper(this.requestSpec, this.responseSpec);
+
+        OffsetDateTime now2 = Utils.getAuditDateTimeToCompare();
         this.clientHelper.activateClient(clientID);
         auditFieldsResponse = ClientHelper.getClientAuditFields(requestSpec, responseSpec, clientID, "");
 
-        createdDate = OffsetDateTime.parse((String) auditFieldsResponse.get(CREATED_DATE), DateTimeFormatter.ISO_OFFSET_DATE_TIME);
-
+        OffsetDateTime createdDate2 = OffsetDateTime.parse((String) auditFieldsResponse.get(CREATED_DATE),
+                DateTimeFormatter.ISO_OFFSET_DATE_TIME);
         lastModifiedDate = OffsetDateTime.parse((String) auditFieldsResponse.get(LAST_MODIFIED_DATE),
                 DateTimeFormatter.ISO_OFFSET_DATE_TIME);
 
         LOG.info("-------------------------Check Audit dates---------------------------");
         assertEquals(1, auditFieldsResponse.get(CREATED_BY));
-        assertEquals(now.getYear(), createdDate.getYear());
-        assertEquals(now.getMonth(), createdDate.getMonth());
-        assertEquals(now.getDayOfMonth(), createdDate.getDayOfMonth());
-        assertEquals(now.getHour(), createdDate.getHour());
-        assertEquals(now.getMinute(), createdDate.getMinute());
-
-        now = OffsetDateTime.now(ZoneId.of("Asia/Kolkata"));
+        assertTrue(DateUtils.isEqual(now, createdDate2, ChronoUnit.MINUTES));
+        assertTrue(DateUtils.isEqual(createdDate, createdDate2));
 
         assertEquals(userId, auditFieldsResponse.get(LAST_MODIFIED_BY));
-        assertEquals(now.getYear(), lastModifiedDate.getYear());
-        assertEquals(now.getMonth(), lastModifiedDate.getMonth());
-        assertEquals(now.getDayOfMonth(), lastModifiedDate.getDayOfMonth());
-        assertEquals(now.getHour(), lastModifiedDate.getHour());
-        assertEquals(now.getMinute(), lastModifiedDate.getMinute());
+        assertTrue(DateUtils.isEqual(now2, lastModifiedDate, ChronoUnit.MINUTES));
     }
-
 }