Skip to content

Commit

Permalink
feat: Add store tests for all sql execution paths [DHIS2-18321]
Browse files Browse the repository at this point in the history
  • Loading branch information
@david-mackessy
david-mackessy committed Jan 10, 2025
1 parent e5db611 commit 22d0070
Show file tree
Hide file tree
Showing 3 changed files with 231 additions and 53 deletions.
20 changes: 20 additions & 0 deletions dhis-2/dhis-api/src/main/java/org/hisp/dhis/datavalue/DataValueStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,26 @@ DataValue getDataValue(
void mergeDataValuesWithCategoryOptionCombos(
@Nonnull CategoryOptionCombo target, @Nonnull Collection<CategoryOptionCombo> sources);

/**
* SQL for handling merging {@link DataValue}s. There may be multiple potential {@link DataValue}
* duplicates. Duplicate {@link DataValue}s with the latest {@link DataValue#lastUpdated} values
* are kept, the rest are deleted. Only one of these entries can exist due to the composite key
* constraint. <br>
* The 3 execution paths are:
*
* <p>1. If the source {@link DataValue} is not a duplicate, it simply gets its {@link
* DataValue#attributeOptionCombo} updated to that of the target.
*
* <p>2. If the source {@link DataValue} is a duplicate and has an earlier {@link
* DataValue#lastUpdated} value, it is deleted.
*
* <p>3. If the source {@link DataValue} is a duplicate and has a later {@link
* DataValue#lastUpdated} value, the target {@link DataValue} is deleted. The source is kept and
* has its {@link DataValue#attributeOptionCombo} updated to that of the target.
*
* @param target target {@link CategoryOptionCombo}
* @param sources source {@link CategoryOptionCombo}s
*/
void mergeDataValuesWithAttributeOptionCombos(
@Nonnull CategoryOptionCombo target, @Nonnull Collection<CategoryOptionCombo> sources);
}
20 changes: 0 additions & 20 deletions ...service-core/src/main/java/org/hisp/dhis/datavalue/hibernate/HibernateDataValueStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,26 +424,6 @@ -- loop through each record with a source CategoryOptionCombo
jdbcTemplate.update(plpgsql);
}

/**
* SQL for handling merging {@link DataValue}s. There may be multiple potential {@link DataValue}
* duplicates. Duplicate {@link DataValue}s with the latest {@link DataValue#lastUpdated} values
* are kept, the rest are deleted. Only one of these entries can exist due to the composite key
* constraint. <br>
* The 3 execution paths are:
*
* <p>1. If the source {@link DataValue} is not a duplicate, it simply gets its {@link
* DataValue#attributeOptionCombo} updated to that of the target.
*
* <p>2. If the source {@link DataValue} is a duplicate and has an earlier {@link
* DataValue#lastUpdated} value, it is deleted.
*
* <p>3. If the source {@link DataValue} is a duplicate and has a later {@link
* DataValue#lastUpdated} value, the target {@link DataValue} is deleted. The source is kept and
* has its {@link DataValue#attributeOptionCombo} updated to that of the target.
*
* @param target target {@link CategoryOptionCombo}
* @param sources source {@link CategoryOptionCombo}s
*/
@Override
public void mergeDataValuesWithAttributeOptionCombos(
@Nonnull CategoryOptionCombo target, @Nonnull Collection<CategoryOptionCombo> sources) {
Expand Down
244 changes: 211 additions & 33 deletions dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/datavalue/DataValueStoreTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,8 @@ void getDataValuesByAoc() {

@Test
@DisplayName(
"Merging duplicate DataValues (cat opt combos) leaves only the last updated value remaining")
void mergeDvWithDuplicates() {
"Merging duplicate DataValues (cat opt combos) leaves only the last updated (source) value remaining")
void mergeDvWithDuplicatesKeepSource() {
// given
TestCategoryMetadata categoryMetadata = setupCategoryMetadata();

Expand Down Expand Up @@ -270,60 +270,238 @@ void mergeDvWithDuplicates() {
dv4.setSource(ou);
dv4.setLastUpdated(DateUtils.parseDate("2024-11-02"));

dataValueStore.addDataValue(dv1);
dataValueStore.addDataValue(dv2);
dataValueStore.addDataValue(dv3);
dataValueStore.addDataValue(dv4);
addDataValues(dv1, dv2, dv3, dv4);

// check pre merge state
List<DataValue> preMergeState = dataValueStore.getAllDataValuesByDataElement(List.of(de));

assertEquals(4, preMergeState.size(), "there should be 4 data values");
assertTrue(
preMergeState.stream()
.map(dv -> dv.getCategoryOptionCombo().getId())
.collect(Collectors.toSet())
.containsAll(
List.of(
categoryMetadata.coc1().getId(),
categoryMetadata.coc2().getId(),
categoryMetadata.coc3().getId(),
categoryMetadata.coc4().getId())),
"All data values have different category option combos");
checkCocIdsPresent(
preMergeState,
List.of(
categoryMetadata.coc1().getId(),
categoryMetadata.coc2().getId(),
categoryMetadata.coc3().getId(),
categoryMetadata.coc4().getId()));

entityManager.flush();
// when
mergeDataValues(
categoryMetadata.coc3(), List.of(categoryMetadata.coc1(), categoryMetadata.coc2()));

// then
List<DataValue> postMergeState = dataValueStore.getAllDataValuesByDataElement(List.of(de));

assertEquals(2, postMergeState.size(), "there should be 2 data values");
checkCocIdsPresent(
preMergeState, List.of(categoryMetadata.coc3().getId(), categoryMetadata.coc4().getId()));

checkDataValuesPresent(
postMergeState, List.of("dv test 2 - last updated", "dv test 4, untouched"));

checkDatesPresent(
postMergeState,
List.of(DateUtils.parseDate("2025-01-08"), DateUtils.parseDate("2024-11-02")));
}

@Test
@DisplayName(
"Merging duplicate DataValues (cat opt combos) leaves only the last updated (target) value remaining")
void mergeDvWithDuplicatesKeepTarget() {
// given
TestCategoryMetadata categoryMetadata = setupCategoryMetadata();

Period p1 = createPeriod(DateUtils.getDate(2024, 1, 1), DateUtils.getDate(2023, 2, 1));

DataElement de = createDataElement('z');
manager.persist(de);

OrganisationUnit ou = createOrganisationUnit("org u 1");
manager.persist(ou);

// data values with same period, org unit, data element and attr opt combo
// which will be identified as duplicates during merging
DataValue dv1 = createDataValue('1', p1, "dv test 1");
dv1.setCategoryOptionCombo(categoryMetadata.coc1());
dv1.setAttributeOptionCombo(categoryMetadata.coc4());
dv1.setDataElement(de);
dv1.setSource(ou);
dv1.setLastUpdated(DateUtils.parseDate("2024-12-01"));

DataValue dv2 = createDataValue('2', p1, "dv test 2");
dv2.setCategoryOptionCombo(categoryMetadata.coc2());
dv2.setAttributeOptionCombo(categoryMetadata.coc4());
dv2.setDataElement(de);
dv2.setSource(ou);
dv2.setLastUpdated(DateUtils.parseDate("2025-01-02"));

DataValue dv3 = createDataValue('3', p1, "dv test 3 - last updated");
dv3.setCategoryOptionCombo(categoryMetadata.coc3());
dv3.setAttributeOptionCombo(categoryMetadata.coc4());
dv3.setDataElement(de);
dv3.setSource(ou);
dv3.setLastUpdated(DateUtils.parseDate("2025-01-06"));

DataValue dv4 = createDataValue('4', p1, "dv test 4, untouched");
dv4.setCategoryOptionCombo(categoryMetadata.coc4());
dv4.setAttributeOptionCombo(categoryMetadata.coc4());
dv4.setDataElement(de);
dv4.setSource(ou);
dv4.setLastUpdated(DateUtils.parseDate("2024-11-02"));

addDataValues(dv1, dv2, dv3, dv4);

// check pre merge state
List<DataValue> preMergeState = dataValueStore.getAllDataValuesByDataElement(List.of(de));

assertEquals(4, preMergeState.size(), "there should be 4 data values");
checkCocIdsPresent(
preMergeState,
List.of(
categoryMetadata.coc1().getId(),
categoryMetadata.coc2().getId(),
categoryMetadata.coc3().getId(),
categoryMetadata.coc4().getId()));

// when
dataValueStore.mergeDataValuesWithCategoryOptionCombos(
mergeDataValues(
categoryMetadata.coc3(), List.of(categoryMetadata.coc1(), categoryMetadata.coc2()));
entityManager.flush();
entityManager.clear();

// then
List<DataValue> postMergeState = dataValueStore.getAllDataValuesByDataElement(List.of(de));

assertEquals(2, postMergeState.size(), "there should be 2 data values");
checkCocIdsPresent(
postMergeState, List.of(categoryMetadata.coc3().getId(), categoryMetadata.coc4().getId()));

checkDataValuesPresent(
postMergeState, List.of("dv test 3 - last updated", "dv test 4, untouched"));

checkDatesPresent(
postMergeState,
List.of(DateUtils.parseDate("2025-01-06"), DateUtils.parseDate("2024-11-02")));
}

@Test
@DisplayName(
"Merging non-duplicate DataValues (cat opt combos) updates the cat opt combo value only")
void mergeDvWithNoDuplicates() {
// given
TestCategoryMetadata categoryMetadata = setupCategoryMetadata();

Period p1 = createPeriod(DateUtils.getDate(2024, 1, 1), DateUtils.getDate(2023, 2, 1));
Period p2 = createPeriod(DateUtils.getDate(2024, 2, 1), DateUtils.getDate(2023, 3, 1));
Period p3 = createPeriod(DateUtils.getDate(2024, 3, 1), DateUtils.getDate(2023, 4, 1));
Period p4 = createPeriod(DateUtils.getDate(2024, 4, 1), DateUtils.getDate(2023, 5, 1));

DataElement de = createDataElement('z');
manager.persist(de);

OrganisationUnit ou = createOrganisationUnit("org u 1");
manager.persist(ou);

// data values with different period, so no duplicates detected during merging
DataValue dv1 = createDataValue('1', p1, "dv test 1");
dv1.setCategoryOptionCombo(categoryMetadata.coc1());
dv1.setAttributeOptionCombo(categoryMetadata.coc4());
dv1.setDataElement(de);
dv1.setSource(ou);
dv1.setLastUpdated(DateUtils.parseDate("2024-12-01"));

DataValue dv2 = createDataValue('2', p2, "dv test 2 - last updated");
dv2.setCategoryOptionCombo(categoryMetadata.coc2());
dv2.setAttributeOptionCombo(categoryMetadata.coc4());
dv2.setDataElement(de);
dv2.setSource(ou);
dv2.setLastUpdated(DateUtils.parseDate("2025-01-08"));

DataValue dv3 = createDataValue('3', p3, "dv test 3");
dv3.setCategoryOptionCombo(categoryMetadata.coc3());
dv3.setAttributeOptionCombo(categoryMetadata.coc4());
dv3.setDataElement(de);
dv3.setSource(ou);
dv3.setLastUpdated(DateUtils.parseDate("2024-12-06"));

DataValue dv4 = createDataValue('4', p4, "dv test 4, untouched");
dv4.setCategoryOptionCombo(categoryMetadata.coc4());
dv4.setAttributeOptionCombo(categoryMetadata.coc4());
dv4.setDataElement(de);
dv4.setSource(ou);
dv4.setLastUpdated(DateUtils.parseDate("2024-11-02"));

addDataValues(dv1, dv2, dv3, dv4);

// check pre merge state
List<DataValue> preMergeState = dataValueStore.getAllDataValuesByDataElement(List.of(de));

assertEquals(4, preMergeState.size(), "there should be 4 data values");
checkCocIdsPresent(
preMergeState,
List.of(
categoryMetadata.coc1().getId(),
categoryMetadata.coc2().getId(),
categoryMetadata.coc3().getId(),
categoryMetadata.coc4().getId()));

// when
mergeDataValues(
categoryMetadata.coc3(), List.of(categoryMetadata.coc1(), categoryMetadata.coc2()));

// then
List<DataValue> postMergeState = dataValueStore.getAllDataValuesByDataElement(List.of(de));

assertEquals(4, postMergeState.size(), "there should still be 4 data values");
checkCocIdsPresent(
postMergeState, List.of(categoryMetadata.coc3().getId(), categoryMetadata.coc4().getId()));

checkDataValuesPresent(
postMergeState,
List.of("dv test 1", "dv test 2 - last updated", "dv test 3", "dv test 4, untouched"));

checkDatesPresent(
postMergeState,
List.of(
DateUtils.parseDate("2025-01-08"),
DateUtils.parseDate("2024-11-02"),
DateUtils.parseDate("2024-12-01"),
DateUtils.parseDate("2024-12-06")));
}

private void checkDatesPresent(List<DataValue> dataValues, List<Date> dates) {
assertTrue(
postMergeState.stream()
.map(dv -> dv.getCategoryOptionCombo().getId())
dataValues.stream()
.map(DataValue::getLastUpdated)
.collect(Collectors.toSet())
.containsAll(List.of(categoryMetadata.coc3().getId(), categoryMetadata.coc4().getId())),
"Only 2 expected cat opt combos should be present");
.containsAll(dates),
"Expected dates should be present");
}

private void checkDataValuesPresent(List<DataValue> dataValues, List<String> values) {
assertTrue(
postMergeState.stream()
dataValues.stream()
.map(DataValue::getValue)
.collect(Collectors.toSet())
.containsAll(List.of("dv test 2 - last updated", "dv test 4, untouched")),
"Only latest DataValue and untouched DataValue should be present");
.containsAll(values),
"Expected DataValues should be present");
}

private void checkCocIdsPresent(List<DataValue> dataValues, List<Long> cocIds) {
assertTrue(
postMergeState.stream()
.map(DataValue::getLastUpdated)
dataValues.stream()
.map(dv -> dv.getCategoryOptionCombo().getId())
.collect(Collectors.toSet())
.containsAll(
List.of(DateUtils.parseDate("2025-01-08"), DateUtils.parseDate("2024-11-02"))),
"Only latest lastUpdated value and untouched lastUpdated should exist");
.containsAll(cocIds),
"Data values have expected category option combos");
}

private void mergeDataValues(CategoryOptionCombo target, List<CategoryOptionCombo> sources) {
dataValueStore.mergeDataValuesWithCategoryOptionCombos(target, sources);
entityManager.flush();
entityManager.clear();
}

private void addDataValues(DataValue... dvs) {
for (DataValue dv : dvs) dataValueStore.addDataValue(dv);
entityManager.flush();
}

private DataValue createDataValue(char uniqueChar, Period period, String value) {
Expand Down

0 comments on commit 22d0070

Please sign in to comment.