Skip to content

Commit

Permalink
Move dataset views mgmt into Horreum logic
Browse files Browse the repository at this point in the history
As part of this refactoring I removed table
view_recalc_queue and:

Triggers
- dsv_after_delete
- dsv_after_update
- recalc_dataset_view
- dsv_after_insert

Functions:
- dsv_after_vc_delete_func
- dsv_after_vc_update_func
- recalc_dataset_view
- dsv_after_lv_insert_func

Signed-off-by: Andrea Lamparelli <[email protected]>
  • Loading branch information
lampajr committed Dec 6, 2024
1 parent 1d550ce commit acba64d
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import jakarta.ws.rs.DefaultValue;

import org.hibernate.Hibernate;
import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.Session;
import org.hibernate.query.NativeQuery;
import org.hibernate.type.StandardBasicTypes;
Expand Down Expand Up @@ -445,9 +447,8 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i
"Evaluation of label %s failed: '%s' Code:<pre>%s</pre>", row[0], e.getMessage(), jsCode),
(out) -> logMessage(datasetId, PersistentLogDAO.DEBUG, "Output while calculating labels: <pre>%s</pre>", out));

//Create new dataset views from the recently created label values
em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
em.createNativeQuery("call calc_dataset_view(?1);").setParameter(1, datasetId).executeUpdate();
// create new dataset views from the recently created label values
calcDatasetViews(datasetId);

createFingerprint(datasetId, testId);
mediator.updateLabels(new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation));
Expand All @@ -456,10 +457,48 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i
testId, new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation)));
}

@Transactional
public void calcDatasetViews(int datasetId) {
// TODO(user) move calc_dataset_view into Horreum business logic see https://github.com/hibernate/hibernate-orm/pull/7457
em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
em.createNativeQuery("call calc_dataset_view(?1, NULL);").setParameter(1, datasetId).executeUpdate();
}

@Transactional
@SuppressWarnings("unchecked")
public void calcDatasetViewsByTestAndView(int testId, int viewId) {
// delete all dataset views associated to the provided viewId and testId
// for new views it won't delete anything
em.createNativeQuery(
"DELETE FROM dataset_view WHERE view_id = ?1 AND dataset_id IN (SELECT id FROM dataset WHERE testid = ?2)")
.setParameter(1, viewId)
.setParameter(2, testId)
.executeUpdate();

// re-create dataset views associated to the provided viewId
try (ScrollableResults<Integer> datasetIds = em
.createNativeQuery("SELECT id FROM dataset WHERE testid = ?1")
.setParameter(1, testId)
.unwrap(NativeQuery.class)
.setReadOnly(false)
.setFetchSize(100)
.scroll(ScrollMode.FORWARD_ONLY)) {
while (datasetIds.next()) {
int datasetId = datasetIds.get();
log.tracef("Recalculate dataset views for view %d and dataset %d", viewId, datasetId);
em.createNativeQuery("call calc_dataset_view(?1, ?2);")
.setParameter(1, datasetId)
.setParameter(2, viewId)
.executeUpdate();
}
}
}

@Transactional
public void deleteDataset(int datasetId) {
em.createNativeQuery("DELETE FROM label_values WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
em.createNativeQuery("DELETE FROM dataset_schemas WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
em.createNativeQuery("DELETE FROM fingerprint WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
em.createNativeQuery("DELETE FROM dataset WHERE id = ?1").setParameter(1, datasetId).executeUpdate();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1087,9 +1087,8 @@ public Map<Integer, String> updateSchema(int id, String path, String schemaUri)
@Transactional
@Override
public List<Integer> recalculateDatasets(int runId) {
transform(runId, true);
return session.createNativeQuery("SELECT id FROM dataset WHERE runid = ? ORDER BY ordinal", Integer.class)
.setParameter(1, runId).getResultList();
log.infof("Transforming run id %d", runId);
return transform(runId, true);
}

@RolesAllowed(Roles.ADMIN)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,7 @@ public void recalculateDatasets(int testId) {
status.finished++;
status.datasets += newDatasets;
if (status.finished == status.totalRuns) {
log.infof("Datasets recalculation for test %d (%s) completed", testId, test.name);
recalculations.remove(testId, status);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ public class UIServiceImpl implements UIService {
@Inject
TestServiceImpl testService;

@Inject
DatasetServiceImpl datasetService;

@Override
@RolesAllowed("tester")
@WithRoles
Expand Down Expand Up @@ -64,6 +67,9 @@ private View doUpdate(TestDAO test, ViewDAO view) {
test.views.add(view);
test.persist();
em.flush();

// update datasets views
datasetService.calcDatasetViewsByTestAndView(test.id, view.id);
return ViewMapper.from(view);
}

Expand All @@ -80,6 +86,8 @@ public void deleteView(int testId, int viewId) {
if (!test.views.removeIf(v -> v.id == viewId)) {
throw ServiceException.badRequest("Test does not contain this view!");
}
// remove dataset views records linked to this view
em.createNativeQuery("DELETE FROM dataset_view WHERE view_id = ?1").setParameter(1, viewId).executeUpdate();
// the orphan removal doesn't work for some reason, we need to remove if manually
ViewDAO.deleteById(viewId);
test.persist();
Expand Down
39 changes: 39 additions & 0 deletions horreum-backend/src/main/resources/db/changeLog.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4688,4 +4688,43 @@
$$ LANGUAGE plpgsql;
</sql>
</changeSet>
<changeSet id="126" author="lampajr">
<validCheckSum>ANY</validCheckSum>
<sql>
-- drop triggers
DROP TRIGGER IF EXISTS dsv_after_delete ON viewcomponent;
DROP TRIGGER IF EXISTS dsv_after_update ON viewcomponent;
DROP TRIGGER IF EXISTS recalc_dataset_view ON view_recalc_queue;
DROP TRIGGER IF EXISTS dsv_after_insert ON label_values;

-- drop functions
DROP FUNCTION dsv_after_vc_delete_func;
DROP FUNCTION dsv_after_vc_update_func;
DROP FUNCTION recalc_dataset_view;
DROP FUNCTION dsv_after_lv_insert_func;

-- drop view_recalc_queue table as not needed anymore
DROP TABLE view_recalc_queue;

-- still need db procedure until https://hibernate.atlassian.net/browse/HHH-17314 is fixed in quarkus
-- viewId can be NULL
CREATE OR REPLACE PROCEDURE calc_dataset_view(datasetId bigint, viewId bigint DEFAULT NULL) AS $$
BEGIN
WITH view_agg AS (
SELECT
vc.view_id, vc.id as vcid, array_agg(DISTINCT label.id) as label_ids, jsonb_object_agg(label.name, lv.value) as value FROM dataset_schemas ds
JOIN label ON label.schema_id = ds.schema_id
JOIN viewcomponent vc ON vc.labels ? label.name
JOIN label_values lv ON lv.label_id = label.id AND lv.dataset_id = ds.dataset_id
WHERE ds.dataset_id = datasetId AND (viewId IS NULL OR vc.view_id = viewId)
AND vc.view_id IN (SELECT view.id FROM view JOIN dataset ON view.test_id = dataset.testid WHERE dataset.id = datasetId)
GROUP BY vc.view_id, vcid
)
INSERT INTO dataset_view (dataset_id, view_id, label_ids, value)
SELECT datasetId, view_id, array_agg(DISTINCT label_id), jsonb_object_agg(vcid, value) FROM view_agg, unnest(label_ids) as label_id
GROUP BY view_id;
END
$$ LANGUAGE plpgsql;
</sql>
</changeSet>
</databaseChangeLog>

0 comments on commit acba64d

Please sign in to comment.