From acba64dc77cc6bd6c51fc9be8eb8dd074bd872d6 Mon Sep 17 00:00:00 2001 From: Andrea Lamparelli Date: Thu, 5 Dec 2024 17:33:30 +0100 Subject: [PATCH] Move dataset views mgmt into Horreum logic As part of this refactoring I removed table view_recalc_queue and: Triggers - dsv_after_delete - dsv_after_update - recalc_dataset_view - dsv_after_insert Functions: - dsv_after_vc_delete_func - dsv_after_vc_update_func - recalc_dataset_view - dsv_after_lv_insert_func Signed-off-by: Andrea Lamparelli --- .../tools/horreum/svc/DatasetServiceImpl.java | 45 +++++++++++++++++-- .../tools/horreum/svc/RunServiceImpl.java | 5 +-- .../tools/horreum/svc/TestServiceImpl.java | 1 + .../tools/horreum/svc/UIServiceImpl.java | 8 ++++ .../src/main/resources/db/changeLog.xml | 39 ++++++++++++++++ 5 files changed, 92 insertions(+), 6 deletions(-) diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/DatasetServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/DatasetServiceImpl.java index cfed11c91..10ae3bad3 100644 --- a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/DatasetServiceImpl.java +++ b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/DatasetServiceImpl.java @@ -18,6 +18,8 @@ import jakarta.ws.rs.DefaultValue; import org.hibernate.Hibernate; +import org.hibernate.ScrollMode; +import org.hibernate.ScrollableResults; import org.hibernate.Session; import org.hibernate.query.NativeQuery; import org.hibernate.type.StandardBasicTypes; @@ -445,9 +447,8 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i "Evaluation of label %s failed: '%s' Code:
%s
", row[0], e.getMessage(), jsCode), (out) -> logMessage(datasetId, PersistentLogDAO.DEBUG, "Output while calculating labels: 
%s
", out)); - //Create new dataset views from the recently created label values - em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate(); - em.createNativeQuery("call calc_dataset_view(?1);").setParameter(1, datasetId).executeUpdate(); + // create new dataset views from the recently created label values + calcDatasetViews(datasetId); createFingerprint(datasetId, testId); mediator.updateLabels(new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation)); @@ -456,10 +457,48 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i testId, new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation))); } + @Transactional + public void calcDatasetViews(int datasetId) { + // TODO(user) move calc_dataset_view into Horreum business logic see https://github.com/hibernate/hibernate-orm/pull/7457 + em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate(); + em.createNativeQuery("call calc_dataset_view(?1, NULL);").setParameter(1, datasetId).executeUpdate(); + } + + @Transactional + @SuppressWarnings("unchecked") + public void calcDatasetViewsByTestAndView(int testId, int viewId) { + // delete all dataset views associated to the provided viewId and testId + // for new views it won't delete anything + em.createNativeQuery( + "DELETE FROM dataset_view WHERE view_id = ?1 AND dataset_id IN (SELECT id FROM dataset WHERE testid = ?2)") + .setParameter(1, viewId) + .setParameter(2, testId) + .executeUpdate(); + + // re-create dataset views associated to the provided viewId + try (ScrollableResults datasetIds = em + .createNativeQuery("SELECT id FROM dataset WHERE testid = ?1") + .setParameter(1, testId) + .unwrap(NativeQuery.class) + .setReadOnly(false) + .setFetchSize(100) + .scroll(ScrollMode.FORWARD_ONLY)) { + while (datasetIds.next()) { + int datasetId = datasetIds.get(); + log.tracef("Recalculate dataset views for view %d and dataset %d", viewId, datasetId); + em.createNativeQuery("call calc_dataset_view(?1, ?2);") + .setParameter(1, datasetId) + .setParameter(2, viewId) + .executeUpdate(); + } + } + } + @Transactional public void deleteDataset(int datasetId) { em.createNativeQuery("DELETE FROM label_values WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate(); em.createNativeQuery("DELETE FROM dataset_schemas WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate(); + em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate(); em.createNativeQuery("DELETE FROM fingerprint WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate(); em.createNativeQuery("DELETE FROM dataset WHERE id = ?1").setParameter(1, datasetId).executeUpdate(); } diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java index 4d65d03e6..88961da86 100644 --- a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java +++ b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java @@ -1087,9 +1087,8 @@ public Map updateSchema(int id, String path, String schemaUri) @Transactional @Override public List recalculateDatasets(int runId) { - transform(runId, true); - return session.createNativeQuery("SELECT id FROM dataset WHERE runid = ? ORDER BY ordinal", Integer.class) - .setParameter(1, runId).getResultList(); + log.infof("Transforming run id %d", runId); + return transform(runId, true); } @RolesAllowed(Roles.ADMIN) diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java index f2c158124..46eef8d91 100644 --- a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java +++ b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java @@ -669,6 +669,7 @@ public void recalculateDatasets(int testId) { status.finished++; status.datasets += newDatasets; if (status.finished == status.totalRuns) { + log.infof("Datasets recalculation for test %d (%s) completed", testId, test.name); recalculations.remove(testId, status); } } diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/UIServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/UIServiceImpl.java index b3c28f8ac..d8692018e 100644 --- a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/UIServiceImpl.java +++ b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/UIServiceImpl.java @@ -25,6 +25,9 @@ public class UIServiceImpl implements UIService { @Inject TestServiceImpl testService; + @Inject + DatasetServiceImpl datasetService; + @Override @RolesAllowed("tester") @WithRoles @@ -64,6 +67,9 @@ private View doUpdate(TestDAO test, ViewDAO view) { test.views.add(view); test.persist(); em.flush(); + + // update datasets views + datasetService.calcDatasetViewsByTestAndView(test.id, view.id); return ViewMapper.from(view); } @@ -80,6 +86,8 @@ public void deleteView(int testId, int viewId) { if (!test.views.removeIf(v -> v.id == viewId)) { throw ServiceException.badRequest("Test does not contain this view!"); } + // remove dataset views records linked to this view + em.createNativeQuery("DELETE FROM dataset_view WHERE view_id = ?1").setParameter(1, viewId).executeUpdate(); // the orphan removal doesn't work for some reason, we need to remove if manually ViewDAO.deleteById(viewId); test.persist(); diff --git a/horreum-backend/src/main/resources/db/changeLog.xml b/horreum-backend/src/main/resources/db/changeLog.xml index ae409e624..59532b782 100644 --- a/horreum-backend/src/main/resources/db/changeLog.xml +++ b/horreum-backend/src/main/resources/db/changeLog.xml @@ -4688,4 +4688,43 @@ $$ LANGUAGE plpgsql; + + ANY + + -- drop triggers + DROP TRIGGER IF EXISTS dsv_after_delete ON viewcomponent; + DROP TRIGGER IF EXISTS dsv_after_update ON viewcomponent; + DROP TRIGGER IF EXISTS recalc_dataset_view ON view_recalc_queue; + DROP TRIGGER IF EXISTS dsv_after_insert ON label_values; + + -- drop functions + DROP FUNCTION dsv_after_vc_delete_func; + DROP FUNCTION dsv_after_vc_update_func; + DROP FUNCTION recalc_dataset_view; + DROP FUNCTION dsv_after_lv_insert_func; + + -- drop view_recalc_queue table as not needed anymore + DROP TABLE view_recalc_queue; + + -- still need db procedure until https://hibernate.atlassian.net/browse/HHH-17314 is fixed in quarkus + -- viewId can be NULL + CREATE OR REPLACE PROCEDURE calc_dataset_view(datasetId bigint, viewId bigint DEFAULT NULL) AS $$ + BEGIN + WITH view_agg AS ( + SELECT + vc.view_id, vc.id as vcid, array_agg(DISTINCT label.id) as label_ids, jsonb_object_agg(label.name, lv.value) as value FROM dataset_schemas ds + JOIN label ON label.schema_id = ds.schema_id + JOIN viewcomponent vc ON vc.labels ? label.name + JOIN label_values lv ON lv.label_id = label.id AND lv.dataset_id = ds.dataset_id + WHERE ds.dataset_id = datasetId AND (viewId IS NULL OR vc.view_id = viewId) + AND vc.view_id IN (SELECT view.id FROM view JOIN dataset ON view.test_id = dataset.testid WHERE dataset.id = datasetId) + GROUP BY vc.view_id, vcid + ) + INSERT INTO dataset_view (dataset_id, view_id, label_ids, value) + SELECT datasetId, view_id, array_agg(DISTINCT label_id), jsonb_object_agg(vcid, value) FROM view_agg, unnest(label_ids) as label_id + GROUP BY view_id; + END + $$ LANGUAGE plpgsql; + +