Move dataset views mgmt into Horreum logic

As part of this refactoring I removed table view_recalc_queue and: Triggers - dsv_after_delete - dsv_after_update - recalc_dataset_view - dsv_after_insert Functions: - dsv_after_vc_delete_func - dsv_after_vc_update_func - recalc_dataset_view - dsv_after_lv_insert_func Signed-off-by: Andrea Lamparelli <[email protected]>
Hyperfoil · Dec 9, 2024 · 7f2ba80 · 7f2ba80
1 parent 1d550ce
commit 7f2ba80
Show file tree

Hide file tree

Showing 6 changed files with 109 additions and 13 deletions.
diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/DatasetServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/DatasetServiceImpl.java
@@ -18,6 +18,8 @@
 import jakarta.ws.rs.DefaultValue;
 
 import org.hibernate.Hibernate;
+import org.hibernate.ScrollMode;
+import org.hibernate.ScrollableResults;
 import org.hibernate.Session;
 import org.hibernate.query.NativeQuery;
 import org.hibernate.type.StandardBasicTypes;
@@ -445,9 +447,8 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i
                         "Evaluation of label %s failed: '%s' Code:<pre>%s</pre>", row[0], e.getMessage(), jsCode),
                 (out) -> logMessage(datasetId, PersistentLogDAO.DEBUG, "Output while calculating labels: <pre>%s</pre>", out));
 
-        //Create new dataset views from the recently created label values
-        em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
-        em.createNativeQuery("call calc_dataset_view(?1);").setParameter(1, datasetId).executeUpdate();
+        // create new dataset views from the recently created label values
+        calcDatasetViews(datasetId);
 
         createFingerprint(datasetId, testId);
         mediator.updateLabels(new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation));
@@ -456,10 +457,48 @@ void calculateLabelValues(int testId, int datasetId, int queryLabelId, boolean i
                     testId, new Dataset.LabelsUpdatedEvent(testId, datasetId, isRecalculation)));
     }
 
+    @Transactional
+    public void calcDatasetViews(int datasetId) {
+        // TODO(user) move calc_dataset_view into Horreum business logic see https://github.com/hibernate/hibernate-orm/pull/7457
+        em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
+        em.createNativeQuery("call calc_dataset_view(?1, NULL);").setParameter(1, datasetId).executeUpdate();
+    }
+
+    @Transactional
+    @SuppressWarnings("unchecked")
+    public void calcDatasetViewsByTestAndView(int testId, int viewId) {
+        // delete all dataset views associated to the provided viewId and testId
+        // for new views it won't delete anything
+        em.createNativeQuery(
+                "DELETE FROM dataset_view WHERE view_id = ?1 AND dataset_id IN (SELECT id FROM dataset WHERE testid = ?2)")
+                .setParameter(1, viewId)
+                .setParameter(2, testId)
+                .executeUpdate();
+
+        // re-create dataset views associated to the provided viewId
+        try (ScrollableResults<Integer> datasetIds = em
+                .createNativeQuery("SELECT id FROM dataset WHERE testid = ?1")
+                .setParameter(1, testId)
+                .unwrap(NativeQuery.class)
+                .setReadOnly(false)
+                .setFetchSize(100)
+                .scroll(ScrollMode.FORWARD_ONLY)) {
+            while (datasetIds.next()) {
+                int datasetId = datasetIds.get();
+                log.tracef("Recalculate dataset views for view %d and dataset %d", viewId, datasetId);
+                em.createNativeQuery("call calc_dataset_view(?1, ?2);")
+                        .setParameter(1, datasetId)
+                        .setParameter(2, viewId)
+                        .executeUpdate();
+            }
+        }
+    }
+
     @Transactional
     public void deleteDataset(int datasetId) {
         em.createNativeQuery("DELETE FROM label_values WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
         em.createNativeQuery("DELETE FROM dataset_schemas WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
+        em.createNativeQuery("DELETE FROM dataset_view WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
         em.createNativeQuery("DELETE FROM fingerprint WHERE dataset_id = ?1").setParameter(1, datasetId).executeUpdate();
         em.createNativeQuery("DELETE FROM dataset WHERE id = ?1").setParameter(1, datasetId).executeUpdate();
     }

diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java
@@ -1087,9 +1087,8 @@ public Map<Integer, String> updateSchema(int id, String path, String schemaUri)
     @Transactional
     @Override
     public List<Integer> recalculateDatasets(int runId) {
-        transform(runId, true);
-        return session.createNativeQuery("SELECT id FROM dataset WHERE runid = ? ORDER BY ordinal", Integer.class)
-                .setParameter(1, runId).getResultList();
+        log.infof("Transforming run id %d", runId);
+        return transform(runId, true);
     }
 
     @RolesAllowed(Roles.ADMIN)

diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/TestServiceImpl.java
@@ -669,6 +669,7 @@ public void recalculateDatasets(int testId) {
                             status.finished++;
                             status.datasets += newDatasets;
                             if (status.finished == status.totalRuns) {
+                                log.infof("Datasets recalculation for test %d (%s) completed", testId, test.name);
                                 recalculations.remove(testId, status);
                             }
                         }

diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/UIServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/UIServiceImpl.java
@@ -25,6 +25,9 @@ public class UIServiceImpl implements UIService {
     @Inject
     TestServiceImpl testService;
 
+    @Inject
+    DatasetServiceImpl datasetService;
+
     @Override
     @RolesAllowed("tester")
     @WithRoles
@@ -64,6 +67,9 @@ private View doUpdate(TestDAO test, ViewDAO view) {
         test.views.add(view);
         test.persist();
         em.flush();
+
+        // update datasets views
+        datasetService.calcDatasetViewsByTestAndView(test.id, view.id);
         return ViewMapper.from(view);
     }
 
@@ -80,6 +86,8 @@ public void deleteView(int testId, int viewId) {
         if (!test.views.removeIf(v -> v.id == viewId)) {
             throw ServiceException.badRequest("Test does not contain this view!");
         }
+        // remove dataset views records linked to this view
+        em.createNativeQuery("DELETE FROM dataset_view WHERE view_id = ?1").setParameter(1, viewId).executeUpdate();
         // the orphan removal doesn't work for some reason, we need to remove if manually
         ViewDAO.deleteById(viewId);
         test.persist();

diff --git a/horreum-backend/src/main/resources/db/changeLog.xml b/horreum-backend/src/main/resources/db/changeLog.xml
@@ -4688,4 +4688,43 @@
             $$ LANGUAGE plpgsql;
         </sql>
     </changeSet>
+    <changeSet id="126" author="lampajr">
+        <validCheckSum>ANY</validCheckSum>
+        <sql>
+            -- drop triggers
+            DROP TRIGGER IF EXISTS dsv_after_delete ON viewcomponent;
+            DROP TRIGGER IF EXISTS dsv_after_update ON viewcomponent;
+            DROP TRIGGER IF EXISTS recalc_dataset_view ON view_recalc_queue;
+            DROP TRIGGER IF EXISTS dsv_after_insert ON label_values;
+
+            -- drop functions
+            DROP FUNCTION dsv_after_vc_delete_func;
+            DROP FUNCTION dsv_after_vc_update_func;
+            DROP FUNCTION recalc_dataset_view;
+            DROP FUNCTION dsv_after_lv_insert_func;
+
+            -- drop view_recalc_queue table as not needed anymore
+            DROP TABLE view_recalc_queue;
+
+            -- still need db procedure until https://hibernate.atlassian.net/browse/HHH-17314 is fixed in quarkus
+            -- viewId can be NULL
+            CREATE OR REPLACE PROCEDURE calc_dataset_view(datasetId bigint, viewId bigint DEFAULT NULL) AS $$
+            BEGIN
+            WITH view_agg AS (
+                SELECT
+                    vc.view_id, vc.id as vcid, array_agg(DISTINCT label.id) as label_ids, jsonb_object_agg(label.name, lv.value) as value FROM dataset_schemas ds
+                JOIN label ON label.schema_id = ds.schema_id
+                JOIN viewcomponent vc ON vc.labels ? label.name
+                JOIN label_values lv ON lv.label_id = label.id AND lv.dataset_id = ds.dataset_id
+                WHERE ds.dataset_id = datasetId AND (viewId IS NULL OR vc.view_id = viewId)
+                    AND vc.view_id IN (SELECT view.id FROM view JOIN dataset ON view.test_id = dataset.testid WHERE dataset.id = datasetId)
+                GROUP BY vc.view_id, vcid
+            )
+            INSERT INTO dataset_view (dataset_id, view_id, label_ids, value)
+                SELECT datasetId, view_id, array_agg(DISTINCT label_id), jsonb_object_agg(vcid, value) FROM view_agg, unnest(label_ids) as label_id
+                GROUP BY view_id;
+            END
+            $$ LANGUAGE plpgsql;
+        </sql>
+    </changeSet>
 </databaseChangeLog>
diff --git a/horreum-backend/src/test/java/io/hyperfoil/tools/horreum/svc/DatasetServiceTest.java b/horreum-backend/src/test/java/io/hyperfoil/tools/horreum/svc/DatasetServiceTest.java
@@ -34,13 +34,15 @@
 import io.hyperfoil.tools.horreum.entity.data.*;
 import io.hyperfoil.tools.horreum.entity.data.ViewComponentDAO;
 import io.hyperfoil.tools.horreum.mapper.LabelMapper;
+import io.hyperfoil.tools.horreum.mapper.ViewMapper;
 import io.hyperfoil.tools.horreum.server.CloseMe;
 import io.hyperfoil.tools.horreum.test.HorreumTestProfile;
 import io.hyperfoil.tools.horreum.test.PostgresResource;
 import io.quarkus.test.common.QuarkusTestResource;
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.TestProfile;
 import io.quarkus.test.oidc.server.OidcWiremockTestResource;
+import io.restassured.response.Response;
 
 @QuarkusTest
 @QuarkusTestResource(PostgresResource.class)
@@ -380,13 +382,21 @@ public void testDatasetView() {
                 assertTrue(ids.contains(labelA));
                 assertTrue(ids.contains(labelB));
 
-                Util.withTx(tm, () -> {
-                    try (CloseMe ignored = roleManager.withRoles(Arrays.asList(TESTER_ROLES))) {
-                        int vcs = em.createNativeQuery("UPDATE viewcomponent SET labels = '[\"a\",\"b\"]'").executeUpdate();
-                        assertEquals(2, vcs);
-                    }
-                    return null;
-                });
+                // there must be only one result here!
+                ViewDAO dbView = (ViewDAO) ViewDAO.findAll().list().get(0);
+                assertEquals(2, dbView.components.size());
+
+                View updateView = ViewMapper.from(dbView);
+                ArrayNode vcLabels = (ArrayNode) updateView.components.stream().filter(vc -> vc.labels.size() == 1).findFirst()
+                        .orElseThrow().labels;
+                vcLabels.add("b");
+
+                Response response = jsonRequest()
+                        .auth()
+                        .oauth2(getTesterToken())
+                        .body(updateView)
+                        .post("/api/ui/view");
+                assertEquals(200, response.getStatusCode());
 
                 JsonNode updated = fetchDatasetsByTest(test.id);
                 JsonNode updatedView = updated.get("datasets").get(0).get("view");