#4916 - Support for MS SQL Server

- Added test for deleting document that has learning records attached - Added project import and delete integration test - Fix issue that search service is hanging at shutdown
inception-project · Aug 6, 2024 · fcfaed9 · fcfaed9
1 parent ccaec01
commit fcfaed9
Show file tree

Hide file tree

Showing 10 changed files with 160 additions and 31 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -13,6 +13,7 @@
 
 # Make sure that these files are treated as binary so that newlines are preserved.
 #   overrides GIT's determination if a file is text or not
+*.zip binary
 *.gz binary
 *.bin binary
 *.dump binary

diff --git a/...c/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/LearningRecordTest.java b/...c/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/LearningRecordTest.java
@@ -17,23 +17,21 @@
  */
 package de.tudarmstadt.ukp.inception.recommendation.api.model;
 
+import static java.util.Arrays.fill;
 import static org.assertj.core.api.Assertions.assertThat;
 
-import java.util.Arrays;
-
 import org.junit.jupiter.api.Test;
 
 public class LearningRecordTest
 {
-
     @Test
     public void thatTokenTextIsTruncated()
     {
-        char[] charArray = new char[300];
-        Arrays.fill(charArray, 'X');
-        String longTokenText = new String(charArray);
+        var charArray = new char[300];
+        fill(charArray, 'X');
+        var longTokenText = new String(charArray);
 
-        LearningRecord sut = new LearningRecord();
+        var sut = new LearningRecord();
         sut.setTokenText(longTokenText);
 
         assertThat(sut.getTokenText()).as("TokenText has been truncated").hasSize(255);

diff --git a/inception/inception-app-webapp/pom.xml b/inception/inception-app-webapp/pom.xml
@@ -184,6 +184,30 @@
       <groupId>de.tudarmstadt.ukp.inception.app</groupId>
       <artifactId>inception-sharing</artifactId>
     </dependency>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-project</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-project-export</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-recommendation-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-api-annotation</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-export-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>de.tudarmstadt.ukp.inception.app</groupId>
+      <artifactId>inception-schema-api</artifactId>
+    </dependency>
     <!--
     <dependency>
       <groupId>de.tudarmstadt.ukp.inception.app</groupId>
@@ -957,6 +981,12 @@
               <dependency>de.tudarmstadt.ukp.inception.app:inception-project</dependency>
               <dependency>de.tudarmstadt.ukp.inception.app:inception-project-api</dependency>
               <dependency>de.tudarmstadt.ukp.inception.app:inception-project-initializers</dependency>
+              <dependency>de.tudarmstadt.ukp.inception.app:inception-project</dependency>
+              <dependency>de.tudarmstadt.ukp.inception.app:inception-project-export</dependency>
+              <dependency>de.tudarmstadt.ukp.inception.app:inception-recommendation-api</dependency>
+              <dependency>de.tudarmstadt.ukp.inception.app:inception-api-annotation</dependency>
+              <dependency>de.tudarmstadt.ukp.inception.app:inception-export-api</dependency>
+              <dependency>de.tudarmstadt.ukp.inception.app:inception-schema-api</dependency>
             </ignoredNonTestScopedDependencies>
             <usedDependencies>
               <!-- INCEpTION backend modules - used via Spring -->

diff --git a/...bapp/src/test/java/de/tudarmstadt/ukp/inception/db/InceptionIntegrationTest_ImplBase.java b/...bapp/src/test/java/de/tudarmstadt/ukp/inception/db/InceptionIntegrationTest_ImplBase.java
@@ -23,6 +23,8 @@
 import static de.tudarmstadt.ukp.inception.support.deployment.DeploymentModeService.PROFILE_PRODUCTION_MODE;
 import static java.util.Arrays.asList;
 
+import java.util.zip.ZipFile;
+
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -36,13 +38,22 @@
 import org.springframework.test.context.DynamicPropertySource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import de.tudarmstadt.ukp.clarin.webanno.api.export.ProjectImportRequest;
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
 import de.tudarmstadt.ukp.clarin.webanno.model.Project;
+import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;
 import de.tudarmstadt.ukp.clarin.webanno.project.initializers.webanno.StandardProjectInitializer;
 import de.tudarmstadt.ukp.inception.INCEpTION;
+import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport;
 import de.tudarmstadt.ukp.inception.app.config.InceptionApplicationContextInitializer;
+import de.tudarmstadt.ukp.inception.documents.api.DocumentService;
 import de.tudarmstadt.ukp.inception.documents.api.RepositoryProperties;
 import de.tudarmstadt.ukp.inception.project.api.ProjectInitializationRequest;
 import de.tudarmstadt.ukp.inception.project.api.ProjectService;
+import de.tudarmstadt.ukp.inception.project.export.ProjectExportService;
+import de.tudarmstadt.ukp.inception.recommendation.api.LearningRecordService;
+import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecord;
+import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService;
 import de.tudarmstadt.ukp.inception.support.logging.Logging;
 
 @ActiveProfiles({ //
@@ -58,12 +69,24 @@ public abstract class InceptionIntegrationTest_ImplBase
     @Autowired
     ProjectService projectService;
 
+    @Autowired
+    ProjectExportService projectExportService;
+
     @Autowired
     StandardProjectInitializer standardProjectInitializer;
 
     @Autowired
     RepositoryProperties repositoryProperties;
 
+    @Autowired
+    DocumentService documentService;
+
+    @Autowired
+    AnnotationSchemaService schemaService;
+
+    @Autowired
+    LearningRecordService learningRecordService;
+
     @BeforeEach
     void setupClass()
     {
@@ -89,7 +112,7 @@ void testContextStartsUpSuccessfully() throws Exception
     }
 
     @Test
-    void testCreatingAndRemovingProject() throws Exception
+    void testCreatingAndDeletingProject() throws Exception
     {
         var project = Project.builder().withName("test").withSlug("test").build();
         projectService.createProject(project);
@@ -103,4 +126,54 @@ void testCreatingAndRemovingProject() throws Exception
 
         projectService.removeProject(project);
     }
+
+    @Test
+    void testImportingAndDeletingRemovingProject() throws Exception
+    {
+        var request = ProjectImportRequest.builder() //
+                .withCreateMissingUsers(true) //
+                .withImportPermissions(true) //
+                .build();
+
+        Project project;
+        try (var zipFile = new ZipFile("src/test/resources/test-project-with-recommenders.zip")) {
+            project = projectExportService.importProject(request, zipFile);
+        }
+
+        projectService.removeProject(project);
+    }
+
+    @Test
+    void testDeletingSourceDocumentWithLearningRecordAttached() throws Exception
+    {
+        var project = Project.builder() //
+                .withName("test") //
+                .build();
+        var layer = AnnotationLayer.builder() //
+                .withProject(project) //
+                .withName("Layer") //
+                .withUiName("Layer") //
+                .withType(SpanLayerSupport.TYPE) //
+                .build();
+        var doc = SourceDocument.builder() //
+                .withProject(project) //
+                .withName("Blah") //
+                .build();
+        var learningRecord = LearningRecord.builder() //
+                .withLayer(layer) //
+                .withSourceDocument(doc) //
+                .withSuggestionType(layer.getType()) //
+                .build();
+        try {
+            projectService.createProject(project);
+            schemaService.createOrUpdateLayer(layer);
+            documentService.createSourceDocument(doc);
+            learningRecordService.createLearningRecord(learningRecord);
+
+            documentService.removeSourceDocument(doc);
+        }
+        finally {
+            projectService.removeProject(project);
+        }
+    }
 }
diff --git a/inception/inception-app-webapp/src/test/resources/test-project-with-recommenders.zip b/inception/inception-app-webapp/src/test/resources/test-project-with-recommenders.zip
diff --git a/...ents/src/main/java/de/tudarmstadt/ukp/inception/documents/DocumentStorageServiceImpl.java b/...ents/src/main/java/de/tudarmstadt/ukp/inception/documents/DocumentStorageServiceImpl.java
@@ -81,7 +81,9 @@ public void removeSourceDocumentFile(SourceDocument aDocument) throws IOExceptio
         Validate.notNull(aDocument, "Parameter [sourceDocument] must be specified");
 
         var path = getSourceDocumentFolder(aDocument);
-        FileUtils.forceDelete(path.toFile());
+        if (Files.exists(path)) {
+            FileUtils.forceDelete(path.toFile());
+        }
     }
 
     @Override

diff --git a/...n/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java b/...n/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java
@@ -853,6 +853,7 @@ public void onDocumentCreated(AfterDocumentCreatedEvent aEvent)
     public void onDocumentRemoval(BeforeDocumentRemovedEvent aEvent)
     {
         resetState(aEvent.getDocument().getProject());
+        deleteLearningRecords(aEvent.getDocument());
     }
 
     @EventListener
@@ -1605,6 +1606,13 @@ public void removeLearningRecords(String aDataOwner, SourceDocument aDocument)
             }
         }
 
+        public void removeLearningRecords(SourceDocument aDocument)
+        {
+            for (var records : learningRecords.values()) {
+                records.removeIf(r -> Objects.equals(r.getSourceDocument(), aDocument));
+            }
+        }
+
         public void removeLearningRecords(LearningRecord aRecord)
         {
             var records = learningRecords.get(aRecord.getLayer());
@@ -1949,6 +1957,20 @@ public void createLearningRecords(LearningRecord... aRecords)
         }
     }
 
+    private void deleteLearningRecords(SourceDocument aDocument)
+    {
+        synchronized (states) {
+            for (var state : states.values()) {
+                state.removeLearningRecords(aDocument);
+            }
+        }
+
+        var sql = "DELETE FROM LearningRecord l where l.sourceDocument = :document";
+        entityManager.createQuery(sql) //
+                .setParameter("document", aDocument) //
+                .executeUpdate();
+    }
+
     private void deleteLearningRecords(SourceDocument aDocument, String aDataOwner)
     {
         var state = getState(aDataOwner, aDocument.getProject());

diff --git a/...tion-search-core/src/main/java/de/tudarmstadt/ukp/inception/search/SearchServiceImpl.java b/...tion-search-core/src/main/java/de/tudarmstadt/ukp/inception/search/SearchServiceImpl.java
@@ -176,7 +176,7 @@ public void destroy()
         // all the indexes
         schedulingService.stopAllTasksMatching(task -> task instanceof IndexingTask_ImplBase);
         long t0 = currentTimeMillis();
-        while (isBusy() && currentTimeMillis() - t0 < 10_000) {
+        while (isBusy() && (currentTimeMillis() - t0) < 10_000) {
             try {
                 Thread.sleep(500);
             }
@@ -187,9 +187,15 @@ public void destroy()
 
         while (!indexes.isEmpty()) {
             synchronized (indexes) {
-                List<PooledIndex> pooledIndexesSnapshot = new ArrayList<>(indexes.values());
+                var pooledIndexesSnapshot = indexes.values() //
+                        .stream().filter(i -> !i.isDead()) //
+                        .toList();
 
-                for (PooledIndex pooledIndex : pooledIndexesSnapshot) {
+                if (pooledIndexesSnapshot.isEmpty()) {
+                    break;
+                }
+
+                for (var pooledIndex : pooledIndexesSnapshot) {
                     unloadIndex(pooledIndex);
                 }
             }

diff --git a/...roject/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/ProjectImportPanel.java b/...roject/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/ProjectImportPanel.java
@@ -26,8 +26,6 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
@@ -40,7 +38,6 @@
 import org.apache.wicket.feedback.IFeedback;
 import org.apache.wicket.markup.html.form.CheckBox;
 import org.apache.wicket.markup.html.form.Form;
-import org.apache.wicket.markup.html.form.upload.FileUpload;
 import org.apache.wicket.markup.html.panel.Panel;
 import org.apache.wicket.model.CompoundPropertyModel;
 import org.apache.wicket.model.IModel;
@@ -82,18 +79,18 @@ public ProjectImportPanel(String aId, IModel<Project> aModel)
         preferences = Model.of(new Preferences());
         selectedModel = aModel;
 
-        Form<Preferences> form = new Form<>("form", CompoundPropertyModel.of(preferences));
+        var form = new Form<>("form", CompoundPropertyModel.of(preferences));
 
         // Only administrators who can access user management can also use the "create missing
         // users" checkbox. Also, the option is only available when importing permissions in the
         // first place.
-        CheckBox generateUsers = new CheckBox("generateUsers");
+        var generateUsers = new CheckBox("generateUsers");
         generateUsers.setOutputMarkupPlaceholderTag(true);
         generateUsers.add(visibleWhen(() -> preferences.getObject().importPermissions));
         form.add(generateUsers);
         authorize(generateUsers, Component.RENDER, ROLE_ADMIN.name());
 
-        CheckBox importPermissions = new CheckBox("importPermissions");
+        var importPermissions = new CheckBox("importPermissions");
         importPermissions.add(new LambdaAjaxFormComponentUpdatingBehavior("change", _target -> {
             if (!preferences.getObject().importPermissions) {
                 preferences.getObject().generateUsers = false;
@@ -116,11 +113,11 @@ public ProjectImportPanel(String aId, IModel<Project> aModel)
 
     private void actionImport(AjaxRequestTarget aTarget, Form<Preferences> aForm)
     {
-        List<FileUpload> exportedProjects = fileUpload.getFileUploads();
+        var exportedProjects = fileUpload.getFileUploads();
 
-        User currentUser = userRepository.getCurrentUser();
-        boolean currentUserIsAdministrator = userRepository.isAdministrator(currentUser);
-        boolean currentUserIsProjectCreator = userRepository.isProjectCreator(currentUser);
+        var currentUser = userRepository.getCurrentUser();
+        var currentUserIsAdministrator = userRepository.isAdministrator(currentUser);
+        var currentUserIsProjectCreator = userRepository.isProjectCreator(currentUser);
 
         boolean createMissingUsers;
         boolean importPermissions;
@@ -156,15 +153,14 @@ else if (currentUserIsProjectCreator) {
         }
 
         List<Project> importedProjects = new ArrayList<>();
-        for (FileUpload exportedProject : exportedProjects) {
-            ProjectImportRequest request = new ProjectImportRequest(createMissingUsers,
-                    importPermissions, manager);
+        for (var exportedProject : exportedProjects) {
+            var request = new ProjectImportRequest(createMissingUsers, importPermissions, manager);
 
             try {
                 // Workaround for WICKET-6425
-                File tempFile = File.createTempFile("project-import", null);
-                try (InputStream is = new BufferedInputStream(exportedProject.getInputStream());
-                        OutputStream os = new FileOutputStream(tempFile);) {
+                var tempFile = File.createTempFile("project-import", null);
+                try (var is = new BufferedInputStream(exportedProject.getInputStream());
+                        var os = new FileOutputStream(tempFile);) {
                     if (!ZipUtils.isZipStream(is)) {
                         throw new IOException("Invalid ZIP file");
                     }

diff --git a/...main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/documents/SourceDocumentTable.java b/...main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/documents/SourceDocumentTable.java
@@ -215,10 +215,11 @@ private String renderDocumentSize(SourceDocument aDocumnent)
     private String renderInitialCasSize(SourceDocument aDocument)
     {
         try {
-            return documentService.getInitialCasFileSize(aDocument)
-                    .map(FileUtils::byteCountToDisplaySize).orElse("unknown");
+            return documentService.getInitialCasFileSize(aDocument) //
+                    .map(FileUtils::byteCountToDisplaySize) //
+                    .orElse("unknown");
         }
-        catch (IOException e) {
+        catch (Exception e) {
             LOG.error("Unable to get size of INITIAL CAS file for {}", aDocument, e);
             return "error";
         }