Merge branch 'develop' into 7492_muting_notifications IQSS#7492

doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst

@@ -16,4 +16,6 @@ Reporting tools and queries created by members of the Dataverse community.
 
 * dataverse-metrics (<https://github.com/IQSS/dataverse-metrics>): Aggregates and visualizes metrics for Dataverse installations around the world or a single Dataverse installation.
 
-* Useful queries from the Dataverse Community (<https://docs.google.com/document/d/1-Y_iUduSxdDNeK1yiGUxe7t-Md7Fy965jp4o4m1XEoE/edit#heading=h.avuoo5kf0mdt>): A community-generated and maintained document of postgresql queries for getting information about users and dataverse collections, datasets, and files in your Dataverse installation. If you are trying to find out some information from your Dataverse installation, chances are that someone else has had the same questions and it's now listed in this document. If it's not listed, please feel free to add it to the document.
+* Automated Dataverse Metrics Reports (<https://github.com/QualitativeDataRepository/dataverse-metrics>): Create beautiful, customizable reports about your Dataverse installation at the click of a button, using R-Markdown and GitHub Actions.
+
+* Useful queries from the Dataverse Community (<https://docs.google.com/document/d/1-Y_iUduSxdDNeK1yiGUxe7t-Md7Fy965jp4o4m1XEoE/edit#heading=h.avuoo5kf0mdt>): A community-generated and maintained document of postgresql queries for getting information about users and dataverse collections, datasets, and files in your Dataverse installation. If you are trying to find out some information from your Dataverse installation, chances are that someone else has had the same questions and it's now listed in this document. If it's not listed, please feel free to add it to the document.

doc/sphinx-guides/source/api/sword.rst

@@ -82,7 +82,7 @@ New features as of v1.1
 
 - "Contributor" can now be populated and the "Type" (Editor, Funder, Researcher, etc.) can be specified with an XML attribute. For example: ``<dcterms:contributor type="Funder">CaffeineForAll</dcterms:contributor>``
 
-- "License" can now be set with ``dcterms:license`` and the possible values determined by the installation ("CC0 1.0" and "CC BY 4.0" by default). "License" interacts with "Terms of Use" (``dcterms:rights``) in that if you include ``dcterms:rights`` in the XML and don't include ``dcterms:license``, the license will be "Custom Dataset Terms" and "Terms of Use" will be populated. If you don't include ``dcterms:rights``, the default license will be used. It is invalid to specify a license and also include ``dcterms:rights``; an error will be returned. For backwards compatibility, ``dcterms:rights`` is allowed to be blank (i.e. ``<dcterms:rights></dcterms:rights>``) but blank values will not be persisted to the database and the license will be set to "Custom Dataset Terms".
+- "License" can now be set with ``dcterms:license`` and the possible values determined by the installation ("CC0 1.0" and "CC BY 4.0" by default). "License" interacts with "Terms of Use" (``dcterms:rights``) in that if you include ``dcterms:rights`` in the XML and don't include ``dcterms:license``, the license will be "Custom Dataset Terms" and "Terms of Use" will be populated. If you don't include ``dcterms:rights``, the default license will be used. It is invalid to specify a license and also include ``dcterms:rights``; an error will be returned. For backwards compatibility, ``dcterms:rights`` is allowed to be blank (i.e. ``<dcterms:rights></dcterms:rights>``) but blank values will not be persisted to the database and the license will be set to "Custom Dataset Terms". Note that if admins of an installation have disabled "Custom Dataset Terms" you will get an error if you try to pass ``dcterms:rights``.
 
 - "Contact E-mail" is automatically populated from dataset owner's email.
 

src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordServiceBean.java

@@ -12,6 +12,7 @@
 import edu.harvard.iq.dataverse.license.License;
 import edu.harvard.iq.dataverse.license.LicenseServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.util.ArrayList;
 
 import java.util.List;
@@ -36,6 +37,8 @@ public class SwordServiceBean {
     DatasetFieldServiceBean datasetFieldService;
     @Inject
     LicenseServiceBean licenseServiceBean;
+    @EJB
+    SystemConfig systemConfig;
 
     /**
      * Mutate the dataset version, adding a datasetContact (email address) from
@@ -150,6 +153,11 @@ public void addDatasetSubjectIfMissing(DatasetVersion datasetVersion) {
     public void setDatasetLicenseAndTermsOfUse(DatasetVersion datasetVersionToMutate, SwordEntry swordEntry) throws SwordError {
         Map<String, List<String>> dcterms = swordEntry.getDublinCore();
         List<String> listOfLicensesProvided = dcterms.get("license");
+        List<String> rights = dcterms.get("rights");
+        if (rights != null && !systemConfig.isAllowCustomTerms()) {
+            throw new SwordError("Custom Terms (dcterms:rights) are not allowed.");
+        }
+
         TermsOfUseAndAccess terms = new TermsOfUseAndAccess();
         datasetVersionToMutate.setTermsOfUseAndAccess(terms);
         terms.setDatasetVersion(datasetVersionToMutate);

src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java

@@ -13,6 +13,7 @@
 import edu.harvard.iq.dataverse.api.dto.*;  
 import edu.harvard.iq.dataverse.api.dto.FieldDTO;
 import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO;
+import edu.harvard.iq.dataverse.license.LicenseServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
@@ -67,6 +68,9 @@ public class ImportGenericServiceBean {
     @EJB
     SettingsServiceBean settingsService;
 
+    @EJB
+    LicenseServiceBean licenseService;
+
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
 
@@ -103,7 +107,7 @@ public void importXML(String xmlToParse, String foreignFormat, DatasetVersion da
             logger.fine(json);
             JsonReader jsonReader = Json.createReader(new StringReader(json));
             JsonObject obj = jsonReader.readObject();
-            DatasetVersion dv = new JsonParser(datasetFieldSvc, blockService, settingsService).parseDatasetVersion(obj, datasetVersion);
+            DatasetVersion dv = new JsonParser(datasetFieldSvc, blockService, settingsService, licenseService).parseDatasetVersion(obj, datasetVersion);
         } catch (XMLStreamException ex) {
             //Logger.getLogger("global").log(Level.SEVERE, null, ex);
             throw new EJBException("ERROR occurred while parsing XML fragment  ("+xmlToParse.substring(0, 64)+"...); ", ex);
@@ -115,50 +119,6 @@ public void importXML(String xmlToParse, String foreignFormat, DatasetVersion da
             } catch (XMLStreamException ex) {}
         }
     }
-
-    public void importXML(File xmlFile, String foreignFormat, DatasetVersion datasetVersion) {
-
-        FileInputStream in = null;
-        XMLStreamReader xmlr = null;
-
-        // look up the foreign metadata mapping for this format:
-
-        ForeignMetadataFormatMapping mappingSupported = findFormatMappingByName (foreignFormat);
-        if (mappingSupported == null) {
-            throw new EJBException("Unknown/unsupported foreign metadata format "+foreignFormat);
-        }
-
-        try {
-            in = new FileInputStream(xmlFile);
-            XMLInputFactory xmlFactory = javax.xml.stream.XMLInputFactory.newInstance();
-            xmlr =  xmlFactory.createXMLStreamReader(in);
-            DatasetDTO datasetDTO = processXML(xmlr, mappingSupported);
-
-            Gson gson = new Gson();
-            String json = gson.toJson(datasetDTO.getDatasetVersion());
-            logger.info("Json:\n"+json);
-            JsonReader jsonReader = Json.createReader(new StringReader(json));
-            JsonObject obj = jsonReader.readObject();
-            DatasetVersion dv = new JsonParser(datasetFieldSvc, blockService, settingsService).parseDatasetVersion(obj, datasetVersion);
-        } catch (FileNotFoundException ex) {
-            //Logger.getLogger("global").log(Level.SEVERE, null, ex);
-            throw new EJBException("ERROR occurred in mapDDI: File Not Found!");
-        } catch (XMLStreamException ex) {
-            //Logger.getLogger("global").log(Level.SEVERE, null, ex);
-            throw new EJBException("ERROR occurred while parsing XML (file "+xmlFile.getAbsolutePath()+"); ", ex);
-        } catch (JsonParseException ex) {
-            Logger.getLogger(ImportGenericServiceBean.class.getName()).log(Level.SEVERE, null, ex);
-        } finally {
-            try {
-                if (xmlr != null) { xmlr.close(); }
-            } catch (XMLStreamException ex) {}
-
-            try {
-                if (in != null) { in.close();}
-            } catch (IOException ex) {}
-        }
-
-    }
 
     public DatasetDTO processXML( XMLStreamReader xmlr, ForeignMetadataFormatMapping foreignFormatMapping) throws XMLStreamException {
         // init - similarly to what I'm doing in the metadata extraction code? 
@@ -551,34 +511,6 @@ public DatasetDTO doImport(String xmlToParse) throws XMLStreamException {
         }
         return datasetDTO;
     }
-
-     public void importDCTerms(String xmlToParse, DatasetVersion datasetVersion, DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService) {
-        DatasetDTO datasetDTO = this.initializeDataset();
-        try {
-            // Read docDescr and studyDesc into DTO objects.
-            Map<String, String> fileMap = mapDCTerms(xmlToParse, datasetDTO);
-            // 
-            // convert DTO to Json, 
-            Gson gson = new Gson();
-            String json = gson.toJson(datasetDTO.getDatasetVersion());
-            JsonReader jsonReader = Json.createReader(new StringReader(json));
-            JsonObject obj = jsonReader.readObject();
-            //and call parse Json to read it into a datasetVersion
-            DatasetVersion dv = new JsonParser(datasetFieldSvc, blockService, settingsService).parseDatasetVersion(obj, datasetVersion);
-        } catch (XMLStreamException | JsonParseException e) {
-            // EMK TODO: exception handling
-            e.printStackTrace();
-        }
-
-        //EMK TODO:  Call methods for reading FileMetadata and related objects from xml, return list of FileMetadata objects.
-        /*try {
-            
-         Map<String, DataTable> dataTableMap = new DataTableImportDDI().processDataDscr(xmlr);
-         } catch(Exception e) {
-            
-         }*/
-        // Save Dataset and DatasetVersion in database
-    }
 
     public Map<String, String> mapDCTerms(String xmlToParse, DatasetDTO datasetDTO) throws XMLStreamException {
 

src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java

@@ -282,11 +282,22 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto)
     }
 
     private static void writeOtherStudyMaterial(XMLStreamWriter xmlw , DatasetVersionDTO version) throws XMLStreamException {
+        List<String> relMaterials;
+        List<String> relDatasets;
+        List<String> relReferences;
+        try {
+            relMaterials = dto2PrimitiveList(version, DatasetFieldConstant.relatedMaterial);
+            relDatasets = dto2PrimitiveList(version, DatasetFieldConstant.relatedDatasets);
+            relReferences = dto2PrimitiveList(version, DatasetFieldConstant.otherReferences); 
+        } catch (Exception e) {
+            logger.warning("Exporting dataset to DDI failed for related materials element: " + e.getMessage());
+            return;
+        }
         xmlw.writeStartElement("othrStdyMat");
-        writeFullElementList(xmlw, "relMat", dto2PrimitiveList(version, DatasetFieldConstant.relatedMaterial));
-        writeFullElementList(xmlw, "relStdy", dto2PrimitiveList(version, DatasetFieldConstant.relatedDatasets));
+        writeFullElementList(xmlw, "relMat", relMaterials);
+        writeFullElementList(xmlw, "relStdy", relDatasets);
         writeRelPublElement(xmlw, version);
-        writeFullElementList(xmlw, "othRefs", dto2PrimitiveList(version, DatasetFieldConstant.otherReferences));
+        writeFullElementList(xmlw, "othRefs", relReferences);
         xmlw.writeEndElement(); //othrStdyMat
     }
 

src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java

@@ -75,6 +75,7 @@ public class JsonParser {
      */
     boolean lenient = false;  
 
+    @Deprecated
     public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService) {
         this.datasetFieldSvc = datasetFieldSvc;
         this.blockService = blockService;

src/test/java/edu/harvard/iq/dataverse/api/SwordIT.java

@@ -5,6 +5,7 @@
 import com.jayway.restassured.response.Response;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Paths;
@@ -725,6 +726,19 @@ public void testCustomTerms() {
                 .statusCode(OK.getStatusCode())
                 .body("data.latestVersion.termsOfUse", equalTo("Call me"))
                 .body("data.latestVersion.license", equalTo(null));
+
+        UtilIT.setSetting(SettingsServiceBean.Key.AllowCustomTermsOfUse, "false")
+                .then().assertThat().statusCode(OK.getStatusCode());
+
+        Response createDatasetCustomTermsDisabled = UtilIT.createDatasetViaSwordApi(dataverseAlias, title, description, license, rights, apiToken);
+        createDatasetCustomTermsDisabled.prettyPrint();
+        createDatasetCustomTermsDisabled.then().assertThat()
+                //  <summary>Custom Terms (dcterms:rights) are not allowed.</summary>
+                .statusCode(BAD_REQUEST.getStatusCode());
+
+        // cleanup, allow custom terms again (delete because it defaults to true)
+        UtilIT.deleteSetting(SettingsServiceBean.Key.AllowCustomTermsOfUse);
+
     }
 
     @Test
@@ -958,6 +972,8 @@ public void testDeleteFiles() {
 
     @AfterClass
     public static void tearDownClass() {
+        // cleanup, allow custom terms again (delete because it defaults to true)
+        UtilIT.deleteSetting(SettingsServiceBean.Key.AllowCustomTermsOfUse);
         UtilIT.deleteUser(superuser);
     }