Merge branch 'develop' into 10888-add-api-for-comparing-dataset-versions

.env

@@ -1,5 +1,5 @@
 APP_IMAGE=gdcc/dataverse:unstable
-POSTGRES_VERSION=16
+POSTGRES_VERSION=17
 DATAVERSE_DB_USER=dataverse
 SOLR_VERSION=9.3.0
-SKIP_DEPLOY=0
+SKIP_DEPLOY=0

.github/workflows/deploy_beta_testing.yml

@@ -10,9 +10,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: 'zulu'
           java-version: '17'
@@ -32,7 +32,7 @@ jobs:
         run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
 
       - name: Upload war artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: built-app
           path: ./target/${{ env.war_file }}
@@ -42,10 +42,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Download war artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: built-app
           path: ./

.github/workflows/maven_unit_test.yml

@@ -32,9 +32,9 @@ jobs:
         steps:
           # TODO: As part of #10618 change to setup-maven custom action
           # Basic setup chores
-          - uses: actions/checkout@v3
+          - uses: actions/checkout@v4
           - name: Set up JDK ${{ matrix.jdk }}
-            uses: actions/setup-java@v3
+            uses: actions/setup-java@v4
             with:
                 java-version: ${{ matrix.jdk }}
                 distribution: temurin
@@ -57,7 +57,7 @@ jobs:
 
           # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
           # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
-          - uses: actions/upload-artifact@v3
+          - uses: actions/upload-artifact@v4
             with:
                 name: dataverse-java${{ matrix.jdk }}.war
                 path: target/dataverse*.war
@@ -67,7 +67,7 @@ jobs:
           - run: |
                 tar -cvf java-builddir.tar target
                 tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-*
-          - uses: actions/upload-artifact@v3
+          - uses: actions/upload-artifact@v4
             with:
                 name: java-artifacts
                 path: |
@@ -98,16 +98,16 @@ jobs:
         steps:
             # TODO: As part of #10618 change to setup-maven custom action
             # Basic setup chores
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
             - name: Set up JDK ${{ matrix.jdk }}
-              uses: actions/setup-java@v3
+              uses: actions/setup-java@v4
               with:
                   java-version: ${{ matrix.jdk }}
                   distribution: temurin
                   cache: maven
 
             # Get the build output from the unit test job
-            - uses: actions/download-artifact@v3
+            - uses: actions/download-artifact@v4
               with:
                   name: java-artifacts
             - run: |
@@ -119,7 +119,7 @@ jobs:
 
             # Wrap up and send to coverage job
             - run: tar -cvf java-reportdir.tar target/site
-            - uses: actions/upload-artifact@v3
+            - uses: actions/upload-artifact@v4
               with:
                   name: java-reportdir
                   path: java-reportdir.tar
@@ -132,15 +132,15 @@ jobs:
         steps:
             # TODO: As part of #10618 change to setup-maven custom action
             # Basic setup chores
-          - uses: actions/checkout@v3
-          - uses: actions/setup-java@v3
+          - uses: actions/checkout@v4
+          - uses: actions/setup-java@v4
             with:
                 java-version: '17'
                 distribution: temurin
                 cache: maven
 
           # Get the build output from the integration test job
-          - uses: actions/download-artifact@v3
+          - uses: actions/download-artifact@v4
             with:
                 name: java-reportdir
           - run: tar -xvf java-reportdir.tar

doc/release-notes/10117-guestbook-question-size-limit-raised.md

@@ -0,0 +1 @@
+Custom questions in Guestbooks can now be more than 255 characters and the bug causing a silent failure when questions were longer than this limit has been fixed.

doc/release-notes/10661-guestbook-email-bug-fix.md

@@ -0,0 +1,4 @@
+
+### Guestbook Email Validation Bug fix
+
+Guestbook UI Form: Email address is now checked for valid email format

doc/release-notes/10708 - MDC Citation and DOI parsing improvements.md

@@ -1,3 +1,3 @@
 MDC Citation retrieval with the PID settings has been fixed.
-DOI parsing in Dataverse is case insensitive, improving interaction with services that may change the case.
+PID parsing in Dataverse is now case insensitive, improving interaction with services that may change the case of PIDs.
 Warnings related to managed/excluded PID lists for PID providers have been reduced

doc/release-notes/10742-newest-oldest-sort-order-backwards.md

@@ -0,0 +1,3 @@
+## Minor bug fix to UI to fix the order of the files on the Dataset Files page when ordering by Date
+
+A fix was made to the ui to fix the ordering 'Newest' and 'Oldest' which were reversed

doc/release-notes/10814-Differencing improvement.md

@@ -0,0 +1,3 @@
+### More Scalable Dataset Version Differencing
+
+Differencing between dataset versions, which is done during dataset edit operations and to populate the dataset page versions table has been made signficantly more scalable.

doc/release-notes/10837-exclude-others-ns-harvesting-oai-dc.md

@@ -0,0 +1,3 @@
+Some repository extend the "oai_dc" metadata prefix with specific namespaces. In this case, harvesting of these datasets is not possible, as an XML parsing error is raised.
+
+The PR [#10837](https://github.com/IQSS/dataverse/pull/10837) allows the harvesting of these datasets by excluding tags with namespaces that are not "dc:", and harvest only metadata with the "dc" namespace.

doc/release-notes/10889_bump_PG17_FlyWay10.md

@@ -0,0 +1,7 @@
+This release bumps both the Postgres JDBC driver and Flyway versions. This should better support Postgres version 17, and as of version 10 Flyway no longer requires a paid subscription to support older versions of Postgres.
+
+While we don't encourage the use of older Postgres versions, this flexibility may benefit some of our long-standing installations in their upgrade paths. Postgres 13 remains the version used with automated testing.
+
+As part of this update, the containerized development environment now uses Postgres 17 instead of 16. Developers must delete their data (`rm -rf docker-dev-volumes`) and start with an empty database. They can rerun the quickstart in the dev guide.
+
+The Docker compose file used for [evaluations or demos](https://dataverse-guide--10912.org.readthedocs.build/en/10912/container/running/demo.html) has been upgraded from Postgres 13 to 17.

doc/release-notes/11012-get-dataverse-api-ext.md

@@ -0,0 +1 @@
+The JSON payload of the getDataverse endpoint has been extended to include properties isMetadataBlockRoot and isFacetRoot.

doc/sphinx-guides/source/api/native-api.rst

@@ -126,6 +126,8 @@ Same as in :ref:`create-dataverse-api`, the request JSON supports an optional ``
 
 To obtain an example of how these objects are included in the JSON file, download :download:`dataverse-complete-optional-params.json <../_static/api/dataverse-complete-optional-params.json>` file and modify it to suit your needs.
 
+See also :ref:`collection-attributes-api`.
+
 .. _view-dataverse:
 
 View a Dataverse Collection
@@ -1058,6 +1060,8 @@ The following attributes are supported:
 * ``affiliation`` Affiliation
 * ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
 
+See also :ref:`update-dataverse-api`.
+
 .. _collection-storage-quotas:
 
 Update Collection Input Levels
@@ -3635,7 +3639,8 @@ The fully expanded example above (without environment variables) looks like this
 Currently the following methods are used to detect file types:
 
 - The file type detected by the browser (or sent via API).
-- JHOVE: https://jhove.openpreservation.org
+- Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, this method of file type detection is not utilized during direct upload to S3, since by nature of direct upload Dataverse never sees the contents of the file. However, this code is utilized when the "redetect" API is used.
+- JHOVE: https://jhove.openpreservation.org . Note that the same applies about direct upload to S3 and the "redetect" API.
 - The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``.
 - The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``.
 

doc/sphinx-guides/source/developers/big-data-support.rst

@@ -44,6 +44,7 @@ Features that are Disabled if S3 Direct Upload is Enabled
 The following features are disabled when S3 direct upload is enabled.
 
 - Unzipping of zip files. (See :ref:`compressed-files`.)
+- Detection of file type based on JHOVE and custom code that reads the first few bytes. (See :ref:`redetect-file-type`.)
 - Extraction of metadata from FITS files. (See :ref:`fits`.)
 - Creation of NcML auxiliary files (See :ref:`netcdf-and-hdf5`.)
 - Extraction of a geospatial bounding box from NetCDF and HDF5 files (see :ref:`netcdf-and-hdf5`) unless :ref:`dataverse.netcdf.geo-extract-s3-direct-upload` is set to true.

doc/sphinx-guides/source/developers/making-releases.rst

@@ -10,10 +10,43 @@ Introduction
 
 This document is about releasing the main Dataverse app (https://github.com/IQSS/dataverse). See :doc:`making-library-releases` for how to release our various libraries. Other projects have their own release documentation.
 
-The steps below describe making both regular releases and hotfix releases.
-
 Below you'll see branches like "develop" and "master" mentioned. For more on our branching strategy, see :doc:`version-control`.
 
+Regular or Hotfix?
+------------------
+
+Early on, make sure it's clear what type of release this is. The steps below describe making both regular releases and hotfix releases.
+
+- regular
+
+  - e.g. 6.5 (minor)
+  - e.g. 7.0 (major)
+
+- hotfix
+
+  - e.g. 6.4.1 (patch)
+  - e.g. 7.0.1 (patch)
+
+Ensure Issues Have Been Created
+-------------------------------
+
+In advance of a release, GitHub issues should have been created already that capture certain steps. See https://github.com/IQSS/dataverse-pm/issues/335 for examples.
+
+Declare a Code Freeze
+---------------------
+
+The following steps are made more difficult if code is changing in the "develop" branch. Declare a code freeze until the release is out. Do not allow pull requests to be merged.
+
+Conduct Performance Testing
+---------------------------
+
+See :doc:`/qa/performance-tests` for details.
+
+Conduct Smoke Testing
+---------------------
+
+See :doc:`/qa/testing-approach` for details.
+
 .. _write-release-notes:
 
 Write Release Notes
@@ -23,26 +56,51 @@ Developers express the need for an addition to release notes by creating a "rele
 
 The task at or near release time is to collect these snippets into a single file.
 
-- Create an issue in GitHub to track the work of creating release notes for the upcoming release.
+- Find the issue in GitHub that tracks the work of creating release notes for the upcoming release.
 - Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the release note snippets mentioned above. Snippets may not include any issue number or pull request number in the text so be sure copy the number from the filename of the snippet into the final release note.
 - Delete (``git rm``) the release note snippets as the content is added to the main release notes file.
 - Include instructions describing the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure.
-- Take the release notes .md through the regular Code Review and QA process. That is, make a pull request.
+- Take the release notes .md through the regular Code Review and QA process. That is, make a pull request. Here's an example: https://github.com/IQSS/dataverse/pull/10866
 
-Create a GitHub Issue and Branch for the Release
-------------------------------------------------
+Upgrade Instructions for Internal
+---------------------------------
+
+To upgrade internal, go to /doc/release-notes, open the release-notes.md file for the current release and perform all the steps under "Upgrade Instructions".
+
+Deploy Release Candidate to Demo
+--------------------------------
+
+First, build the release candidate.
+
+ssh into the dataverse-internal server and undeploy the current war file.
+
+Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the following adjustments to the config:
+
+- Repository URL: ``https://github.com/IQSS/dataverse.git``
+- Branch Specifier (blank for 'any'): ``*/develop``
+- Execute shell: Update version in filenames to ``dataverse-5.10.war`` (for example)
+
+Click "Save" then "Build Now".
+
+This will build the war file, and then automatically deploy it on dataverse-internal. Verify that the application has deployed successfully. 
+
+You can scp the war file to the demo server or download it from https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ws/target/
+
+ssh into the demo server and follow the upgrade instructions in the release notes.
+
+Prepare Release Branch
+----------------------
+
+The release branch will have the final changes such as bumping the version number.
 
 Usually we branch from the "develop" branch to create the release branch. If we are creating a hotfix for a particular version (5.11, for example), we branch from the tag (e.g. ``v5.11``).
 
-Use the GitHub issue number and the release tag for the name of the branch. (e.g. ``8583-update-version-to-v5.10.1``
+Create a release branch named after the issue that tracks bumping the version with a descriptive name like "10852-bump-to-6.4" from https://github.com/IQSS/dataverse/pull/10871.
 
 **Note:** the changes below must be the very last commits merged into the develop branch before it is merged into master and tagged for the release!
 
 Make the following changes in the release branch.
 
-Bump Version Numbers and Prepare Container Tags
------------------------------------------------
-
 Increment the version number to the milestone (e.g. 5.10.1) in the following two files:
 
 - modules/dataverse-parent/pom.xml -> ``<properties>`` -> ``<revision>`` (e.g. `pom.xml commit <https://github.com/IQSS/dataverse/commit/3943aa0>`_)
@@ -58,14 +116,11 @@ Return to the parent pom and make the following change, which is necessary for p
 
 (Before you make this change the value should be ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}``. Later on, after cutting a release, we'll change it back to that value.)
 
-Check in the Changes Above into a Release Branch and Merge It
--------------------------------------------------------------
-
 For a regular release, make the changes above in the release branch you created, make a pull request, and merge it into the "develop" branch. Like usual, you can safely delete the branch after the merge is complete.
 
 If you are making a hotfix release, make the pull request against the "master" branch. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later.
 
-Either way, as usual, you should ensure that all tests are passing. Please note that you will need to bump the version in `jenkins.yml <https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible/blob/develop/tests/group_vars/jenkins.yml>`_ in dataverse-ansible to get the tests to pass. Consider doing this before making the pull request. Alternatively, you can bump jenkins.yml after making the pull request and re-run the Jenkins job to make sure tests pass.
+Either way, as usual, you should ensure that all tests are passing. Please note that you will need to bump the version in `jenkins.yml <https://github.com/gdcc/dataverse-ansible/blob/develop/tests/group_vars/jenkins.yml>`_ in dataverse-ansible to get the tests to pass. Consider doing this before making the pull request. Alternatively, you can bump jenkins.yml after making the pull request and re-run the Jenkins job to make sure tests pass.
 
 Merge "develop" into "master"
 -----------------------------
@@ -94,7 +149,7 @@ After the "master" branch has been updated and the GitHub Action to build and pu
 
 To test these images against our API test suite, go to the "alpha" workflow at https://github.com/gdcc/api-test-runner/actions/workflows/alpha.yml and run it.
 
-If there are failures, additional dependencies or settings may have been added to the "develop" workflow. Copy them over and try again.
+Don't be surprised if there are failures. The test runner is a work in progress! Additional dependencies or settings may have been added to the "develop" workflow. Copy them over and try again.
 
 .. _build-guides:
 
@@ -186,11 +241,6 @@ Upload the following artifacts to the draft release you created:
   - metadata block tsv files
   - config files
 
-Deploy on Demo
---------------
-
-Now that you have the release ready to go, consider giving it one final test by deploying it on https://demo.dataverse.org. Note that this is also an opportunity to re-test the upgrade checklist as described in the release note.
-
 Publish the Release
 -------------------
 
@@ -228,7 +278,21 @@ Create a new branch (any name is fine but ``prepare-next-iteration`` is suggeste
 
 Now create a pull request and merge it.
 
-For more background, see :ref:`base-supported-image-tags`.
+For more background, see :ref:`base-supported-image-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896
+
+Deploy Final Release on Demo
+----------------------------
+
+Above you already did the hard work of deploying a release candidate to https://demo.dataverse.org. It should be relatively straightforward to undeploy the release candidate and deploy the final release.
+
+Update SchemaSpy
+----------------
+
+We maintain SchemaSpy at URLs like https://guides.dataverse.org/en/6.3/schemaspy/index.html
+
+Get the attention of the core team and ask someone to update it for the new release.
+
+Consider updating `the thread <https://groups.google.com/g/dataverse-community/c/f95DQU-wlVM/m/cvUp3E9OBgAJ>`_ on the mailing list once the update is in place.
 
 Add the Release to the Dataverse Roadmap
 ----------------------------------------

doc/sphinx-guides/source/developers/selinux.rst

@@ -8,7 +8,7 @@ SELinux
 Introduction
 ------------
 
-The ``shibboleth.te`` file below that is mentioned in the :doc:`/installation/shibboleth` section of the Installation Guide was created on CentOS 6 as part of https://github.com/IQSS/dataverse/issues/3406 but may need to be revised for future versions of RHEL/CentOS (pull requests welcome!). The file is versioned with the docs and can be found in the following location:
+The ``shibboleth.te`` file below that was mentioned in the :doc:`/installation/shibboleth` section of the Installation Guide was created on CentOS 6 as part of https://github.com/IQSS/dataverse/issues/3406 but may need to be revised for future versions of RHEL/CentOS (pull requests welcome!). The file is versioned with the docs and can be found in the following location:
 
 ``doc/sphinx-guides/source/_static/installation/files/etc/selinux/targeted/src/policy/domains/misc/shibboleth.te``
 

doc/sphinx-guides/source/installation/config.rst

@@ -236,6 +236,10 @@ Dataverse automatically manages assigning PIDs and making them findable when dat
 allow updating the PID target URLs and metadata of already-published datasets manually if needed <send-metadata-to-pid-provider>`, e.g. if a Dataverse instance is
 moved to a new URL or when the software is updated to generate additional metadata or address schema changes at the PID service.
 
+Note that while some forms of PIDs (Handles, PermaLinks) are technically case sensitive, common practice is to avoid creating PIDs that differ only by case.
+Dataverse treats PIDs of all types as case-insensitive (as DOIs are by definition). This means that Dataverse will find datasets (in search, to display dataset pages, etc.) 
+when the PIDs entered do not match the case of the original but will have a problem if two PIDs that differ only by case exist in one instance.
+
 Testing PID Providers
 +++++++++++++++++++++