diff --git a/conf/solr/4.6.0/schema.xml b/conf/solr/4.6.0/schema.xml index 10f9b07be5c..323429b62da 100644 --- a/conf/solr/4.6.0/schema.xml +++ b/conf/solr/4.6.0/schema.xml @@ -298,6 +298,8 @@ <field name="unf" type="string" stored="true" indexed="true" multiValued="false"/> <field name="fileSizeInBytes" type="long" stored="true" indexed="true" multiValued="false"/> <field name="fileMd5" type="string" stored="true" indexed="true" multiValued="false"/> + <field name="fileChecksumType" type="string" stored="true" indexed="true" multiValued="false"/> + <field name="fileChecksumValue" type="string" stored="true" indexed="true" multiValued="false"/> <field name="fileContentType" type="string" stored="true" indexed="true" multiValued="false"/> <field name="deaccessionReason" type="string" stored="true" indexed="false" multiValued="false"/> diff --git a/doc/Sphinx/source/img/image1institutional.png b/doc/Sphinx/source/img/image1institutional.png deleted file mode 100755 index 39852ea1c23..00000000000 Binary files a/doc/Sphinx/source/img/image1institutional.png and /dev/null differ diff --git a/doc/Sphinx/source/img/image2institutional.png b/doc/Sphinx/source/img/image2institutional.png deleted file mode 100755 index a14fded0cc3..00000000000 Binary files a/doc/Sphinx/source/img/image2institutional.png and /dev/null differ diff --git a/doc/Sphinx/source/img/image3institutional.png b/doc/Sphinx/source/img/image3institutional.png deleted file mode 100755 index bc213a656f9..00000000000 Binary files a/doc/Sphinx/source/img/image3institutional.png and /dev/null differ diff --git a/doc/Sphinx/source/img/image4institutional.png b/doc/Sphinx/source/img/image4institutional.png deleted file mode 100755 index 40f1c03d1d7..00000000000 Binary files a/doc/Sphinx/source/img/image4institutional.png and /dev/null differ diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst index e7dfb0bf46a..28a46cf58e5 100755 --- a/doc/sphinx-guides/source/admin/index.rst +++ b/doc/sphinx-guides/source/admin/index.rst @@ -13,9 +13,8 @@ These "superuser" tasks are managed via the new page called the Dashboard. A use Contents: .. toctree:: - :maxdepth: 2 - harvestclients - harvestserver - metadataexport - timers + harvestclients + harvestserver + metadataexport + timers diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 7bb2805b99a..66ea551d446 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -60,11 +60,11 @@ Multiple File ("bundle") download Returns the files listed, zipped. Parameters: -~~~~~~~~~~ +~~~~~~~~~~~ none. "All Formats" bundled access for Tabular Files. ----------------------------------------------- +----------------------------------------------- ``/api/access/datafile/bundle/$id`` @@ -78,7 +78,7 @@ It returns a zipped bundle that contains the data in the following formats: * File citation, in Endnote and RIS formats. Parameters: -~~~~~~~~~~ +~~~~~~~~~~~ none. Data Variable Metadata Access diff --git a/doc/sphinx-guides/source/api/index.rst b/doc/sphinx-guides/source/api/index.rst index 6f299ea91df..b9d30d20e91 100755 --- a/doc/sphinx-guides/source/api/index.rst +++ b/doc/sphinx-guides/source/api/index.rst @@ -16,11 +16,10 @@ Rather than using a production installation of Dataverse, API users are welcome Contents: .. toctree:: - :maxdepth: 2 - sword - search - dataaccess - native-api - client-libraries - apps + sword + search + dataaccess + native-api + client-libraries + apps diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a28c1d43b45..8b686df66cd 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -89,7 +89,7 @@ Publish the Dataverse pointed by ``identifier``, which can either by the dataver Datasets ~~~~~~~~ -**Note** Creation of new datasets is done by ``POST``ing them onto dataverses. See dataverse section. +**Note** Creation of new datasets is done with a ``POST`` onto dataverses. See dataverse section. **Note** In all commands below, dataset versions can be referred to as: @@ -185,7 +185,7 @@ Delete a Private URL from a dataset (if it exists):: DELETE http://$SERVER/api/datasets/$id/privateUrl?key=$apiKey Builtin Users -~~~~~ +~~~~~~~~~~~~~ This endopint deals with users of the built-in authentication provider. Note that users may come from other authentication services as well, such as Shibboleth. For this service to work, the setting ``BuiltinUsers.KEY`` has to be set, and its value passed as ``key`` to diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index e9c09e0c61b..ca7b40d4dc6 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -14,6 +14,7 @@ import sys import os +from datetime import datetime sys.path.insert(0, os.path.abspath('../../')) import sphinx_bootstrap_theme @@ -56,16 +57,16 @@ # General information about the project. project = u'Dataverse' -copyright = u'2016, The President & Fellows of Harvard College' +copyright = u'%d, The President & Fellows of Harvard College' % datetime.now().year # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '4.5' +version = '4.5.1' # The full version, including alpha/beta/rc tags. -release = '4.5' +release = '4.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -358,7 +359,7 @@ epub_title = u'Dataverse' epub_author = u'Dataverse Team' epub_publisher = u'Dataverse Team' -epub_copyright = u'2014, Dataverse Team' +epub_copyright = u'%d, The President & Fellows of Harvard College' % datetime.now().year # The basename for the epub file. It defaults to the project name. #epub_basename = u'Consilience Documentation' diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index d3871380f52..532f2f75d7a 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -7,7 +7,7 @@ Development Environment Assumptions ----------- -This guide assumes you are using a Mac but we do have pages for :doc:`/developers/windows` and :doc:`/developers/ubuntu`. +This guide assumes you are using a Mac. If you are using Windows or Linux, please reach out to other developers at https://groups.google.com/forum/#!forum/dataverse-dev Requirements ------------ diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index 9225d00dcd6..a9daa12b16c 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -9,17 +9,14 @@ Developer Guide Contents: .. toctree:: - :maxdepth: 2 - - intro - dev-environment - branching-strategy - testing - documentation - debugging - coding-style - making-releases - tools - unf/index - + intro + dev-environment + branching-strategy + testing + documentation + debugging + coding-style + making-releases + tools + unf/index diff --git a/doc/sphinx-guides/source/developers/ubuntu.rst b/doc/sphinx-guides/source/developers/ubuntu.rst deleted file mode 100755 index 9204a6171eb..00000000000 --- a/doc/sphinx-guides/source/developers/ubuntu.rst +++ /dev/null @@ -1,51 +0,0 @@ -====== -Ubuntu -====== - -Requirements ------------- - -Tested on Ubuntu 14.04. - -Java 8 -~~~~~~ - -- ``sudo apt-get install openjdk-8-jdk openjdk-8-jre`` - - -Maven -~~~~~ - -- ``sudo apt-get install maven`` - - -Glassfish -~~~~~~~~~ - -- ``wget http://download.java.net/glassfish/4.1/release/glassfish-4.1.zip`` - -- ``unzip glassfish-4.1*zip`` - - -PostgreSQL -~~~~~~~~~~ - -- ``sudo apt-get install postgresql postgresql-contrib`` - - -jq -~~ - -- ``sudo apt-get install jq`` - - -Curl -~~~~ - -- ``sudo apt-get install curl`` - - -Recommendations and Dev Environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Please visit :doc:`/developers/dev-environment/` diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst index 3980909fc0b..dc2f37d0ba9 100644 --- a/doc/sphinx-guides/source/developers/unf/index.rst +++ b/doc/sphinx-guides/source/developers/unf/index.rst @@ -1,17 +1,17 @@ .. _unf: -==================================== +===================================== Universal Numerical Fingerprint (UNF) -==================================== +===================================== Contents: .. toctree:: :maxdepth: 2 - unf-v3 - unf-v5 - unf-v6 + unf-v3 + unf-v5 + unf-v6 .. figure:: ./img/unf-diagram.png :align: center diff --git a/doc/sphinx-guides/source/developers/unf/unf-v5.rst b/doc/sphinx-guides/source/developers/unf/unf-v5.rst index 8606ff06ebc..4fb160c20ea 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v5.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v5.rst @@ -10,6 +10,3 @@ UNF Version 5 **To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation.** **Going forward, we are going to offer a preserved version of the Version 5 library and, possibly, an online UNF v5 calculator, for the purposes of validating vectors and data sets for which published Version 5 UNFs exist.** - ------ - diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst deleted file mode 100755 index 9ed6f10dcd5..00000000000 --- a/doc/sphinx-guides/source/developers/windows.rst +++ /dev/null @@ -1,5 +0,0 @@ -======= -Windows -======= - -Developers using Windows who have trouble setting up their :doc:`/developers/dev-environment/` should reach out to over Dataverse developers per https://github.com/IQSS/dataverse/blob/master/CONTRIBUTING.md diff --git a/doc/sphinx-guides/source/index.rst b/doc/sphinx-guides/source/index.rst index 1c3a88dcdef..8fd7056c032 100755 --- a/doc/sphinx-guides/source/index.rst +++ b/doc/sphinx-guides/source/index.rst @@ -3,10 +3,10 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Dataverse 4.5 Guides +Dataverse 4.5.1 Guides ====================== -These guides are for the most recent version of Dataverse. For the guides for **version 4.3.1** please go `here <http://guides.dataverse.org/en/4.3.1/>`_. +These guides are for the most recent version of Dataverse. For the guides for **version 4.5** please go `here <http://guides.dataverse.org/en/4.5/>`_. .. toctree:: :glob: diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 347a86e92db..14dc908e0ab 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -90,7 +90,7 @@ Persistent identifiers are a required and integral part of the Dataverse platfor JVM Options: :ref:`doi.baseurlstring`, :ref:`doi.username`, :ref:`doi.password` -Database Settings: :ref:`:DoiProvider`, :ref:`:Protocol`, :ref:`:Authority`, :ref:`:DoiSeparator` +Database Settings: :ref:`:DoiProvider <:DoiProvider>`, :ref:`:Protocol <:Protocol>`, :ref:`:Authority <:Authority>`, :ref:`:DoiSeparator <:DoiSeparator>` Please note that any datasets creating using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured. @@ -104,8 +104,9 @@ Once this configuration is complete, your Dataverse installation should be ready JVM Options ----------- -JVM stands Java Virtual Machine and as a Java application, Glassfish can read JVM options when it is started. A number of JVM options are configured by the installer below is a complete list of the Dataverse-specific JVM options. You can inspect the configured options by running ``asadmin list-jvm-options | egrep 'dataverse|doi' -``. +JVM stands Java Virtual Machine and as a Java application, Glassfish can read JVM options when it is started. A number of JVM options are configured by the installer below is a complete list of the Dataverse-specific JVM options. You can inspect the configured options by running: + +``asadmin list-jvm-options | egrep 'dataverse|doi'`` When changing values these values with ``asadmin``, you'll need to delete the old value before adding a new one, like this: @@ -188,9 +189,11 @@ dataverse.dataAccess.thumbnail.pdf.limit For limiting the size of thumbnail images generated from files. +.. _doi.baseurlstring: + doi.baseurlstring +++++++++++++++++ -.. _doi.baseurlstring: + As of this writing "https://ezid.cdlib.org" and "https://mds.datacite.org" are the only valid values. See also these related database settings below: - :DoiProvider @@ -198,14 +201,18 @@ As of this writing "https://ezid.cdlib.org" and "https://mds.datacite.org" are t - :Authority - :DoiSeparator +.. _doi.username: + doi.username ++++++++++++ -.. _doi.username: + Used in conjuction with ``doi.baseurlstring``. +.. _doi.password: + doi.password ++++++++++++ -.. _doi.password: + Used in conjuction with ``doi.baseurlstring``. dataverse.handlenet.admcredfile @@ -265,30 +272,45 @@ This is the email address that "system" emails are sent from such as password re ``curl -X PUT -d "Support <support@example.edu>" http://localhost:8080/api/admin/settings/:SystemEmail`` +:FooterCopyright +++++++++++++++++ + +By default the footer says "Copyright © [YYYY]" but you can add text after the year, as in the example below. + +``curl -X PUT -d ", The President & Fellows of Harvard College" http://localhost:8080/api/admin/settings/:FooterCopyright`` + +.. _:DoiProvider: + :DoiProvider ++++++++++++ -.. _:DoiProvider: + As of this writing "EZID" and "DataCite" are the only valid options. ``curl -X PUT -d EZID http://localhost:8080/api/admin/settings/:DoiProvider`` +.. _:Protocol: + :Protocol +++++++++ -.. _:Protocol: + As of this writing "doi" is the only valid option for the protocol for a persistent ID. ``curl -X PUT -d doi http://localhost:8080/api/admin/settings/:Protocol`` +.. _:Authority: + :Authority ++++++++++ -.. _:Authority: + Use the DOI authority assigned to you by your DoiProvider. ``curl -X PUT -d 10.xxxx http://localhost:8080/api/admin/settings/:Authority`` +.. _:DoiSeparator: + :DoiSeparator +++++++++++++ -.. _:DoiSeparator: + It is recommended that you keep this as a slash ("/"). ``curl -X PUT -d "/" http://localhost:8080/api/admin/settings/:DoiSeparator`` @@ -370,7 +392,9 @@ Limit the number of files in a zip that Dataverse will accept. :GoogleAnalyticsCode ++++++++++++++++++++ -For setting up Google Analytics for your Dataverse installation. +Set your Google Analytics Tracking ID thusly: + +``curl -X PUT -d 'trackingID' http://localhost:8080/api/admin/settings/:GoogleAnalyticsCode`` :SolrHostColonPort ++++++++++++++++++ @@ -392,7 +416,7 @@ The relative path URL to which users will be sent after signup. The default sett The location of your TwoRavens installation. Activation of TwoRavens also requires the setting below, ``TwoRavensTabularView`` :TwoRavensTabularView -+++++++++++++++++++ ++++++++++++++++++++++ Set ``TwoRavensTabularView`` to true to allow a user to view tabular files via the TwoRavens application. This boolean affects whether a user will see the "Explore" button. @@ -459,3 +483,28 @@ This setting is experimental per :doc:`/installation/shibboleth`. ++++++++++++ Set to false to disallow local accounts to be created if you are using :doc:`shibboleth` but not for production use until https://github.com/IQSS/dataverse/issues/2838 has been fixed. + +:PiwikAnalyticsId +++++++++++++++++++++ + +Site identifier created in your Piwik instance. Example: + +``curl -X PUT -d 42 http://localhost:8080/api/admin/settings/:PiwikAnalyticsId`` + +:PiwikAnalyticsHost +++++++++++++++++++++ + +Host FQDN or URL of your Piwik instance before the ``/piwik.php``. Examples: + +``curl -X PUT -d stats.domain.tld http://localhost:8080/api/admin/settings/:PiwikAnalyticsHost`` + +or + +``curl -X PUT -d hostname.domain.tld/stats http://localhost:8080/api/admin/settings/:PiwikAnalyticsHost`` + +:FileFixityChecksumAlgorithm +++++++++++++++++++++++++++++ + +Dataverse calculates checksums for uploaded files so that users can determine if their file was corrupted via upload or download. This is sometimes called "file fixity": https://en.wikipedia.org/wiki/File_Fixity + +The default checksum algorithm used is MD5 and should be sufficient for establishing file fixity. "SHA-1" is an experimental alternate value for this setting. diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index ba2992c5ec4..b418370f908 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -9,15 +9,13 @@ Installation Guide Contents: .. toctree:: - :titlesonly: - :maxdepth: 2 - intro - prep - prerequisites - installation-main - config - administration - upgrading - r-rapache-tworavens - shibboleth + intro + prep + prerequisites + installation-main + config + administration + upgrading + r-rapache-tworavens + shibboleth diff --git a/doc/sphinx-guides/source/installation/installer-script.rst b/doc/sphinx-guides/source/installation/installer-script.rst deleted file mode 100644 index 72881587917..00000000000 --- a/doc/sphinx-guides/source/installation/installer-script.rst +++ /dev/null @@ -1 +0,0 @@ -This content has been moved to :doc:`/installation/installation-main`. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 9530860dce9..4a30c5b9a12 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -98,7 +98,7 @@ The standard init script that ships RHEL 6 and similar should work fine. Enable Configuring Database Access for the Dataverse Application (and the Dataverse Installer) -===================================================================================== +======================================================================================= - The application and the installer script will be connecting to PostgreSQL over TCP/IP, using password authentication. In this section we explain how to configure PostgreSQL to accept these connections. diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 1fc31bdb3fe..a88ffa114d2 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -6,7 +6,7 @@ Eventually, this document may be split into several parts, dedicated to individu such as R, rApache and the TwoRavens applications. Particularly, if the TwoRavens team creates an "official" distribution with their own installation manual. 0. PREREQUISITES -+++++++++++++++ +++++++++++++++++ a. httpd (Apache): ------------------ @@ -194,7 +194,7 @@ Note that some of these packages have their own dependencies, and additional ins install.pl script: ++++++++++++++++++ -I. Configure the TwoRavens web (Javascript) application. +I. Configure the TwoRavens web (Javascript) application ------------------------------------------------------- Edit the file ``/var/www/html/dataexplore/app_ddi.js``. diff --git a/doc/sphinx-guides/source/user/data-exploration/index.rst b/doc/sphinx-guides/source/user/data-exploration/index.rst index b6be872249c..708f774bb46 100755 --- a/doc/sphinx-guides/source/user/data-exploration/index.rst +++ b/doc/sphinx-guides/source/user/data-exploration/index.rst @@ -9,10 +9,6 @@ Data Exploration Guide Contents: .. toctree:: - :titlesonly: - :maxdepth: 2 - - tworavens - worldmap - + tworavens + worldmap diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 0a6200583e5..d0259286af9 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -89,8 +89,8 @@ For example, if these files were included within a .zip, the “Map Data” butt * subway_line.dbf Once you publish your dataset with your shape files, you will be able to use the "Map Data" button using `GeoConnect <https://github.com/IQSS/geoconnect>`_ to visualize and manipulate these files -for users to Explore this geospatial data using the `WorldMap <http://worldmap.harvard.edu/>`_ interface. -Please note: In order to map your data file, a copy will be sent to Harvard's `WorldMap <http://worldmap.harvard.edu/>`_ platform. You have the ability to delete any maps, and associated data, from the Harvard WorldMap platform, at any time. +for users to Explore this geospatial data using the `WorldMap <http://worldmap.harvard.edu/>`__ interface. +Please note: In order to map your data file, a copy will be sent to Harvard's `WorldMap <http://worldmap.harvard.edu/>`__ platform. You have the ability to delete any maps, and associated data, from the Harvard WorldMap platform, at any time. Astronomy (FITS) -------------------- @@ -222,7 +222,7 @@ The file permissions page has two sections: Users/Groups and Files. To give someone access to your restricted files, click on the Grant Access to Users/Groups button in the Users/Groups section. -.. _widgets: +.. _dataset-widgets: Widgets ============================= @@ -310,7 +310,8 @@ a file, your dataset will automatically be bumped up to a major version (example |image3| -**Dataset Versions Tab** +Version Details +------------------------------------- To view what has exactly changed starting from the originally published version to any subsequent published versions: click on the Versions tab on the dataset page to see all versions and changes made for that particular dataset. Once you have more than one version (can be version 1 and a draft), you can click the Show Details link in the Versions tab to learn more about the metadata fields and files that were either added or edited. @@ -334,6 +335,6 @@ If you deaccession the most recently published version of the dataset but not al :class: img-responsive .. |image2| image:: ./img/data-download.png :class: img-responsive -.. |image3| image:: http://static.projects.iq.harvard.edu/files/styles/os_files_xxlarge/public/datascience/files/data_publishing_version_workflow.png?itok=8Z0PM-QC +.. |image3| image:: ./img/data_publishing_version_workflow.png :class: img-responsive diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index 6d45a055ecf..bac42305119 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -35,12 +35,13 @@ Edit Dataverse To edit your dataverse, navigate to your dataverse homepage and select the "Edit Dataverse" button, where you will be presented with the following editing options: -- :ref:`General Information <general-information>` : edit name, identifier, category, contact email, affiliation, description, Metadata Elements, and facets for your dataverse. -- :ref:`Theme + Widgets <theme-widgets>` : upload a logo for your dataverse, add a link to your department or personal website, and select colors for your dataverse in order to brand it. Also, you can get code to add to your website to have your dataverse display on it. -- :ref:`Permissions <dataverse-permissions>` : give Dataverse users permissions to your dataverse, i.e.-can edit datasets, and see which users already have which permissions for your dataverse -- :ref:`Dataset Templates <dataset-templates>` : these are useful when you have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in -- :ref:`Dataset Guestbooks <dataset-guestbooks>` : allows you to collect data about who is downloading the files from your datasets -- :ref:`Featured Dataverses <featured-dataverses>` : if you have one or more dataverses, you can use this option to show them at the top of your dataverse page to help others easily find interesting or important dataverses +- :ref:`General Information <general-information>`: edit name, identifier, category, contact email, affiliation, description, Metadata Elements, and facets for your dataverse +- :ref:`Theme <theme>`: upload a logo for your dataverse, add a link to your department or personal website, and select colors for your dataverse in order to brand it +- :ref:`Widgets <dataverse-widgets>`: get code to add to your website to have your dataverse display on it +- :ref:`Permissions <dataverse-permissions>`: give Dataverse users permissions to your dataverse, i.e.-can edit datasets, and see which users already have which permissions for your dataverse +- :ref:`Dataset Templates <dataset-templates>`: these are useful when you have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in +- :ref:`Dataset Guestbooks <dataset-guestbooks>`: allows you to collect data about who is downloading the files from your datasets +- :ref:`Featured Dataverses <featured-dataverses>`: if you have one or more dataverses, you can use this option to show them at the top of your dataverse page to help others easily find interesting or important dataverses - **Delete Dataverse**: you are able to delete your dataverse as long as it is not published and does not have any draft datasets .. _general-information: @@ -52,14 +53,14 @@ The General Information page is how you edit the information you filled in while Tip: The metadata fields you select as required, will appear on the Create Dataset form when someone goes to add a dataset to the dataverse. -.. _widgets: +.. _theme: Theme ==================================================== The Theme feature provides you with a way to customize the look of your dataverse. You can decide either to use the customization from the dataverse above yours or upload your own image file. Supported image types are JPEG, TIFF, or PNG and should be no larger than 500 KB. The maximum display size for an image file in a dataverse's theme is 940 pixels wide by 120 pixels high. Additionally, you can select the colors for the header of your dataverse and the text that appears in your dataverse. You can also add a link to your personal website, the website for your organization or institution, your department, journal, etc. -.. _widgets: +.. _dataverse-widgets: Widgets ================================================= diff --git a/doc/sphinx-guides/source/user/img/data_publishing_version_workflow.png b/doc/sphinx-guides/source/user/img/data_publishing_version_workflow.png new file mode 100644 index 00000000000..6ef11f31750 Binary files /dev/null and b/doc/sphinx-guides/source/user/img/data_publishing_version_workflow.png differ diff --git a/doc/sphinx-guides/source/user/index.rst b/doc/sphinx-guides/source/user/index.rst index 604ad4c2071..9d231cb5f6d 100755 --- a/doc/sphinx-guides/source/user/index.rst +++ b/doc/sphinx-guides/source/user/index.rst @@ -9,12 +9,11 @@ User Guide Contents: .. toctree:: - :maxdepth: 2 - account - find-use-data - dataverse-management - dataset-management - tabulardataingest/index - data-exploration/index - appendix + account + find-use-data + dataverse-management + dataset-management + tabulardataingest/index + data-exploration/index + appendix diff --git a/doc/sphinx-guides/source/user/super-user.rst b/doc/sphinx-guides/source/user/super-user.rst deleted file mode 100755 index 3586d0ea827..00000000000 --- a/doc/sphinx-guides/source/user/super-user.rst +++ /dev/null @@ -1,6 +0,0 @@ -Super User -+++++++++++++++++++++++ - -[Note: Documentation to be added about features available for super admins of -the Dataverse, which provides several options for configuring and -customizing your application.] diff --git a/doc/sphinx-guides/source/user/tabulardataingest/index.rst b/doc/sphinx-guides/source/user/tabulardataingest/index.rst index 0ca316502e7..a190710bdab 100755 --- a/doc/sphinx-guides/source/user/tabulardataingest/index.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/index.rst @@ -9,15 +9,11 @@ Tabular Data File Ingest Contents: .. toctree:: - :titlesonly: - :maxdepth: 2 - - supportedformats - ingestprocess - spss - stata - rdata - excel - csv - + supportedformats + ingestprocess + spss + stata + rdata + excel + csv diff --git a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst index b5d9311d603..ae2cc6cf7fe 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst @@ -92,10 +92,10 @@ the latter reserved for longer, descriptive text. With variables ingested from R data frames the variable name will be used for both the "name" and the "label". -| *Optional R packages exist for providing descriptive variable labels; - in one of the future versions support may be added for such a - mechanism. It would of course work only for R files that were - created with such optional packages*. +*Optional R packages exist for providing descriptive variable labels; +in one of the future versions support may be added for such a +mechanism. It would of course work only for R files that were +created with such optional packages*. Similarly, R categorical values (factors) lack descriptive labels too. **Note:** This is potentially confusing, since R factors do @@ -132,7 +132,7 @@ value: unless the time zone was explicitly defined, R will adjust the value to the current time zone. The resulting behavior is often counter-intuitive: if you create a time value, for example: - timevalue<-as.POSIXct("03/19/2013 12:57:00", format = "%m/%d/%Y %H:%M:%OS"); +``timevalue<-as.POSIXct("03/19/2013 12:57:00", format = "%m/%d/%Y %H:%M:%OS");`` on a computer configured for the San Francisco time zone, the value will be differently displayed on computers in different time zones; @@ -143,9 +143,11 @@ If it is important that the values are always displayed the same way, regardless of the current time zones, it is recommended that the time zone is explicitly defined. For example: - attr(timevalue,"tzone")<-"PST" +``attr(timevalue,"tzone")<-"PST"`` + or - timevalue<-as.POSIXct("03/19/2013 12:57:00", format = "%m/%d/%Y %H:%M:%OS", tz="PST"); + +``timevalue<-as.POSIXct("03/19/2013 12:57:00", format = "%m/%d/%Y %H:%M:%OS", tz="PST");`` Now the value will always be displayed as "15:57 PST", regardless of the time zone that is current for the OS ... **BUT ONLY** if the OS @@ -189,7 +191,7 @@ wasn't defined explicitly, it implicitly becomes a time value in the "UTC" zone!), this means that it is **impossible** to have 2 time value vectors, in Stata/SPSS and R, that produce the same UNF. -| **A pro tip:** if it is important to produce SPSS/Stata and R versions of +**A pro tip:** if it is important to produce SPSS/Stata and R versions of the same data set that result in the same UNF when ingested, you may define the time variables as **strings** in the R data frame, and use the "YYYY-MM-DD HH:mm:ss" formatting notation. This is the formatting used by the UNF @@ -198,4 +200,4 @@ the same UNF as the vector of the same time values in Stata. Note: date values (dates only, without time) should be handled the exact same way as those in SPSS and Stata, and should produce the same -UNFs. \ No newline at end of file +UNFs. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst new file mode 100644 index 00000000000..764bc815a2f --- /dev/null +++ b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst @@ -0,0 +1,9 @@ +Stata +++++++++ + +Of all the third party statistical software providers, Stata does the best job at documenting the internal format of their files, by far. And at making that documentation freely and easily available to developers (yes, we are looking at you, SPSS). Because of that, Stata is the best supported format for tabular data ingest. + + +**New in Dataverse 4.0:** support for Stata v.13 has been added. + + diff --git a/doc/sphinx_bootstrap_theme/bootstrap/layout.html b/doc/sphinx_bootstrap_theme/bootstrap/layout.html index 3478e807b30..2193b142bb9 100755 --- a/doc/sphinx_bootstrap_theme/bootstrap/layout.html +++ b/doc/sphinx_bootstrap_theme/bootstrap/layout.html @@ -133,7 +133,7 @@ {%- if hasdoc('copyright') %} {% trans path=pathto('copyright'), copyright=copyright|e %}© <a href="{{ path }}">Copyright</a> {{ copyright }}.{% endtrans %}<br/> {%- else %} - {% trans copyright=copyright|e %}<a href="http://datascience.iq.harvard.edu/" target="_blank">Data Science</a> at <a href="http://www.iq.harvard.edu/" target="_blank">The Institute for Quantitative Social Science</a> | Code available at <a href="https://github.com/IQSS/dataverse" title="Dataverse on GitHub" target="_blank"><img src="_static/images/githubicon.png" width="20" alt="Dataverse on GitHub"/></a> | Created using <a href="http://sphinx.pocoo.org/" target="_blank">Sphinx</a> {{ sphinx_version }}<br/>Version {{ version }} | Last updated on {{ last_updated }}<br/>© Copyright {{ copyright }} {% endtrans %}<br/> + {% trans copyright=copyright|e %}<a href="http://datascience.iq.harvard.edu/" target="_blank">Data Science</a> at <a href="http://www.iq.harvard.edu/" target="_blank">The Institute for Quantitative Social Science</a> | Code available at <a href="https://github.com/IQSS/dataverse" title="Dataverse on GitHub" target="_blank"><img src="/en/latest/_static/images/githubicon.png" width="20" alt="Dataverse on GitHub"/></a> | Created using <a href="http://sphinx.pocoo.org/" target="_blank">Sphinx</a> {{ sphinx_version }}<br/>Version {{ version }} | Last updated on {{ last_updated }}<br/>© Copyright {{ copyright }} {% endtrans %}<br/> {%- endif %} {%- endif %} </p> diff --git a/doc/sphinx_bootstrap_theme/bootstrap/static/images/githubicon.png b/doc/sphinx_bootstrap_theme/bootstrap/static/images/githubicon.png index 468574c26a9..65581c832f1 100644 Binary files a/doc/sphinx_bootstrap_theme/bootstrap/static/images/githubicon.png and b/doc/sphinx_bootstrap_theme/bootstrap/static/images/githubicon.png differ diff --git a/pom.xml b/pom.xml index 1d66fc226f5..7812b866812 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ <groupId>edu.harvard.iq</groupId> <artifactId>dataverse</artifactId> - <version>4.5</version> + <version>4.5.1</version> <packaging>war</packaging> <name>dataverse</name> diff --git a/scripts/api/setup-optional-harvard.sh b/scripts/api/setup-optional-harvard.sh index 52caa31c1e0..3433e823014 100755 --- a/scripts/api/setup-optional-harvard.sh +++ b/scripts/api/setup-optional-harvard.sh @@ -22,6 +22,7 @@ curl -s -X PUT -d true "$SERVER/admin/settings/:GeoconnectCreateEditMaps" curl -s -X PUT -d true "$SERVER/admin/settings/:GeoconnectViewMaps" echo "- Setting system email" curl -X PUT -d "Dataverse Support <support@dataverse.org>" http://localhost:8080/api/admin/settings/:SystemEmail +curl -X PUT -d ", The President & Fellows of Harvard College" http://localhost:8080/api/admin/settings/:FooterCopyright echo "- Setting up the Harvard Shibboleth institutional group" curl -s -X POST -H 'Content-type:application/json' --upload-file data/shibGroupHarvard.json "$SERVER/admin/groups/shib?key=$adminKey" echo diff --git a/scripts/database/upgrades/3354-alt-checksum.sql b/scripts/database/upgrades/3354-alt-checksum.sql new file mode 100644 index 00000000000..42956fcc65d --- /dev/null +++ b/scripts/database/upgrades/3354-alt-checksum.sql @@ -0,0 +1,7 @@ +ALTER TABLE datafile ADD COLUMN checksumtype character varying(255); +UPDATE datafile SET checksumtype = 'MD5'; +ALTER TABLE datafile ALTER COLUMN checksumtype SET NOT NULL; +-- alternate statement for sbgrid.org and others interested in SHA-1 support +-- note that in the database we use "SHA1" (no hyphen) but the GUI will show "SHA-1" +--UPDATE datafile SET checksumtype = 'SHA1'; +ALTER TABLE datafile RENAME md5 TO checksumvalue; diff --git a/scripts/issues/3354/createDatasetWithSha1Files.sh b/scripts/issues/3354/createDatasetWithSha1Files.sh new file mode 100755 index 00000000000..1792a9e33a3 --- /dev/null +++ b/scripts/issues/3354/createDatasetWithSha1Files.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# existing, works, no files, commenting out +#curl -s -X POST -H "Content-type:application/json" -d @scripts/search/tests/data/dataset-finch1.json "http://localhost:8080/api/dataverses/root/datasets/?key=$API_TOKEN" +# new, has files +curl -s -X POST -H "Content-type:application/json" -d @scripts/issues/3354/datasetWithSha1Files.json "http://localhost:8080/api/dataverses/root/datasets/?key=$API_TOKEN" diff --git a/scripts/issues/3354/datasetWithSha1Files.json b/scripts/issues/3354/datasetWithSha1Files.json new file mode 100644 index 00000000000..95a4d3b88d0 --- /dev/null +++ b/scripts/issues/3354/datasetWithSha1Files.json @@ -0,0 +1,86 @@ +{ + "datasetVersion": { + "files": [ + { + "label": "foo.txt", + "dataFile": { + "filename": "foo.txt", + "contentType": "text/plain", + "storageIdentifier": "157484f9d6c-c36006fa39e5", + "originalFormatLabel": "UNKNOWN", + "checksum": { + "type": "SHA-1", + "value": "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" + } + } + } + ], + "metadataBlocks": { + "citation": { + "fields": [ + { + "value": "Dataset with SHA-1 files", + "typeClass": "primitive", + "multiple": false, + "typeName": "title" + }, + { + "value": [ + { + "authorName": { + "value": "Finch, Fiona", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorName" + }, + "authorAffiliation": { + "value": "Birds Inc.", + "typeClass": "primitive", + "multiple": false, + "typeName": "authorAffiliation" + } + } + ], + "typeClass": "compound", + "multiple": true, + "typeName": "author" + }, + { + "value": [ + { "datasetContactEmail" : { + "typeClass": "primitive", + "multiple": false, + "typeName": "datasetContactEmail", + "value" : "finch@mailinator.com" + } + }], + "typeClass": "compound", + "multiple": true, + "typeName": "datasetContact" + }, + { + "value": [ { + "dsDescriptionValue":{ + "value": "Some people prefer SHA-1 to MD5 for file fixity.", + "multiple":false, + "typeClass": "primitive", + "typeName": "dsDescriptionValue" + }}], + "typeClass": "compound", + "multiple": true, + "typeName": "dsDescription" + }, + { + "value": [ + "Other" + ], + "typeClass": "controlledVocabulary", + "multiple": true, + "typeName": "subject" + } + ], + "displayName": "Citation Metadata" + } + } + } +} diff --git a/scripts/issues/3354/mydata b/scripts/issues/3354/mydata new file mode 100755 index 00000000000..eb76d06e3f9 --- /dev/null +++ b/scripts/issues/3354/mydata @@ -0,0 +1,3 @@ +#!/bin/sh +# FIXME: Make this into a REST Assured test. +curl -s "http://localhost:8080/api/mydata/retrieve?key=$API_TOKEN&role_ids=1&dvobject_types=DataFile&published_states=Published&published_states=Unpublished&published_states=Draft&published_states=In+Review&published_states=Deaccessioned" | jq .data.items diff --git a/scripts/search/search b/scripts/search/search index b058dbdacb9..ac14596ac2a 100755 --- a/scripts/search/search +++ b/scripts/search/search @@ -1,11 +1,11 @@ #!/bin/sh if [ -z "$1" ]; then - curl -s 'http://localhost:8080/api/search?q=*' + curl -H "X-Dataverse-key: $API_TOKEN" -s 'http://localhost:8080/api/search?q=*' #curl -s 'http://localhost:8080/api/search?q=*&key=pete' else # i.e. ./search 'q=*&fq=filetype_s:"image"&fq=dvtype:files' # i.e. ./search 'q=*&start=10' # i.e. ./search 'q=*&sort=name_sort&order=asc' # i.e. ./search 'q=*&sort=name_sort&order=asc' | jq '.itemsJson[] | {name_sort}' - curl -s "http://localhost:8080/api/search?$1" + curl -H "X-Dataverse-key: $API_TOKEN" -s "http://localhost:8080/api/search?$1" fi diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index e07c10cc112..f5120a87c0e 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -92,7 +92,7 @@ footer.dataverseOnGitHub=Dataverse On GitHub footer.dataverseProjectOn=Dataverse Project on footer.Twitter=Twitter footer.dataScienceIQSS=Developed at the <a href="http://www.iq.harvard.edu/" title="Institute for Quantitative Social Science" target="_blank">Institute for Quantitative Social Science</a> -footer.copyright=Copyright © 2016, The President & Fellows of Harvard College +footer.copyright=Copyright © {0} footer.widget.datastored=Data is stored at {0}. footer.widget.login=Log in to footer.privacyPolicy=Privacy Policy @@ -1104,7 +1104,6 @@ dataset.metadata.persistentId.tip=The unique persistent identifier for a Dataset dataset.versionDifferences.termsOfUseAccess=Terms of Use and Access dataset.versionDifferences.termsOfUseAccessChanged=Terms of Use/Access Changed file.viewDiffDialog.restricted=Restricted -file.viewDiffDialog.md5=MD5 dataset.template.tip=Changing the template will clear any fields you may have entered data into. dataset.noTemplate.label=None @@ -1162,9 +1161,8 @@ file.download.header=Download file.preview=Preview: file.fileName=File Name file.type.tabularData=Tabular Data -file.MD5=MD5 -file.MD5.origal=Original File MD5 -file.MD5.exists.tip=A file with this MD5 already exists in the dataset. +file.originalChecksumType=Original File {0} +file.checksum.exists.tip=A file with this checksum already exists in the dataset. file.selectedThumbnail=Thumbnail file.selectedThumbnail.tip=The thumbnail for this file is used as the default thumbnail for the dataset. Click 'Advanced Options' button of another file to select that file. @@ -1400,7 +1398,6 @@ file.tags.label=Tags file.metadataTab.fileMetadata.header=File Metadata file.metadataTab.fileMetadata.persistentid.label=Data File Persistent ID -file.metadataTab.fileMetadata.md5.label=MD5 file.metadataTab.fileMetadata.unf.label=UNF file.metadataTab.fileMetadata.size.label=Size file.metadataTab.fileMetadata.type.label=Type diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index b937eecb624..65d98a0b761 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.dataaccess.DataFileIO; import edu.harvard.iq.dataverse.ingest.IngestReport; import edu.harvard.iq.dataverse.ingest.IngestRequest; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.ShapefileHandler; import java.io.IOException; @@ -16,11 +17,14 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.Files; +import java.util.Arrays; import javax.persistence.Entity; import javax.persistence.OneToMany; import javax.persistence.OneToOne; import javax.persistence.CascadeType; import javax.persistence.Column; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.Index; import javax.persistence.JoinColumn; import javax.persistence.JoinTable; @@ -63,9 +67,56 @@ public class DataFile extends DvObject implements Comparable { @Column( nullable = false ) private String fileSystemName; - - @Column( nullable = false ) - private String md5; + + /** + * End users will see "SHA-1" (with a hyphen) rather than "SHA1" in the GUI + * and API but in the "datafile" table we persist "SHA1" (no hyphen) for + * type safety (using keys of the enum). In the "setting" table, we persist + * "SHA-1" (with a hyphen) to match the GUI and the "Algorithm Name" list at + * https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#MessageDigest + * + * The list of types should be limited to the list above in the technote + * because the string gets passed into MessageDigest.getInstance() and you + * can't just pass in any old string. + */ + public enum ChecksumType { + + MD5("MD5"), + SHA1("SHA-1"); + + private final String text; + + private ChecksumType(final String text) { + this.text = text; + } + + public static ChecksumType fromString(String text) { + if (text != null) { + for (ChecksumType checksumType : ChecksumType.values()) { + if (text.equals(checksumType.text)) { + return checksumType; + } + } + } + throw new IllegalArgumentException("ChecksumType must be one of these values: " + Arrays.asList(ChecksumType.values()) + "."); + } + + @Override + public String toString() { + return text; + } + } + + @Column(nullable = false) + @Enumerated(EnumType.STRING) + private ChecksumType checksumType; + + /** + * Examples include "f622da34d54bdc8ee541d6916ac1c16f" as an MD5 value or + * "3a484dfdb1b429c2e15eb2a735f1f5e4d5b04ec6" as a SHA-1 value" + */ + @Column(nullable = false) + private String checksumValue; /* start: FILE REPLACE ATTRIBUTES */ @@ -403,23 +454,35 @@ public void setRestricted(boolean restricted) { this.restricted = restricted; } - /** - * Fill in until sha1 branch checked in. - * - * @return - */ - public String getCheckSum(){ - return this.md5; + + + + + + + + + + public ChecksumType getChecksumType() { + return checksumType; } - public String getmd5() { - return this.md5; + public void setChecksumType(ChecksumType checksumType) { + this.checksumType = checksumType; } - - public void setmd5(String md5) { - this.md5 = md5; + + public String getChecksumValue() { + return this.checksumValue; } - + + public void setChecksumValue(String checksumValue) { + this.checksumValue = checksumValue; + } + + public String getOriginalChecksumType() { + return BundleUtil.getStringFromBundle("file.originalChecksumType", Arrays.asList(this.checksumType.toString()) ); + } + public DataFileIO getAccessObject() throws IOException { DataFileIO dataAccess = DataAccess.createDataAccessObject(this); @@ -673,6 +736,7 @@ public boolean hasGeospatialTag(){ } return false; } + /** * Set rootDataFileId @@ -707,4 +771,6 @@ public Long getPreviousDataFileId(){ return this.previousDataFileId; } + + } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 7afce6b9241..8b545804607 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -256,7 +256,7 @@ public DataFile findCheapAndEasy(Long id) { Object[] result = null; try { - result = (Object[]) em.createNativeQuery("SELECT t0.ID, t0.CREATEDATE, t0.INDEXTIME, t0.MODIFICATIONTIME, t0.PERMISSIONINDEXTIME, t0.PERMISSIONMODIFICATIONTIME, t0.PUBLICATIONDATE, t0.CREATOR_ID, t0.RELEASEUSER_ID, t0.PREVIEWIMAGEAVAILABLE, t1.CONTENTTYPE, t1.FILESYSTEMNAME, t1.FILESIZE, t1.INGESTSTATUS, t1.MD5, t1.RESTRICTED, t3.ID, t3.AUTHORITY, t3.IDENTIFIER FROM DVOBJECT t0, DATAFILE t1, DVOBJECT t2, DATASET t3 WHERE ((t0.ID = " + id + ") AND (t0.OWNER_ID = t2.ID) AND (t2.ID = t3.ID) AND (t1.ID = t0.ID))").getSingleResult(); + result = (Object[]) em.createNativeQuery("SELECT t0.ID, t0.CREATEDATE, t0.INDEXTIME, t0.MODIFICATIONTIME, t0.PERMISSIONINDEXTIME, t0.PERMISSIONMODIFICATIONTIME, t0.PUBLICATIONDATE, t0.CREATOR_ID, t0.RELEASEUSER_ID, t0.PREVIEWIMAGEAVAILABLE, t1.CONTENTTYPE, t1.FILESYSTEMNAME, t1.FILESIZE, t1.INGESTSTATUS, t1.CHECKSUMVALUE, t1.RESTRICTED, t3.ID, t3.AUTHORITY, t3.IDENTIFIER, t1.CHECKSUMTYPE FROM DVOBJECT t0, DATAFILE t1, DVOBJECT t2, DATASET t3 WHERE ((t0.ID = " + id + ") AND (t0.OWNER_ID = t2.ID) AND (t2.ID = t3.ID) AND (t1.ID = t0.ID))").getSingleResult(); } catch (Exception ex) { return null; } @@ -347,14 +347,14 @@ public DataFile findCheapAndEasy(Long id) { String md5 = (String) result[14]; if (md5 != null) { - dataFile.setmd5(md5); + dataFile.setChecksumValue(md5); } Boolean restricted = (Boolean) result[15]; if (restricted != null) { dataFile.setRestricted(restricted); } - + Dataset owner = new Dataset(); @@ -363,6 +363,17 @@ public DataFile findCheapAndEasy(Long id) { owner.setId((Long)result[16]); owner.setAuthority((String)result[17]); owner.setIdentifier((String)result[18]); + + String checksumType = (String) result[19]; + if (checksumType != null) { + try { + // In the database we store "SHA1" rather than "SHA-1". + DataFile.ChecksumType typeFromStringInDatabase = DataFile.ChecksumType.valueOf(checksumType); + dataFile.setChecksumType(typeFromStringInDatabase); + } catch (IllegalArgumentException ex) { + logger.info("Exception trying to convert " + checksumType + " to enum: " + ex); + } + } dataFile.setOwner(owner); @@ -466,7 +477,7 @@ public void findFileMetadataOptimizedExperimental(Dataset owner, DatasetVersion i = 0; - List<Object[]> fileResults = em.createNativeQuery("SELECT t0.ID, t0.CREATEDATE, t0.INDEXTIME, t0.MODIFICATIONTIME, t0.PERMISSIONINDEXTIME, t0.PERMISSIONMODIFICATIONTIME, t0.PUBLICATIONDATE, t0.CREATOR_ID, t0.RELEASEUSER_ID, t1.CONTENTTYPE, t1.FILESYSTEMNAME, t1.FILESIZE, t1.INGESTSTATUS, t1.MD5, t1.RESTRICTED FROM DVOBJECT t0, DATAFILE t1 WHERE ((t0.OWNER_ID = " + owner.getId() + ") AND ((t1.ID = t0.ID) AND (t0.DTYPE = 'DataFile')))").getResultList(); + List<Object[]> fileResults = em.createNativeQuery("SELECT t0.ID, t0.CREATEDATE, t0.INDEXTIME, t0.MODIFICATIONTIME, t0.PERMISSIONINDEXTIME, t0.PERMISSIONMODIFICATIONTIME, t0.PUBLICATIONDATE, t0.CREATOR_ID, t0.RELEASEUSER_ID, t1.CONTENTTYPE, t1.FILESYSTEMNAME, t1.FILESIZE, t1.INGESTSTATUS, t1.CHECKSUMVALUE, t1.RESTRICTED, t1.CHECKSUMTYPE FROM DVOBJECT t0, DATAFILE t1 WHERE ((t0.OWNER_ID = " + owner.getId() + ") AND ((t1.ID = t0.ID) AND (t0.DTYPE = 'DataFile')))").getResultList(); for (Object[] result : fileResults) { Integer file_id = (Integer) result[0]; @@ -545,13 +556,24 @@ public void findFileMetadataOptimizedExperimental(Dataset owner, DatasetVersion String md5 = (String) result[13]; if (md5 != null) { - dataFile.setmd5(md5); + dataFile.setChecksumValue(md5); } Boolean restricted = (Boolean) result[14]; if (restricted != null) { dataFile.setRestricted(restricted); } + + String checksumType = (String) result[15]; + if (checksumType != null) { + try { + // In the database we store "SHA1" rather than "SHA-1". + DataFile.ChecksumType typeFromStringInDatabase = DataFile.ChecksumType.valueOf(checksumType); + dataFile.setChecksumType(typeFromStringInDatabase); + } catch (IllegalArgumentException ex) { + logger.info("Exception trying to convert " + checksumType + " to enum: " + ex); + } + } // TODO: // - if ingest status is "bad", look up the ingest report; @@ -749,12 +771,8 @@ private void msgt(String m){ */ public DataFile setAndCheckFileReplaceAttributes(DataFile savedDataFile){ - - msgt("setAndCheckFileReplaceAttributes: " + savedDataFile); - msgt("setAndCheckFileReplaceAttributes: getCheckSum()" + savedDataFile.getCheckSum()); - + // Is this the initial version of a file? - msg("savedDataFile.getRootDataFileId(): " + savedDataFile.getRootDataFileId()); if ((savedDataFile.getRootDataFileId() == null)|| (savedDataFile.getRootDataFileId().equals(DataFile.ROOT_DATAFILE_ID_DEFAULT))){ diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index 5fe3c9cd61e..09cc4ad50b2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -711,7 +711,8 @@ public int compare(FileMetadata l1, FileMetadata l2) { if (fileMetadataIsDifferent(fm1, fm2)) { datasetFileDifferenceItem fdi = selectFileMetadataDiffs(fm1, fm2); fdi.setFileId(fm1.getDataFile().getId().toString()); - fdi.setFileMD5(fm1.getDataFile().getmd5()); + fdi.setFileChecksumType(fm1.getDataFile().getChecksumType()); + fdi.setFileChecksumValue(fm1.getDataFile().getChecksumValue()); datasetFilesDiffList.add(fdi); } i++; @@ -719,14 +720,16 @@ public int compare(FileMetadata l1, FileMetadata l2) { } else if (fm2.getDataFile().getId() != null && fm1.getDataFile().getId().compareTo(fm2.getDataFile().getId()) > 0) { datasetFileDifferenceItem fdi = selectFileMetadataDiffs(null, fm2); fdi.setFileId(fm2.getDataFile().getId().toString()); - fdi.setFileMD5(fm2.getDataFile().getmd5()); + fdi.setFileChecksumType(fm2.getDataFile().getChecksumType()); + fdi.setFileChecksumValue(fm2.getDataFile().getChecksumValue()); datasetFilesDiffList.add(fdi); j++; } else if (fm2.getDataFile().getId() == null || fm1.getDataFile().getId().compareTo(fm2.getDataFile().getId()) < 0) { datasetFileDifferenceItem fdi = selectFileMetadataDiffs(fm1, null); fdi.setFileId(fm1.getDataFile().getId().toString()); - fdi.setFileMD5(fm1.getDataFile().getmd5()); + fdi.setFileChecksumType(fm1.getDataFile().getChecksumType()); + fdi.setFileChecksumValue(fm1.getDataFile().getChecksumValue()); datasetFilesDiffList.add(fdi); i++; @@ -740,7 +743,8 @@ public int compare(FileMetadata l1, FileMetadata l2) { fm1 = originalVersion.getFileMetadatas().get(i); datasetFileDifferenceItem fdi = selectFileMetadataDiffs(fm1, null); fdi.setFileId(fm1.getDataFile().getId().toString()); - fdi.setFileMD5(fm1.getDataFile().getmd5()); + fdi.setFileChecksumType(fm1.getDataFile().getChecksumType()); + fdi.setFileChecksumValue(fm1.getDataFile().getChecksumValue()); datasetFilesDiffList.add(fdi); i++; @@ -754,10 +758,18 @@ public int compare(FileMetadata l1, FileMetadata l2) { } else { fdi.setFileId("[UNASSIGNED]"); } - if (fm2.getDataFile().getmd5() != null) { - fdi.setFileMD5(fm2.getDataFile().getmd5()); + if (fm2.getDataFile().getChecksumValue() != null) { + fdi.setFileChecksumType(fm2.getDataFile().getChecksumType()); + fdi.setFileChecksumValue(fm2.getDataFile().getChecksumValue()); } else { - fdi.setFileMD5("[UNASSIGNED]"); + /** + * @todo What should we do here? checksumValue is set to + * "nullable = false" so it should never be non-null. Let's set + * it to "null" and see if this code path is ever reached. If + * not, the null check above can probably be safely removed. + */ + fdi.setFileChecksumType(null); + fdi.setFileChecksumValue("[UNASSIGNED]"); } datasetFilesDiffList.add(fdi); @@ -993,7 +1005,8 @@ public datasetFileDifferenceItem() { } private String fileId; - private String fileMD5; + private DataFile.ChecksumType fileChecksumType; + private String fileChecksumValue; private String fileName1; private String fileType1; @@ -1131,13 +1144,21 @@ public void setFile1Empty(boolean state) { public void setFile2Empty(boolean state) { file2Empty = state; } - - public String getFileMD5() { - return fileMD5; + + public DataFile.ChecksumType getFileChecksumType() { + return fileChecksumType; + } + + public void setFileChecksumType(DataFile.ChecksumType fileChecksumType) { + this.fileChecksumType = fileChecksumType; + } + + public String getFileChecksumValue() { + return fileChecksumValue; } - public void setFileMD5(String fileMD5) { - this.fileMD5 = fileMD5; + public void setFileChecksumValue(String fileChecksumValue) { + this.fileChecksumValue = fileChecksumValue; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 3c676f5d665..8c495b65a0b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -181,7 +181,10 @@ public Response createDataset( String jsonBody, @PathParam("identifier") String } try { try { - DatasetVersion version = jsonParser().parseDatasetVersion(jsonVersion); + DatasetVersion version = new DatasetVersion(); + version.setDataset(ds); + // Use the two argument version so that the version knows which dataset it's associated with. + version = jsonParser().parseDatasetVersion(jsonVersion, version); // force "initial version" properties version.setMinorVersionNumber(null); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java b/src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java index 616d5c6ccce..58787fa5423 100755 --- a/src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java @@ -494,7 +494,7 @@ public Response getWorldMapDatafileInfo(String jsonTokenData, @Context HttpServl jsonData.add("datafile_id", dfile.getId()); jsonData.add("datafile_label", dfile_meta.getLabel()); //jsonData.add("filename", dfile_meta.getLabel()); - jsonData.add("datafile_expected_md5_checksum", dfile.getmd5()); + jsonData.add("datafile_expected_md5_checksum", dfile.getChecksumValue()); Long fsize = dfile.getFilesize(); if (fsize == null){ fsize= new Long(-1); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserPage.java index 6f684d07d83..5dddaf6432f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinUserPage.java @@ -501,6 +501,8 @@ public String save() { String expTime = confirmEmailUtil.friendlyExpirationTime(systemConfig.getMinutesUntilConfirmEmailTokenExpires()); msg = msg + " Your email address has changed and must be re-verified. Please check your inbox at " + currentUser.getEmail() + " and follow the link we've sent. \n\nAlso, please note that the link will only work for the next " + expTime + " before it has expired."; boolean sendEmail = true; + // delete unexpired token, if it exists (clean slate) + confirmEmailService.deleteTokenForUser(currentUser); try { ConfirmEmailInitResponse confirmEmailInitResponse = confirmEmailService.beginConfirm(currentUser); } catch (ConfirmEmailException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java index fe54dd79529..e21ba1f98b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java @@ -200,6 +200,19 @@ private long deleteAllExpiredTokens() { return numDeleted; } + /** + * @param authenticatedUser + * @return True if token is deleted. False otherwise. + */ + public boolean deleteTokenForUser(AuthenticatedUser authenticatedUser) { + ConfirmEmailData confirmEmailData = findSingleConfirmEmailDataByUser(authenticatedUser); + if (confirmEmailData != null) { + em.remove(confirmEmailData); + return true; + } + return false; + } + public ConfirmEmailData createToken(AuthenticatedUser au) { ConfirmEmailData confirmEmailData = new ConfirmEmailData(au, systemConfig.getMinutesUntilConfirmEmailTokenExpires()); em.persist(confirmEmailData); diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index ab8a0ba7591..5e111ed2eaa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -877,7 +877,7 @@ private boolean step_045_auto_checkForFileReplaceDuplicate(){ for (DataFile df : finalFileList){ - if (Objects.equals(df.getCheckSum(), fileToReplace.getCheckSum())){ + if (Objects.equals(df.getChecksumValue(), fileToReplace.getChecksumValue())){ this.addError(getBundleErr("replace.new_file_same_as_replacement")); } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/DuplicateFileChecker.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/DuplicateFileChecker.java index da318f67128..817cebad949 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/DuplicateFileChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/DuplicateFileChecker.java @@ -59,7 +59,7 @@ public boolean isFileInSavedDatasetVersion(DatasetVersion datasetVersion, FileMe if (fileMetadata == null){ throw new NullPointerException("fileMetadata cannot be null"); } - return this.isFileInSavedDatasetVersion(datasetVersion, fileMetadata.getDataFile().getCheckSum()); + return this.isFileInSavedDatasetVersion(datasetVersion, fileMetadata.getDataFile().getChecksumValue()); } /** @@ -101,7 +101,7 @@ public Map<String, Integer> getDatasetHashesFromDatabase(DatasetVersion datasetV List<FileMetadata> fileMetadatas = new ArrayList<>(datasetVersion.getFileMetadatas()); for (FileMetadata fm : fileMetadatas){ - String checkSum = fm.getDataFile().getCheckSum(); + String checkSum = fm.getDataFile().getChecksumValue(); if (checksumHashCounts.get(checkSum) != null){ checksumHashCounts.put(checkSum, checksumHashCounts.get(checkSum).intValue() + 1); }else{ @@ -127,7 +127,7 @@ public static boolean isDuplicateOriginalWay(DatasetVersion workingVersion, File throw new NullPointerException("datasetVersion cannot be null"); } - String selectedCheckSum = fileMetadata.getDataFile().getCheckSum(); + String selectedCheckSum = fileMetadata.getDataFile().getChecksumValue(); if (selectedCheckSum == null) { return false; } @@ -149,7 +149,7 @@ public static boolean isDuplicateOriginalWay(DatasetVersion workingVersion, File while (fmIt.hasNext()) { FileMetadata fm = fmIt.next(); - String currentCheckSum = fm.getDataFile().getCheckSum(); + String currentCheckSum = fm.getDataFile().getChecksumValue(); if (currentCheckSum != null) { if (checkSumMap.get(currentCheckSum) != null) { checkSumMap.put(currentCheckSum, checkSumMap.get(currentCheckSum).intValue() + 1); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 77dbeebdf22..8f5c0fc99c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -62,7 +62,6 @@ import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReader; import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReaderSpi; import edu.harvard.iq.dataverse.util.FileUtil; -import edu.harvard.iq.dataverse.util.MD5Checksum; import edu.harvard.iq.dataverse.util.ShapefileHandler; import edu.harvard.iq.dataverse.util.SumStatCalculator; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -600,11 +599,12 @@ public List<DataFile> createDataFiles(DatasetVersion version, InputStream inputS if (!tempFile.toFile().renameTo(new File(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier()))) { return null; } - - // MD5: - MD5Checksum md5Checksum = new MD5Checksum(); + try { - datafile.setmd5(md5Checksum.CalculateMD5(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier())); + // We persist "SHA1" rather than "SHA-1". + datafile.setChecksumType(systemConfig.getFileFixityChecksumAlgorithm()); + FileUtil fileUtil = new FileUtil(); + datafile.setChecksumValue(fileUtil.CalculateCheckSum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType())); } catch (Exception md5ex) { logger.warning("Could not calculate MD5 signature for new file " + fileName); } @@ -853,12 +853,17 @@ private DataFile createSingleDataFile(DatasetVersion version, InputStream inputS outputStream.close(); } catch (IOException ioex) {} } - - // MD5: + + /** + * @todo Can this block and the similar block above be refactored + * into a common code path? + */ if (datafile != null) { - MD5Checksum md5Checksum = new MD5Checksum(); + FileUtil fileUtil = new FileUtil(); try { - datafile.setmd5(md5Checksum.CalculateMD5(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier())); + // We persist "SHA1" rather than "SHA-1". + datafile.setChecksumType(systemConfig.getFileFixityChecksumAlgorithm()); + datafile.setChecksumValue(fileUtil.CalculateCheckSum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType())); } catch (Exception md5ex) { logger.warning("Could not calculate MD5 signature for new file " + fileName); } diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index 73a413ecef4..f82e93ccbb6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -1,7 +1,5 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. +/** + * @todo Shouldn't this be in the "edu.harvard.iq.dataverse.api" package? Is the only one that isn't. */ package edu.harvard.iq.dataverse.mydata; @@ -268,7 +266,11 @@ private String getJSONErrorString(String jsonMsg, String optionalLoggerMsg){ } - + + /** + * @todo This should support the "X-Dataverse-key" header like the other + * APIs. + */ @Path(retrieveDataPartialAPIPath) @GET @Produces({"application/json"}) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 5da94dc34df..648bebaa4c7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -950,7 +950,16 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset) { datafileSolrInputDocument.addField(SearchFields.FILE_TYPE, FileUtil.getFacetFileType(fileMetadata.getDataFile())); datafileSolrInputDocument.addField(SearchFields.FILE_TYPE_SEARCHABLE, FileUtil.getFacetFileType(fileMetadata.getDataFile())); datafileSolrInputDocument.addField(SearchFields.FILE_SIZE_IN_BYTES, fileMetadata.getDataFile().getFilesize()); - datafileSolrInputDocument.addField(SearchFields.FILE_MD5, fileMetadata.getDataFile().getmd5()); + if (DataFile.ChecksumType.MD5.equals(fileMetadata.getDataFile().getChecksumType())) { + /** + * @todo Someday we should probably deprecate this + * FILE_MD5 in favor of a combination of + * FILE_CHECKSUM_TYPE and FILE_CHECKSUM_VALUE. + */ + datafileSolrInputDocument.addField(SearchFields.FILE_MD5, fileMetadata.getDataFile().getChecksumValue()); + } + datafileSolrInputDocument.addField(SearchFields.FILE_CHECKSUM_TYPE, fileMetadata.getDataFile().getChecksumType().toString()); + datafileSolrInputDocument.addField(SearchFields.FILE_CHECKSUM_VALUE, fileMetadata.getDataFile().getChecksumValue()); datafileSolrInputDocument.addField(SearchFields.DESCRIPTION, fileMetadata.getDescription()); datafileSolrInputDocument.addField(SearchFields.FILE_DESCRIPTION, fileMetadata.getDescription()); datafileSolrInputDocument.addField(SearchFields.UNF, fileMetadata.getDataFile().getUnf()); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java index 871b28e7858..0dddfd3b22e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java @@ -147,6 +147,8 @@ public class SearchFields { public static final String FILE_TYPE = "fileTypeGroupFacet"; public static final String FILE_SIZE_IN_BYTES = "fileSizeInBytes"; public static final String FILE_MD5 = "fileMd5"; + public static final String FILE_CHECKSUM_TYPE = "fileChecksumType"; + public static final String FILE_CHECKSUM_VALUE = "fileChecksumValue"; public static final String FILENAME_WITHOUT_EXTENSION = "fileNameWithoutExtension"; /** * Indexed as a string so we can facet on it. diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 7f0cd51cad4..d3496fc8a1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1048,13 +1048,15 @@ public String dataFileSizeDisplay(DataFile datafile) { } - public String dataFileMD5Display(DataFile datafile) { + public String dataFileChecksumDisplay(DataFile datafile) { if (datafile == null) { return ""; } - if (datafile.getmd5() != null && datafile.getmd5() != "") { - return " MD5: " + datafile.getmd5() + " "; + if (datafile.getChecksumValue() != null && datafile.getChecksumValue() != "") { + if (datafile.getChecksumType() != null) { + return " " + datafile.getChecksumType() + ": " + datafile.getChecksumValue() + " "; + } } return ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index fc7644dfdd1..e4b136cdfaf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.search; +import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; @@ -527,6 +528,12 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat } } solrSearchResult.setFileMd5((String) solrDocument.getFieldValue(SearchFields.FILE_MD5)); + try { + solrSearchResult.setFileChecksumType((DataFile.ChecksumType) DataFile.ChecksumType.fromString((String) solrDocument.getFieldValue(SearchFields.FILE_CHECKSUM_TYPE))); + } catch (IllegalArgumentException ex) { + logger.info("Exception setting setFileChecksumType: " + ex); + } + solrSearchResult.setFileChecksumValue((String) solrDocument.getFieldValue(SearchFields.FILE_CHECKSUM_VALUE)); solrSearchResult.setUnf((String) solrDocument.getFieldValue(SearchFields.UNF)); solrSearchResult.setDatasetVersionId(datasetVersionId); List<String> fileCategories = (ArrayList) solrDocument.getFieldValues(SearchFields.FILE_TAG); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index fa738f0ea6d..bb084c3c69f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.api.Util; +import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.util.ArrayList; import java.util.Date; @@ -15,6 +16,8 @@ import javax.json.JsonObject; import javax.json.JsonObjectBuilder; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; +import java.math.BigDecimal; +import javax.json.JsonValue; public class SolrSearchResult { @@ -67,7 +70,12 @@ public class SolrSearchResult { private String filetype; private String fileContentType; private Long fileSizeInBytes; + /** + * fileMD5 is here for legacy and backward-compatibility reasons. It might be deprecated some day in favor of "fileChecksumType" and "fileChecksumValue" + */ private String fileMd5; + private DataFile.ChecksumType fileChecksumType; + private String fileChecksumValue; private String dataverseAlias; private String dataverseParentAlias; // private boolean statePublished; @@ -494,7 +502,13 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool .add("file_type", this.filetype) .add("file_content_type", this.fileContentType) .add("size_in_bytes", getFileSizeInBytes()) + /** + * "md5" was the only possible value so it's hard-coded here but + * we might want to deprecate it someday since we now put the + * MD5 or SHA-1 in "checksum". + */ .add("md5", getFileMd5()) + .add("checksum", JsonPrinter.getChecksumTypeAndValue(getFileChecksumType(), getFileChecksumValue())) .add("unf", getUnf()) .add("dataset_citation", datasetCitation) .add("deaccession_reason", this.deaccessionReason) @@ -807,13 +821,33 @@ public void setFileSizeInBytes(Long fileSizeInBytes) { } public String getFileMd5() { - return fileMd5; + if (DataFile.ChecksumType.MD5.equals(getFileChecksumType())) { + return fileMd5; + } else { + return null; + } } public void setFileMd5(String fileMd5) { this.fileMd5 = fileMd5; } + public DataFile.ChecksumType getFileChecksumType() { + return fileChecksumType; + } + + public void setFileChecksumType(DataFile.ChecksumType fileChecksumType) { + this.fileChecksumType = fileChecksumType; + } + + public String getFileChecksumValue() { + return fileChecksumValue; + } + + public void setFileChecksumValue(String fileChecksumValue) { + this.fileChecksumValue = fileChecksumValue; + } + public String getNameSort() { return nameSort; } @@ -1020,4 +1054,5 @@ public void setUserRole(List<String> userRole) { this.userRole = userRole; } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index c4aedb81a71..96fe1da403a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -33,6 +33,8 @@ public class SettingsServiceBean { * So there. */ public enum Key { + FooterCopyright, + FileFixityChecksumAlgorithm, MinutesUntilConfirmEmailTokenExpires, /** * Override Solr highlighting "fragsize" diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 3addc9513fa..5e6766a3dcf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -21,10 +21,12 @@ package edu.harvard.iq.dataverse.util; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.ingest.IngestableDataChecker; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; @@ -33,6 +35,8 @@ import java.util.MissingResourceException; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map; import java.util.logging.Logger; @@ -403,4 +407,52 @@ public static String getFriendlySize(Long filesize) { return displaySize; } + + // from MD5Checksum.java + public synchronized String CalculateCheckSum(String datafile, ChecksumType checksumType) { + + FileInputStream fis = null; + try { + fis = new FileInputStream(datafile); + } catch (FileNotFoundException ex) { + throw new RuntimeException(ex); + } + + return CalculateChecksum(fis, checksumType); + } + + // from MD5Checksum.java + public synchronized String CalculateChecksum(InputStream in, ChecksumType checksumType) { + MessageDigest md = null; + try { + // Use "SHA-1" (toString) rather than "SHA1", for example. + md = MessageDigest.getInstance(checksumType.toString()); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + + byte[] dataBytes = new byte[1024]; + + int nread; + try { + while ((nread = in.read(dataBytes)) != -1) { + md.update(dataBytes, 0, nread); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } finally { + try { + in.close(); + } catch (Exception e) { + } + } + + byte[] mdbytes = md.digest(); + StringBuilder sb = new StringBuilder(""); + for (int i = 0; i < mdbytes.length; i++) { + sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1)); + } + return sb.toString(); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MD5Checksum.java b/src/main/java/edu/harvard/iq/dataverse/util/MD5Checksum.java deleted file mode 100644 index 2b4d8755743..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/util/MD5Checksum.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.util; - -import java.io.InputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -/** - * (from v.3.6) - * @author xyang - */ -public class MD5Checksum implements java.io.Serializable { - - public MD5Checksum() { - } - - public synchronized String CalculateMD5 (String datafile) { - - FileInputStream fis = null; - try { - fis = new FileInputStream(datafile); - } catch (FileNotFoundException ex) { - throw new RuntimeException(ex); - } - - return CalculateMD5(fis); - /* - byte[] dataBytes = new byte[1024]; - - int nread; - try { - while ((nread = fis.read(dataBytes)) != -1) { - md.update(dataBytes, 0, nread); - } - } catch (IOException ex) { - throw new RuntimeException(ex); - } - - byte[] mdbytes = md.digest(); - StringBuilder sb = new StringBuilder(""); - for (int i = 0; i < mdbytes.length; i++) { - sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1)); - } - return sb.toString(); - */ - } - - public synchronized String CalculateMD5 (InputStream in) { - MessageDigest md = null; - try { - md = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - - byte[] dataBytes = new byte[1024]; - - int nread; - try { - while ((nread = in.read(dataBytes)) != -1) { - md.update(dataBytes, 0, nread); - } - } catch (IOException ex) { - throw new RuntimeException(ex); - } finally { - try {in.close();} catch (Exception e) {} - } - - byte[] mdbytes = md.digest(); - StringBuilder sb = new StringBuilder(""); - for (int i = 0; i < mdbytes.length; i++) { - sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1)); - } - return sb.toString(); - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 5246be6acd2..f262be3ff81 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1,12 +1,15 @@ package edu.harvard.iq.dataverse.util; import com.ocpsoft.pretty.PrettyContext; +import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.net.UnknownHostException; +import java.time.Year; +import java.util.Arrays; import java.util.MissingResourceException; import java.util.Properties; import java.util.ResourceBundle; @@ -528,4 +531,20 @@ public boolean isTimerServer() { } return false; } + + public String getFooterCopyrightAndYear() { + return BundleUtil.getStringFromBundle("footer.copyright", Arrays.asList(Year.now().getValue() + "")); + } + + public DataFile.ChecksumType getFileFixityChecksumAlgorithm() { + DataFile.ChecksumType saneDefault = DataFile.ChecksumType.MD5; + String checksumStringFromDatabase = settingsService.getValueForKey(SettingsServiceBean.Key.FileFixityChecksumAlgorithm, saneDefault.toString()); + try { + DataFile.ChecksumType checksumTypeFromDatabase = DataFile.ChecksumType.fromString(checksumStringFromDatabase); + return checksumTypeFromDatabase; + } catch (IllegalArgumentException ex) { + logger.info("The setting " + SettingsServiceBean.Key.FileFixityChecksumAlgorithm + " is misconfigured. " + ex.getMessage() + " Returning sane default: " + saneDefault + "."); + return saneDefault; + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 15961748511..ce9b66c81b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -35,6 +35,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObject; @@ -49,6 +50,8 @@ */ public class JsonParser { + private static final Logger logger = Logger.getLogger(JsonParser.class.getCanonicalName()); + DatasetFieldServiceBean datasetFieldSvc; MetadataBlockServiceBean blockService; SettingsServiceBean settingsService; @@ -384,18 +387,44 @@ public DataFile parseDataFile(JsonObject datafileJson) { contentType = "application/octet-stream"; } String storageIdentifier = datafileJson.getString("storageIdentifier"); - String md5 = datafileJson.getString("md5", null); - - if (md5 == null) { - md5 = "unknown"; + JsonObject checksum = datafileJson.getJsonObject("checksum"); + if (checksum != null) { + // newer style that allows for SHA-1 rather than MD5 + /** + * @todo Add more error checking. Do we really expect people to set + * file metadata without uploading files? Some day we'd like to work + * on a "native" API that allows for multipart upload of the JSON + * describing the files (this "parseDataFile" method) and the bits + * of the files themselves. See + * https://github.com/IQSS/dataverse/issues/1612 + */ + String type = checksum.getString("type"); + if (type != null) { + String value = checksum.getString("value"); + if (value != null) { + try { + dataFile.setChecksumType(DataFile.ChecksumType.fromString(type)); + dataFile.setChecksumValue(value); + } catch (IllegalArgumentException ex) { + logger.info("Invalid"); + } + } + } + } else { + // older, MD5 logic, still her for backward compatibility + String md5 = datafileJson.getString("md5", null); + if (md5 == null) { + md5 = "unknown"; + } + dataFile.setChecksumType(DataFile.ChecksumType.MD5); + dataFile.setChecksumValue(md5); } - + // TODO: // unf (if available)... etc.? dataFile.setContentType(contentType); dataFile.setStorageIdentifier(storageIdentifier); - dataFile.setmd5(md5); return dataFile; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 1666f7cd265..efbce0709ec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -30,6 +30,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; +import edu.harvard.iq.dataverse.search.SolrSearchResult; import edu.harvard.iq.dataverse.util.DatasetFieldWalker; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; @@ -494,9 +495,14 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) { .add("originalFileFormat", df.getOriginalFileFormat()) .add("originalFormatLabel", df.getOriginalFormatLabel()) .add("UNF", df.getUnf()) - .add("md5", df.getmd5()) .add("rootDataFileId", df.getRootDataFileId()) .add("previousDataFileId", df.getPreviousDataFileId()) + /** + * @todo Should we deprecate "md5" now that it's under + * "checksum" (which may also be a SHA-1 rather than an MD5)? + */ + .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue())) + .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue())) .add("description", df.getDescription()); } @@ -688,4 +694,22 @@ public Set<Collector.Characteristics> characteristics() { }; } + public static String getMd5IfItExists(DataFile.ChecksumType checksumType, String checksumValue) { + if (DataFile.ChecksumType.MD5.equals(checksumType)) { + return checksumValue; + } else { + return null; + } + } + + public static JsonObjectBuilder getChecksumTypeAndValue(DataFile.ChecksumType checksumType, String checksumValue) { + if (checksumType != null) { + return Json.createObjectBuilder() + .add("type", checksumType.toString()) + .add("value", checksumValue); + } else { + return null; + } + } + } diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index eb3db1ffcaf..cb8a74063b2 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -825,7 +825,7 @@ <p:column styleClass="versionValue"> <h:outputText value="#{bundle['file.viewDiffDialog.fileID']} #{fileDiff.fileId}"/> <br></br> - <h:outputText value="#{bundle['file.viewDiffDialog.md5']} #{fileDiff.fileMD5}"/> + <h:outputText value="#{fileDiff.fileChecksumType} #{fileDiff.fileChecksumValue}"/> </p:column> <p:column styleClass="versionDetails" rendered="#{! fileDiff.file1Empty}"> <h:outputText value="#{bundle['file.viewDiffDialog.fileName']}: #{fileDiff.fileName1}" styleClass="diffDetailBlock" rendered="#{fileDiff.fileName1 != null}"/> diff --git a/src/main/webapp/dataverse_footer.xhtml b/src/main/webapp/dataverse_footer.xhtml index 97141f69b69..285c4408f19 100644 --- a/src/main/webapp/dataverse_footer.xhtml +++ b/src/main/webapp/dataverse_footer.xhtml @@ -17,7 +17,7 @@ <h:outputText value="#{bundle['footer.dataScienceIQSS']}" escape="false"/>  |  #{bundle['footer.dataverseProjectOn']} <a href="https://twitter.com/dataverseorg" target="_blank" title="#{bundle['footer.dataverseProjectOn']} #{bundle['footer.Twitter']}"><span class="socicon socicon-twitter" title="#{bundle['footer.dataverseProjectOn']} #{bundle['footer.Twitter']}"/></a>  |  #{bundle['footer.codeAvailable']} <a href="https://github.com/IQSS/dataverse" target="_blank" title="#{bundle['footer.dataverseOnGitHub']}"><span class="socicon socicon-github" title="#{bundle['footer.dataverseOnGitHub']}"/></a> </p> <p> - <h:outputText value="#{bundle['footer.copyright']}" escape="false"/> <ui:fragment rendered="#{!empty settingsWrapper.get(':ApplicationPrivacyPolicyUrl')}"> |  <h:outputLink value="#{settingsWrapper.get(':ApplicationPrivacyPolicyUrl')}" target="_blank">#{bundle['footer.privacyPolicy']}</h:outputLink></ui:fragment> + <h:outputText value="#{systemConfig.footerCopyrightAndYear}#{settingsWrapper.get(':FooterCopyright')}" escape="false"/> <ui:fragment rendered="#{!empty settingsWrapper.get(':ApplicationPrivacyPolicyUrl')}"> |  <h:outputLink value="#{settingsWrapper.get(':ApplicationPrivacyPolicyUrl')}" target="_blank">#{bundle['footer.privacyPolicy']}</h:outputLink></ui:fragment> </p> </div> <div class="col-xs-7 small" jsf:rendered="#{widgetView}"> diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index d5e83590a82..fa8b48994e5 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -124,5 +124,6 @@ } </script> <ui:include src="google-analytics-snippet.xhtml"></ui:include> + <ui:include src="piwik-analytics-snippet.xhtml"></ui:include> </h:body> </html> diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 3ff3f10f459..317c7e6c0a9 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -149,16 +149,16 @@ <p:inputText id="fileName" value="#{fileMetadata.label}" style="width:60%; margin-bottom:.5em;"/> <p:message for="fileName"/> </ui:fragment> - <!-- TYPE + SIZE + DATE + MD5 --> + <!-- TYPE + SIZE + DATE + CHECKSUM --> <div class="text-muted small"> <h:outputText id="fileTypeOutputRegular" value="#{fileMetadata.dataFile.friendlyType}" rendered="#{!(fileMetadata.dataFile.tabularData)}"/> <h:outputText id="fileTypeOutputTabular" value="#{bundle['file.type.tabularData']}" rendered="#{fileMetadata.dataFile.tabularData}"/> <h:outputText id="fileCreatePublishDate" value=" - #{EditDatafilesPage.getFileDateToDisplay(fileMetadata)}" rendered="#{!(empty fileMetadata.id)}"/> - <div class="mD5-block"> - <h:outputText id="fileMD5" value="#{fileMetadata.dataFile.tabularData ? bundle['file.MD5.origal'] : bundle['file.MD5']}: #{fileMetadata.dataFile.md5};" rendered="#{!(empty fileMetadata.dataFile.md5) and (!EditDatafilesPage.isDuplicate(fileMetadata))}"/> - <h:outputText id="duplicateFileMD5" styleClass="text-danger" value="#{fileMetadata.dataFile.tabularData ? bundle['file.MD5.origal']: bundle['file.MD5']}: #{fileMetadata.dataFile.md5}" rendered="#{!(empty fileMetadata.dataFile.md5) and EditDatafilesPage.isDuplicate(fileMetadata)}"/> + <div class="checksum-block"> + <h:outputText id="fileChecksum" value="#{fileMetadata.dataFile.tabularData ? fileMetadata.dataFile.originalChecksumType : fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue};" rendered="#{!(empty fileMetadata.dataFile.checksumValue) and (!EditDatafilesPage.isDuplicate(fileMetadata))}"/> + <h:outputText id="duplicateFileChecksum" styleClass="text-danger" value="#{fileMetadata.dataFile.tabularData ? fileMetadata.dataFile.originalChecksumType: fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue}" rendered="#{!(empty fileMetadata.dataFile.checksumValue) and EditDatafilesPage.isDuplicate(fileMetadata)}"/> <ui:fragment rendered="#{EditDatafilesPage.isDuplicate(fileMetadata) and (empty fileMetadata.id)}"> - <div style="display:inline-block;" class="ui-message ui-message-error ui-widget ui-corner-all fileDuplicateWarning"><span class="ui-message-error-detail">#{bundle['file.MD5.exists.tip']}</span></div> + <div style="display:inline-block;" class="ui-message ui-message-error ui-widget ui-corner-all fileDuplicateWarning"><span class="ui-message-error-detail">#{bundle['file.checksum.exists.tip']}</span></div> </ui:fragment> </div> </div> diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index ee3d4f71af4..da8fbc3f48b 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -194,8 +194,9 @@ </h:outputFormat> <div class="mD5-block" jsf:rendered="#{!FilePage.file.tabularData}"> - <h:outputText id="fileMD5" value="#{FilePage.file.tabularData ? bundle['file.MD5.origal'] : bundle['file.MD5']}: #{FilePage.file.md5};" - rendered="#{!(empty FilePage.file.md5)}"/> + <!-- FIXME: When this page is fixed (no more "Method isThumbnailAvailable not found") also replace "file.MD5" with FilePage.file.checksumType and "file.MD5.origal" with "file.originalChecksumType" --> + <h:outputText id="fileMD5" value="#{FilePage.file.tabularData ? bundle['file.MD5.origal'] : bundle['file.MD5']}: #{FilePage.file.checksumValue};" + rendered="#{!(empty FilePage.file.checksumValue)}"/> </div> <div class="text-muted" jsf:rendered="#{FilePage.file.tabularData}"> @@ -236,12 +237,13 @@ </div> </div> </ui:remove> - <div class="form-group" jsf:rendered="#{!(empty FilePage.file.md5)}"> + <div class="form-group" jsf:rendered="#{!(empty FilePage.file.checksumValue)}"> <label class="col-sm-3 control-label" for="metadata_md5"> + <!-- FIXME: When this page is fixed (no more "Method isThumbnailAvailable not found") also replace "file.MD5" with FilePage.file.checksumType and "file.MD5.origal" with "file.originalChecksumType" --> #{FilePage.file.tabularData ? bundle['file.MD5.origal'] : bundle['file.MD5']} </label> <div class="col-sm-9"> - <h:outputText value="#{FilePage.file.md5}"/> + <h:outputText value="#{FilePage.file.checksumValue}"/> </div> </div> <div class="form-group" jsf:rendered="#{!(empty FilePage.file.unf)}"> @@ -304,8 +306,9 @@ FITS stuff.... </li> <li> + <!-- FIXME: When this page is fixed (no more "Method isThumbnailAvailable not found") use FilePage.file.checksumType rather than "file.metadataTab.fileMetadata.md5.label" --> #{bundle['file.metadataTab.fileMetadata.md5.label']} - <h:outputText value="#{FilePage.file.md5};" rendered="#{!(empty FilePage.file.md5)}"/> + <h:outputText value="#{FilePage.file.checksumValue};" rendered="#{!(empty FilePage.file.checksumValue)}"/> </li> </div> </div> diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 0a87c89198f..1af77330d03 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -264,7 +264,7 @@ </h:outputFormat> <div class="mD5-block" jsf:rendered="#{!fileMetadata.dataFile.tabularData}"> - <h:outputText id="fileMD5" value="#{bundle['file.MD5']}: #{fileMetadata.dataFile.md5};" rendered="#{!(empty fileMetadata.dataFile.md5) and ((DatasetPage.editMode != 'FILE' and DatasetPage.editMode != 'CREATE') or !DatasetPage.isDuplicate(fileMetadata))}"/> + <h:outputText id="fileMD5" value="#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue};" rendered="#{!(empty fileMetadata.dataFile.checksumValue) and ((DatasetPage.editMode != 'FILE' and DatasetPage.editMode != 'CREATE') or !DatasetPage.isDuplicate(fileMetadata))}"/> </div> </div> <!-- UNF + Variables, Obsersvations --> diff --git a/src/main/webapp/mydata_templates/cards.html b/src/main/webapp/mydata_templates/cards.html deleted file mode 100644 index 1d12409e7cd..00000000000 --- a/src/main/webapp/mydata_templates/cards.html +++ /dev/null @@ -1,82 +0,0 @@ -{% if data.selected_filters %} - {% for pub_stat in data.selected_filters.publication_statuses %} - <a class="lnk_cbox_remove btn btn-primary btn-xs" rel="id_cbox_{{ pub_stat|lower }}">Status: {{ pub_stat }} <span class="glyphicon glyphicon-remove" aria-hidden="true"></span></a> - {% endfor %} - {% for role_name in data.selected_filters.role_names %} - <a class="lnk_cbox_remove btn btn-info btn-xs" rel="id_cbox_{{ role_name|lower }}">Role: {{ role_name }} <span class="glyphicon glyphicon-remove" aria-hidden="true"></span></a> - {% endfor %} -{% endif %} - -{# <!-- Iterate through and render cards --> #} - -{% set card_bg_color = cycler("", "background-color:#f7f7f7;") %} -{% for sdoc in data.items -%} - {# <!-- set border color --> #} - {% if sdoc.type == "dataverse" %} - {% set card_border_color = "#c55b28" %} - {% elif sdoc.type == "dataset" %} - {% set card_border_color = "#428bca" %} - {% elif sdoc.type == "file" %} - {% set card_border_color = "#ccc" %} - {% endif %} - - <div style="border:1px solid {{ card_border_color }};padding:4px 10px; margin:20px; width:700px;{{ card_bg_color.next() }}"> - <!-- publication status --> - {#({{ loop.index }})#} {% if sdoc.is_draft_state %}<span class="label label-primary">Draft</span> {% endif %} - {% if sdoc.is_unpublished_state %}<span class="label label-primary">Unpublished</span> {% endif %} - {% if sdoc.is_published %} <span class="label label-success">Published</span> {% endif %} - <!-- roles --> - {% for role_name in sdoc.user_roles %} - <span class="label label-info">{{ role_name }}</span> - {% endfor %} - - {% if sdoc.type == "dataverse" %} - <div class="card-title-icon-block"> - <span class="icon-dataverse text-brand pull-right" title="Dataverse"></span> - <a href="/dataverse/{{ sdoc.identifier }}"><span style="padding:4px 0;">{{ sdoc.name }}</span></a> - <span class="text-muted" style="margin-left: .5em;">(affiliation?)</span> - </div> - <div class="card-preview-icon-block text-center"> - <a href="/dataverse/{{ sdoc.identifier }}"><span class="icon-dataverse text-brand"></span></a> - {# <a href="/dataverse/{{ sdoc.identifier }}"><img src="{{ sdoc.image_url | replace("https://RAD-rprasad", "") }}" alt="icon image" /></a> #} - </div> - <span class="text-muted" style="margin-right:.5em;">{{ sdoc.date_to_display_on_card }}</span> - {% if sdoc.parentId %} - {% if sdoc.parentId != sdoc.identifier %} - - <a href="/dataverse.xhtml?id={{ sdoc.parentId }}">{{ sdoc.parentName }}</a> - {% endif %} - {% endif %} - - {% if parent_alias %} - <a href="/dataverse/{{ parent_alias }}">{{ parent_alias }}</a> - {% endif %} - <hr style="margin:.5em;border:0;" />{{ sdoc.description }} - - {% elif sdoc.type == "dataset" %} - - <div class="card-title-icon-block"><span class="icon-dataset text-info pull-right" title="Dataset"></span> - <a href="/dataset.xhtml?persistentId={{ sdoc.global_id }}"><span style="padding:4px 0;">{{ sdoc.name }}</span></a> - </div> - <div class="card-preview-icon-block text-center"> - <a href="/dataset.xhtml?persistentId={{ sdoc.global_id }}"><span class="icon-dataset text-info"></span></a> - </div><span class="text-muted">{{ sdoc.date_to_display_on_card }}</span> - - <a href="/dataverse.xhtml?id={{ sdoc.parentId }}">{{ sdoc.parentName }}</a> - <br /><span class="resultDatasetCitationBlock bg-citation">{{ sdoc.citation|safe }}</span> - <br />{{ sdoc.description|safe }} - - {% elif sdoc.type == "file" %} - <div class="card-title-icon-block"><span class="icon-file text-muted pull-right" title="File"></span> - - <a href="/dataset.xhtml?persistentId={{ sdoc.parentIdentifier }}"><span style="padding:4px 0;">{{ sdoc.name }}</span></a> - </div> - <div class="card-preview-icon-block text-center"> - <img src="{{ sdoc.image_url | replace("https://RAD-rprasad", "") }}?pfdrid_c=true" alt="" /> - <a href="/dataset.xhtml?persistentId={{ sdoc.parentIdentifier }}"><span class="icon-dataset text-info"></span></a> - </div><span class="text-muted">{{ sdoc.date_to_display_on_card }}</span> - - <a href="/dataset.xhtml?persistentId={{ sdoc.parentIdentifier }}">{{ sdoc.parentName }}</a> - {#<br /><span class="resultDatasetCitationBlock bg-citation">{{ sdoc.dataset_citation|safe }}</span>#} - <br /><span class="text-muted">{{ sdoc.file_type }}</span><span class="text-muted"> - {{ sdoc.size_in_bytes }} KB - </span><span class="text-muted"> MD5: {{ sdoc.md5 }} </span> -{% endif %} -</div> - -{%- endfor %} diff --git a/src/main/webapp/mydata_templates/cards_minimum.html b/src/main/webapp/mydata_templates/cards_minimum.html index 6fa4066d48b..c0dd50db7f1 100644 --- a/src/main/webapp/mydata_templates/cards_minimum.html +++ b/src/main/webapp/mydata_templates/cards_minimum.html @@ -83,7 +83,7 @@ </span> <br /><span class="text-muted">{{ card_info.file_type }}</span> <span class="text-muted"> - {{ card_info.size_in_bytes }} KB - </span> - <span class="text-muted"> MD5: {{ card_info.md5 }} </span> + <span class="text-muted"> {{ card_info.checksum.type }}: {{ card_info.checksum.value }} </span> {% endif %} <hr style="margin:.5em;border:0;"/> diff --git a/src/main/webapp/piwik-analytics-snippet.xhtml b/src/main/webapp/piwik-analytics-snippet.xhtml new file mode 100644 index 00000000000..dad71bc43e7 --- /dev/null +++ b/src/main/webapp/piwik-analytics-snippet.xhtml @@ -0,0 +1,26 @@ +<ui:composition xmlns="http://www.w3.org/1999/xhtml" + xmlns:h="http://java.sun.com/jsf/html" + xmlns:f="http://java.sun.com/jsf/core" + xmlns:ui="http://java.sun.com/jsf/facelets" + xmlns:c="http://java.sun.com/jsp/jstl/core" + xmlns:p="http://primefaces.org/ui" + xmlns:o="http://omnifaces.org/ui" + xmlns:jsf="http://xmlns.jcp.org/jsf"> + <c:if test="#{(not empty settingsWrapper.get(':PiwikAnalyticsId')) and (not empty settingsWrapper.get(':PiwikAnalyticsHost'))}"> + <!-- Piwik --> + <script type="text/javascript"> + var _paq = _paq || []; + _paq.push(['trackPageView']); + _paq.push(['enableLinkTracking']); + (function() { + var u="//#{settingsWrapper.get(':PiwikAnalyticsHost')}/"; + _paq.push(['setTrackerUrl', u+'piwik.php']); + _paq.push(['setSiteId', #{settingsWrapper.get(':PiwikAnalyticsId')}]); + var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; + g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); + })(); + </script> + <noscript><p><img src="//#{settingsWrapper.get(':PiwikAnalyticsHost')}/piwik.php?idsite=#{settingsWrapper.get(':PiwikAnalyticsId')}" style="border:0;" alt="" /></p></noscript> + <!-- End Piwik Code --> + </c:if> +</ui:composition> diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 22336165006..40e689c9869 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -608,7 +608,7 @@ <h:outputText styleClass="text-muted" value=" - #{SearchIncludeFragment.dataFileSizeDisplay(result.entity)}" rendered="#{!result.harvested}" /> - <h:outputText styleClass="text-muted" value=" - #{SearchIncludeFragment.dataFileMD5Display(result.entity)}" rendered="#{!result.harvested and !SearchIncludeFragment.isTabular(result.entity)}"/> + <h:outputText styleClass="text-muted" value=" - #{SearchIncludeFragment.dataFileChecksumDisplay(result.entity)}" rendered="#{!result.harvested and !SearchIncludeFragment.isTabular(result.entity)}"/> <!-- if this is a tabular data file, extra information, such as the numbers of variables and observations and the unf, is displayed: --> <h:outputText styleClass="text-muted" value=" - #{SearchIncludeFragment.tabularDataDisplayInfo(result.entity)}" diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 57114710ca7..6bca29aff12 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -18,13 +18,17 @@ import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static com.jayway.restassured.path.json.JsonPath.with; +import edu.harvard.iq.dataverse.DataFile; import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.util.UUID; +import static javax.ws.rs.core.Response.Status.NO_CONTENT; import static junit.framework.Assert.assertEquals; import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; public class DatasetsIT { @@ -389,4 +393,76 @@ public void testPrivateUrl() { */ } + @Test + public void testFileChecksum() { + + Response createUser = UtilIT.createRandomUser(); +// createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + System.out.println("dataset id: " + datasetId); + + Response getDatasetJsonNoFiles = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJsonNoFiles.prettyPrint(); + String protocol1 = JsonPath.from(getDatasetJsonNoFiles.getBody().asString()).getString("data.protocol"); + String authority1 = JsonPath.from(getDatasetJsonNoFiles.getBody().asString()).getString("data.authority"); + String identifier1 = JsonPath.from(getDatasetJsonNoFiles.getBody().asString()).getString("data.identifier"); + String dataset1PersistentId = protocol1 + ":" + authority1 + "/" + identifier1; + + Response makeSureSettingIsDefault = UtilIT.deleteSetting(SettingsServiceBean.Key.FileFixityChecksumAlgorithm); + makeSureSettingIsDefault.prettyPrint(); + makeSureSettingIsDefault.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Setting :FileFixityChecksumAlgorithm deleted.")); + Response getDefaultSetting = UtilIT.getSetting(SettingsServiceBean.Key.FileFixityChecksumAlgorithm); + getDefaultSetting.prettyPrint(); + getDefaultSetting.then().assertThat() + .body("message", equalTo("Setting :FileFixityChecksumAlgorithm not found")); + + Response uploadMd5File = UtilIT.uploadRandomFile(dataset1PersistentId, apiToken); + uploadMd5File.prettyPrint(); + assertEquals(CREATED.getStatusCode(), uploadMd5File.getStatusCode()); + Response getDatasetJsonAfterMd5File = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJsonAfterMd5File.prettyPrint(); + getDatasetJsonAfterMd5File.then().assertThat() + .body("data.latestVersion.files[0].dataFile.md5", equalTo("0386269a5acb2c57b4eade587ff4db64")) + .body("data.latestVersion.files[0].dataFile.checksum.type", equalTo("MD5")) + .body("data.latestVersion.files[0].dataFile.checksum.value", equalTo("0386269a5acb2c57b4eade587ff4db64")); + + int fileId = JsonPath.from(getDatasetJsonAfterMd5File.getBody().asString()).getInt("data.latestVersion.files[0].dataFile.id"); + Response deleteFile = UtilIT.deleteFile(fileId, apiToken); + deleteFile.prettyPrint(); + deleteFile.then().assertThat() + .statusCode(NO_CONTENT.getStatusCode()); + + Response setToSha1 = UtilIT.setSetting(SettingsServiceBean.Key.FileFixityChecksumAlgorithm, DataFile.ChecksumType.SHA1.toString()); + setToSha1.prettyPrint(); + setToSha1.then().assertThat() + .statusCode(OK.getStatusCode()); + Response getNonDefaultSetting = UtilIT.getSetting(SettingsServiceBean.Key.FileFixityChecksumAlgorithm); + getNonDefaultSetting.prettyPrint(); + getNonDefaultSetting.then().assertThat() + .body("data.message", equalTo("SHA-1")) + .statusCode(OK.getStatusCode()); + + Response uploadSha1File = UtilIT.uploadRandomFile(dataset1PersistentId, apiToken); + uploadSha1File.prettyPrint(); + assertEquals(CREATED.getStatusCode(), uploadSha1File.getStatusCode()); + Response getDatasetJsonAfterSha1File = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJsonAfterSha1File.prettyPrint(); + getDatasetJsonAfterSha1File.then().assertThat() + .body("data.latestVersion.files[0].dataFile.md5", nullValue()) + .body("data.latestVersion.files[0].dataFile.checksum.type", equalTo("SHA-1")) + .body("data.latestVersion.files[0].dataFile.checksum.value", equalTo("17ea9225aa0e96ae6ff61c256237d6add6c197d1")) + .statusCode(OK.getStatusCode()); + + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 6fefbdcd8a7..fa097182407 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -537,6 +537,16 @@ static Response deleteSetting(SettingsServiceBean.Key settingKey) { return response; } + static Response getSetting(SettingsServiceBean.Key settingKey) { + Response response = given().when().get("/api/admin/settings/" + settingKey); + return response; + } + + static Response setSetting(SettingsServiceBean.Key settingKey, String value) { + Response response = given().body(value).when().put("/api/admin/settings/" + settingKey); + return response; + } + static Response getRoleAssignmentsOnDataverse(String dataverseAliasOrId, String apiToken) { String url = "/api/dataverses/" + dataverseAliasOrId + "/assignments"; System.out.println("URL: " + url);