diff --git a/migration/v11.0/README.md b/migration/v11.0/README.md new file mode 100644 index 00000000..95e8da10 --- /dev/null +++ b/migration/v11.0/README.md @@ -0,0 +1,67 @@ +# Migrating to gist v11.0 from v10.0 + +See also additional documentation in the file `docs/MajorVersionMigration.md`. + +This file gives a quick overview of using the migration utilities provided +by the gist team to help migrate from the previous version of gist. These +scripts are not cumulative; you must run the migrations for each version +in sequence. + +These utilities are a starting point. You should review them before running +them to make sure you approve of the actions prior to using them. + +These utilities do not handle everything, in some cases we can only warn you +about changes that were made in gist and you will need to decide how you want +to handle the changes. + +## Migration Queries + +All of our migration tools are SPARQL queries. + +You will note that some files have very similar names. Here is what the suffixes mean: + +- `_default.rq` : These queries only work on the default graph`*` +- `_ngraphs.rq` : These queries only work on named graphs +- `.rq` : These queries work on both the default graph and named graphs + +`*` the handling of these queries will be dependent upon which Triplestore +you are using and how it was configured. Some Triplestores will only use +triples in the default graph, some will use all triples in all named graphs. + +## Requirements + +These migration scripts are meant to be run using the open source onto_tool +program. Information about onto_tool and how to install it, is available at +[github](https://github.com/semanticarts/ontology-toolkit). + +## Run on local files + +NOTE: A known limitation of `onto_tool` is that it does not support named graphs when run against local files. +It does support named graphs when run against a SPARQL endpoint. + +Start in this directory. + +Put source data files in the ./input/ directory. See the onto_tool documentation for which file formats are supported. + +Execute: + +```shell +onto_tool bundle migrate_local.yaml +``` + +Output files will be created in the ./output/ directory. + +Report files will be created in the ./reports/ directory. + +## Run on a SPARQL Endpoint + +You will need to tell onto_tool the URL of your SPARQL endpoint. You can do that +by editing the migration_endpoint.yaml file, or you can put it into the command +line execution like this: + +```shell +onto_tool bundle -v user USER -v password PWD + -v endpoint ENDPOINT-URI + [ -v update_endpoint UPDATE-URI ] + -v report REPORT-DIR migrate_endpoint.yaml +``` diff --git a/migration/v11.0/action_rename_default.rq b/migration/v11.0/action_rename_default.rq new file mode 100644 index 00000000..8cbbf305 --- /dev/null +++ b/migration/v11.0/action_rename_default.rq @@ -0,0 +1,27 @@ +prefix gist: + +DELETE { + ?oldTerm ?sp ?so . + ?ps ?oldTerm ?po . + ?os ?op ?oldTerm . +} +INSERT { + ?newTerm ?sp ?so . + ?ps ?newTerm ?po . + ?os ?op ?newTerm . +} +where { + values (?oldTerm ?newTerm) { + (gist:ContemporaneousEvent gist:ContemporaryEvent) + (gist:hasOrderedMember gist:hasMember) + (gist:baseConversionFactor gist:conversionFactor) + (gist:standardConversionFactor gist:conversionFactor) + } + { + ?oldTerm ?sp ?so + } UNION { + ?ps ?oldTerm ?po + } UNION { + ?os ?op ?oldTerm + } +} diff --git a/migration/v11.0/action_rename_ngraphs.rq b/migration/v11.0/action_rename_ngraphs.rq new file mode 100644 index 00000000..58513c4f --- /dev/null +++ b/migration/v11.0/action_rename_ngraphs.rq @@ -0,0 +1,33 @@ +prefix gist: + +DELETE { + graph ?graph { + ?oldTerm ?sp ?so . + ?ps ?oldTerm ?po . + ?os ?op ?oldTerm . + } +} +INSERT { + graph ?graph { + ?newTerm ?sp ?so . + ?ps ?newTerm ?po . + ?os ?op ?newTerm . + } +} +where { + values (?oldTerm ?newTerm) { + (gist:ContemporaneousEvent gist:ContemporaryEvent) + (gist:hasOrderedMember gist:hasMember) + (gist:baseConversionFactor gist:conversionFactor) + (gist:standardConversionFactor gist:conversionFactor) + } + graph ?graph { + { + ?oldTerm ?sp ?so + } UNION { + ?ps ?oldTerm ?po + } UNION { + ?os ?op ?oldTerm + } + } +} diff --git a/migration/v11.0/detect_domain_changed.rq b/migration/v11.0/detect_domain_changed.rq new file mode 100644 index 00000000..b37cf73e --- /dev/null +++ b/migration/v11.0/detect_domain_changed.rq @@ -0,0 +1,44 @@ +prefix skos: +prefix rdfs: +prefix sh: +PREFIX owl: +prefix gist: + +CONSTRUCT { + ?report a sh:ValidationReport ; + sh:conforms false ; + sh:result + [ + a sh:ValidationResult ; + sh:focusNode ?modified ; + sh:resultMessage ?error ; + sh:resultSeverity sh:Warning ; + sh:sourceConstraintComponent + ] . +} +WHERE { + { + select distinct ?modified where { + values ?modified { + gist:usesTimeZoneStandard + } + { + ?s ?modified ?o + } UNION { + ?s ?p ?modified + } UNION { + ?modified ?p ?o + } UNION { + graph ?g { ?s ?modified ?o } + } UNION { + graph ?g { ?s ?p ?modified } + } UNION { + graph ?g { ?modified ?p ?o } + } + } + } + bind( as ?report) + bind(CONCAT("WARNING: a rdfs:domain of property gist:", + REPLACE(STR(?modified), '^.*[/#]', ''), " was removed. If you were relying on ", + "inference from the removed domain you will need to address this change." ) as ?error) +} diff --git a/migration/v11.0/detect_domain_changed_default.rq b/migration/v11.0/detect_domain_changed_default.rq new file mode 100644 index 00000000..a99c3f00 --- /dev/null +++ b/migration/v11.0/detect_domain_changed_default.rq @@ -0,0 +1,44 @@ +prefix skos: +prefix rdfs: +prefix sh: +PREFIX owl: +prefix gist: + +CONSTRUCT { + ?report a sh:ValidationReport ; + sh:conforms false ; + sh:result + [ + a sh:ValidationResult ; + sh:focusNode ?modified ; + sh:resultMessage ?error ; + sh:resultSeverity sh:Warning ; + sh:sourceConstraintComponent + ] . +} +WHERE { + { + select distinct ?modified where { + values ?modified { + gist:usesTimeZoneStandard + } + { + ?s ?modified ?o + } UNION { + ?s ?p ?modified + } UNION { + ?modified ?p ?o +# } UNION { +# graph ?g { ?s ?modified ?o } +# } UNION { +# graph ?g { ?s ?p ?modified } +# } UNION { +# graph ?g { ?modified ?p ?o } + } + } + } + bind( as ?report) + bind(CONCAT("WARNING: a rdfs:domain of property gist:", + REPLACE(STR(?modified), '^.*[/#]', ''), " was removed. If you were relying on ", + "inference from the removed domain you will need to address this change." ) as ?error) +} diff --git a/migration/v11.0/detect_removed.rq b/migration/v11.0/detect_removed.rq new file mode 100644 index 00000000..95b474c9 --- /dev/null +++ b/migration/v11.0/detect_removed.rq @@ -0,0 +1,75 @@ +prefix skos: +prefix rdfs: +prefix sh: +PREFIX owl: +prefix gist: + +CONSTRUCT { + ?report a sh:ValidationReport ; + sh:conforms false ; + sh:result + [ + a sh:ValidationResult ; + sh:focusNode ?deleted ; + sh:resultMessage ?error ; + sh:resultSeverity sh:Warning ; + sh:sourceConstraintComponent + ] . +} +WHERE { + { + select distinct ?deleted where { + values ?deleted { + ## Time Model Classes + gist:TimeInstant + gist:DateInstant + gist:GreenwichInstant + gist:HumanInstant + gist:LocalInstant + gist:SystemInstant + + ## Time Model Properties + gist:hasEnd + gist:hasActual + gist:hasActualEnd + gist:hasPlannedEnd + gist:hasDeathDate + gist:hasStart + gist:hasActualStart + gist:hasPlanned + gist:hasBirthDate + gist:hasPlannedStart + gist:epoch + gist:localDate + gist:localDateTime + gist:localTime + gist:universalDate + gist:universalDateTime + gist:universalTime + gist:isSameTimeAs + gist:wasLastModifiedAt + + ## Misc + # gist:hasOrderedMember + gist:hasBiologicalOffspring + } + { + ?s ?deleted ?o + } UNION { + ?s ?p ?deleted + } UNION { + ?deleted ?p ?o + } UNION { + graph ?g { ?s ?deleted ?o } + } UNION { + graph ?g { ?s ?p ?deleted } + } UNION { + graph ?g { ?deleted ?p ?o } + } + } + } + bind( as ?report) + bind(CONCAT("Removed entity gist:", + REPLACE(STR(?deleted), '^.*[/#]', ''), " referenced in data. ", + "Review release notes for information about the changes.") as ?error) +} diff --git a/migration/v11.0/detect_removed_default.rq b/migration/v11.0/detect_removed_default.rq new file mode 100644 index 00000000..0062467a --- /dev/null +++ b/migration/v11.0/detect_removed_default.rq @@ -0,0 +1,75 @@ +prefix skos: +prefix rdfs: +prefix sh: +PREFIX owl: +prefix gist: + +CONSTRUCT { + ?report a sh:ValidationReport ; + sh:conforms false ; + sh:result + [ + a sh:ValidationResult ; + sh:focusNode ?deleted ; + sh:resultMessage ?error ; + sh:resultSeverity sh:Warning ; + sh:sourceConstraintComponent + ] . +} +WHERE { + { + select distinct ?deleted where { + values ?deleted { + ## Time Model Classes + gist:TimeInstant + gist:DateInstant + gist:GreenwichInstant + gist:HumanInstant + gist:LocalInstant + gist:SystemInstant + + ## Time Model Properties + gist:hasEnd + gist:hasActual + gist:hasActualEnd + gist:hasPlannedEnd + gist:hasDeathDate + gist:hasStart + gist:hasActualStart + gist:hasPlanned + gist:hasBirthDate + gist:hasPlannedStart + gist:epoch + gist:localDate + gist:localDateTime + gist:localTime + gist:universalDate + gist:universalDateTime + gist:universalTime + gist:isSameTimeAs + gist:wasLastModifiedAt + + ## Misc + # gist:hasOrderedMember + gist:hasBiologicalOffspring + } + { + ?s ?deleted ?o + } UNION { + ?s ?p ?deleted + } UNION { + ?deleted ?p ?o +# } UNION { +# graph ?g { ?s ?deleted ?o } +# } UNION { +# graph ?g { ?s ?p ?deleted } +# } UNION { +# graph ?g { ?deleted ?p ?o } + } + } + } + bind( as ?report) + bind(CONCAT("Removed entity gist:", + REPLACE(STR(?deleted), '^.*[/#]', ''), " referenced in data. ", + "Review release notes for information about the changes.") as ?error) +} diff --git a/migration/v11.0/detect_renamed.rq b/migration/v11.0/detect_renamed.rq new file mode 100644 index 00000000..2c0088d3 --- /dev/null +++ b/migration/v11.0/detect_renamed.rq @@ -0,0 +1,45 @@ +prefix skos: +prefix rdfs: +prefix sh: +PREFIX owl: +prefix gist: + +CONSTRUCT { + ?report a sh:ValidationReport ; + sh:conforms false ; + sh:result + [ + a sh:ValidationResult ; + sh:focusNode ?renamed ; + sh:resultMessage ?error ; + sh:resultSeverity sh:Warning ; + sh:sourceConstraintComponent + ] . +} +WHERE { + { + select distinct ?renamed where { + values ?renamed { + gist:hasJurisdictionOver + } + { + ?s ?renamed ?o + } UNION { + ?s ?p ?renamed + } UNION { + ?renamed ?p ?o + } UNION { + graph ?g { ?s ?renamed ?o } + } UNION { + graph ?g { ?s ?p ?renamed } + } UNION { + graph ?g { ?renamed ?p ?o } + } + } + } + bind( as ?report) + bind(CONCAT("WARNING: incorrectly named property gist:", + REPLACE(STR(?renamed), '^.*[/#]', ''), " was changed to gist:isUnderJurisdictionOf. ", + "The previous name did not match the defintion. We won't automatically update it ", + "because you might have applied it based on the name, not the definition." ) as ?error) +} diff --git a/migration/v11.0/detect_renamed_default.rq b/migration/v11.0/detect_renamed_default.rq new file mode 100644 index 00000000..6f68f24c --- /dev/null +++ b/migration/v11.0/detect_renamed_default.rq @@ -0,0 +1,45 @@ +prefix skos: +prefix rdfs: +prefix sh: +PREFIX owl: +prefix gist: + +CONSTRUCT { + ?report a sh:ValidationReport ; + sh:conforms false ; + sh:result + [ + a sh:ValidationResult ; + sh:focusNode ?renamed ; + sh:resultMessage ?error ; + sh:resultSeverity sh:Warning ; + sh:sourceConstraintComponent + ] . +} +WHERE { + { + select distinct ?renamed where { + values ?renamed { + gist:hasJurisdictionOver + } + { + ?s ?renamed ?o + } UNION { + ?s ?p ?renamed + } UNION { + ?renamed ?p ?o +# } UNION { +# graph ?g { ?s ?renamed ?o } +# } UNION { +# graph ?g { ?s ?p ?renamed } +# } UNION { +# graph ?g { ?renamed ?p ?o } + } + } + } + bind( as ?report) + bind(CONCAT("WARNING: incorrectly named property gist:", + REPLACE(STR(?renamed), '^.*[/#]', ''), " was changed to gist:isUnderJurisdictionOf. ", + "The previous name did not match the defintion. We won't automatically update it ", + "because you might have applied it based on the name, not the definition." ) as ?error) +} diff --git a/migration/v11.0/input/detect.ttl b/migration/v11.0/input/detect.ttl new file mode 100644 index 00000000..f332da4d --- /dev/null +++ b/migration/v11.0/input/detect.ttl @@ -0,0 +1,37 @@ +@prefix skos: . +@prefix rdfs: . +@prefix sh: . +@prefix owl: . +@prefix gist: . +@prefix rdf: . +@prefix ex: . +@prefix xsd: . + +gist:ContemporaneousEvent ex:_predicate ex:_object . +ex:_subject gist:ContemporaneousEvent ex:_object . +ex:_subject ex:_predicate gist:ContemporaneousEvent . + +gist:usesTimeZoneStandard ex:_predicate ex:_object . +ex:_subject gist:usesTimeZoneStandard ex:_object . +ex:_subject ex:_predicate gist:usesTimeZoneStandard . + +gist:hasActualStart ex:_predicate ex:_object . +ex:_subject gist:hasActualStart ex:_object . +ex:_subject ex:_predicate gist:hasActualStart . + +gist:hasJurisdictionOver ex:_predicate ex:_object . +ex:_subject gist:hasJurisdictionOver ex:_object . +ex:_subject ex:_predicate gist:hasJurisdictionOver . + +gist:hasOrderedMember ex:_predicate ex:_object . +ex:_subject gist:hasOrderedMember ex:_object . +ex:_subject ex:_predicate gist:hasOrderedMember . + +gist:baseConversionFactor ex:_predicate ex:_object . +ex:_subject gist:baseConversionFactor ex:_object . +ex:_subject ex:_predicate gist:baseConversionFactor . + +gist:standardConversionFactor ex:_predicate ex:_object . +ex:_subject gist:standardConversionFactor ex:_object . +ex:_subject ex:_predicate gist:standardConversionFactor . + diff --git a/migration/v11.0/input/rename.ttl b/migration/v11.0/input/rename.ttl new file mode 100644 index 00000000..731747fb --- /dev/null +++ b/migration/v11.0/input/rename.ttl @@ -0,0 +1,24 @@ +@prefix skos: . +@prefix rdfs: . +@prefix sh: . +@prefix owl: . +@prefix gist: . +@prefix rdf: . +@prefix ex: . +@prefix xsd: . + +gist:ContemporaneousEvent ex:_predicate_1 ex:_object_1 . +ex:_subject_1 gist:ContemporaneousEvent ex:_object_1 . +ex:_subject_1 ex:_predicate_1 gist:ContemporaneousEvent . + +gist:hasOrderedMember ex:_predicate_2 ex:_object_2 . +ex:_subject_2 gist:hasOrderedMember ex:_object_2 . +ex:_subject_2 ex:_predicate_2 gist:hasOrderedMember . + +gist:baseConversionFactor ex:_predicate_3 ex:_object_3 . +ex:_subject_3 gist:baseConversionFactor ex:_object_3 . +ex:_subject_3 ex:_predicate_3 gist:baseConversionFactor . + +gist:standardConversionFactor ex:_predicate_4 ex:_object_4 . +ex:_subject_4 gist:standardConversionFactor ex:_object_4 . +ex:_subject_4 ex:_predicate_4 gist:standardConversionFactor . diff --git a/migration/v11.0/migrate_endpoint.yaml b/migration/v11.0/migrate_endpoint.yaml new file mode 100644 index 00000000..c7e5db3e --- /dev/null +++ b/migration/v11.0/migrate_endpoint.yaml @@ -0,0 +1,41 @@ +bundle: gist-v11.0-migration +variables: + file_path: "." + endpoint: "" + update_endpoint: "{endpoint}" + user: "UPDATE_USER" + password: "UPDATE_PASSWORD" + report: "./reports" +actions: +- action: "sparql" + message: "Migrate renamed terms in named graphs." + query: "{file_path}/action_rename_ngraphs.rq" + endpoint: + query_uri: "{endpoint}" + update_uri: "{update_endpoint}" + user: "{user}" + password: "{password}" +- action: "sparql" + message: "Migrate renamed terms in default graph." + query: "{file_path}/action_rename_default.rq" + endpoint: + query_uri: "{endpoint}" + update_uri: "{update_endpoint}" + user: "{user}" + password: "{password}" +- action: "verify" + message: "Check for other items that should be reviewed." + type: "construct" + inference: "none" + stopOnFail: false + target: '{report}' + endpoint: + query_uri: "{endpoint}" + user: "{user}" + password: "{password}" + queries: + source: '{file_path}/' + includes: + - 'detect_domain_changed.rq' + - 'detect_removed.rq' + - 'detect_renamed.rq' diff --git a/migration/v11.0/migrate_local.yaml b/migration/v11.0/migrate_local.yaml new file mode 100644 index 00000000..cc17a443 --- /dev/null +++ b/migration/v11.0/migrate_local.yaml @@ -0,0 +1,41 @@ +bundle: gist-v11.0-migration +variables: + file_path: "." + input: "./input" + output: "./output" + report: "./reports" +tools: +- name: "rename-terms" + type: "sparql" + query: "{file_path}/action_rename_default.rq" +actions: +- action: "transform" + message: "Migrate renamed terms." + tool: "rename-terms" + source: "{input}" + target: "{output}" + includes: + - "*.ttl" + - "*.owl" +#- action: "copy" +# message: "Make a copy of source files." +# source: "{input}" +# target: "{output}" +# - "*.ttl" +# - "*.owl" +- action: "verify" + message: "Check for issues that should be reviewed." + type: "construct" + inference: "none" + stopOnFail: false + source: "{output}" + target: "{report}" + includes: + - "*.ttl" + - "*.owl" + queries: + source: '{file_path}/' + includes: + - 'detect_domain_changed_default.rq' + - 'detect_removed_default.rq' + - 'detect_renamed_default.rq'