Merge pull request #32 from monarch-initiative/exclusion-terms

Feature: Exclusion tables

docs/odk-workflows/RepoManagement.md

@@ -11,6 +11,12 @@ sh run.sh make update_repo
 
 There are a large number of options that can be set to configure your ODK, but we will only discuss a few of them here.
 
+NOTE for Windows users:
+
+You may get a cryptic failure such as `Set Illegal Option -` if the update script located in `src/scripts/update_repo.sh` 
+was saved using Windows Line endings. These need to change to unix line endings. In Notepad++, for example, you can 
+click on Edit->EOL Conversion->Unix LF to change this.
+
 ## Managing imports
 
 You can use the update repository worflow described on this page to perform the following operations to your imports:

docs/odk-workflows/RepositoryFileStructure.md

@@ -31,7 +31,7 @@ These are the components in MONDO-INGEST
 | Filename | URL |
 | -------- | --- |
 | omim.owl | https://github.com/monarch-initiative/omim/releases/latest/download/omim.ttl |
-| ordo.owl | http://www.orphadata.org/data/ORDO/ORDO_en_4.0.owl |
+| ordo.owl | http://www.orphadata.org/data/ORDO/ORDO_en_4.1.owl |
 | ncit.owl | http://purl.obolibrary.org/obo/ncit.owl |
 | doid.owl | http://purl.obolibrary.org/obo/doid.owl |
 | icd10cm.owl | https://data.bioontology.org/ontologies/ICD10CM/submissions/21/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb |

requirements-unlocked.txt

@@ -1,3 +1,5 @@
 jinja2
+oaklib
 pandas
+pyyaml
 sssom

requirements.txt

@@ -1,15 +1,32 @@
+aiohttp==3.8.1
+aiosignal==1.2.0
 alabaster==0.7.12
+antlr4-python3-runtime==4.9.3
+appdirs==1.4.4
+async-timeout==4.0.2
 attrs==21.4.0
 Babel==2.10.1
+bcp47==0.0.4
+bioregistry==0.5.29
+bleach==5.0.0
 certifi==2021.10.8
+CFGraph==0.2.1
+chardet==4.0.0
 charset-normalizer==2.0.12
-click==8.1.2
+click==8.1.3
 commonmark==0.9.1
 decorator==5.1.1
 Deprecated==1.2.13
+deprecation==2.1.0
 distlib==0.3.4
 docutils==0.17.1
+et-xmlfile==1.1.0
+fastobo==0.11.1
 filelock==3.6.0
+frozenlist==1.3.0
+fsspec==2022.5.0
+funowl==0.1.12
+graphviz==0.20
 greenlet==1.1.2
 hbreader==0.9.1
 idna==3.3
@@ -19,47 +36,96 @@ isodate==0.6.1
 Jinja2==3.1.1
 joblib==1.1.0
 json-flattener==0.1.9
+jsonasobj==1.3.1
 jsonasobj2==1.0.4
+jsonpatch==1.32
+jsonpath-ng==1.5.3
+jsonpointer==2.3
 jsonschema==4.4.0
-linkml-runtime==1.2.7
+keyring==23.6.0
+kgcl==0.1.0
+kgcl-schema==0.1.2
+lark==1.1.2
+linkml==1.2.14
+linkml-dataops==0.1.0
+linkml-runtime==1.2.16
+markdown-it-py==2.1.0
 MarkupSafe==2.1.1
+mdit-py-plugins==0.3.0
+mdurl==0.1.1
+more-click==0.1.1
+multidict==6.0.2
+myst-parser==0.18.0
 networkx==2.8
 numpy==1.22.3
+nxontology==0.4.1
+oaklib==0.1.23
+openpyxl==3.0.10
 packaging==21.3
 pandas==1.4.2
 pandasql==0.7.3
+parse==1.19.0
 pbr==5.8.1
+pkginfo==1.8.3
 platformdirs==2.5.2
+ply==3.11
 prefixcommons==0.1.9
+pronto==2.4.6
+pydantic==1.9.1
 Pygments==2.12.0
+PyJSG==0.11.10
 pyparsing==2.4.7
 pyrsistent==0.18.1
+PyShEx==0.8.1
+PyShExC==0.9.1
+pystow==0.4.4
 python-dateutil==2.8.2
 pytz==2022.1
 PyYAML==6.0
+ratelimit==2.2.1
 rdflib==6.1.1
+rdflib-jsonld==0.6.1
+rdflib-shim==1.0.3
+readme-renderer==35.0
 recommonmark==0.7.1
 requests==2.27.1
+requests-toolbelt==0.9.1
+rfc3986==2.0.0
+rfc3987==1.3.8
+rich==12.4.4
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.6
 scikit-learn==1.0.2
 scipy==1.8.0
+semsql==0.1.7
+ShExJSG==0.8.2
 six==1.16.0
 snowballstemmer==2.2.0
+sparqlslurper==0.5.1
 SPARQLWrapper==2.0.0
 Sphinx==4.5.0
+sphinx-click==4.2.0
 sphinxcontrib-applehelp==1.0.2
 sphinxcontrib-devhelp==1.0.2
 sphinxcontrib-htmlhelp==2.0.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.5
 SQLAlchemy==1.4.36
-sssom==0.3.8
+SQLAlchemy-Utils==0.38.2
+sssom==0.3.11
 stevedore==3.5.0
 threadpoolctl==3.1.0
+tqdm==4.64.0
+twine==4.0.1
+typing_extensions==4.2.0
 urllib3==1.26.9
 validators==0.18.2
 virtualenv==20.14.1
 virtualenv-clone==0.5.7
 virtualenvwrapper==4.8.4
+watchdog==2.1.9
+webencodings==0.5.1
 wrapt==1.14.0
+yarl==1.7.2
 zipp==3.8.0

src/ontology/Makefile

@@ -68,11 +68,14 @@ all_odk: odkversion test all_assets
 
 .PHONY: test
 test: odkversion sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt
-	$(ROBOT) reason --input $(SRC) --reasoner ELK  --equivalent-classes-allowed asserted-only --exclude-tautologies structural --output test.owl && rm test.owl && echo "Success"
+	$(ROBOT) reason --input $(SRC) --reasoner ELK  --equivalent-classes-allowed asserted-only \
+                  --exclude-tautologies structural --output test.owl && rm test.owl &&\
+  echo "Success"
 
 .PHONY: odkversion
 odkversion:
-	echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, not the version of the ODK you are running)" &&\
+	echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, \
+        not the version of the ODK you are running)" &&\
 	echo "ROBOT version (ODK): " && $(ROBOT) --version
 
 $(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR):
@@ -164,10 +167,10 @@ endif
 # ----------------------------------------
 
 $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR)
-	$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON)  --print 5 -o $@
+	$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@
 
 $(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR)
-	$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON)  --print 5 -o $@
+	$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@
 
 # ----------------------------------------
 # Release assets
@@ -203,7 +206,8 @@ CLEANFILES=$(MAIN_FILES) $(SRCMERGED)
 prepare_release: all_odk
 	rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\
 	rm -f $(CLEANFILES) &&\
-	echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab"
+	echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release \
+        on your git hosting site such as GitHub or GitLab"
 
 .PHONY: prepare_initial_release
 prepare_initial_release: all_assets
@@ -327,10 +331,9 @@ $(COMPONENTSDIR)/omim.owl:  component-download-omim.owl
 .PRECIOUS: $(COMPONENTSDIR)/omim.owl
 
 
-
 .PHONY: component-download-ordo.owl
 component-download-ordo.owl: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I http://www.orphadata.org/data/ORDO/ORDO_en_4.0.owl \
+	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I http://www.orphadata.org/data/ORDO/ORDO_en_4.1.owl \
 	annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $(TMPDIR)/$@.owl; fi
 
 $(COMPONENTSDIR)/ordo.owl:  component-download-ordo.owl
@@ -339,7 +342,6 @@ $(COMPONENTSDIR)/ordo.owl:  component-download-ordo.owl
 .PRECIOUS: $(COMPONENTSDIR)/ordo.owl
 
 
-
 .PHONY: component-download-ncit.owl
 component-download-ncit.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I http://purl.obolibrary.org/obo/ncit.owl \
@@ -351,7 +353,6 @@ $(COMPONENTSDIR)/ncit.owl:  component-download-ncit.owl
 .PRECIOUS: $(COMPONENTSDIR)/ncit.owl
 
 
-
 .PHONY: component-download-doid.owl
 component-download-doid.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I http://purl.obolibrary.org/obo/doid.owl \
@@ -363,7 +364,6 @@ $(COMPONENTSDIR)/doid.owl:  component-download-doid.owl
 .PRECIOUS: $(COMPONENTSDIR)/doid.owl
 
 
-
 .PHONY: component-download-icd10cm.owl
 component-download-icd10cm.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I https://data.bioontology.org/ontologies/ICD10CM/submissions/21/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb \
@@ -375,7 +375,6 @@ $(COMPONENTSDIR)/icd10cm.owl:  component-download-icd10cm.owl
 .PRECIOUS: $(COMPONENTSDIR)/icd10cm.owl
 
 
-
 .PHONY: component-download-icd10who.owl
 component-download-icd10who.owl: | $(TMPDIR)
 	if [ $(MIR) = true ] && [ $(COMP) = true ]; then $(ROBOT) merge -I https://github.com/monarch-initiative/icd10who/releases/download/latest/icd10who.ttl \
@@ -385,9 +384,6 @@ $(COMPONENTSDIR)/icd10who.owl:  component-download-icd10who.owl
 	if [ $(COMP) = true ] ; then if cmp -s $(TMPDIR)/component-download-icd10who.owl.owl $@ ; then echo "Component identical."; else echo "Component is different, updating." && cp $(TMPDIR)/component-download-icd10who.owl.owl $@; fi; fi
 
 .PRECIOUS: $(COMPONENTSDIR)/icd10who.owl
-
-
-
 # ----------------------------------------
 # Mirroring upstream ontologies
 # ----------------------------------------
@@ -402,18 +398,23 @@ IMP_LARGE=true # Global parameter to bypass handling of large imports
 .PHONY: mirror-ro
 .PRECIOUS: $(MIRRORDIR)/ro.owl
 mirror-ro: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
+	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 &&\
+		$(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl &&\
+		mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
 
 
 ## ONTOLOGY: omo
 .PHONY: mirror-omo
 .PRECIOUS: $(MIRRORDIR)/omo.owl
 mirror-omo: | $(TMPDIR)
-	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/omo.owl --create-dirs -o $(MIRRORDIR)/omo.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/omo.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
+	if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/omo.owl --create-dirs -o $(MIRRORDIR)/omo.owl --retry 4 --max-time 200 &&\
+		$(ROBOT) convert -i $(MIRRORDIR)/omo.owl -o $@.tmp.owl &&\
+		mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
 
 
 $(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
-	if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." && cp $(TMPDIR)/mirror-$*.owl $@; fi; fi
+	if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
+		cp $(TMPDIR)/mirror-$*.owl $@; fi; fi
 
 
 
@@ -469,8 +470,6 @@ $(ONT)-full.json: $(ONT)-full.owl
 	$(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
 		convert --check false -f json -o $@.tmp.json &&\
 	jq -S 'walk(if type == "array" then sort else . end)' $@.tmp.json > $@ && rm $@.tmp.json
-# We always want a base - even if it is not explicitly configured..
-# We always want a full release - even if it is not explicitly configured..
 # ----------------------------------------
 # Release artefacts: main release artefacts
 # ----------------------------------------
@@ -494,15 +493,6 @@ $(ONTOLOGYTERMS): $(SRC) $(OTHER_SRC)
 	touch $(ONTOLOGYTERMS) && \
 	$(ROBOT) query --use-graphs true -f csv -i $< --query ../sparql/mondo-ingest_terms.sparql $@
 
-
-
-
-
-
-
-
-
-
 # base: OTHER sources of interest, such as definitions owl
 $(ONT)-base.owl: $(SRC) $(OTHER_SRC)
 	$(ROBOT) remove --input $< --select imports --trim false \