Merge pull request #180 from microbiomedata/176-review-srcdata-paths-…

…and-remove-everything-besides-valid-and-invalid-if-possible remove everything from src/data besides valid and invalid
microbiomedata · Oct 28, 2024 · 7f0b83d · 7f0b83d
2 parents 93e452b + f8d8562
commit 7f0b83d
Show file tree

Hide file tree

Showing 24 changed files with 26 additions and 250 deletions.
diff --git a/examples/README.md b/examples/README.md
@@ -2,4 +2,4 @@
 
 This folder contains example data conforming to nmdc_submission_schema
 
-The source for these is in [src/data](../src/data/examples)
+The source for these is in [src/data](../src/data/)
diff --git a/project.Makefile b/project.Makefile
@@ -84,7 +84,7 @@ local/with_shuttles_yq.yaml: local/with_shuttles.yaml
 	yq -i '(.classes.[].slot_usage.[] | select(.name=="chem_administration") | .examples) = [{"value": "agar [CHEBI:2509];2018-05-11|agar [CHEBI:2509];2018-05-22"}, {"value": "agar [CHEBI:2509];2018-05"}]' $@
 
 # use yq to add patterns with a secondary condition like mutivalued
-	yq -i '(.classes.[].slot_usage.[] | select(.range == "GeolocationValue")  | .pattern) = "^[-+]?([1-8]?\d(\.\d+)?|90(\.0+)?)\s[-+]?(180(\.0+)?|((1[0-7]\d)|([1-9]?\d))(\.\d+)?)$$"' $@
+	yq -i '(.classes.[].slot_usage.[] | select(.range == "GeolocationValue")  | .pattern) = "^[-+]?([1-8]?\d(\.\d{1,8})?|90(\.0{1,8})?)\s[-+]?(180(\.0{1,8})?|((1[0-7]\d)|([1-9]?\d))(\.\d{1,8})?)$$"' $@
 	yq -i '(.classes.[].slot_usage.[] | select(.range == "GeolocationValue")  | .range) = "string"' $@
 
 	yq -i '(.classes.[].slot_usage.[] | select(.range == "QuantityValue") | .pattern) = "^[-+]?[0-9]*\.?[0-9]+ +\S.*$$"' $@
@@ -181,6 +181,8 @@ local/nmdc.yaml
 
 src/nmdc_submission_schema/schema/nmdc_submission_schema.yaml: local/with_modifications.yaml project/thirdparty/GoldEcosystemTree.json
 	$(RUN) inject-gold-pathway-terms -g $(word 2,$^) -i $< -o $@
+	#cp $< $@
+
 # remove the multivalued true annotation from these gloabl slot definitions for the sake of linkml-convert
 # follow the .string_serialization=="{text};{float} {unit}" and .multivalued == true pattern?
 
@@ -356,17 +358,18 @@ local/abp_pvs.txt: local/abp_tree_down.txt
 # human construction
 # astronomical body part
 
-src/data/data_harmonizer_io/soil_for_linkml.json: src/data/data_harmonizer_io/soil_from_dh.json
-	$(RUN) dh-json2linkml \
-		--input-file $< \
-		--output-file $@ \
-		--key soil_data
-
+#src/data/data_harmonizer_io/soil_for_linkml.json: src/data/data_harmonizer_io/soil_from_dh.json
+#	$(RUN) dh-json2linkml \
+#		--input-file $< \
+#		--output-file $@ \
+#		--key soil_data
 
-src/data/data_harmonizer_io/soil_data.json: src/data/data_harmonizer_io/soil_for_linkml.json
-	$(RUN) linkml-json2dh \
-		--input-file $< \
-		--output-dir $(dir $@)
+## todo frozen content in src/data/data_harmonizer_io has been removed
+## todo find a better home for the se scripts if they are still of any use
+#src/data/data_harmonizer_io/soil_data.json: src/data/data_harmonizer_io/soil_for_linkml.json
+#	$(RUN) linkml-json2dh \
+#		--input-file $< \
+#		--output-dir $(dir $@)
 
 local/usage_template.tsv: src/nmdc_submission_schema/schema/nmdc_submission_schema.yaml
 	mkdir -p $(@D)

diff --git a/sheets_and_friends/tsv_in/modifications_long.tsv b/sheets_and_friends/tsv_in/modifications_long.tsv
@@ -66,6 +66,7 @@ JgiMgInterface|JgiMgLrInterface	dna_sample_format	replace_attribute	required			t
 JgiMgInterface|JgiMgLrInterface	dna_sample_format	replace_attribute	recommended			false
 JgiMgInterface|JgiMgLrInterface	dna_sample_name	replace_attribute	required			true
 JgiMgInterface|JgiMgLrInterface	dna_sample_name	replace_attribute	recommended			false
+JgiMgInterface|JgiMgLrInterface	dna_sample_name	replace_attribute	pattern			^[_a-zA-Z0-9-]*$
 JgiMgInterface|JgiMgLrInterface	dna_seq_project	replace_attribute	required			true
 JgiMgInterface|JgiMgLrInterface	dna_seq_project	replace_attribute	recommended			false
 JgiMgInterface|JgiMgLrInterface	dna_seq_project_name	replace_attribute	required			true
@@ -191,6 +192,7 @@ JgiMtInterface	rna_sample_format	replace_attribute	required			true
 JgiMtInterface	rna_sample_format	replace_attribute	recommended			false
 JgiMtInterface	rna_sample_name	replace_attribute	required			true
 JgiMtInterface	rna_sample_name	replace_attribute	recommended			false
+JgiMtInterface	rna_sample_name	replace_attribute	pattern			^[_a-zA-Z0-9-]*$
 JgiMtInterface	rna_seq_project	replace_attribute	required			true
 JgiMtInterface	rna_seq_project	replace_attribute	recommended			false
 JgiMtInterface	rna_seq_project_name	replace_attribute	required			true

diff --git a/sheets_and_friends/tsv_in/validation_converter.tsv b/sheets_and_friends/tsv_in/validation_converter.tsv
@@ -7,7 +7,7 @@ from_val	from_type	to_type	to_val	len
 {float}-{float}	MIxS string serialization	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\s*-\s*[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$	
 {float}|{float}-{float}	MIxS string serialization	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?(\s*-\s*[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)?$	
 {integer}	MIxS string serialization	DH datatype	integer	7
-{lat lon}	linkml string_serialization	DH pattern regex	^[-+]?([1-8]?\d(\.\d+)?|90(\.0+)?)\s[-+]?(180(\.0+)?|((1[0-7]\d)|([1-9]?\d))(\.\d+)?)$	86
+{lat lon}	linkml string_serialization	DH pattern regex	^[-+]?([1-8]?\d(\.\d{1,8})?|90(\.0{1,8})?)\s[-+]?(180(\.0{1,8})?|((1[0-7]\d)|([1-9]?\d))(\.\d{1,8})?)$	86
 {termLabel} {[termID]}	MIxS string serialization	DH pattern regex	^\S+.*\S+ \[[a-zA-Z]{2,}:\d+\]$	
 {termLabel} {[termID]}; {timestamp}	MIxS string serialization	DH pattern regex	^\S+.*\S+ \[[a-zA-Z]{2,}:\d+\]; ([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$	314
 {termLabel} {[termID]};{timestamp}	MIxS string serialization	DH pattern regex	^\S+.*\S+ \[[a-zA-Z]{2,}:\d+\];([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$	313
@@ -24,7 +24,7 @@ ControlledIdentifiedTermValue	linkml range	DH datatype	string
 ControlledTermValue	linkml range	DH datatype	string	
 ControlledTermValue	linkml range	DH pattern regex	^\S+.*\S+ \[[a-zA-Z]{2,}:\d+\]$	
 GeolocationValue	linkml range	DH datatype	string	
-GeolocationValue	linkml range	DH pattern regex	^[-+]?([1-8]?\d(\.\d+)?|90(\.0+)?)\s[-+]?(180(\.0+)?|((1[0-7]\d)|([1-9]?\d))(\.\d+)?)$	
+GeolocationValue	linkml range	DH pattern regex	^[-+]?([1-8]?\d(\.\d{1,8})?|90(\.0{1,8})?)\s[-+]?(180(\.0{1,8})?|((1[0-7]\d)|([1-9]?\d))(\.\d{1,8})?)$	
 OntologyClass	linkml range	DH datatype	string	
 QuantityValue	linkml range	DH datatype	string	
 TextValue	linkml range	DH datatype	string	

diff --git a/src/data/data_harmonizer_io/soil_from_dh.json b/src/data/data_harmonizer_io/soil_from_dh.json
diff --git a/src/data/data_harmonizer_io/soil_from_dh.tsv b/src/data/data_harmonizer_io/soil_from_dh.tsv
diff --git a/src/data/data_harmonizer_io/soil_from_dh.xlsx b/src/data/data_harmonizer_io/soil_from_dh.xlsx
diff --git a/src/data/data_harmonizer_io/soil_validatable.json b/src/data/data_harmonizer_io/soil_validatable.json
diff --git a/src/data/examples/SampleData-water-data-exhaustive.yaml b/src/data/examples/SampleData-water-data-exhaustive.yaml
diff --git a/...ta-jgi_mg_data-colon-dna_sample_name.yaml → ...ta-jgi_mg_data-colon-dna_sample_name.yaml b/...ta-jgi_mg_data-colon-dna_sample_name.yaml → ...ta-jgi_mg_data-colon-dna_sample_name.yaml
@@ -11,7 +11,7 @@ jgi_mg_data:
     dna_project_contact: xxx
     dna_samp_id: xxx
     dna_sample_format: DNAStable
-    dna_sample_name: DNA:0546789 # contains colon and should not be allowed 'a-z, A-Z, 0-9, - and _ only'
+    dna_sample_name: DNA:0546789 # description says "Sample names must ... contain a-z, A-Z, 0-9, - and _ only." but there's no patten constraint
     dna_seq_project: xxx
     dna_seq_project_name: xxx
     dna_seq_project_pi: xxx

diff --git a/...ta-jgi_mg_data-in-plate-invalid-well.yaml → ...ta-jgi_mg_data-in-plate-invalid-well.yaml b/...ta-jgi_mg_data-in-plate-invalid-well.yaml → ...ta-jgi_mg_data-in-plate-invalid-well.yaml
diff --git a/...ta-jgi_mg_data-in-plate-missing-well.yaml → ...ta-jgi_mg_data-in-plate-missing-well.yaml b/...ta-jgi_mg_data-in-plate-missing-well.yaml → ...ta-jgi_mg_data-in-plate-missing-well.yaml
diff --git a/...leData-jgi_mg_data-in-tube-with-well.yaml → ...leData-jgi_mg_data-in-tube-with-well.yaml b/...leData-jgi_mg_data-in-tube-with-well.yaml → ...leData-jgi_mg_data-in-tube-with-well.yaml
diff --git a/...gi_mt_data-in-plate-invalid-well-val.yaml → ...gi_mt_data-in-plate-invalid-well-val.yaml b/...gi_mt_data-in-plate-invalid-well-val.yaml → ...gi_mt_data-in-plate-invalid-well-val.yaml
diff --git a/...gi_mt_data-in-plate-missing-well-val.yaml → ...gi_mt_data-in-plate-missing-well-val.yaml b/...gi_mt_data-in-plate-missing-well-val.yaml → ...gi_mt_data-in-plate-missing-well-val.yaml
diff --git a/...ta-jgi_mt_data-in-tube-with-well-val.yaml → ...ta-jgi_mt_data-in-tube-with-well-val.yaml b/...ta-jgi_mt_data-in-tube-with-well-val.yaml → ...ta-jgi_mt_data-in-tube-with-well-val.yaml
diff --git a/...tion-rna_cont_well-and-rna_cont_type.yaml → ...tion-rna_cont_well-and-rna_cont_type.yaml b/...tion-rna_cont_well-and-rna_cont_type.yaml → ...tion-rna_cont_well-and-rna_cont_type.yaml
diff --git a/...i_mt_data-too-large-value-rna_volume.yaml → ...i_mt_data-too-large-value-rna_volume.yaml b/...i_mt_data-too-large-value-rna_volume.yaml → ...i_mt_data-too-large-value-rna_volume.yaml
diff --git a/...eData-soil-data-long-decimal-lat_lon.yaml → ...eData-soil-data-long-decimal-lat_lon.yaml b/...eData-soil-data-long-decimal-lat_lon.yaml → ...eData-soil-data-long-decimal-lat_lon.yaml
diff --git a/src/data/unexpected_pass/SampleData-jgi_mt_data-illegal-rna_seq_project_name.yaml b/src/data/unexpected_pass/SampleData-jgi_mt_data-illegal-rna_seq_project_name.yaml
diff --git a/src/data/unexpected_pass/SampleData-jgi_mt_data-illegal-rna_seq_project_pi.yaml b/src/data/unexpected_pass/SampleData-jgi_mt_data-illegal-rna_seq_project_pi.yaml
diff --git a/src/data/unexpected_pass/SampleData-jgi_mt_data-too-small-value-rna_volume.yaml b/src/data/unexpected_pass/SampleData-jgi_mt_data-too-small-value-rna_volume.yaml
diff --git a/src/data/unexpected_pass/SampleData-soil-data-illegal-env_package.yaml b/src/data/unexpected_pass/SampleData-soil-data-illegal-env_package.yaml
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,4 @@

		This folder contains example data conforming to nmdc_submission_schema

		The source for these is in [src/data](../src/data/examples)
		The source for these is in [src/data](../src/data/)