From bc84d491e3266cbc1baa9fe466331814a0e186c2 Mon Sep 17 00:00:00 2001 From: dachafra Date: Tue, 29 Oct 2024 11:44:44 +0100 Subject: [PATCH] adding translation for in-memory RML --- VERSION | 2 +- src/yatter/__init__.py | 6 +- src/yatter/constants.py | 18 ++++- src/yatter/predicateobject.py | 2 +- src/yatter/source.py | 63 ++++++++++++++++- test/rml-in-memory/IMTC001/mapping.ttl | 46 +++++++++++++ .../IMTC001/{mappings.yml => mapping.yml} | 10 +-- test/rml-in-memory/IMTC001/mappings.rml.ttl | 43 ------------ test/rml-in-memory/IMTC001/test_imtc001.py | 26 +++++++ .../mappings.rml.ttl => IMTC002/mapping.ttl} | 7 +- .../IMTC002/{mappings.yml => mapping.yml} | 6 +- test/rml-in-memory/IMTC002/test_imtc002.py | 26 +++++++ .../mappings.rml.ttl => IMTC003/mapping.ttl} | 7 +- .../IMTC003/{mappings.yml => mapping.yml} | 2 +- test/rml-in-memory/IMTC003/test_imtc003.py | 26 +++++++ .../IMTC004/{mappings.rml.ttl => mapping.ttl} | 7 +- .../IMTC004/{mappings.yml => mapping.yml} | 8 +-- test/rml-in-memory/IMTC004/test_imtc004.py | 26 +++++++ .../IMTC005/{mappings.rml.ttl => mapping.ttl} | 7 +- .../IMTC005/{mappings.yml => mapping.yml} | 2 +- test/rml-in-memory/IMTC005/test_imtc005.py | 26 +++++++ .../IMTC006/{mappings.rml.ttl => mapping.ttl} | 63 ++++++++++++++--- .../IMTC006/{mappings.yml => mapping.yml} | 6 +- test/rml-in-memory/IMTC006/test_imtc006.py | 26 +++++++ .../IMTC007/{mappings.rml.ttl => mapping.ttl} | 68 +++++++++++++++---- .../IMTC007/{mappings.yml => mapping.yml} | 4 +- test/rml-in-memory/IMTC007/test_imtc007.py | 26 +++++++ .../IMTC008/{mappings.rml.ttl => mapping.ttl} | 66 ++++++++++++++---- .../IMTC008/{mappings.yml => mapping.yml} | 6 +- test/rml-in-memory/IMTC008/test_imtc008.py | 26 +++++++ 30 files changed, 538 insertions(+), 119 deletions(-) create mode 100644 test/rml-in-memory/IMTC001/mapping.ttl rename test/rml-in-memory/IMTC001/{mappings.yml => mapping.yml} (73%) delete mode 100644 test/rml-in-memory/IMTC001/mappings.rml.ttl create mode 100644 test/rml-in-memory/IMTC001/test_imtc001.py rename test/rml-in-memory/{IMTC003/mappings.rml.ttl => IMTC002/mapping.ttl} (85%) rename test/rml-in-memory/IMTC002/{mappings.yml => mapping.yml} (83%) create mode 100644 test/rml-in-memory/IMTC002/test_imtc002.py rename test/rml-in-memory/{IMTC002/mappings.rml.ttl => IMTC003/mapping.ttl} (85%) rename test/rml-in-memory/IMTC003/{mappings.yml => mapping.yml} (91%) create mode 100644 test/rml-in-memory/IMTC003/test_imtc003.py rename test/rml-in-memory/IMTC004/{mappings.rml.ttl => mapping.ttl} (86%) rename test/rml-in-memory/IMTC004/{mappings.yml => mapping.yml} (79%) create mode 100644 test/rml-in-memory/IMTC004/test_imtc004.py rename test/rml-in-memory/IMTC005/{mappings.rml.ttl => mapping.ttl} (85%) rename test/rml-in-memory/IMTC005/{mappings.yml => mapping.yml} (92%) create mode 100644 test/rml-in-memory/IMTC005/test_imtc005.py rename test/rml-in-memory/IMTC006/{mappings.rml.ttl => mapping.ttl} (61%) rename test/rml-in-memory/IMTC006/{mappings.yml => mapping.yml} (86%) create mode 100644 test/rml-in-memory/IMTC006/test_imtc006.py rename test/rml-in-memory/IMTC007/{mappings.rml.ttl => mapping.ttl} (60%) rename test/rml-in-memory/IMTC007/{mappings.yml => mapping.yml} (89%) create mode 100644 test/rml-in-memory/IMTC007/test_imtc007.py rename test/rml-in-memory/IMTC008/{mappings.rml.ttl => mapping.ttl} (58%) rename test/rml-in-memory/IMTC008/{mappings.yml => mapping.yml} (80%) create mode 100644 test/rml-in-memory/IMTC008/test_imtc008.py diff --git a/VERSION b/VERSION index 65087b4..5625e59 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.4 +1.2 diff --git a/src/yatter/__init__.py b/src/yatter/__init__.py index f586a89..e32272d 100644 --- a/src/yatter/__init__.py +++ b/src/yatter/__init__.py @@ -17,13 +17,14 @@ def translate(yarrrml_data, mapping_format=RML_URI): rml_mapping.extend(generate_database_connections(yarrrml_data, list_initial_sources)) rml_mapping.extend(add_logical_targets(yarrrml_data)) rml_mapping.extend(add_functions(yarrrml_data)) + external_refs = [] try: mappings, mapping_format = get_non_asserted_mappings(yarrrml_data, mapping_format) for mapping in yarrrml_data.get(YARRRML_MAPPINGS): if mapping_format == R2RML_URI: source_list = add_table(yarrrml_data, mapping, list_initial_sources) else: - source_list = add_source(yarrrml_data, mapping, list_initial_sources) + source_list, external_refs = add_source(yarrrml_data, mapping, list_initial_sources) subject_list = add_subject(yarrrml_data, mapping, mapping_format) pred = add_predicate_object_maps(yarrrml_data, mapping, mapping_format) it = 0 @@ -37,6 +38,9 @@ def translate(yarrrml_data, mapping_format=RML_URI): rml_mapping[len(rml_mapping) - 1] = rml_mapping[len(rml_mapping) - 1][:-2] rml_mapping.append(".\n\n\n") it = it + 1 + external_refs = list(dict.fromkeys(external_refs)) + for ref in external_refs: + rml_mapping.append(ref) logger.info("RML content is created!") rml_mapping_string = "".join(rml_mapping) diff --git a/src/yatter/constants.py b/src/yatter/constants.py index c5aad0a..6f363dd 100644 --- a/src/yatter/constants.py +++ b/src/yatter/constants.py @@ -21,6 +21,7 @@ VOID_URI = 'http://rdfs.org/ns/void#' FNML_URI = 'http://semweb.mmlab.be/ns/fnml#' GREL_URI = 'http://users.ugent.be/~bjdmeest/function/grel.ttl#' +SD_URI = 'https://w3id.org/okn/o/sd/' RML_PREFIX = '@prefix' RML_BASE = '@base' @@ -28,6 +29,7 @@ RML_LOGICAL_SOURCE = 'rml:logicalSource' RML_SOURCE = 'rml:source' RML_REFERENCE_FORMULATION = 'rml:referenceFormulation' +RML_REFERENCE_FORMULATION_CLASS = 'rml:ReferenceFormulation' RML_ITERATOR = 'rml:iterator' RML_REFERENCE = 'rml:reference' RML_LANGUAGE_MAP = 'rml:languageMap' @@ -103,7 +105,16 @@ D2RQ_USER = 'd2rq:username' D2RQ_PASS = 'd2rq:password' - +############################################################################## +############################# SD CONSTANTS ########################### +############################################################################## +SD_DATASET_SPEC = 'sd:DatasetSpecification' +SD_NAME = 'sd:name' +SD_HAS_DATA_TRANSFORMATION = 'sd:hasDataTransformation' +SD_HAS_SOFTWARE_REQUIREMENTS = 'sd:hasSoftwareRequirements' +SD_HAS_SOURCE_CODE= 'sd:hasSourceCode' +SD_PROGRAMMING_LANGUAGE = 'sd:programmingLanguage' +KG4DI_DEFINED_BY = 'kg4di:definedBy' ############################################################################## ############################# YARRRML CONSTANTS ########################### ############################################################################## @@ -122,6 +133,11 @@ YARRRML_USERNAME = 'username' YARRRML_PASSWORD = 'password' +YARRRML_STRUCTURE_DEFINER = 'structureDefiner' +YARRRML_SOFTWARE_SPECIFICATION = 'softwareSpecification' +YARRRML_PROGRAMMING_LANGUAGE = 'programmingLanguage' +YARRRML_SOFTWARE_REQUIREMENTS = 'softwareRequirements' + YARRRML_MAPPINGS = 'mappings' # used for mappings in conditions and mappings main key YARRRML_MAPPING = 'mapping' YARRRRL_MAPPINGS_SHORTCUT = 'm' diff --git a/src/yatter/predicateobject.py b/src/yatter/predicateobject.py index bc2434b..0e2a059 100644 --- a/src/yatter/predicateobject.py +++ b/src/yatter/predicateobject.py @@ -288,7 +288,7 @@ def ref_mapping(data, mapping, om, yarrrml_key, ref_type_property, mapping_forma else: if mapping_format == STAR_URI: object = STAR_OBJECT - source_list = add_source(data, mapping_join, list_initial_sources) + source_list, external_refs = add_source(data, mapping_join, list_initial_sources) number_joins_rml = len(subject_list) * len(source_list) for i in range(number_joins_rml): diff --git a/src/yatter/source.py b/src/yatter/source.py index f5fb009..222acd5 100644 --- a/src/yatter/source.py +++ b/src/yatter/source.py @@ -30,6 +30,7 @@ def add_source(data, mapping, list_initial_sources): source_template = "\t" + RML_LOGICAL_SOURCE + " [\n\t\ta " + RML_LOGICAL_SOURCE_CLASS + \ ";\n\t\t" + RML_SOURCE + " " final_list = [] + external_references_list = [] sources = get_sources(data, mapping) for source in sources: db_identifier = mapping @@ -40,13 +41,24 @@ def add_source(data, mapping, list_initial_sources): if YARRRML_ACCESS in source: if YARRRML_QUERY in source: final_list.append(source_template + database_source(mapping, source, db_identifier)) + elif YARRRML_STRUCTURE_DEFINER in source: + source, external_references = add_in_memory_source(mapping,source) + final_list.append(source_template + source) + if external_references is not None: + external_references_list.append(external_references) else: final_list.append(source_template + add_source_full(mapping, source)) elif type(source) is list: - final_list.append(source_template + add_source_simplified(mapping, source)) + if "$(" in source[0]: + source, external_references = add_in_memory_source(mapping, source) + final_list.append(source_template + source) + if external_references is not None: + external_references_list.append(external_references) + else: + final_list.append(source_template + add_source_simplified(mapping, source)) else: raise Exception("ERROR: source " + source + " in mapping " + mapping + " not valid") - return final_list + return final_list, external_references_list def add_table(data, mapping, list_initial_sources): @@ -105,6 +117,53 @@ def add_source_simplified(mapping, source): + source[1] + "\";\n\t];\n" return source_rdf +def add_in_memory_source(mapping, source): + external_reference_formulation = None + if type(source) is list: + source = extend_in_memory(source) + source_rdf = "[\n\t\t\ta " + SD_DATASET_SPEC + ";\n\t\t\t" + + access = str(source.get(YARRRML_ACCESS)).replace("$(","").replace(")","") + source_rdf += SD_NAME + " \"" + access + "\";\n" + + if YARRRML_SOFTWARE_SPECIFICATION in source: + source_rdf += "\t\t\t" + SD_HAS_DATA_TRANSFORMATION + "[\n\t\t\t\t" + + if YARRRML_SOFTWARE_REQUIREMENTS in source.get(YARRRML_SOFTWARE_SPECIFICATION): + software_requirements = str(source.get(YARRRML_SOFTWARE_SPECIFICATION)[YARRRML_SOFTWARE_REQUIREMENTS]) + source_rdf += SD_HAS_SOFTWARE_REQUIREMENTS + " \""+ software_requirements +"\";\n\t\t\t\t" + + if YARRRML_PROGRAMMING_LANGUAGE in source.get(YARRRML_SOFTWARE_SPECIFICATION): + programming_language = str(source.get(YARRRML_SOFTWARE_SPECIFICATION)[YARRRML_PROGRAMMING_LANGUAGE]) + source_rdf += SD_HAS_SOURCE_CODE +"[\n\t\t\t\t\t" + SD_PROGRAMMING_LANGUAGE + " \"" + programming_language + "\";" + source_rdf +="\n\t\t\t\t];\n" + + source_rdf += "\t\t\t];\n" + source_rdf += "\t\t];\n" + + if YARRRML_ITERATOR in source: + source_rdf += "\t\t" + RML_ITERATOR + " \"" + source.get(YARRRML_ITERATOR) + "\";\n" + + if YARRRML_REFERENCE_FORMULATION in source: + reference_formulation = str(source.get(YARRRML_REFERENCE_FORMULATION)) + source_rdf += "\t\t" + RML_REFERENCE_FORMULATION + " ql:"+ reference_formulation + ";\n" + external_reference_formulation = "ql:" + reference_formulation + " a " + RML_REFERENCE_FORMULATION_CLASS +";\n" + external_reference_formulation +="\t" + KG4DI_DEFINED_BY +" \""+source.get(YARRRML_STRUCTURE_DEFINER) +"\"." + source_rdf += "\t];\n" + + + + return source_rdf, external_reference_formulation + +def extend_in_memory(source): + features = source[0].split("~") + access = features[0] + defined_by = features[1].split("-")[0] + reference_formulation = features[1].split("-")[1] + extended_source = {"access": access, "structureDefiner": defined_by, "referenceFormulation": reference_formulation} + if len(source) == 2: + extended_source["iterator"] = source[1] + return extended_source def add_source_full(mapping, source): source_rdf = "" diff --git a/test/rml-in-memory/IMTC001/mapping.ttl b/test/rml-in-memory/IMTC001/mapping.ttl new file mode 100644 index 0000000..16303fb --- /dev/null +++ b/test/rml-in-memory/IMTC001/mapping.ttl @@ -0,0 +1,46 @@ +@prefix insta: . +@prefix rr: . +@prefix rml: . +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix sd: . +@prefix kg4di: . +@prefix ql: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + a rml:LogicalSource; + rml:source [ + a sd:DatasetSpecification; + sd:name "output_dataframe"; + sd:hasDataTransformation[ + sd:hasSoftwareRequirements "pandas>=1.1.0"; + sd:hasSourceCode[ + sd:programmingLanguage "Python3.9"; + ]; + ]; + ]; + rml:referenceFormulation ql:DataFrame; + ]; + rr:subjectMap [ + a rr:SubjectMap; + rr:template "http://instagram.com/data/user{Id}"; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant rdf:type; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:constant insta:User; + ]; + ]. + + +ql:DataFrame a rml:ReferenceFormulation; + kg4di:definedBy "Pandas". diff --git a/test/rml-in-memory/IMTC001/mappings.yml b/test/rml-in-memory/IMTC001/mapping.yml similarity index 73% rename from test/rml-in-memory/IMTC001/mappings.yml rename to test/rml-in-memory/IMTC001/mapping.yml index e236f5e..d7551b1 100644 --- a/test/rml-in-memory/IMTC001/mappings.yml +++ b/test/rml-in-memory/IMTC001/mapping.yml @@ -14,10 +14,10 @@ mappings: sources: access: $(output_dataframe) referenceFormulation: DataFrame - structureDefiner: Pandas - softwareSpecification: - programmingLanguage: Python3.9 - softwareRequirements: “pandas>=1.1.0” - s: http://instagram.com/data/user{Id} + structureDefiner: Pandas + softwareSpecification: + programmingLanguage: Python3.9 + softwareRequirements: pandas>=1.1.0 + s: http://instagram.com/data/user$(Id) po: - [a, insta:User] \ No newline at end of file diff --git a/test/rml-in-memory/IMTC001/mappings.rml.ttl b/test/rml-in-memory/IMTC001/mappings.rml.ttl deleted file mode 100644 index ab5a0bf..0000000 --- a/test/rml-in-memory/IMTC001/mappings.rml.ttl +++ /dev/null @@ -1,43 +0,0 @@ -@prefix insta: . -@prefix rr: . -@prefix rml: . -@prefix rdf: . -@prefix rdfs: . -@prefix xsd: . -@prefix sd: . -@prefix kg4di: . - - - a rr:TriplesMap; - - rml:logicalSource [ - a rml:LogicalSource; - rml:source [ - a sd:DatasetSpecification; - sd:name "output_dataframe"; - sd:hasDataTransformation [ - sd:hasSoftwareRequirements "pandas>=1.1.0"; - sd:hasSourceCode [ - sd:programmingLanguage "Python3.9"; - ]; - ]; - ]; - rml:referenceFormulation ql:DataFrame; - ]; - rr:subjectMap [ - a rr:SubjectMap; - rr:template "http://instagram.com/data/user{Id}"; - ]; - rr:predicateObjectMap [ - rr:predicateMap [ - a rr:PredicateMap; - rr:constant rdf:type; - ]; - rr:objectMap [ - a rr:ObjectMap; - rr:constant insta:User; - ]; - ]. - -ql:DataFrame a rml:ReferenceFormulation; - kg4di:definedBy "Pandas". \ No newline at end of file diff --git a/test/rml-in-memory/IMTC001/test_imtc001.py b/test/rml-in-memory/IMTC001/test_imtc001.py new file mode 100644 index 0000000..17cd890 --- /dev/null +++ b/test/rml-in-memory/IMTC001/test_imtc001.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc001(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file diff --git a/test/rml-in-memory/IMTC003/mappings.rml.ttl b/test/rml-in-memory/IMTC002/mapping.ttl similarity index 85% rename from test/rml-in-memory/IMTC003/mappings.rml.ttl rename to test/rml-in-memory/IMTC002/mapping.ttl index 6eb2920..6d3f2b7 100644 --- a/test/rml-in-memory/IMTC003/mappings.rml.ttl +++ b/test/rml-in-memory/IMTC002/mapping.ttl @@ -5,10 +5,11 @@ @prefix rdfs: . @prefix xsd: . @prefix sd: . -@prefix kg4di: . +@prefix kg4di: . +@prefix ql: . +@base . - - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; diff --git a/test/rml-in-memory/IMTC002/mappings.yml b/test/rml-in-memory/IMTC002/mapping.yml similarity index 83% rename from test/rml-in-memory/IMTC002/mappings.yml rename to test/rml-in-memory/IMTC002/mapping.yml index cbb281c..51528fb 100644 --- a/test/rml-in-memory/IMTC002/mappings.yml +++ b/test/rml-in-memory/IMTC002/mapping.yml @@ -14,7 +14,7 @@ mappings: sources: access: $(output_dataframe) referenceFormulation: DataFrame - structureDefiner: Pandas - s: http://instagram.com/data/user{Id} + structureDefiner: Pandas + s: http://instagram.com/data/user$(Id) po: - - [a, insta:User] \ No newline at end of file + - [a, insta:User] \ No newline at end of file diff --git a/test/rml-in-memory/IMTC002/test_imtc002.py b/test/rml-in-memory/IMTC002/test_imtc002.py new file mode 100644 index 0000000..003c344 --- /dev/null +++ b/test/rml-in-memory/IMTC002/test_imtc002.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc002(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file diff --git a/test/rml-in-memory/IMTC002/mappings.rml.ttl b/test/rml-in-memory/IMTC003/mapping.ttl similarity index 85% rename from test/rml-in-memory/IMTC002/mappings.rml.ttl rename to test/rml-in-memory/IMTC003/mapping.ttl index 6eb2920..6d3f2b7 100644 --- a/test/rml-in-memory/IMTC002/mappings.rml.ttl +++ b/test/rml-in-memory/IMTC003/mapping.ttl @@ -5,10 +5,11 @@ @prefix rdfs: . @prefix xsd: . @prefix sd: . -@prefix kg4di: . +@prefix kg4di: . +@prefix ql: . +@base . - - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; diff --git a/test/rml-in-memory/IMTC003/mappings.yml b/test/rml-in-memory/IMTC003/mapping.yml similarity index 91% rename from test/rml-in-memory/IMTC003/mappings.yml rename to test/rml-in-memory/IMTC003/mapping.yml index 3928ada..78cb547 100644 --- a/test/rml-in-memory/IMTC003/mappings.yml +++ b/test/rml-in-memory/IMTC003/mapping.yml @@ -13,6 +13,6 @@ mappings: df_map: sources: - [$(output_dataframe)~Pandas-DataFrame] - s: http://instagram.com/data/user{Id} + s: http://instagram.com/data/user$(Id) po: - [a, insta:User] \ No newline at end of file diff --git a/test/rml-in-memory/IMTC003/test_imtc003.py b/test/rml-in-memory/IMTC003/test_imtc003.py new file mode 100644 index 0000000..abe26ca --- /dev/null +++ b/test/rml-in-memory/IMTC003/test_imtc003.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc003(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file diff --git a/test/rml-in-memory/IMTC004/mappings.rml.ttl b/test/rml-in-memory/IMTC004/mapping.ttl similarity index 86% rename from test/rml-in-memory/IMTC004/mappings.rml.ttl rename to test/rml-in-memory/IMTC004/mapping.ttl index 34c6e92..e56a280 100644 --- a/test/rml-in-memory/IMTC004/mappings.rml.ttl +++ b/test/rml-in-memory/IMTC004/mapping.ttl @@ -5,10 +5,11 @@ @prefix rdfs: . @prefix xsd: . @prefix sd: . -@prefix kg4di: . +@prefix kg4di: . +@prefix ql: . +@base . - - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; diff --git a/test/rml-in-memory/IMTC004/mappings.yml b/test/rml-in-memory/IMTC004/mapping.yml similarity index 79% rename from test/rml-in-memory/IMTC004/mappings.yml rename to test/rml-in-memory/IMTC004/mapping.yml index 80c8c15..b919fb7 100644 --- a/test/rml-in-memory/IMTC004/mappings.yml +++ b/test/rml-in-memory/IMTC004/mapping.yml @@ -14,10 +14,10 @@ mappings: sources: access: $(output_dict) referenceFormulation: Dictionary - structureDefiner: Python + structureDefiner: Python iterator: $.users[*] - softwareSpecification: - programmingLanguage: Python3.9 - s: http://instagram.com/data/user{Id} + softwareSpecification: + programmingLanguage: Python3.9 + s: http://instagram.com/data/user$(id) po: - [a, insta:User] \ No newline at end of file diff --git a/test/rml-in-memory/IMTC004/test_imtc004.py b/test/rml-in-memory/IMTC004/test_imtc004.py new file mode 100644 index 0000000..420f78e --- /dev/null +++ b/test/rml-in-memory/IMTC004/test_imtc004.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc004(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file diff --git a/test/rml-in-memory/IMTC005/mappings.rml.ttl b/test/rml-in-memory/IMTC005/mapping.ttl similarity index 85% rename from test/rml-in-memory/IMTC005/mappings.rml.ttl rename to test/rml-in-memory/IMTC005/mapping.ttl index a80e050..54f3dba 100644 --- a/test/rml-in-memory/IMTC005/mappings.rml.ttl +++ b/test/rml-in-memory/IMTC005/mapping.ttl @@ -5,10 +5,11 @@ @prefix rdfs: . @prefix xsd: . @prefix sd: . -@prefix kg4di: . +@prefix kg4di: . +@prefix ql: . +@base . - - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; diff --git a/test/rml-in-memory/IMTC005/mappings.yml b/test/rml-in-memory/IMTC005/mapping.yml similarity index 92% rename from test/rml-in-memory/IMTC005/mappings.yml rename to test/rml-in-memory/IMTC005/mapping.yml index 38ed3f3..5a3dba6 100644 --- a/test/rml-in-memory/IMTC005/mappings.yml +++ b/test/rml-in-memory/IMTC005/mapping.yml @@ -13,6 +13,6 @@ mappings: dict_map: sources: - [$(output_dict)~Python-Dictionary, "$.users[*]"] - s: http://instagram.com/data/user{Id} + s: http://instagram.com/data/user$(id) po: - [a, insta:User] \ No newline at end of file diff --git a/test/rml-in-memory/IMTC005/test_imtc005.py b/test/rml-in-memory/IMTC005/test_imtc005.py new file mode 100644 index 0000000..b243732 --- /dev/null +++ b/test/rml-in-memory/IMTC005/test_imtc005.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc005(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file diff --git a/test/rml-in-memory/IMTC006/mappings.rml.ttl b/test/rml-in-memory/IMTC006/mapping.ttl similarity index 61% rename from test/rml-in-memory/IMTC006/mappings.rml.ttl rename to test/rml-in-memory/IMTC006/mapping.ttl index 5415dbb..10bcbeb 100644 --- a/test/rml-in-memory/IMTC006/mappings.rml.ttl +++ b/test/rml-in-memory/IMTC006/mapping.ttl @@ -5,19 +5,20 @@ @prefix rdfs: . @prefix xsd: . @prefix sd: . -@prefix kg4di: . +@prefix kg4di: . +@prefix ql: . +@base . - - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; rml:source [ a sd:DatasetSpecification; sd:name "variable1"; - sd:hasDataTransformation [ + sd:hasDataTransformation[ sd:hasSoftwareRequirements "pandas>=1.1.0"; - sd:hasSourceCode [ + sd:hasSourceCode[ sd:programmingLanguage "Python3.9"; ]; ]; @@ -26,7 +27,7 @@ ]; rr:subjectMap [ a rr:SubjectMap; - rr:template "http://instagram.com/data/user{Id}"; + rr:template "http://instagram.com/data/user{id}"; ]; rr:predicateObjectMap [ rr:predicateMap [ @@ -56,21 +57,30 @@ rr:objectMap [ a rr:ObjectMap; rr:template "{Name} {Surname}"; - rr:datatype rdfs:Literal + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:followersNumber; + ]; + rr:objectMap [ + a rr:ObjectMap; + rml:reference "Followers"; ]; ]. - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; rml:source [ a sd:DatasetSpecification; sd:name "variable2"; - sd:hasDataTransformation [ + sd:hasDataTransformation[ sd:hasSoftwareRequirements "pandas>=1.1.0"; - sd:hasSourceCode [ + sd:hasSourceCode[ sd:programmingLanguage "Python3.9"; ]; ]; @@ -79,7 +89,37 @@ ]; rr:subjectMap [ a rr:SubjectMap; - rr:template "http://instagram.com/data/user{Id}"; + rr:template "http://instagram.com/data/user{id}"; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant rdf:type; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:constant insta:User; + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:username; + ]; + rr:objectMap [ + a rr:ObjectMap; + rml:reference "Username"; + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:name; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:template "{Name} {Surname}"; + ]; ]; rr:predicateObjectMap [ rr:predicateMap [ @@ -92,5 +132,6 @@ ]; ]. + ql:DataFrame a rml:ReferenceFormulation; kg4di:definedBy "Pandas". \ No newline at end of file diff --git a/test/rml-in-memory/IMTC006/mappings.yml b/test/rml-in-memory/IMTC006/mapping.yml similarity index 86% rename from test/rml-in-memory/IMTC006/mappings.yml rename to test/rml-in-memory/IMTC006/mapping.yml index 78ff649..81d3d52 100644 --- a/test/rml-in-memory/IMTC006/mappings.yml +++ b/test/rml-in-memory/IMTC006/mapping.yml @@ -18,14 +18,14 @@ mappings: structureDefiner: Pandas softwareSpecification: programmingLanguage: Python3.9 - softwareRequirements: “pandas>=1.1.0” + softwareRequirements: pandas>=1.1.0 - access: $(variable2) referenceFormulation: DataFrame structureDefiner: Pandas softwareSpecification: programmingLanguage: Python3.9 - softwareRequirements: “pandas>=1.1.0” - s: http://instagram.com/data/user{Id} + softwareRequirements: pandas>=1.1.0 + s: http://instagram.com/data/user$(id) po: - [a, insta:User] - [insta:username, $(Username)] diff --git a/test/rml-in-memory/IMTC006/test_imtc006.py b/test/rml-in-memory/IMTC006/test_imtc006.py new file mode 100644 index 0000000..1c48aaa --- /dev/null +++ b/test/rml-in-memory/IMTC006/test_imtc006.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc006(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file diff --git a/test/rml-in-memory/IMTC007/mappings.rml.ttl b/test/rml-in-memory/IMTC007/mapping.ttl similarity index 60% rename from test/rml-in-memory/IMTC007/mappings.rml.ttl rename to test/rml-in-memory/IMTC007/mapping.ttl index 54c6afe..496db03 100644 --- a/test/rml-in-memory/IMTC007/mappings.rml.ttl +++ b/test/rml-in-memory/IMTC007/mapping.ttl @@ -5,28 +5,29 @@ @prefix rdfs: . @prefix xsd: . @prefix sd: . -@prefix kg4di: . +@prefix kg4di: . +@prefix ql: . +@base . - - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; rml:source [ a sd:DatasetSpecification; sd:name "variable1"; - sd:hasDataTransformation [ - sd:hasSourceCode [ + sd:hasDataTransformation[ + sd:hasSourceCode[ sd:programmingLanguage "Python3.9"; ]; ]; ]; - rml:referenceFormulation ql:Dictionary; rml:iterator "$.users[*]"; + rml:referenceFormulation ql:Dictionary; ]; rr:subjectMap [ a rr:SubjectMap; - rr:template "http://instagram.com/data/user{Id}"; + rr:template "http://instagram.com/data/user{id}"; ]; rr:predicateObjectMap [ rr:predicateMap [ @@ -56,30 +57,70 @@ rr:objectMap [ a rr:ObjectMap; rr:template "{Name} {Surname}"; - rr:datatype rdfs:Literal + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:follows; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:template "http://instagram.com/data/user{Follows}"; + rr:termType rr:IRI ]; ]. - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; rml:source [ a sd:DatasetSpecification; sd:name "variable2"; - sd:hasDataTransformation [ - sd:hasSourceCode [ + sd:hasDataTransformation[ + sd:hasSourceCode[ sd:programmingLanguage "Python3.9"; ]; ]; ]; - rml:referenceFormulation ql:Dictionary; rml:iterator "$.followers[*]"; + rml:referenceFormulation ql:Dictionary; ]; rr:subjectMap [ a rr:SubjectMap; - rr:template "http://instagram.com/data/user{Id}"; + rr:template "http://instagram.com/data/user{id}"; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant rdf:type; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:constant insta:User; + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:username; + ]; + rr:objectMap [ + a rr:ObjectMap; + rml:reference "Username"; + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:name; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:template "{Name} {Surname}"; + ]; ]; rr:predicateObjectMap [ rr:predicateMap [ @@ -93,5 +134,6 @@ ]; ]. + ql:Dictionary a rml:ReferenceFormulation; kg4di:definedBy "Python". diff --git a/test/rml-in-memory/IMTC007/mappings.yml b/test/rml-in-memory/IMTC007/mapping.yml similarity index 89% rename from test/rml-in-memory/IMTC007/mappings.yml rename to test/rml-in-memory/IMTC007/mapping.yml index bc5ec84..619cbb0 100644 --- a/test/rml-in-memory/IMTC007/mappings.yml +++ b/test/rml-in-memory/IMTC007/mapping.yml @@ -25,9 +25,9 @@ mappings: iterator: $.followers[*] softwareSpecification: programmingLanguage: Python3.9 - s: http://instagram.com/data/user{Id} + s: http://instagram.com/data/user$(id) po: - [a, insta:User] - [insta:username, $(Username)] - [insta:name, "$(Name) $(Surname)"] - - [insta:follows, http://instagram.com/data/user${Follows}~iri] + - [insta:follows, http://instagram.com/data/user$(Follows)~iri] diff --git a/test/rml-in-memory/IMTC007/test_imtc007.py b/test/rml-in-memory/IMTC007/test_imtc007.py new file mode 100644 index 0000000..e889bf3 --- /dev/null +++ b/test/rml-in-memory/IMTC007/test_imtc007.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc007(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file diff --git a/test/rml-in-memory/IMTC008/mappings.rml.ttl b/test/rml-in-memory/IMTC008/mapping.ttl similarity index 58% rename from test/rml-in-memory/IMTC008/mappings.rml.ttl rename to test/rml-in-memory/IMTC008/mapping.ttl index e64944a..a6e0541 100644 --- a/test/rml-in-memory/IMTC008/mappings.rml.ttl +++ b/test/rml-in-memory/IMTC008/mapping.ttl @@ -5,28 +5,29 @@ @prefix rdfs: . @prefix xsd: . @prefix sd: . -@prefix kg4di: . +@prefix kg4di: . +@prefix ql: . +@base . - - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; rml:source [ a sd:DatasetSpecification; sd:name "variable1"; - sd:hasDataTransformation [ - sd:hasSourceCode [ + sd:hasDataTransformation[ + sd:hasSourceCode[ sd:programmingLanguage "Python3.9"; ]; ]; ]; - rml:referenceFormulation ql:Dictionary; rml:iterator "$.users[*]"; + rml:referenceFormulation ql:Dictionary; ]; rr:subjectMap [ a rr:SubjectMap; - rr:template "http://instagram.com/data/user{Id}"; + rr:template "http://instagram.com/data/user{id}"; ]; rr:predicateObjectMap [ rr:predicateMap [ @@ -56,22 +57,62 @@ rr:objectMap [ a rr:ObjectMap; rr:template "{Name} {Surname}"; - rr:datatype rdfs:Literal + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:follows; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:template "http://instagram.com/data/user{Follows}"; + rr:termType rr:IRI ]; ]. - a rr:TriplesMap; + a rr:TriplesMap; rml:logicalSource [ a rml:LogicalSource; - rml:source "followers.json"; + rml:source "followersperson.json"; rml:referenceFormulation ql:JSONPath; rml:iterator "$.followers[*]"; ]; rr:subjectMap [ a rr:SubjectMap; - rr:template "http://instagram.com/data/user{Id}"; + rr:template "http://instagram.com/data/user{id}"; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant rdf:type; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:constant insta:User; + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:username; + ]; + rr:objectMap [ + a rr:ObjectMap; + rml:reference "Username"; + ]; + ]; + rr:predicateObjectMap [ + rr:predicateMap [ + a rr:PredicateMap; + rr:constant insta:name; + ]; + rr:objectMap [ + a rr:ObjectMap; + rr:template "{Name} {Surname}"; + ]; ]; rr:predicateObjectMap [ rr:predicateMap [ @@ -85,5 +126,6 @@ ]; ]. + ql:Dictionary a rml:ReferenceFormulation; - kg4di:definedBy "Python". + kg4di:definedBy "Python". \ No newline at end of file diff --git a/test/rml-in-memory/IMTC008/mappings.yml b/test/rml-in-memory/IMTC008/mapping.yml similarity index 80% rename from test/rml-in-memory/IMTC008/mappings.yml rename to test/rml-in-memory/IMTC008/mapping.yml index ce19422..7fb2e38 100644 --- a/test/rml-in-memory/IMTC008/mappings.yml +++ b/test/rml-in-memory/IMTC008/mapping.yml @@ -19,10 +19,10 @@ mappings: iterator: $.users[*] softwareSpecification: programmingLanguage: Python3.9 - - source: [followersperson.json~jsonpath, "$.followers[*]"] - s: http://instagram.com/data/user{Id} + - [followersperson.json~jsonpath, "$.followers[*]"] + s: http://instagram.com/data/user$(id) po: - [a, insta:User] - [insta:username, $(Username)] - [insta:name, "$(Name) $(Surname)"] - - [insta:follows, http://instagram.com/data/user${Follows}~iri] + - [insta:follows, http://instagram.com/data/user$(Follows)~iri] diff --git a/test/rml-in-memory/IMTC008/test_imtc008.py b/test/rml-in-memory/IMTC008/test_imtc008.py new file mode 100644 index 0000000..1e96394 --- /dev/null +++ b/test/rml-in-memory/IMTC008/test_imtc008.py @@ -0,0 +1,26 @@ +__author__ = "Ioannis Dasoulas" +__credits__ = ["Ioannis Dasoulas"] + +__license__ = "Apache-2.0" +__maintainer__ = "David Chaves-Fraga" +__email__ = "david.chaves@upm.es" + + +import os +from ruamel.yaml import YAML +import yatter +from rdflib.graph import Graph +from rdflib import compare +RML_URI = 'http://semweb.mmlab.be/ns/rml#' + + +def test_imtc008(): + expected_mapping = Graph() + expected_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl") + + translated_mapping = Graph() + yaml = YAML(typ='safe', pure=True) + mapping_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml') + translated_mapping.parse(data=yatter.translate(yaml.load(open(mapping_path)), mapping_format=RML_URI), format="ttl") + + assert compare.isomorphic(expected_mapping, translated_mapping) \ No newline at end of file