Merge pull request #81 from oeg-upm/main

Include last updates in dev

.github/workflows/pypi-publish.yml

@@ -0,0 +1,50 @@
+name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
+
+on: push
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel
+    - name: Build a binary wheel and a source tarball
+      run: |
+        cd src
+        python setup.py -k rel sdist bdist_wheel
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v3
+      with:
+        name: python-package-distributions
+        path: src/dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/yatter  # Replace <package-name> with your PyPI project name
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v3
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/testgha.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine pytest DeepDiff
+        pip install setuptools wheel pytest DeepDiff
         pip install -r requirements.txt
     - name: Build yatter
       run: |

VERSION

@@ -1 +1 @@
-1.1.0
+1.1.4

src/setup.py

@@ -26,15 +26,15 @@
     name="yatter",
     version=vers_taged,
     author="David Chaves-Fraga",
-    author_email="david.chaves@upm.es",
+    author_email="david.chaves@usc.es",
     license="Apache 2.0",
     description="A translator from YARRRML to RML mappings.",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url="https://github.com/oeg-upm/yarrrml-translator",
+    url="https://github.com/oeg-upm/yatter",
     project_urls={
-        'Source code': 'https://github.com/oeg-upm/yarrrml-translator',
-        'Issue tracker': 'https://github.com/oeg-upm/yarrrml-translator/issues',
+        'Source code': 'https://github.com/oeg-upm/yatter',
+        'Issue tracker': 'https://github.com/oeg-upm/yatter/issues',
     },
     include_package_data=True,
     packages=setuptools.find_packages(),

src/yatter/__init__.py

@@ -69,7 +69,7 @@ def inverse_translation(rdf_mapping, mapping_format=RML_URI):
     triples_map = [tm[rdflib.Variable('triplesMap')] for tm in rdf_mapping.query(query).bindings]
 
     for tm in triples_map:
-        tm_name = tm.split("/")[-1]
+        tm_name = tm.split("/")[-1].split("#")[-1]
         yarrrml_tm = {YARRRML_SOURCES: [add_inverse_source(tm, rdf_mapping, mapping_format)]}
         subject, classes = add_inverse_subject(tm, rdf_mapping)
         yarrrml_tm.update(subject)

src/yatter/__main__.py

@@ -14,6 +14,7 @@ def write_results(mapping):
     elif type(mapping) is dict:
         with open(args.output_mapping_path, "wb") as f:
             yaml = YAML()
+            yaml.width = 3000
             yaml.default_flow_style = False
             yaml.dump(mapping, f)
 

src/yatter/constants.py

@@ -81,6 +81,7 @@
 R2RML_TERMTYPE = 'rr:termType'
 R2RML_LANGUAGE = 'rr:language'
 R2RML_IRI = 'rr:IRI'
+R2RML_BLANK_NODE = 'rr:BlankNode'
 R2RML_LITERAL = 'rr:Literal'
 R2RML_REFOBJECT_CLASS = 'rr:RefObjectMap'
 R2RML_PARENT_TRIPLESMAP = 'rr:parentTriplesMap'
@@ -92,6 +93,7 @@
 R2RML_TABLE_NAME = 'rr:tableName'
 R2RML_COLUMN = 'rr:column'
 
+
 ##############################################################################
 #############################   D2RQ CONSTANTS  ###########################
 ##############################################################################
@@ -154,6 +156,7 @@
 
 YARRRML_IRI = '~iri'
 YARRRML_LANG = '~lang'
+YARRRML_BLANK = 'blank'
 
 YARRRML_QUOTED = 'quoted'
 YARRRML_NON_ASSERTED = 'quotedNonAsserted'
@@ -212,7 +215,8 @@
     'json': 'JSONPath',
     'xpath': 'XPath',
     'jsonpath': 'JSONPath',
-    "shp": "SHP"
+    "shp": "SHP",
+    "xlsx": "CSV"
 }
 
 YARRRML_DATABASES_DRIVER = {

src/yatter/function.py

@@ -43,7 +43,7 @@ def add_internal_function(mapping_id, mapping_data, functions):
 def transform_function(mapping_id, function_value, functions):
     global local_id
     function_return = None
-    if YARRRML_JOIN in function_value[YARRRML_FUNCTION]:
+    if function_value[YARRRML_FUNCTION].replace(" ","").startswith(YARRRML_JOIN+"("):
         function_return = generate_extended_join(function_value[YARRRML_FUNCTION])
     elif function_value[YARRRML_FUNCTION] != YARRRML_EQUAL:
         function_id = "function_" + mapping_id

src/yatter/mapping.py

@@ -1,7 +1,6 @@
 from .import *
 prefixes = {}
 
-
 def add_mapping(mapping, mappings, it):
     map_template = "<" + mapping + "_" + str(it) + "> a "
     if mappings[mapping]:
@@ -99,7 +98,7 @@ def add_inverse_prefix(rdf_mapping):
 
 
 def get_non_asserted_mappings(yarrrml_data, mapping_format):
-    mappings =  dict.fromkeys(list(yarrrml_data.get(YARRRML_MAPPINGS).keys()))
+    mappings = dict.fromkeys(list(yarrrml_data.get(YARRRML_MAPPINGS).keys()))
     for mapping in yarrrml_data.get(YARRRML_MAPPINGS):
         keys = yarrrml_data.get(YARRRML_MAPPINGS).get(mapping).keys()
         for key in keys:

src/yatter/predicateobject.py

@@ -154,27 +154,28 @@ def add_predicate_object(data, mapping, predicate_object, mapping_format=RML_URI
             execution = True
         template += generate_rml_termmap(R2RML_PREDICATE, R2RML_PREDICATE_CLASS, pm_value, "\t\t\t")
         if execution:
-            template = template.replace(R2RML_CONSTANT + " " + pm_value, RML_EXECUTION + " <" + pm_value + ">")
+            template = template.replace(R2RML_CONSTANT + " \"" + pm_value + "\"", RML_EXECUTION + " <" + pm_value + ">")
         if YARRRML_TARGETS in pm:
             template = template[0:-3] + "\t" + RML_LOGICAL_TARGET + " <" + pm[YARRRML_TARGETS] + ">\n\t\t];\n"
 
     for om in object_list:
         iri = False
+        blank = False
         if type(om) == list:
             object_value = om[0]
             if YARRRML_IRI in om[0]:
-                object_value = om[0].split(YARRRML_IRI)[0]
+                object_value = om[0].split("~")[0]
                 iri = True
+            if YARRRML_BLANK in om[0]:
+                object_value = om[0].split("~")[0]
+                blank = True
             if mapping_format == STAR_URI:
                 template += generate_rml_termmap(STAR_OBJECT, R2RML_OBJECT_CLASS,
                                                  object_value, "\t\t\t", mapping_format)
             else:
                 object_map = generate_rml_termmap(R2RML_OBJECT, R2RML_OBJECT_CLASS,
                                                   object_value, "\t\t\t", mapping_format)
 
-                if object_value not in prefixes and ":" not in object_value and R2RML_CONSTANT in object_map:
-                    object_map = object_map.replace(object_value, f"\"{object_value}\"")
-
                 template += object_map
             if len(om) == 2:
                 types = check_type(om[1])
@@ -204,6 +205,9 @@ def add_predicate_object(data, mapping, predicate_object, mapping_format=RML_URI
             if iri:
                 template = template[0:len(template) - 5] + "\t\t\t" + R2RML_TERMTYPE + " " \
                            + R2RML_IRI + "\n\t\t];\n"
+            if blank:
+                template = template[0:len(template) - 5] + "\t\t\t" + R2RML_TERMTYPE + " " \
+                           + R2RML_BLANK_NODE + "\n\t\t];\n"
         elif YARRRML_MAPPING in om or YARRRML_NON_ASSERTED in om or YARRRML_QUOTED in om:
             if YARRRML_MAPPING in om:
                 template += ref_mapping(data, mapping, om, YARRRML_MAPPING, R2RML_PARENT_TRIPLESMAP, mapping_format)
@@ -226,10 +230,8 @@ def add_predicate_object(data, mapping, predicate_object, mapping_format=RML_URI
                 template += generate_rml_termmap(STAR_OBJECT, R2RML_OBJECT_CLASS,
                                                  object_value, "\t\t\t", mapping_format)
             elif YARRRML_FUNCTION in om:
-                template += generate_rml_termmap(R2RML_OBJECT, R2RML_OBJECT_CLASS, om[YARRRML_FUNCTION], "\t\t\t",
-                                                 mapping_format)
-                template = template.replace(R2RML_CONSTANT + " " + om[YARRRML_FUNCTION], RML_EXECUTION + " <" + om.get(
-                    YARRRML_FUNCTION) + ">")
+                template += generate_rml_termmap(R2RML_OBJECT, R2RML_OBJECT_CLASS, om[YARRRML_FUNCTION], "\t\t\t", mapping_format)
+                template = template.replace(R2RML_CONSTANT+" \""+om[YARRRML_FUNCTION]+ "\"", RML_EXECUTION + " <" + om.get(YARRRML_FUNCTION) + ">")
             else:
                 template += generate_rml_termmap(R2RML_OBJECT, R2RML_OBJECT_CLASS,
                                                  object_value, "\t\t\t", mapping_format)
@@ -246,6 +248,9 @@ def add_predicate_object(data, mapping, predicate_object, mapping_format=RML_URI
                     elif om.get(YARRRML_TYPE) == "literal":
                         template = template[0:len(template) - 5] + "\t\t\t" + R2RML_TERMTYPE + " " \
                                    + R2RML_LITERAL + "\n\t\t];\n"
+                    elif om.get(YARRRML_TYPE) == YARRRML_BLANK:
+                        template = template[0:len(template) - 5] + "\t\t\t" + R2RML_TERMTYPE + " " \
+                                   + R2RML_BLANK_NODE + "\n\t\t];\n"
                 if YARRRML_TARGETS in om:
                     template = template[0:len(template) - 5] + "\t\t\t" + RML_LOGICAL_TARGET + " <" + om.get(
                         YARRRML_TARGETS) + ">\n\t\t];\n"
@@ -323,7 +328,7 @@ def add_inverse_pom(mapping_id, rdf_mapping, classes, prefixes):
     yarrrml_poms = []
     yaml = YAML()
     for c in classes:
-        yarrrml_pom = yaml.seq(['rdf:type', c.toPython()])
+        yarrrml_pom = yaml.seq(['rdf:type', find_prefixes(c.toPython(),prefixes)])
         yarrrml_pom.fa.set_flow_style()
         yarrrml_poms.append(yarrrml_pom)
 
@@ -351,9 +356,11 @@ def add_inverse_pom(mapping_id, rdf_mapping, classes, prefixes):
             f' ?object {RML_LANGUAGE_MAP} ?languageMap .' \
             f' ?languageMap {R2RML_TEMPLATE}|{R2RML_CONSTANT}|{RML_REFERENCE} ?languageMapValue .}} }} ' \
             f'OPTIONAL {{ ?object {R2RML_PARENT_TRIPLESMAP} ?parentTriplesMap .' \
-            f'?object {R2RML_JOIN_CONITION} ?join_condition .' \
-            f'?join_condition {R2RML_CHILD} ?child .' \
-            f'?join_condition {R2RML_PARENT} ?parent }} }}'
+            f'OPTIONAL {{ '\
+                f'?object {R2RML_JOIN_CONITION} ?join_condition .' \
+                f'?join_condition {R2RML_CHILD} ?child .' \
+                f'?join_condition {R2RML_PARENT} ?parent }} }}'\
+            f'}}'
 
     for tm in rdf_mapping.query(query):
         yarrrml_pom = []
@@ -373,16 +380,22 @@ def add_inverse_pom(mapping_id, rdf_mapping, classes, prefixes):
         elif prefix:
             predicate = tm['predicateValue'].toPython().replace(prefixes[prefix[0]], prefix[0] + ":")
 
+        predicate = find_prefixes(predicate,prefixes)
+
         if tm['parentTriplesMap']:
-            yarrrml_pom = {'p': predicate, 'o': {'mapping': None, 'condition':
-                {'function': 'equal', 'parameters': []}}}
-            yarrrml_pom['o']['mapping'] = tm['parentTriplesMap'].split("/")[-1]
-            child = yaml.seq(['str1', '$(' + tm['child'] + ')'])
-            child.fa.set_flow_style()
-            parent = yaml.seq(['str2', '$(' + tm['parent'] + ')'])
-            parent.fa.set_flow_style()
-            yarrrml_pom['o']['condition']['parameters'].append(child)
-            yarrrml_pom['o']['condition']['parameters'].append(parent)
+            if tm['child']:
+                yarrrml_pom = {'p': predicate, 'o': {'mapping': None, 'condition':
+                    {'function': 'equal', 'parameters': []}}}
+                yarrrml_pom['o']['mapping'] = tm['parentTriplesMap'].split("/")[-1]
+                child = yaml.seq(['str1', '$(' + tm['child'] + ')'])
+                child.fa.set_flow_style()
+                parent = yaml.seq(['str2', '$(' + tm['parent'] + ')'])
+                parent.fa.set_flow_style()
+                yarrrml_pom['o']['condition']['parameters'].append(child)
+                yarrrml_pom['o']['condition']['parameters'].append(parent)
+            else:
+                yarrrml_pom = {'p': predicate, 'o': {'mapping': tm['parentTriplesMap'].split("/")[-1]}}
+
 
         else:
             datatype = None
@@ -401,9 +414,7 @@ def add_inverse_pom(mapping_id, rdf_mapping, classes, prefixes):
             if not object.startswith("http"):
                 object = '$(' + object + ')'
             elif object.startswith("http") and "{" not in object:
-                prefix = list({i for i in prefixes if object.startswith(prefixes[i])})
-                if prefix:
-                    object = object.replace(prefixes[prefix[0]], prefix[0] + ":")
+               object = find_prefixes(object,prefixes)
             else:
                 object = object.replace('{', '$(').replace('}', ')')
 
@@ -441,11 +452,13 @@ def add_inverse_pom(mapping_id, rdf_mapping, classes, prefixes):
 
             if type(yarrrml_pom) is list:
                 if datatype:
+                    datatype = find_prefixes(datatype,prefixes)
                     yarrrml_pom.append(datatype)
                 if language:
                     yarrrml_pom.append(language)
             elif type(yarrrml_pom) is dict:
                 if datatype:
+                    datatype = find_prefixes(datatype, prefixes)
                     yarrrml_pom[YARRRML_DATATYPE] = datatype
                 if language:
                     yarrrml_pom[YARRRML_LANGUAGE] = language
@@ -456,3 +469,9 @@ def add_inverse_pom(mapping_id, rdf_mapping, classes, prefixes):
         yarrrml_poms.append(yarrrml_pom)
 
     return yarrrml_poms
+
+def find_prefixes(text, prefixes):
+    prefix = list({i for i in prefixes if text.startswith(prefixes[i])})
+    if len(prefix) > 0:
+        text = text.replace(prefixes[prefix[0]], prefix[0] + ":")
+    return text

src/yatter/source.py

@@ -89,12 +89,12 @@ def add_source_simplified(mapping, source):
     source_extension = os.path.splitext(file_path)[1].replace(".","")
     ref_formulation_rml = YARRRML_REFERENCE_FORMULATIONS[reference_formulation]
 
-    if switch_in_reference_formulation(reference_formulation) != source_extension:
+    if switch_in_reference_formulation(reference_formulation, source_extension) != source_extension:
         raise Exception(
             "ERROR: mismatch extension and referenceFormulation in source " + source + " in mapping " + mapping)
     else:
         if len(source) == 1:  # do not have iterator
-            if source_extension == "csv" or source_extension == "SQL2008":
+            if source_extension == "csv" or source_extension == "SQL2008" or source_extension == "xlsx":
                 source_rdf += '"' + file_path + '"' + ";\n" + "\t\t" + RML_REFERENCE_FORMULATION + " ql:" \
                               + ref_formulation_rml + "\n" + "\t];\n"
             else:
@@ -169,7 +169,7 @@ def database_source(mapping, source, db_identifier):
     return source_rdf
 
 
-def switch_in_reference_formulation(value):
+def switch_in_reference_formulation(value, source_extension=None):
     value = value.lower()
     if "json" in value:
         if "path" in value:
@@ -181,6 +181,11 @@ def switch_in_reference_formulation(value):
             switcher = "xml"
         else:
             switcher = "xpath"
+    elif source_extension:
+        if source_extension == "xlsx":
+            switcher = "xlsx"
+        else:
+            switcher = value
     else:
         switcher = value
     return switcher
@@ -277,7 +282,7 @@ def get_logical_source(logical_source_id, rdf_mapping):
         raise Exception()
 
     if source and reference_formulation and iterator:
-        yarrrml_source =  [source.value + '~' + reference_formulation.toPython().replace(QL_URI, '').lower(), iterator.value]
+        yarrrml_source = [source.value + '~' + reference_formulation.toPython().replace(QL_URI, '').lower(), iterator.value]
     elif source and sql_query:
         # this means a database source
         source_dict = {"query": sql_query.value, "source": source.value}