diff --git a/.github/docker/gen_dockerfile.sh b/.github/docker/gen_dockerfile.sh index 54a470efe..ba382a792 100755 --- a/.github/docker/gen_dockerfile.sh +++ b/.github/docker/gen_dockerfile.sh @@ -7,7 +7,7 @@ # # Arguments: # SYSTEM: system type. Ex: manylinux, musllinux -# SYSTEM_TYPE: Ex: 2010, 2014, _2_24, 2_28 (manylinux), _1_1, _1_2 (musllinux) +# SYSTEM_TYPE: Ex: 2010, 2014, _2_24, _2_28 (manylinux), _1_1, _1_2 (musllinux) # ARCH: Ex: x86_64, i686 set -eu diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 4713a250a..a4d9d3f01 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -1,5 +1,20 @@ name: CI tests +# This CI builds DLite and runs (almost) all tests. +# +# The following plugins are not tested here, since they depends on an external service: +# +# Protocol plugins: +# - sftp +# +# Storage plugins: +# - postgresql +# - mongodb +# - redis +# +# Please remember to update respective plugin docstring if this list changes. +# + on: [push] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index e25972506..9044d7607 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,6 +118,9 @@ if(MSVC) # Do not complain about strdup() and similar POSIX functions to be # deprecated in MSVC add_definitions("-D_CRT_NONSTDC_NO_DEPRECATE") + + # Do not complain about standard library functions + add_definitions("-D_CRT_SECURE_NO_WARNINGS") endif() # Uncomment the lines below to compile with AddressSanitizer @@ -127,13 +130,6 @@ endif() # Install paths # ------------- -# DLite install paths (CMAKE_INSTALL_PREFIX) is prepended to these -set(DLITE_TEMPLATE_DIRS "share/dlite/templates") -set(DLITE_STORAGE_PLUGIN_DIRS "share/dlite/storage-plugins") -set(DLITE_MAPPING_PLUGIN_DIRS "share/dlite/mapping-plugins") -set(DLITE_PYTHON_MAPPING_PLUGIN_DIRS "share/dlite/python-mapping-plugins") -set(DLITE_PYTHON_STORAGE_PLUGIN_DIRS "share/dlite/python-storage-plugins") -set(DLITE_STORAGES "share/dlite/storages") # Install path for CMake files if(WIN32 AND NOT CYGWIN) @@ -159,6 +155,7 @@ include(GNUInstallDirs) # Installation paths set(DLITE_ROOT ${CMAKE_INSTALL_PREFIX}) +set(DLITE_BUILD_ROOT ${dlite_BINARY_DIR}) set(DLITE_INCLUDE_DIRS include/dlite) set(DLITE_LIBRARY_DIR lib) set(DLITE_RUNTIME_DIR bin) @@ -168,6 +165,7 @@ include(MakePlatformPaths) make_platform_paths( PATHS DLITE_ROOT + DLITE_BUILD_ROOT DLITE_INCLUDE_DIRS DLITE_LIBRARY_DIR DLITE_RUNTIME_DIR @@ -231,7 +229,7 @@ if(WITH_PYTHON) set(Python3_USE_STATIC_LIBS TRUE) endif() - # + # Find Python version if(DEFINED ENV{VIRTUAL_ENV}) message(STATUS "Detected virtual environment $ENV{VIRTUAL_ENV}") @@ -267,27 +265,83 @@ if(WITH_PYTHON) endif() unset(CMAKE_CROSSCOMPILING_EMULATOR) + # Find Python installation directory (relative to CMAKE_INSTALL_PREFIX) + if(NOT Python3_PKGDIR) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import site; print(site.getsitepackages()[0])" + OUTPUT_VARIABLE Python3_PKGDIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endif() + + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import sys; print(sys.prefix)" + OUTPUT_VARIABLE Python3_prefix + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + file(RELATIVE_PATH Python3_SITE ${Python3_prefix} ${Python3_PKGDIR}) + if(Python3_SITE) + set(Python3_PREFIX "${Python3_SITE}/dlite/") + else() + set(Python3_PREFIX "dlite/") + endif() + + # Add linker flags for linking against Python + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import sysconfig; print(sysconfig.get_config_var('PY_LDFLAGS'))" + OUTPUT_VARIABLE Python3_LDFLAGS + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(Python3_LDFLAGS) + list(APPEND extra_link_libraries ${Python3_LDFLAGS}) + endif() + + # Find python-config + find_program( + Python3_config + NAMES ${Python3_EXECUTABLE}-config python3-config python-config + HINTS ${Python3_prefix}/bin + ) + # Link libraries when compiling against static Python (e.g. manylinux) if(WITH_STATIC_PYTHON) execute_process( - COMMAND ${Python3_EXECUTABLE}-config --ldflags + COMMAND ${Python3_config} --ldflags --embed OUTPUT_VARIABLE Python3_LDFLAGS OUTPUT_STRIP_TRAILING_WHITESPACE ) - # TODO: find a more portable way to add the "-Xlinker -export-dynamic" options. - # They are needed to ensure that the linker include all symbols in - # the static Python library. + # Linker flags "-Xlinker -export-dynamic" are needed to ensure that + # the linker include all symbols in the static Python library. + include(CheckLinkerFlag) + check_linker_flag(C "-Xlinker -export-dynamic" HAVE_linker_dynexport) + if(HAVE_linker_dynexport) + list(APPEND Python3_LDFLAGS "-Xlinker" "-export-dynamic") + endif() + set(Python3_STATIC_LIBS ${Python3_LDFLAGS} - -Xlinker -export-dynamic ${Python3_LIBRARY} ) - - list(APPEND extra_link_libraries ${Python3_STATIC_LIBS}) + if(Python3_STATIC_LIBS) + list(APPEND extra_link_libraries ${Python3_STATIC_LIBS}) + endif() endif() + # Link libraries when compiling against static Python (e.g. cibuildwheel) + # if(DEFINED ENV{Python3_LIBRARY}) + # set(Python3_STATIC_LIBS $ENV{Python3_LIBRARY}) + # list(APPEND extra_link_libraries $ENV{Python3_LIBRARY}) + # endif() + # message(STATUS "ENV{Python3_LIBRARY}: $ENV{Python3_LIBRARY}") + + message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}") + message(STATUS "Python3_prefix = ${Python3_prefix}") + message(STATUS "Python3_PKGDIR = ${Python3_PKGDIR}") + message(STATUS "Python3_SITE = ${Python3_SITE}") + message(STATUS "Python3_PREFIX = ${Python3_PREFIX}") message(STATUS "Python3_LIBRARIES = ${Python3_LIBRARIES}") message(STATUS "Python3_EXECUTABLE = ${Python3_EXECUTABLE}") message(STATUS "Python3_INCLUDE_DIRS = ${Python3_INCLUDE_DIRS}") @@ -324,9 +378,6 @@ else() set(Python3_LIBRARIES "") endif() -message(STATUS "extra_link_libraries = ${extra_link_libraries}") - - # # Fortran # ======= @@ -341,6 +392,22 @@ if(WITH_FORTRAN) enable_fortran_compiler_flag_if_supported("-Werror") endif() +# Unset extra_link_libraries if it is empty +message(STATUS "extra_link_libraries = ${extra_link_libraries}") +if(extra_link_libraries MATCHES "^[ \t\n\r;]*$") + unset(extra_link_libraries) +endif() + + +# DLite install paths (CMAKE_INSTALL_PREFIX) is prepended to these +set(DLITE_TEMPLATE_DIRS "${Python3_PREFIX}share/dlite/templates") +set(DLITE_STORAGE_PLUGIN_DIRS "${Python3_PREFIX}share/dlite/storage-plugins") +set(DLITE_MAPPING_PLUGIN_DIRS "${Python3_PREFIX}share/dlite/mapping-plugins") +set(DLITE_PYTHON_MAPPING_PLUGIN_DIRS "${Python3_PREFIX}share/dlite/python-mapping-plugins") +set(DLITE_PYTHON_STORAGE_PLUGIN_DIRS "${Python3_PREFIX}share/dlite/python-storage-plugins") +set(DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS "${Python3_PREFIX}share/dlite/python-protocol-plugins") +set(DLITE_STORAGES "${Python3_PREFIX}share/dlite/storages") + # Variables to include in dliteConfig.cmake @@ -364,7 +431,8 @@ if(WITH_PYTHON) Python3_LIBRARIES DLITE_PYTHON_STORAGE_PLUGIN_DIRS DLITE_PYTHON_MAPPING_PLUGIN_DIRS - ) + DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS + ) endif() @@ -611,10 +679,13 @@ list(REMOVE_DUPLICATES dlite_LD_LIBRARY_PATH) set(dlite_PYTHONPATH ${dlite_BINARY_DIR}/bindings/python $ENV{PYTHONPATH} - ) +) if(dlite_PYTHONPATH) list(REMOVE_DUPLICATES dlite_PYTHONPATH) endif() +if(NOT dlite_PYTHONPATH MATCHES "\\\\") + string(REPLACE "\\" "\\\\" dlite_PYTHONPATH ${dlite_PYTHONPATH}) +endif() # DLITE_STORAGE_PLUGIN_DIRS - search path for DLite storage plugins set(dlite_STORAGE_PLUGINS "") @@ -628,35 +699,35 @@ endif() if(WITH_PYTHON) build_append(dlite_STORAGE_PLUGINS ${dlite_BINARY_DIR}/storages/python) endif() -list(REMOVE_DUPLICATES dlite_STORAGE_PLUGINS) # DLITE_MAPPING_PLUGIN_DIRS - search path for DLite mapping plugins set(dlite_MAPPING_PLUGINS "") -list(REMOVE_DUPLICATES dlite_MAPPING_PLUGINS) # DLITE_PYTHON_STORAGE_PLUGIN_DIRS - search path for Python storage plugins set(dlite_PYTHON_STORAGE_PLUGINS ${dlite_SOURCE_DIR}/storages/python/python-storage-plugins - ) -list(REMOVE_DUPLICATES dlite_PYTHON_STORAGE_PLUGINS) +) # DLITE_PYTHON_MAPPING_PLUGIN_DIRS - search path for Python mapping plugins set(dlite_PYTHON_MAPPING_PLUGINS ${dlite_SOURCE_DIR}/bindings/python/python-mapping-plugins - ) -list(REMOVE_DUPLICATES dlite_PYTHON_MAPPING_PLUGINS) +) + +# DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS - search path for Python protocol plugins +set(dlite_PYTHON_PROTOCOL_PLUGINS + ${dlite_SOURCE_DIR}/bindings/python/python-protocol-plugins +) # DLITE_TEMPLATE_DIRS - search path for DLite templates set(dlite_TEMPLATES ${dlite_SOURCE_DIR}/tools/templates - ) -list(REMOVE_DUPLICATES dlite_TEMPLATES) +) # DLITE_STORAGES - DLite storages (inc. metadata) set(dlite_STORAGES ${dlite_SOURCE_DIR}/examples/storages/*.json - ) -list(REMOVE_DUPLICATES dlite_STORAGES) +) + #if(UNIX) # string(REPLACE ";" ":" dlite_PATH "${dlite_PATH}") @@ -666,6 +737,7 @@ list(REMOVE_DUPLICATES dlite_STORAGES) # string(REPLACE ";" ":" dlite_MAPPING_PLUGINS "${dlite_MAPPING_PLUGINS}") # string(REPLACE ";" ":" dlite_PYTHON_STORAGE_PLUGINS "${dlite_PYTHON_STORAGE_PLUGINS}") # string(REPLACE ";" ":" dlite_PYTHON_MAPPING_PLUGINS "${dlite_PYTHON_MAPPING_PLUGINS}") +# string(REPLACE ";" ":" dlite_PYTHON_PROTOCOL_PLUGINS "${dlite_PYTHON_PROTOCOL_PLUGINS}") # string(REPLACE ";" ":" dlite_TEMPLATES "${dlite_TEMPLATES}") #endif() @@ -675,6 +747,7 @@ make_platform_paths( dlite_PYTHONPATH dlite_PYTHON_STORAGE_PLUGINS dlite_PYTHON_MAPPING_PLUGINS + dlite_PYTHON_PROTOCOL_PLUGINS dlite_TEMPLATES dlite_STORAGES MULTI_CONFIG_PATHS @@ -727,6 +800,7 @@ set(test_env "export DLITE_MAPPING_PLUGIN_DIRS=${dlite_MAPPING_PLUGINS}" "export DLITE_PYTHON_STORAGE_PLUGIN_DIRS=${dlite_PYTHON_STORAGE_PLUGINS}" "export DLITE_PYTHON_MAPPING_PLUGIN_DIRS=${dlite_PYTHON_MAPPING_PLUGINS}" + "export DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS=${dlite_PYTHON_PROTOCOL_PLUGINS}" "export DLITE_TEMPLATE_DIRS=${dlite_TEMPLATES}" "export DLITE_STORAGES='${dlite_STORAGES}'" "" diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index b74707010..edbfcc948 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -6,6 +6,7 @@ set(py_sources options.py utils.py mappings.py + protocol.py datamodel.py rdf.py dataset.py @@ -143,11 +144,11 @@ add_custom_target(python_package ALL DEPENDS ${abs_targets} ${package_targets}) add_custom_command( OUTPUT ${abs_targets} + COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir} COMMAND ${CMAKE_COMMAND} -E copy_if_different ${abs_sources} ${pkgdir} DEPENDS - ${pkgdir} ${abs_sources} dlite-plugins-json ) @@ -186,6 +187,7 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/mapping-plugins COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/python-storage-plugins COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/python-mapping-plugins + COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/python-protocol-plugins COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/storages COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/bin COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -210,6 +212,11 @@ add_custom_command( -DDEST_DIR=${pkgdir}/share/dlite/python-mapping-plugins -DPATTERN="*.py" -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake + COMMAND ${CMAKE_COMMAND} + -DSOURCE_DIR=${dlite_SOURCE_DIR}/bindings/python/python-protocol-plugins + -DDEST_DIR=${pkgdir}/share/dlite/python-protocol-plugins + -DPATTERN="*.py" + -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${dlite_SOURCE_DIR}/storages/python/python-storage-plugins -DDEST_DIR=${pkgdir}/share/dlite/storages @@ -240,21 +247,24 @@ else() set(pyext_ext ".pyd") endif() -execute_process(COMMAND - ${RUNNER} ${Python3_EXECUTABLE} -c "import site, pathlib; print(pathlib.Path(site.getusersitepackages()).relative_to(site.getuserbase()).as_posix())" - OUTPUT_VARIABLE Python3_MODULE_PATH - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - install( DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dlite - DESTINATION ${Python3_MODULE_PATH} + DESTINATION "${Python3_SITE}" + USE_SOURCE_PERMISSIONS PATTERN ".gitignore" EXCLUDE PATTERN "*~" EXCLUDE ) install( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python-mapping-plugins - DESTINATION share/dlite + DESTINATION ${Python3_PREFIX}share/dlite + USE_SOURCE_PERMISSIONS + PATTERN ".gitignore" EXCLUDE + PATTERN "*~" EXCLUDE +) +install( + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python-protocol-plugins + DESTINATION ${Python3_PREFIX}share/dlite + USE_SOURCE_PERMISSIONS PATTERN ".gitignore" EXCLUDE PATTERN "*~" EXCLUDE ) diff --git a/bindings/python/dlite-entity-python.i b/bindings/python/dlite-entity-python.i index b5020c30d..7a572c2ef 100644 --- a/bindings/python/dlite-entity-python.i +++ b/bindings/python/dlite-entity-python.i @@ -2,7 +2,11 @@ /* Python-specific extensions to dlite-entity.i */ %pythoncode %{ +import tempfile import warnings +from typing import Sequence +from pathlib import Path +from urllib.parse import urlparse from uuid import UUID import numpy as np @@ -400,22 +404,95 @@ def get_instance( # Allow metaid to be an Instance if isinstance(metaid, Instance): metaid = metaid.uri - return Instance( + inst = Instance( metaid=metaid, dims=dimensions, id=id, dimensions=(), properties=() # arrays must not be None ) + return instance_cast(inst) + + @classmethod + def load( + cls, protocol, driver, location, options=None, id=None, metaid=None + ): + """Load the instance from storage: + + Arguments: + protocol: Name of protocol plugin used for data transfer. + driver: Name of storage plugin for data parsing. + location: Location of resource. Typically a URL or file path. + options: Options passed to the protocol and driver plugins. + id: ID of instance to load. + metaid: If given, the instance is tried mapped to this metadata + before it is returned. + + Return: + Instance loaded from storage. + """ + from dlite.protocol import Protocol + + with Protocol(protocol, location=location, options=options) as pr: + buffer = pr.load(uuid=id) + try: + return cls.from_bytes( + driver, buffer, id=id, options=options, metaid=metaid + ) + except _dlite.DLiteUnsupportedError: + pass + tmpfile = None + try: + with tempfile.NamedTemporaryFile(delete=False) as f: + tmpfile = f.name + f.write(buffer) + inst = cls.from_location( + driver, tmpfile, options=options, id=id, metaid=metaid + ) + finally: + Path(tmpfile).unlink() + return instance_cast(inst) @classmethod def from_url(cls, url, metaid=None): - """Load the instance from `url`. The URL should be of the form - ``driver://location?options#id``. + """Load the instance from `url`. The URL should be of one of the + the following forms: + + driver://location?options#id + protocol+driver://location?options#id + protocol://location?driver=;options#id + + where `protocol`, `driver`, `location`, `options` and `id are + documented in the load() method. + If `metaid` is provided, the instance is tried mapped to this metadata before it is returned. """ - return Instance( - url=url, metaid=metaid, - dims=(), dimensions=(), properties=() # arrays - ) + from dlite.protocol import Protocol + from dlite.options import parse_query + + p = urlparse(url) + if "driver=" in p.query or "+" in p.scheme: + if "driver=" in p.query: + protocol = p.scheme + driver = parse_query(p.query)["driver"] + elif "+" in p.scheme: + protocol, driver = p.scheme.split("+", 1) + elif Path(p.path).suffix: + protocol = p.scheme + driver = Path(p.path).suffix[1:] + else: + raise _dlite.DLiteParseError( + f"cannot infer driver from URL: {url}" + ) + location = f"{protocol}://{p.netloc}{p.path}" + inst = cls.load( + protocol, driver, location, options=p.query, id=p.fragment, + metaid=metaid + ) + else: + inst = Instance( + url=url, metaid=metaid, + dims=(), dimensions=(), properties=() # arrays + ) + return instance_cast(inst) @classmethod def from_storage(cls, storage, id=None, metaid=None): @@ -425,13 +502,16 @@ def get_instance( If `metaid` is provided, the instance is tried mapped to this metadata before it is returned. """ - return Instance( + inst = Instance( storage=storage, id=id, metaid=metaid, dims=(), dimensions=(), properties=() # arrays ) + return instance_cast(inst) @classmethod - def from_location(cls, driver, location, options=None, id=None): + def from_location( + cls, driver, location, options=None, id=None, metaid=None + ): """Load the instance from storage specified by `driver`, `location` and `options`. `id` is the id of the instance in the storage (not required if the storage only contains more one instance). @@ -439,26 +519,30 @@ def get_instance( from dlite.options import make_query if options and not isinstance(options, str): options = make_query(options) - return Instance( + inst = Instance( driver=driver, location=str(location), options=options, id=id, + metaid=metaid, dims=(), dimensions=(), properties=() # arrays ) + return instance_cast(inst) @classmethod def from_json(cls, jsoninput, id=None, metaid=None): """Load the instance from json input.""" - return Instance( + inst = Instance( jsoninput=jsoninput, id=id, metaid=metaid, dims=(), dimensions=(), properties=() # arrays ) + return instance_cast(inst) @classmethod def from_bson(cls, bsoninput): """Load the instance from bson input.""" - return Instance( + inst = Instance( bsoninput=bsoninput, dims=(), dimensions=(), properties=() # arrays ) + return instance_cast(inst) @classmethod def from_dict(cls, d, id=None, single=None, check_storages=True): @@ -486,30 +570,35 @@ def get_instance( New instance. """ from dlite.utils import instance_from_dict - return instance_from_dict( + inst = instance_from_dict( d, id=id, single=single, check_storages=check_storages, ) + return instance_cast(inst) @classmethod - def from_bytes(cls, driver, buffer, id=None, options=None): + def from_bytes(cls, driver, buffer, options=None, id=None, metaid=None): """Load the instance with ID `id` from bytes `buffer` using the given storage driver. """ from dlite.options import make_query if options and not isinstance(options, str): options = make_query(options) - return _from_bytes(driver, buffer, id=id, options=options) + inst = _from_bytes( + driver, buffer, id=id, options=options, metaid=metaid + ) + return instance_cast(inst) @classmethod def create_metadata(cls, uri, dimensions, properties, description): """Create a new metadata entity (instance of entity schema) casted to an instance. """ - return Instance( + inst = Instance( uri=uri, dimensions=dimensions, properties=properties, description=description, dims=() # arrays ) + return instance_cast(inst) @classmethod def create_from_metaid(cls, metaid, dimensions, id=None): @@ -525,10 +614,11 @@ def get_instance( meta = get_instance(metaid) dimensions = [dimensions[dim.name] for dim in meta.properties['dimensions']] - return Instance( + inst = Instance( metaid=metaid, dims=dimensions, id=id, dimensions=(), properties=() # arrays must not be None ) + return instance_cast(inst) @classmethod def create_from_url(cls, url, metaid=None): @@ -556,10 +646,11 @@ def get_instance( warnings.warn( "create_from_storage() is deprecated, use from_storage() instead.", DeprecationWarning, stacklevel=2) - return Instance( + inst = Instance( storage=storage, id=id, metaid=metaid, dims=(), dimensions=(), properties=() # arrays ) + return instance_cast(inst) @classmethod def create_from_location(cls, driver, location, options=None, id=None): @@ -573,29 +664,137 @@ def get_instance( from dlite.options import make_query if options and not isinstance(options, str): options = make_query(options) - return Instance( + inst = Instance( driver=driver, location=str(location), options=options, id=id, dims=(), dimensions=(), properties=() # arrays ) + return instance_cast(inst) - def save(self, dest, location=None, options=None): + def save(self, *dest, location=None, options=None): """Saves this instance to url or storage. Call signatures: + - save(storage) - save(url) - save(driver, location, options=None) - - save(storage) + - save(protocol, driver, location, options=None) + + Arguments: + storage: A dlite.Storage instance to store the instance to. + url: A URL for the storate to store to. + protocol: Name of protocol plugin to use for transferring the + serialised data to `location`. + driver: Name of storage plugin for serialisation. + location: A string describing where to save the instance. + options: Options to the protocol and driver plugins. Should be + a semicolon- or ampersand-separated string of key=value pairs. + + Notes: + The URL may be given in any of the following forms: + + driver://location?options#id + protocol+driver://location?options#id + protocol://location?driver=;options#id + """ - from dlite.options import make_query + from dlite.protocol import Protocol + from dlite.options import make_query, parse_query + if options and not isinstance(options, str): options = make_query(options) + + # Assign arguments from call signature. + # Far too complicated, but ensures backward compatibility. + storage = url = protocol = driver = None if isinstance(dest, Storage): - self.save_to_storage(storage=dest) - elif location: - with Storage(dest, str(location), options) as storage: - storage.save(self) + storage = dest elif isinstance(dest, str): - self.save_to_url(dest) + if location: + driver = dest + else: + url = dest + elif isinstance(dest, Sequence): + if len(dest) == 1: + if isinstance(dest[0], Storage): + storage, = dest + elif location: + driver, = dest + else: + url, = dest + if len(dest) == 2: + if location: + protocol, driver = dest + else: + driver, location = dest + elif len(dest) == 3: + if not location and options is None: + arg1, arg2, arg3 = dest + if arg2 is None and arg3 is None: + url = arg1 + else: + driver, location, options = dest + elif not location: + protocol, driver, location = dest + else: + raise _dlite.DLiteValueError( + "dlite.Instance.save() got `location` both as " + f"positional ({dest[2]}) and keyword ({location}) " + "argument" + ) + elif len(dest) == 4: + if location or options: + raise _dlite.DLiteValueError( + "dlite.Instance.save() got `location` and/or " + "`options` both as positional and keyword arguments" + ) + protocol, driver, location, options = dest + + # Call lower-level save methods + if protocol: + try: + buf = self.to_bytes(driver, options=options) + except (_dlite.DLiteAttributeError, _dlite.DLiteUnsupportedError): + buf = None + if not buf: + try: + with tempfile.NamedTemporaryFile(delete=False) as f: + tmpfile = f.name + self.save(driver, location=tmpfile, options=options) + with open(tmpfile, "rb") as f: + buf = f.read() + finally: + Path(tmpfile).unlink() + with Protocol(protocol, location=location, options=options) as pr: + pr.save(buf) + elif driver: + with Storage(driver, str(location), options) as storage: + storage.save(self) + elif url: + protocol = driver = None + scheme, loc, options, _ = split_url(url) + if "+" in scheme: + protocol, driver = scheme.split("+", 1) + elif scheme in Protocol.loaded_plugins(): + protocol = scheme + else: + driver = scheme + if "driver=" in options: + driver = parse_query(options)["driver"] + elif not driver: + suffix = Path(loc).suffix + if suffix: + driver = suffix[1:] + else: + raise _dlite.DLiteParseError( + f"cannot infer driver from URL: {url}" + ) + if protocol: + self.save(protocol, driver, location=loc, options=options) + else: + opts = f"?{options}" if options else "" + self.save_to_url(f"{driver}://{loc}{opts}") + elif storage: + self.save_to_storage(storage=storage) else: raise _dlite.DLiteTypeError( 'Arguments to save() do not match any of the call signatures' diff --git a/bindings/python/dlite-entity.i b/bindings/python/dlite-entity.i index 23b78a51f..f1d8e731c 100644 --- a/bindings/python/dlite-entity.i +++ b/bindings/python/dlite-entity.i @@ -168,7 +168,7 @@ struct _DLiteProperty { dliteStringPtr, sizeof(char *), $self->shape); } void set_shape(obj_t *arr) { - int i, n = dlite_swig_length(arr); + int i, n = (int)dlite_swig_length(arr); char **new=NULL; if (!(new = calloc(n, sizeof(char *)))) FAIL("allocation failure"); @@ -478,7 +478,7 @@ Call signatures: uint8_t *hashp = NULL; if (hash) { uint8_t data[DLITE_HASH_SIZE]; - if (strhex_decode(data, sizeof(data), hash, strlen(hash)) < 0) { + if (strhex_decode(data, sizeof(data), hash, (int)strlen(hash)) < 0) { dlite_err(1, "cannot decode hash: %s\n", hash); return; } @@ -770,7 +770,7 @@ Returns: char *get_uri() { char *uri; if ($self->uri) return strdup($self->uri); - int n = strlen($self->meta->uri); + int n = (int)strlen($self->meta->uri); if (!(uri = malloc(n + DLITE_UUID_LENGTH + 2))) return dlite_err(dliteMemoryError, "allocation failure"), NULL; memcpy(uri, $self->meta->uri, n); @@ -841,7 +841,8 @@ bool dlite_instance_has_property(struct _DLiteInstance *inst, const char *name); struct _DLiteInstance * dlite_instance_memload(const char *driver, unsigned char *INPUT_BYTES, size_t LEN, - const char *id=NULL, const char *options=NULL); + const char *id=NULL, const char *options=NULL, + const char *metaid=NULL); /* FIXME - how do we avoid duplicating these constants from dlite-schemas.h? */ diff --git a/bindings/python/dlite-macros.i b/bindings/python/dlite-macros.i index c123d95fb..ddd491340 100644 --- a/bindings/python/dlite-macros.i +++ b/bindings/python/dlite-macros.i @@ -8,7 +8,7 @@ /** Macro for getting rid of unused parameter warnings... */ #define UNUSED(x) (void)(x) -/* Turns macro literal `s` into a C string */ +/** Turns macro literal `s` into a C string */ #define STRINGIFY(s) _STRINGIFY(s) #define _STRINGIFY(s) # s @@ -28,4 +28,16 @@ #define FAIL4(msg, a1, a2, a3, a4) do { \ dlite_err(1, msg, a1, a2, a3, a4); goto fail; } while (0) +/** Failure macros with explicit error codes */ +#define FAILCODE(code, msg) do { \ + dlite_err(code, msg); goto fail; } while (0) +#define FAILCODE1(code, msg, a1) do { \ + dlite_err(code, msg, a1); goto fail; } while (0) +#define FAILCODE2(code, msg, a1, a2) do { \ + dlite_err(code, msg, a1, a2); goto fail; } while (0) +#define FAILCODE3(code, msg, a1, a2, a3) do { \ + dlite_err(code, msg, a1, a2, a3); goto fail; } while (0) +#define FAILCODE4(code, msg, a1, a2, a3, a4) do { \ + dlite_err(code, msg, a1, a2, a3, a4); goto fail; } while (0) + %} diff --git a/bindings/python/dlite-path-python.i b/bindings/python/dlite-path-python.i index e6c6928ac..8b900a62d 100644 --- a/bindings/python/dlite-path-python.i +++ b/bindings/python/dlite-path-python.i @@ -6,13 +6,13 @@ def __contains__(self, value): return value in self.aslist() - def __getitem__(self, key): + def __getitem__(self, index): n = len(self) - if key < 0: - key += n - if key < 0 or key >= n: - raise IndexError(f'key out of range: {key}') - return self.getitem(key) + if index < 0: + index += n + if index < 0 or index >= n: + raise IndexError(f'index out of range: {index}') + return self.getitem(index) def __iter__(self): @@ -42,25 +42,29 @@ storage_plugin_path = FUPath("storage-plugins") mapping_plugin_path = FUPath("mapping-plugins") python_storage_plugin_path = FUPath("python-storage-plugins") python_mapping_plugin_path = FUPath("python-mapping-plugins") +python_protocol_plugin_path = FUPath("python-protocol-plugins") # Update default search paths from pathlib import Path pkgdir = Path(__file__).resolve().parent sharedir = pkgdir / "share" / "dlite" -if (sharedir / "storages").exists(): - storage_path[-1] = sharedir / "storages" - #storage_path.append(sharedir / "storages") -if (sharedir / "storage-plugins").exists(): - storage_plugin_path[-1] = sharedir / "storage-plugins" - #storage_plugin_path.append(sharedir / "storage-plugins") -if (sharedir / "mapping-plugins").exists(): - mapping_plugin_path[-1] = sharedir / "mapping-plugins" - #mapping_plugin_path.append(sharedir / "mapping-plugins") -if (sharedir / "python-storage-plugins").exists(): - python_storage_plugin_path[-1] = sharedir / "python-storage-plugins" - #python_storage_plugin_path.append(sharedir / "python-storage-plugins") -if (sharedir / "python-mapping-plugins").exists(): - python_mapping_plugin_path[-1] = sharedir / "python-mapping-plugins" - #python_mapping_plugin_path.append(sharedir / "python-mapping-plugins") +# if (sharedir / "storages").exists(): +# storage_path[-1] = sharedir / "storages" +# #storage_path.append(sharedir / "storages") +# if (sharedir / "storage-plugins").exists(): +# storage_plugin_path[-1] = sharedir / "storage-plugins" +# #storage_plugin_path.append(sharedir / "storage-plugins") +# if (sharedir / "mapping-plugins").exists(): +# mapping_plugin_path[-1] = sharedir / "mapping-plugins" +# #mapping_plugin_path.append(sharedir / "mapping-plugins") +# if (sharedir / "python-storage-plugins").exists(): +# python_storage_plugin_path[-1] = sharedir / "python-storage-plugins" +# #python_storage_plugin_path.append(sharedir / "python-storage-plugins") +# if (sharedir / "python-mapping-plugins").exists(): +# python_mapping_plugin_path[-1] = sharedir / "python-mapping-plugins" +# #python_mapping_plugin_path.append(sharedir / "python-mapping-plugins") +if (sharedir / "python-protocol-plugins").exists(): + #python_protocol_plugin_path[-1] = sharedir / "python-protocol-plugins" + python_protocol_plugin_path.append(sharedir / "python-protocol-plugins") %} diff --git a/bindings/python/dlite-path.i b/bindings/python/dlite-path.i index 13d0b2eb2..2b4f51a92 100644 --- a/bindings/python/dlite-path.i +++ b/bindings/python/dlite-path.i @@ -26,6 +26,7 @@ //#include "dlite-macros.h" #include "pyembed/dlite-python-storage.h" #include "pyembed/dlite-python-mapping.h" +#include "pyembed/dlite-python-protocol.h" const char *platforms[] = {"native", "unix", "windows", "apple"}; %} @@ -56,6 +57,8 @@ Creates a _Path instance of type `pathtype`. return dlite_python_storage_paths(); } else if (strcmp(pathtype, "python-mapping-plugins") == 0) { return dlite_python_mapping_paths(); + } else if (strcmp(pathtype, "python-protocol-plugins") == 0) { + return dlite_python_protocol_paths(); } else { return dlite_err(1, "invalid pathtype: %s", pathtype), NULL; } diff --git a/bindings/python/dlite-python.i b/bindings/python/dlite-python.i index 7a4184c66..f0fa88e32 100644 --- a/bindings/python/dlite-python.i +++ b/bindings/python/dlite-python.i @@ -214,14 +214,14 @@ int dlite_swig_read_python_blob(PyObject *src, uint8_t *dest, size_t n) Py_ssize_t len; const char *s = PyUnicode_AsUTF8AndSize(src, &len); if (!s) FAIL("failed representing string as UTF-8"); - if (strhex_decode(dest, n, s, len) < 0) + if (strhex_decode(dest, n, s, (int)len) < 0) FAIL("cannot convert Python string to blob"); - retval = len/2; + retval = (int)len/2; } else if (PyObject_CheckBuffer(src)) { Py_buffer view; if (PyObject_GetBuffer(src, &view, PyBUF_SIMPLE)) goto fail; memcpy(dest, view.buf, n); - retval = view.len; + retval = (int)(view.len); PyBuffer_Release(&view); } else { FAIL("Only Python types supporting the buffer protocol " @@ -1098,6 +1098,26 @@ int dlite_swig_set_property_by_index(DLiteInstance *inst, int i, obj_t *obj) return status; } + +/* Expose PyRun_File() to python. Returns NULL on error. */ +PyObject *dlite_run_file(const char *path, PyObject *globals, PyObject *locals) +{ + char *basename=NULL; + FILE *fp=NULL; + PyObject *result=NULL; + if (!(basename = fu_basename(path))) + FAILCODE(dliteMemoryError, "cannot allocate path base name"); + if (!(fp = fopen(path, "rt"))) + FAILCODE1(dliteIOError, "cannot open python file: %s", path); + if (!(result = PyRun_File(fp, basename, Py_file_input, globals, locals))) + dlite_err(dlitePythonError, "cannot run python file: %s", path); + + fail: + if (fp) fclose(fp); + if (basename) free(basename); + return result; +} + %} @@ -1409,8 +1429,14 @@ PyObject *dlite_python_mapping_base(void); %rename(errgetexc) dlite_python_module_error; %feature("docstring", "Returns DLite exception corresponding to error code.") dlite_python_module_error; PyObject *dlite_python_module_error(int code); + int _get_number_of_errors(void); +%rename(run_file) dlite_run_file; +%feature("docstring", "Exposing PyRun_File from the Python C API.") dlite_run_file; +PyObject *dlite_run_file(const char *path, PyObject *globals=NULL, PyObject *locals=NULL); + + %pythoncode %{ for n in range(_dlite._get_number_of_errors()): exc = errgetexc(-n) @@ -1419,6 +1445,11 @@ DLiteStorageBase = _dlite._get_storage_base() DLiteMappingBase = _dlite._get_mapping_base() del n, exc + +class DLiteProtocolBase: + """Base class for Python protocol plugins.""" + + def instance_cast(inst, newtype=None): """Return instance converted to a new instance subclass. diff --git a/bindings/python/dlite-storage.i b/bindings/python/dlite-storage.i index c7aced65e..80d498d4d 100644 --- a/bindings/python/dlite-storage.i +++ b/bindings/python/dlite-storage.i @@ -8,6 +8,7 @@ char *_storage_plugin_help(const char *name) { const DLiteStoragePlugin *api = dlite_storage_plugin_get(name); + if (!api) return NULL; if (api->help) return api->help(api); return dlite_err(dliteUnsupportedError, "\"%s\" storage does not support help", name), NULL; diff --git a/bindings/python/paths.py.in b/bindings/python/paths.py.in index 68bfbbb60..079fea3f3 100644 --- a/bindings/python/paths.py.in +++ b/bindings/python/paths.py.in @@ -8,5 +8,6 @@ dlite_STORAGE_PLUGINS = '@dlite_STORAGE_PLUGINS@' dlite_MAPPING_PLUGINS = '@dlite_MAPPING_PLUGINS@' dlite_PYTHON_STORAGE_PLUGINS = '@dlite_PYTHON_STORAGE_PLUGINS@' dlite_PYTHON_MAPPING_PLUGINS = '@dlite_PYTHON_MAPPING_PLUGINS@' +dlite_PYTHON_PROTOCOL_PLUGINS = '@dlite_PYTHON_PROTOCOL_PLUGINS@' dlite_TEMPLATES = '@dlite_TEMPLATES@' dlite_STORAGES = '@dlite_STORAGES@' diff --git a/bindings/python/protocol.py b/bindings/python/protocol.py new file mode 100644 index 000000000..c8dc071c1 --- /dev/null +++ b/bindings/python/protocol.py @@ -0,0 +1,355 @@ +"""Protocol plugins. + +The module also include a few help functions for calling +functions/class methods and using zipfile to transparently +representing a file directory. + +""" +import inspect +import io +import os +import re +import traceback +import warnings +import zipfile +from glob import glob +from pathlib import Path + +import dlite + + +class Protocol(): + """Provides an interface to protocol plugins. + + Arguments: + protocol: Name of protocol. + location: Location of resource. Typically a URL or file path. + options: Options passed to the protocol plugin. + """ + + def __init__(self, protocol, location, options=None): + self.load_plugins() # Load plugins before anything else + + d = {cls.__name__: cls for cls in dlite.DLiteProtocolBase.__subclasses__()} + if protocol not in d: + if protocol in self._failed_plugins: + raise dlite.DLiteProtocolError( + f"protocol plugin failed to load: {protocol}" + ) + else: + msg = [f"no such protocol plugin: {protocol}"] + if True or os.getenv("DLITE_PYDEBUG"): + msg.append("Checked search path:") + for path in dlite.python_protocol_plugin_path: + from glob import glob + msg.append(f"- {path}: {glob(path+'/*')}") + raise dlite.DLiteProtocolError("\n".join(msg)) + + self.conn = d[protocol]() + self.protocol = protocol + self.closed = False + self._call("open", str(location), options=options) + + def close(self): + """Close connection.""" + self._call("close") + self.closed = True + + def load(self, uuid=None): + """Load data from connection and return it as a bytes object.""" + return self._call("load", uuid=uuid) + + def save(self, data, uuid=None): + """Save bytes object `data` to connection.""" + self._call("save", data, uuid=uuid) + + def delete(self, uuid): + """Delete instance with given `uuid` from connection.""" + return self._call("delete", uuid=uuid) + + def query(self, pattern=None): + """Generator method that iterates over all UUIDs in the connection + who"s metadata URI matches glob pattern `pattern`. + + Arguments: + pattern: Glob pattern for matching metadata URIs. + + Yields: + UUIDs of instances who's metadata URI matches glob pattern `pattern`. + If no `pattern` is given, the UUIDs of all instances in the + connection are yielded. + """ + return self._call("query", pattern=pattern) + + # The stem of protocol plugins that failed to load + _failed_plugins = set() + + @classmethod + def load_plugins(cls): + """Load all protocol plugins. + + The names of all plugin files that have been attempted to load + are cached (regardless whether loading succeeded or + not). Hence, it is safe to call this class method multiple + times. It won't reload already loaded plugins or try to + reload plugins that already failed to load. + + """ + + # This should not be needed when PR #953 has been merged + if not hasattr(dlite, "_plugindict"): + dlite._plugindict = {} + + for path in dlite.python_protocol_plugin_path: + if Path(path).is_dir(): + path = f"{Path(path) / '*.py'}" + for filename in glob(path): + name = Path(filename).stem + scopename = f"{name}_protocol" + if (scopename not in dlite._plugindict + and name not in cls._failed_plugins + ): + dlite._plugindict.setdefault(scopename, {}) + scope = dlite._plugindict[scopename] + try: + dlite.run_file(filename, scope, scope) + except Exception as exc: + msg = ( + f"\n{traceback.format_exc()}" + if os.getenv("DLITE_PYDEBUG") + else f": {exc}" + ) + warnings.warn( + f"cannot load protocol plugin: {name}{msg}" + ) + cls._failed_plugins.add(name) + + @classmethod + def loaded_plugins(cls): + """Return a set with the names of already loaded plugins.""" + return set(p.__name__ for p in dlite.DLiteProtocolBase.__subclasses__()) + + @classmethod + def failed_plugins(cls): + """Return a set with the stem of the plugin files that have failed ot + load. This is typically the same as the plugin names. + """ + return cls._failed_plugins.copy() + + def _call(self, method, *args, **kwargs): + """Call given method usin `call()` if it exists.""" + if self.closed: + raise dlite.DLiteIOError( + f"calling closed connection to '{self.protocol}' protocol " + "plugin" + ) + if hasattr(self.conn, method): + return call(getattr(self.conn, method), *args, **kwargs) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + try: + if not self.closed: + self.close() + except Exception: # Ignore exceptions at shutdown + pass + +# Help functions + +def call(func, *args, **kwargs): + """Call a function. Keyword arguments that are None, will not be passed + to `func` if `func` does not have this argument. + """ + param = inspect.signature(func).parameters + + kw = {} + for name, value in kwargs.items(): + if name in param: + kw[name] = value + elif value is not None: + raise KeyError(f"{func} has not keyword argument: {name}") + + return func(*args, **kw) + + +# Functions for zip archive + +zip_compressions = { + "none": zipfile.ZIP_STORED, + "deflated": zipfile.ZIP_DEFLATED, + "bzip2": zipfile.ZIP_BZIP2, + "lzma": zipfile.ZIP_LZMA, +} + + +def load_path( + path, + include=None, + exclude=None, + compression="lzma", + compresslevel=None, +): + """Returns content of path as bytes. + + If `path` is a directory, it will be zipped first. + + Arguments: + path: File-like object or name to a regular file, directory or + socket to load from . + include: Regular expression matching file names to be included + when path is a directory. + exclude: Regular expression matching file names to be excluded + when path is a directory. May exclude files included with + `include`. + compression: Compression to use if path is a directory. + Should be "none", "deflated", "bzip2" or "lzma". + compresslevel: Integer compression level. + For compresison "none" or "lzma"; no effect. + For compresison "deflated"; 0 to 9 are valid. + For compresison "bzip2"; 1 to 9 are valid. + + Returns: + Bytes object with content of `path`. + """ + if isinstance(path, io.IOBase): + return path.read() + + p = Path(path) + if p.is_file(): + return p.read_bytes() + + if p.is_dir(): + incl = re.compile(include) if include else None + excl = re.compile(exclude) if exclude else None + buf = io.BytesIO() + with zipfile.ZipFile( + file=buf, + mode="w", + compression=zip_compressions[compression], + compresslevel=compresslevel, + ) as fzip: + + def iterdir(dirpath): + """Add all files matching regular expressions to zipfile.""" + for p in dirpath.iterdir(): + name = str(p.relative_to(path)) + if (p.is_file() + and (not incl or incl.match(name)) + and (not excl or not excl.match(name)) + ): + fzip.writestr(name, p.read_bytes()) + elif p.is_dir(): + iterdir(p) + + iterdir(p) + return buf.getvalue() + + if p.is_socket(): + buf = [] + while True: + data = path.recv(1024) + if data: + buf.apend(data) + else: + return b"".join(buf) + + +def save_path(data, path, overwrite=False, include=None, exclude=None): + """Save `data` to file path. + + Arguments: + data: Bytes object to save. + path: File-like object or name to a regular file, directory or + socket to save to. + overwrite: Whether to overwrite an existing path. + include: Regular expression matching file names to be included + when path is a directory. + exclude: Regular expression matching file names to be excluded + when path is a directory. May exclude files included with + `include`. + """ + if isinstance(path, io.IOBase): + path.write(data) + else: + p = Path(path) + + buf = io.BytesIO(data) + iszip = zipfile.is_zipfile(buf) + buf.seek(0) + + if iszip: + if p.exists() and not p.is_dir(): + raise ValueError( + f"cannot write zipped data to non-directory path: {path}" + ) + incl = re.compile(include) if include else None + excl = re.compile(exclude) if exclude else None + with zipfile.ZipFile(file=buf, mode="r") as fzip: + for name in fzip.namelist(): + if ((incl and not incl.match(name)) + or (excl and excl.match(name))): + continue + filepath = p / name + filepath.parent.mkdir(parents=True, exist_ok=True) + if filepath.exists(): + if not overwrite: + continue + if not filepath.is_file(): + raise OSError( + "cannot write to existing non-file path: " + f"{filename}" + ) + fzip.extract(name, path=path) + elif p.exists(): + if p.is_socket(): + path.sendall(data) + elif p.is_file(): + if overwrite: + p.write_bytes(data) + else: + raise ValueError( + f"cannot write data to existing non-file path: {path}" + ) + else: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(data) + + +def is_archive(data): + """Returns true if binary data `data` is an archieve, otherwise false.""" + return zipfile.is_zipfile(io.BytesIO(data)) + + +def archive_check(data): + """Raises an exception if `data` is not an archive. Otherwise return + a io.BytesIO object for data.""" + buf = io.BytesIO(data) + if not zipfile.is_zipfile(buf): + raise TypeError("data is not an archieve") + return buf + +def archive_names(data): + """If `data` is an archive, return a list of archive file names.""" + with zipfile.ZipFile(file=archive_check(data), mode="r") as fzip: + return fzip.namelist() + + +def archive_extract(data, name): + """If `data` is an archieve, return object with given name.""" + with zipfile.ZipFile(file=archive_check(data), mode="r") as fzip: + with fzip.open(name, mode="r") as f: + return f.read() + + +def archive_add(data, name, content): + """If `data` is an archieve, return a new bytes object with + `content` added to `data` using the given name.""" + with zipfile.ZipFile(file=archive_check(data), mode="a") as fzip: + with fzip.open(name, mode="w") as f: + f.write(content.encode() if hasattr(content, "encode") else content) + return buf.getvalue() diff --git a/bindings/python/python-protocol-plugins/file.py b/bindings/python/python-protocol-plugins/file.py new file mode 100644 index 000000000..d8608591b --- /dev/null +++ b/bindings/python/python-protocol-plugins/file.py @@ -0,0 +1,106 @@ +"""DLite protocol plugin for files.""" +import re +from pathlib import Path +from urllib.parse import urlparse + +import dlite +from dlite.options import Options +from dlite.protocol import archive_names, is_archive, load_path, save_path + + +class file(dlite.DLiteProtocolBase): + """DLite protocol plugin for files.""" + + def open(self, location, options=None): + """Opens `location`. + + Arguments: + location: A URL or path to a file or directory. + options: Supported options: + - `mode`: Combination of "r" (read), "w" (write) or "a" (append) + Defaults to "r" if `location` exists and "w" otherwise. + This default avoids accidentially overwriting an existing + file. Use "rw" for reading and writing. + - `url`: Whether `location` is an URL. The default is + read it as an URL if it starts with a scheme, otherwise + as a file path. + - `include`: Regular expression matching file names to be + included if `location` is a directory. + - `exclude`: Regular expression matching file names to be + excluded if `location` is a directory. + - `compression`: Compression to use if path is a directory. + Should be "none", "deflated", "bzip2" or "lzma". + - `compresslevel`: Integer compression level. + For compresison "none" or "lzma"; no effect. + For compresison "deflated"; 0 to 9 are valid. + For compresison "bzip2"; 1 to 9 are valid. + """ + opts = Options(options, "compression=lzma") + isurl = dlite.asbool(opts.url) if "url" in opts else bool( + # The last slash in the regex is to avoid matching a Windows + # path C:\... + re.match(r"^[a-zA-Z][a-zA-Z0-9.+-]*:[^\\]", str(location)) + ) + self.path = Path(urlparse(location).path if isurl else location) + self.mode = ( + opts.mode if "mode" in opts + else "r" if self.path.exists() + else "w" + ) + self.options = opts + + def load(self, uuid=None): + """Return data loaded from file. + + If `location` is a directory, it is returned as a zip archive. + """ + self._required_mode("r", "load") + path = self.path/uuid if uuid else self.path + return load_path( + path=path, + include=self.options.get("include"), + exclude=self.options.get("exclude"), + compression=self.options.get("compression"), + compresslevel=self.options.get("compresslevel"), + ) + + def save(self, data, uuid=None): + """Save `data` to file.""" + self._required_mode("wa", "save") + path = self.path/uuid if uuid else self.path + save_path( + data=data, + path=path, + overwrite=True if "w" in self.mode else False, + include=self.options.get("include"), + exclude=self.options.get("exclude"), + ) + + def delete(self, uuid): + """Delete instance with given `uuid`.""" + self._required_mode("w", "delete") + path = self.path/uuid + if path.exists(): + path.unlink() + elif self.path.exists() and self.path.is_dir(): + save_path( + data=load_path(self.path, exclude=uuid), + path=path, + overwrite=True, + ) + else: + raise dlite.DLiteIOError(f"cannot delete {uuid} in: {self.path}") + + def query(self): + """Iterator over all filenames in the directory referred to by + `location`. If `location` is a file, return its name.""" + data = self.load() + return archive_names(data) if is_archive(data) else self.path.name + + def _required_mode(self, required, operation): + """Raises DLiteIOError if mode does not contain any of the mode + letters in `required`.""" + if not any(c in self.mode for c in required): + raise dlite.DLiteIOError( + f"mode='{self.mode}', cannot {operation}: {self.path}" + ) diff --git a/bindings/python/python-protocol-plugins/http.py b/bindings/python/python-protocol-plugins/http.py new file mode 100644 index 000000000..0da7e026e --- /dev/null +++ b/bindings/python/python-protocol-plugins/http.py @@ -0,0 +1,52 @@ +"""DLite protocol plugin for http connections.""" +from urllib.parse import urlparse + +import requests + +import dlite +from dlite.options import Options + + +class http(dlite.DLiteProtocolBase): + """DLite protocol plugin for http and https connections.""" + + def open(self, location, options=None): + """Opens `location`. + + Arguments: + location: A URL or path to a file or directory. + options: Options will be passed as keyword arguments to + requests. For available options, see + https://requests.readthedocs.io/en/latest/api/#requests.request + A timeout of 1 second will be added by default. + """ + self.location = location + self.options = Options(options, "timeout=1") + + def load(self, uuid=None): + """Return data loaded from file. + + If `location` is a directory, it is returned as a zip archive. + """ + kw = {"params": uuid} + kw.update(self.options) + timeout = float(kw.pop("timeout")) + r = requests.get(self.location, timeout=timeout, **kw) + r.raise_for_status() + return r.content + + def save(self, data, uuid=None): + """Save `data` to file.""" + kw = {"params": uuid} + kw.update(self.options) + timeout = float(kw.pop("timeout")) + r = requests.post(self.location, data=data, timeout=timeout, **kw) + r.raise_for_status() + + def delete(self, uuid): + """Delete instance with given `uuid`.""" + kw = {"params": uuid} + kw.update(self.options) + timeout = float(kw.pop("timeout")) + r = requests.delete(self.location, timeout=timeout, **kw) + r.raise_for_status() diff --git a/bindings/python/python-protocol-plugins/sftp.py b/bindings/python/python-protocol-plugins/sftp.py new file mode 100644 index 000000000..e7a9c1ceb --- /dev/null +++ b/bindings/python/python-protocol-plugins/sftp.py @@ -0,0 +1,203 @@ +"""DLite protocol plugin for sftp. + +Note: Continous testing is not run for this plugin. +""" + +import io +import stat +import zipfile +from pathlib import Path +from urllib.parse import urlparse + +import paramiko + +import dlite +from dlite.options import Options +from dlite.protocol import zip_compressions + + +class sftp(dlite.DLiteProtocolBase): + """DLite protocol plugin for sftp.""" + + def open(self, location, options=None): + """Opens `location`. + + Arguments: + location: SFTP host name. May be just the host name or fully + qualified as `username:password@host:port`. In the latter + case the port/username/password takes precedence over `options`. + options: Supported options: + - `username`: User name. + - `password`: Password. + - `hostname`: Host name. + - `port`: Port number. Default is 22. + - `key_type`: Key type for key-based authorisation, ex: + "ssh-ed25519" + - `key_bytes`: Hex-encoded key bytes for key-based + authorisation. + - `include`: Regular expression matching file names to be + included if `location` is a directory. + - `exclude`: Regular expression matching file names to be + excluded if `location` is a directory. + - `compression`: Zip compression method. One of "none", + "deflated" (default), "bzip2" or "lzma". + - `compresslevel`: Integer compression level. + For compresison "none" or "lzma"; no effect. + For compresison "deflated"; 0 to 9 are valid. + For compresison "bzip2"; 1 to 9 are valid. + + Example: + + # For key-based authorisation, you may get the `key_type` and + # `key_bytes` arguments as follows: + pkey = paramiko.Ed25519Key.from_private_key_file( + "/home/john/.ssh/id_ed25519" + ) + key_type = pkey.name + key_bytes = pkey.asbytes().hex() + + """ + options = Options(options, "port=22;compression=lzma") + + p = urlparse(location) + if not p.scheme: + p = urlparse("sftp://" + location) + username = p.username if p.username else options.pop("username", None) + password = p.password if p.password else options.pop("password", None) + hostname = p.hostname if p.hostname else options.get("hostname") + port = p.port if p.port else int(options.pop("port")) + + transport = paramiko.Transport((hostname, port)) + + if "key_type" in options and "key_bytes" in options: + pkey = paramiko.PKey.from_type_string( + options.key_type, bytes.fromhex(options.key_bytes) + ) + transport.connect(username=username, pkey=pkey) + elif username and password: + transport.connect(username=username, password=password) + else: + transport.connect() + + self.options = options + self.client = paramiko.SFTPClient.from_transport(transport) + self.transport = transport + self.path = p.path + + def close(self): + """Close the connection.""" + self.client.close() + self.transport.close() + + def load(self, uuid=None): + """Load data from remote location and return as a bytes object. + + If the remote location is a directory, all its files are + stored in a zip object and then the bytes content of the zip + object is returned. + """ + path = f"{self.path.rstrip('/')}/{uuid}" if uuid else self.path + opts = self.options + s = self.client.stat(path) + if stat.S_ISDIR(s.st_mode): + incl = re.compile(opts.include) if "include" in opts else None + excl = re.compile(opts.exclude) if "exclude" in opts else None + buf = io.BytesIO() + with zipfile.ZipFile( + file=buf, + mode="w", + compression=zip_compressions[opts.compression], + compresslevel=opts.get("compresslevel"), + ) as fzip: + + def iterdir(dirpath): + """Add all files matching regular expressions.""" + for entry in self.client.listdir_attr(dirpath): + fullpath = Path(dirpath) / entry.filename + name = str(fullpath.relative_to(path)) + if (stat.S_ISREG(entry.st_mode) + and (not incl or incl.match(name)) + and (not excl or not excl.match(name)) + ): + fzip.writestr(name, entry.asbytes()) + elif stat.S_ISDIR(entry.st_mode): + iterdir(str(fullpath)) + + iterdir(path) + data = buf.getvalue() + elif stat.S_ISREG(s.st_mode): + with self.client.open(path, mode="r") as f: + data = f.read() + else: + raise TypeError( + "remote path must either be a directory or a regular " + f"file: {path}" + ) + return data + + def save(self, data, uuid=None): + """Save bytes object `data` to remote location.""" + path = f"{self.path.rstrip('/')}/{uuid}" if uuid else self.path + opts = self.options + + buf = io.BytesIO(data) + iszip = zipfile.is_zipfile(buf) + buf.seek(0) + + if iszip: + incl = re.compile(opts.include) if "include" in opts else None + excl = re.compile(opts.exclude) if "exclude" in opts else None + with zipfile.ZipFile(file=buf, mode="r") as fzip: + for name in fzip.namelist(): + if ((incl and not incl.match(name)) + or (excl and excl.match(name))): + continue + with fzip.open(name, mode="r") as f: + pathname = f"{path}/{name}" + self._create_missing_dirs(pathname) + self.client.putfo(f, pathname) + else: + self._create_missing_dirs(path) + self.client.putfo(buf, path) + + def _create_missing_dirs(self, path): + """Create missing directories in `path` on remote host.""" + p = Path(path).parent + missing_dirs = [] + while True: + try: + self.client.stat(str(p)) + except FileNotFoundError: + missing_dirs.append(p.parts[-1]) + p = p.parent + else: + break + + for dir in reversed(missing_dirs): + p = p / dir + self.client.mkdir(str(p)) + + def delete(self, uuid=None): + """Delete instance with given `uuid`.""" + path = f"{self.path.rstrip('/')}/{uuid}" if uuid else self.path + self.client.remove(path) + + def query(self, pattern=None): + """Generator over all filenames in the directory referred to by + `location`. If `location` is a file, return its name. + + Arguments: + pattern: Glob pattern for matching metadata URIs. Unused. + """ + s = self.client.stat(self.path) + if stat.S_ISDIR(s.st_mode): + for entry in self.client.listdir_attr(self.path): + if stat.S_ISREG(entry): + yield entry.filename + elif stat.S_ISREG(s.st_mode): + yield self.path + else: + raise TypeError( + "remote path must either be a directory or a regular " + f"file: {self.path}" + ) diff --git a/bindings/python/quantity.py b/bindings/python/quantity.py old mode 100755 new mode 100644 diff --git a/bindings/python/scripts/CMakeLists.txt b/bindings/python/scripts/CMakeLists.txt index 1ef15e3ed..bb560f977 100644 --- a/bindings/python/scripts/CMakeLists.txt +++ b/bindings/python/scripts/CMakeLists.txt @@ -28,12 +28,14 @@ endfunction() # Install install( - CODE " - execute_process( - COMMAND ${Python3_EXECUTABLE} -m pip install - --prefix=${CMAKE_INSTALL_PREFIX} --force . - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - )" + PROGRAMS dlite-validate + DESTINATION bin + #CODE " + # execute_process( + # COMMAND ${Python3_EXECUTABLE} -m pip install + # --prefix=${CMAKE_INSTALL_PREFIX} --force . + # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + # )" ) diff --git a/bindings/python/tests/CMakeLists.txt b/bindings/python/tests/CMakeLists.txt index ef0da2b03..7166eb388 100644 --- a/bindings/python/tests/CMakeLists.txt +++ b/bindings/python/tests/CMakeLists.txt @@ -27,8 +27,10 @@ set(tests test_iri test_dataset1_save test_dataset2_load + test_protocol test_isolated_plugins test_options + test_plugin ) foreach(test ${tests}) diff --git a/bindings/python/tests/input/blob.json b/bindings/python/tests/input/blob.json new file mode 100644 index 000000000..3e401848b --- /dev/null +++ b/bindings/python/tests/input/blob.json @@ -0,0 +1,11 @@ +{ + "9d4e0a82-a8ca-41c1-9db3-84908ebf6041": { + "meta": "http://onto-ns.com/meta/0.1/Blob", + "dimensions": { + "n": 3 + }, + "properties": { + "content": [97, 98, 99] + } + } +} \ No newline at end of file diff --git a/bindings/python/tests/input/subdir/blob1.json b/bindings/python/tests/input/subdir/blob1.json new file mode 100644 index 000000000..c3db11dd7 --- /dev/null +++ b/bindings/python/tests/input/subdir/blob1.json @@ -0,0 +1,12 @@ +{ + "de191839-2940-56aa-9fe8-47fa2151a100": { + "uri": "blob1", + "meta": "http://onto-ns.com/meta/0.1/Blob", + "dimensions": { + "n": 3 + }, + "properties": { + "content": [97, 98, 99] + } + } +} diff --git a/bindings/python/tests/input/subdir/blob2.json b/bindings/python/tests/input/subdir/blob2.json new file mode 100644 index 000000000..6470b6e72 --- /dev/null +++ b/bindings/python/tests/input/subdir/blob2.json @@ -0,0 +1,12 @@ +{ + "8255c571-727f-5bcc-adfd-846b73873bd9": { + "uri": "blob2", + "meta": "http://onto-ns.com/meta/0.1/Blob", + "dimensions": { + "n": 3 + }, + "properties": { + "content": [97, 98, 99] + } + } +} diff --git a/bindings/python/tests/plugins/bufftest.py b/bindings/python/tests/plugins/bufftest.py new file mode 100644 index 000000000..0d5b23534 --- /dev/null +++ b/bindings/python/tests/plugins/bufftest.py @@ -0,0 +1,17 @@ +"""Test plugin that only defines to_bytes() and from_bytes().""" + +import dlite + + +class bufftest(dlite.DLiteStorageBase): + """Test plugin that represents instances as byte-encoded json.""" + + @classmethod + def to_bytes(cls, inst, options=None): + """Returns instance as bytes.""" + return str(inst).encode() + + @classmethod + def from_bytes(cls, buffer, id=None, options=None): + """Load instance from buffer.""" + return dlite.Instance.from_json(buffer.decode()) diff --git a/bindings/python/tests/test_plugin.py b/bindings/python/tests/test_plugin.py new file mode 100644 index 000000000..8d374c44c --- /dev/null +++ b/bindings/python/tests/test_plugin.py @@ -0,0 +1,37 @@ +"""Test storages.""" +from pathlib import Path + +import numpy as np + +import dlite +from dlite.testutils import importcheck, raises +from dlite.protocol import Protocol, archive_extract + +yaml = importcheck("yaml") +requests = importcheck("requests") + + +thisdir = Path(__file__).absolute().parent +outdir = thisdir / "output" +indir = thisdir / "input" +entitydir = thisdir / "entities" +plugindir = thisdir / "plugins" +dlite.storage_path.append(entitydir / "*.json") +dlite.python_storage_plugin_path.append(plugindir) + + +# Test plugin that only defines to_bytes() and from_bytes() +dlite.Storage.plugin_help("bufftest") +inst = dlite.Instance.from_metaid( + "http://onto-ns.com/meta/0.1/Blob", dimensions=[3] +) +inst.content = b"abc" +buf = inst.to_bytes("bufftest") + +# Change ID in binary representation before creating a new instance +buf2 = bytearray(buf) +buf2[5:9] = b'0123' +inst2 = dlite.Instance.from_bytes("bufftest", buf2) +assert inst2.uuid != inst.uuid +assert inst2.dimensions == inst.dimensions +assert all(inst2.content == inst.content) diff --git a/bindings/python/tests/test_protocol.py b/bindings/python/tests/test_protocol.py new file mode 100644 index 000000000..b56838564 --- /dev/null +++ b/bindings/python/tests/test_protocol.py @@ -0,0 +1,153 @@ +"""Test protocol plugins.""" + +import getpass +import json +import os +import shutil +from pathlib import Path + +import dlite +from dlite.protocol import ( + Protocol, + load_path, + save_path, + is_archive, archive_names, + archive_extract, + archive_add, +) +from dlite.testutils import importcheck, raises + +yaml = importcheck("yaml") +requests = importcheck("requests") +paramiko = importcheck("paramiko") + + +thisdir = Path(__file__).resolve().parent +outdir = thisdir / "output" / "protocol" +indir = thisdir / "input" + + +# Test load_path() and save_path() +# -------------------------------- +data1 = load_path(indir / "coll.json") +save_path(data1, outdir / "coll.json", overwrite=True) +assert is_archive(data1) is False + +data2 = load_path(indir, exclude=r".*(~|\.bak)$") +save_path(data2, outdir / "indir", overwrite=True, include=r".*\.json$") +assert is_archive(data2) is True +assert "rdf.ttl" in archive_names(data2) +assert archive_extract(data2, "coll.json") == data1 + + +# Test file plugin +# ---------------- +# Test save/load files via the Protocol class +outfile = outdir / "hello.txt" +outfile.unlink(missing_ok=True) +pr = Protocol(protocol="file", location=outfile, options="mode=rw") +pr.save(b"hello world") +assert pr.load() == b"hello world" +assert pr.query() == "hello.txt" +pr.close() +with raises(dlite.DLiteIOError): # double-close raises an DLiteIOError + pr.close() + +# Test loading with dlite.Instance.load() +b = dlite.Instance.load("file", "json", indir/"blob.json") +assert b.content.tolist() == [97, 98, 99] + +# Test saving with dlite.Instance.save() +b.save("file", "json", outdir/"blob.json", "mode=w") + +# Test load directory +# Since a directory plugin would be application-specific, we call +# Protocol explicitly here +with Protocol("file", indir/"subdir") as pr: + data = pr.load() +data1 = archive_extract(data, "blob1.json") +data2 = archive_extract(data, "blob2.json") +b1 = dlite.Instance.from_bytes("json", data1) +b2 = dlite.Instance.from_bytes("json", data2) +assert b1.uri == "blob1" +assert b2.uri == "blob2" +assert b1.content.tolist() == [97, 98, 99] +assert b2.content.tolist() == [97, 98, 99] + +# Test save directory +shutil.rmtree(outdir/"subdir", ignore_errors=True) +with Protocol("file", outdir/"subdir", options={"mode": "w"}) as pr: + pr.save(data) +assert (outdir/"subdir"/"blob1.json").exists() +assert (outdir/"subdir"/"blob2.json").exists() +assert (outdir/"subdir"/"blob1.json").read_bytes() == data1 +assert (outdir/"subdir"/"blob2.json").read_bytes() == data2 + + +# Test http plugin +# ---------------- +if requests: + url = ( + "https://raw.githubusercontent.com/SINTEF/dlite/refs/heads/master/" + "examples/entities/aa6060.json" + ) + with Protocol(protocol="http", location=url) as pr: + s = pr.load() + d = json.loads(s) + assert d["25a1d213-15bb-5d46-9fcc-cbb3a6e0568e"]["uri"] == "aa6060" + +if requests and yaml: + url = ( + "https://raw.githubusercontent.com/SINTEF/dlite/refs/heads/master/" + "storages/python/tests-python/input/test_meta.yaml" + ) + m = dlite.Instance.load(protocol="http", driver="yaml", location=url) + assert m.uri == "http://onto-ns.com/meta/0.1/TestEntity" + assert type(m) is dlite.Metadata + + +# Test sftp plugin +# ---------------- +if False: + host = os.getenv("SFTP_HOST", "localhost") + port = os.getenv("SFTP_PORT", "22") + username = os.getenv("SFTP_USERNAME", getpass.getuser()) + password = os.getenv("SFTP_PASSWORD") # + pkpassword = os.getenv("SFTP_PKPASSWORD") # Private key password + keytype = os.getenv("SFTP_KEYTYPE", "ed25519") # Private key type + if paramiko and (password or pkpassword): + options = {"port": int(port), "username": username} + if pkpassword: + pkey = paramiko.Ed25519Key.from_private_key_file( + f"/home/{username}/.ssh/id_{keytype}", + password=pkpassword, + ) + options["pkey_type"] = pkey.name + options["pkey_bytes"] = pkey.asbytes().hex() + else: + options["password"] = password + + b.save( + "sftp", "json", location=f"{host}/tmp/blob.json", options=options + ) + + + + #v = con.load() + #assert v.decode().startswith("[SemImageFile]") + # + #con = Protocol( + # protocol="sftp", + # location=f"{host}/P_MATCHMACKER_SHARE/SINTEF/test", + # options=f"port={port};username={username};password={password}", + #) + #uuid = dlite.get_uuid() + #con.save(b"hello world", uuid=uuid) + #val = con.load(uuid=uuid) + #assert val.decode() == "hello world" + #con.delete(uuid=uuid) + # + #if False: # Don't polute sftp server + # con.save(data2, uuid="data2") + # data3 = con.load(uuid="data2") + # save_path(data3, outdir/"data3", overwrite=True) diff --git a/bindings/python/tests/test_storage.py b/bindings/python/tests/test_storage.py index 8742c472b..a986b0bac 100755 --- a/bindings/python/tests/test_storage.py +++ b/bindings/python/tests/test_storage.py @@ -1,23 +1,23 @@ -#!/usr/bin/env python +"""Test storages.""" from pathlib import Path import numpy as np import dlite -from dlite.testutils import raises +from dlite.testutils import importcheck, raises +from dlite.protocol import Protocol, archive_extract -try: - import yaml - HAVE_YAML = True -except ModuleNotFoundError: - HAVE_YAML = False +yaml = importcheck("yaml") +requests = importcheck("requests") thisdir = Path(__file__).absolute().parent outdir = thisdir / "output" indir = thisdir / "input" entitydir = thisdir / "entities" +plugindir = thisdir / "plugins" dlite.storage_path.append(entitydir / "*.json") +dlite.python_storage_plugin_path.append(plugindir) url = f"json://{entitydir}/MyEntity.json" @@ -98,9 +98,6 @@ # FIXME: Add test for the `arrays`, `no-parent` and `compact` options. # Should we rename `arrays` to `soft7` for consistency with the Python API? -with raises(dlite.DLiteValueError): - inst.to_bytes("json", options="invalid-opt=").decode() - # Test json print("--- testing json") @@ -114,7 +111,7 @@ # Test yaml -if HAVE_YAML: +if yaml: print('--- testing yaml') inst.save(f"yaml://{outdir}/test_storage_inst.yaml?mode=w") del inst @@ -194,7 +191,7 @@ # Tests for issue #587 -if HAVE_YAML: +if yaml: bytearr = inst.to_bytes("yaml") #print(bytes(bytearr).decode()) @@ -225,7 +222,28 @@ for i in range(6): print(iter2.next()) -print("==========") -for i in range(5): - print(iter.next()) +# Test URL versions of dlite.Instance.save() +Blob = dlite.get_instance("http://onto-ns.com/meta/0.1/Blob") +blob = Blob([3], id="myblob") +blob.content = b'abc' +blob.save(f"file+json://{outdir}/blob1.json?mode=w") +blob.save(f"file://{outdir}/blob2.txt?driver=json;mode=w") +blob.save(f"file://{outdir}/blob3.json?mode=w") +blob.save(f"json://{outdir}/blob4.json?mode=w") +t1 = (outdir/"blob1.json").read_text() +t2 = (outdir/"blob2.txt").read_text() +t3 = (outdir/"blob3.json").read_text() +t4 = (outdir/"blob4.json").read_text() +assert t2 == t1 +assert t3 == t1 +assert ( + t4.replace(" ", "").replace("\n", "") == + t1.replace(" ", "").replace("\n", "") +) + + +# Test plugin that only defines to_bytes() and from_bytes() +#print("===================================") +#dlite.Storage.plugin_help("testbuff") +#buf = inst.to_bytes("bufftest") diff --git a/bindings/python/testutils.py b/bindings/python/testutils.py index 0612bfa78..4cc6fefa3 100644 --- a/bindings/python/testutils.py +++ b/bindings/python/testutils.py @@ -62,13 +62,22 @@ def __exit__(self, exc_type, exc_value, tb): ) from exc_value -def importskip(module_name, exitcode=44): +def importcheck(module_name, package=None): + """Import and return the requested module or None if the module can't + be imported.""" + try: + return importlib.import_module(module_name, package=package) + except ModuleNotFoundError as exc: + return None + + +def importskip(module_name, package=None, exitcode=44): """Import and return the requested module. Calls `sys.exit()` with given exitcode if the module cannot be imported. """ try: - return importlib.import_module(module_name) + return importlib.import_module(module_name, package=package) except ModuleNotFoundError as exc: print(f"{exc}: skipping test", file=sys.stderr) sys.exit(exitcode) diff --git a/cmake/dliteGenEnv-sub.cmake b/cmake/dliteGenEnv-sub.cmake index 72c5396ac..366f99997 100644 --- a/cmake/dliteGenEnv-sub.cmake +++ b/cmake/dliteGenEnv-sub.cmake @@ -6,15 +6,16 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${dlite_SOURCE_DIR}/cmake) # Convert from pipe-separated paths to lists set(dlite_BUILD_ROOT ${dlite_BINARY_DIR}) -string(REPLACE "|" ";" dlite_PATH "${dlite_PATH}") -string(REPLACE "|" ";" dlite_LD_LIBRARY_PATH "${dlite_LD_LIBRARY_PATH}") -string(REPLACE "|" ";" dlite_PYTHONPATH "${dlite_PYTHONPATH}") -string(REPLACE "|" ";" dlite_STORAGE_PLUGINS "${dlite_STORAGE_PLUGINS}") -string(REPLACE "|" ";" dlite_MAPPING_PLUGINS "${dlite_MAPPING_PLUGINS}") -string(REPLACE "|" ";" dlite_PYTHON_STORAGE_PLUGINS "${dlite_PYTHON_STORAGE_PLUGINS}") -string(REPLACE "|" ";" dlite_PYTHON_MAPPING_PLUGINS "${dlite_PYTHON_MAPPING_PLUGINS}") -string(REPLACE "|" ";" dlite_TEMPLATES "${dlite_TEMPLATES}") -string(REPLACE "|" ";" dlite_STORAGES "${dlite_STORAGES}") +string(REPLACE "|" ";" dlite_PATH "${dlite_PATH}") +string(REPLACE "|" ";" dlite_LD_LIBRARY_PATH "${dlite_LD_LIBRARY_PATH}") +string(REPLACE "|" ";" dlite_PYTHONPATH "${dlite_PYTHONPATH}") +string(REPLACE "|" ";" dlite_STORAGE_PLUGINS "${dlite_STORAGE_PLUGINS}") +string(REPLACE "|" ";" dlite_MAPPING_PLUGINS "${dlite_MAPPING_PLUGINS}") +string(REPLACE "|" ";" dlite_PYTHON_STORAGE_PLUGINS "${dlite_PYTHON_STORAGE_PLUGINS}") +string(REPLACE "|" ";" dlite_PYTHON_MAPPING_PLUGINS "${dlite_PYTHON_MAPPING_PLUGINS}") +string(REPLACE "|" ";" dlite_PYTHON_PROTOCOL_PLUGINS "${dlite_PYTHON_PROTOCOL_PLUGINS}") +string(REPLACE "|" ";" dlite_TEMPLATES "${dlite_TEMPLATES}") +string(REPLACE "|" ";" dlite_STORAGES "${dlite_STORAGES}") # Add platform-specific paths @@ -28,6 +29,7 @@ make_platform_paths( dlite_PYTHONPATH dlite_PYTHON_STORAGE_PLUGINS dlite_PYTHON_MAPPING_PLUGINS + dlite_PYTHON_PROTOCOL_PLUGINS dlite_TEMPLATES MULTI_CONFIG_PATHS dlite_PATH diff --git a/cmake/dliteGenEnv.cmake b/cmake/dliteGenEnv.cmake index ef90e4394..f7ffe6b2f 100644 --- a/cmake/dliteGenEnv.cmake +++ b/cmake/dliteGenEnv.cmake @@ -33,6 +33,7 @@ function(dlite_genenv output newline_style) string(REPLACE ";" "|" MAPPING_PLUGINS "${dlite_MAPPING_PLUGINS}") string(REPLACE ";" "|" PYTHON_STORAGE_PLUGINS "${dlite_PYTHON_STORAGE_PLUGINS}") string(REPLACE ";" "|" PYTHON_MAPPING_PLUGINS "${dlite_PYTHON_MAPPING_PLUGINS}") + string(REPLACE ";" "|" PYTHON_PROTOCOL_PLUGINS "${dlite_PYTHON_PROTOCOL_PLUGINS}") string(REPLACE ";" "|" TEMPLATES "${dlite_TEMPLATES}") @@ -54,6 +55,7 @@ function(dlite_genenv output newline_style) -Ddlite_MAPPING_PLUGINS="${MAPPING_PLUGINS}" -Ddlite_PYTHON_STORAGE_PLUGINS="${PYTHON_STORAGE_PLUGINS}" -Ddlite_PYTHON_MAPPING_PLUGINS="${PYTHON_MAPPING_PLUGINS}" + -Ddlite_PYTHON_PROTOCOL_PLUGINS="${PYTHON_PROTOCOL_PLUGINS}" -Ddlite_TEMPLATES="${TEMPLATES}" -Ddlite_STORAGES="${dlite_STORAGES}" -P ${script} diff --git a/doc/_static/storage-protocol.svg b/doc/_static/storage-protocol.svg new file mode 100644 index 000000000..055de0b2c --- /dev/null +++ b/doc/_static/storage-protocol.svg @@ -0,0 +1,4 @@ + + + +
Storage
Storage
Protocol plugin
Protocol p...
Storage plugin
Storage pl...
Encoded data
Encoded da...
Data transfer
Data trans...
DLite instance
DLite inst...
Application
Application
Serialise/parse DLite instance to/from encoded data 
Serialise/parse DLit...
Transfer data to/from remote storage
Transfer data to/from r...
json, bson, rdf, ...
json, bson, rdf, ...
file, http, sftp, ...
file, http, sftp, ...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/doc/user_guide/index.rst b/doc/user_guide/index.rst index 989eb6ec8..c6ef982c6 100644 --- a/doc/user_guide/index.rst +++ b/doc/user_guide/index.rst @@ -8,13 +8,14 @@ User Guide concepts type-system exceptions - mappings collections + storage_plugins + storage_plugins_mongodb + protocol_plugins + mappings transactions tools code_generation - storage_plugins - storage_plugins_mongodb configure_behavior_changes environment_variables features diff --git a/doc/user_guide/protocol_plugins.md b/doc/user_guide/protocol_plugins.md new file mode 100644 index 000000000..c930a6ec6 --- /dev/null +++ b/doc/user_guide/protocol_plugins.md @@ -0,0 +1,47 @@ +Protocol plugins +================ +Protocol plugins is a new feature of DLite that allow a clear separation between serialising/parsing DLite instances to/from external data representations and transferring data to/from external data resources. +This is illustrated in the Figure below. + +![DLite storage and protocol plugins.](../../_static/storage-protocol.svg) + +It allows to mix and match different protocol and storage plugins, thereby reducing the total number of plugins that has to be implemented. + +For example can you combine a *bson* storage plugin with the *file*, *http* or *sftp* protocols to load an instance stored as BSON, from a file, a website or a SFTP server, respectively. + +The `dlite.Instance` Python class, provides a simple interface to protocol and storage plugins via the `dlite.Instance.load()` class method and the `dlite.Instance.save()` method. + + +**Examples** + +Accessing a YAML file using the *http* protocol: + +```python + >>> url = "https://raw.githubusercontent.com/SINTEF/dlite/refs/heads/master/storages/python/tests-python/input/test_meta.yaml" + >>> dlite.Instance.load(protocol="http", driver="yaml", location=url) + +``` + +Directly access to protocols +---------------------------- +Protocol plugins can also be accessed in Python via the `Protocol` class. + +In the example below the data `b"hello world"` is first saved to the file `hello.txt` and then loaded back again: +```python + >>> with Protocol(protocol="file", location="hello.txt", options="mode=rw") as pr: + ... pr.save(b"hello world") + ... s = pr.load() + >>> s + b'hello world' + +``` + +**Note** that all data are bytes objects. + + +Creating protocol plugins +------------------------- +Currently, DLite is shipped with the following protocols: *file*, *http*, *sftp*. +New protocols will be available if the path to the directory is added to the `dlite.python_protocol_plugin` path variable. + +Please see the existing plugins, for how to implement a new plugin. diff --git a/doc/user_guide/storage_plugin.py b/doc/user_guide/storage_plugin.py index e4955543f..e015670f2 100644 --- a/doc/user_guide/storage_plugin.py +++ b/doc/user_guide/storage_plugin.py @@ -1,20 +1,14 @@ """Template for Python storage plugins.""" -# NOTE: -# Please, do not define any variables or functions outside the scope -# of the plugin class. -# -# The reason for this requirement is that all plugins will be loaded -# into the same shared scope within the built-in interpreter. -# Hence, variables or functions outside the plugin class may interfere -# with other plugins, resulting in hard-to-find bugs. - class plugin_driver_name(dlite.DLiteStorageBase): """General description of the Python storage plugin.""" def open(self, location, options=None): - """Open storage. + """Open storage. Optional. + + Must be defined if any of the methods close(), flush(), load(), + save(), delete() or query() are defined. Arguments: location: Path to storage. @@ -54,7 +48,9 @@ def delete(self, uuid): """ def query(self, pattern=None): - """Generator method that iterates over all UUIDs in the storage + """Queries the storage for instance UUIDs. Optional. + + A generator method that iterates over all UUIDs in the storage who"s metadata URI matches glob pattern `pattern`. Arguments: @@ -68,7 +64,7 @@ def query(self, pattern=None): @classmethod def from_bytes(cls, buffer, id=None, options=None): - """Load instance with given `id` from `buffer`. + """Load instance with given `id` from `buffer`. Optional. Arguments: buffer: Bytes or bytearray object to load the instance from. @@ -91,11 +87,3 @@ def to_bytes(cls, inst, options=None): Returns: The bytes (or bytearray) object that the instance is saved to. """ - - def _example_help_method(self, *args): - """Example help method. - - If you need a help function, please make it a class method to - avoid possible naming conflicts with help functions in other - storage plugins. - """ diff --git a/doc/user_guide/storage_plugins.md b/doc/user_guide/storage_plugins.md index 6c4544319..1734fadf1 100644 --- a/doc/user_guide/storage_plugins.md +++ b/doc/user_guide/storage_plugins.md @@ -17,21 +17,73 @@ It can be a file on disk, a local database or a database accessed via a web inte Loading data from a storage into an instance and saving it back again is a key mechanism for interoperability at a syntactic level. DLite provides a plugin system that makes it easy to connect to new data sources via a common interface (using a [strategy design pattern]). -Opening a storage takes three arguments, a `driver` name identifying the storage plugin to use, the `location` of the storage and `options`. +Opening a storage takes in the general case four arguments, a `protocol` name identifying the [protocol plugin] to use for data transfer, a `driver` name identifying the storage plugin to use for parsing/serialisation, the `location` of the storage and a set of storage-specific `options`. + +The `protocol` argument was introduced in v0.5.22 to provide a clear separation between transfer of raw data from/to the storage and parsing/serialisation of the data. Storage plugins can be categorised as either *generic* or *specific*. A generic storage plugin can store and retrieve any type of instance and metadata while a specific storage plugin typically deals with specific instances of one type of entity. DLite comes with a set of generic storage plugins, like json, yaml, rdf, hdf5, postgresql and mongodb. -It also comes with a specific Blob storage plugin, that can load and save instances of a `http://onto-ns.com/meta/0.1/Blob` entity. +It also comes with a specific `Blob` and `Image` storage plugin, that can load and save instances of `http://onto-ns.com/meta/0.1/Blob` and `http://onto-ns.com/meta/0.1/Image`, respectively. Storage plugins can be written in either C or Python. +Using storages implicitly from Python +------------------------------------- +For convenience DLite also has an interface for creating storages implicitly. + +### Loading an instance +If you only want to load a single instance from a storage, you can use one of the following class methods: +* `dlite.Instance.load()`: load from a location using a specific protocol +* `dlite.Instance.from_location()`: load from a location +* `dlite.Instance.from_url()`: load from URL +* `dlite.Instance.from_bytes()`: load from a buffer +* `dlite.Instance.from_dict()`: load from Python dict +* `dlite.Instance.from_json()`: load from a JSON string +* `dlite.Instance.from_bson()`: load from a BSON string +* `dlite.Instance.from_storage()`: load from a storage +* `dlite.Instance.from_metaid()`: create a new empty instance (not loading anything) + +For example + +```python + >>> import dlite + >>> newinst = dlite.Instance.from_location("json", "newfile.json", id="ex:blob1") + >>> newinst.uri + 'ex:blob1' + +``` + +To load a YAML file from a web location, you can combine the `http` [protocol plugin] with the `yaml` storage plugin using `dlite.Instance.load()`: + +```python + >>> url = "https://raw.githubusercontent.com/SINTEF/dlite/refs/heads/master/storages/python/tests-python/input/test_meta.yaml" + >>> dlite.Instance.load(protocol="http", driver="yaml", location=url) + +``` + +### Saving an instance +Similarly, if you want to save an instance, you can use the following methods: +* `dlite.Instance.save()`: save to location, optionally using a specific protocol. + Can also take an open `dlite.Storage` object or an URL as argument. +* `dlite.Instance.to_bytes()`: returns the instance as a bytes object +* `dlite.Instance.asdict()`: returns the instance as a python dict +* `dlite.Instance.asjson()`: returns the instance as a JSON string +* `dlite.Instance.asbson()`: returns the instance as a BSON string + +For example, saving `newinst` to BSON, can be done with: + +```python + >>> newinst.save("bson", "newinst.bson", options="mode=w") + +``` + + Working with storages in Python ------------------------------- For instance, to create a new file-based JSON storage can in Python be done with: ```python - >>> import dlite >>> s = dlite.Storage("json", location="newfile.json", options="mode=w") ``` @@ -137,51 +189,37 @@ In fact, even though `blob1` and `inst1` are different python objects, they shar ``` -Using storages implicitly -------------------------- -For convenience DLite also has an interface for creating storages implicitly. -If you only want to load a single instance from a storage, you can use one of the following class methods: -* `dlite.Instance.from_location()`: to read from a location -* `dlite.Instance.from_url()`: to read from URL - -For example - -```python - >>> newinst = dlite.Instance.from_location( - ... "json", "newfile.json", id="ex:blob1") - >>> newinst.uri - 'ex:blob1' - -``` - -DLite instances also have the methods `save()` and `save_to_url()` for saving to a storage without first creating a `dlite.Storage` object. - -Saving this instance to BSON, can be done in a one-liner: - -```python - >>> newinst.save("bson", "newinst.bson", options="mode=w") - -``` - - Writing Python storage plugins ------------------------------ Storage plugins can be written in either C or Python. -In Python the storage plugin should be a Python module defining a subclass of `dlite.DLiteStorageBase` with a set of methods for opening, closing, reading, writing and searching the storage. -In order for DLite to find the storage plugin, it should be in the search path defined by the `DLITE_PYTHON_STORAGE_PLUGIN_DIRS` environment variable or from Python, in `dlite.python_storage_plugin_path`. - +In Python the storage plugin should be a Python module defining a subclass of `dlite.DLiteStorageBase` defining one or more of the following methods: +- **open()**: Open and initiate the storage. Required if any of load(), save(), delete(), query(), flush(), close() are defined. +- **load()**: Load an instance from the storage and return it. +- **save()**: Save an instance to the storage. +- **delete()**: Delete and an instance from the storage. +- **query()**: Query the storage for instance UUIDs. +- **flush()**: Flushed cached data to the storage. +- **close()**: Close the storage. +- **from_bytes()**: Class method that loads an instance from a buffer. +- **to_bytes()**: Class method that saves an instance to a buffer. + +A plugin can be defined with only the `from_bytes` and `to_bytes` methods. This has the advantage that only serialisation/deserialisation of the data and datamodels is considered in the actual plugin. In such cases Instances must +be instantiated with the dlite.Instance.load method, and the [protocol plugin] must be given. + +All methods are optional. You only have to implement the methods providing the functionality you need. See the [Python storage plugin template] for how to write a Python storage plugin. A complete example can be found in the [Python storage plugin example]. +In order for DLite to find the storage plugin, it should be in the search path defined by the `DLITE_PYTHON_STORAGE_PLUGIN_DIRS` environment variable or from Python, in `dlite.python_storage_plugin_path`. -:::{danger} -**For DLite <0.5.23 storage plugins were executed in the same scope. -Hence, to avoid confusing and hard-to-find bugs due to interference between your plugins, you should not define any variables or functions outside the `DLiteStorageBase` subclass!** -::: -Since DLite v0.5.23, plugins are evaluated in separate scopes (which are available in `dlite._plugindict). +:::{note} +**Prior to DLite v0.5.23 all storage plugins were executed in the same scope.** +This could lead to confusing and hard-to-find bugs due to interference between your plugins. +::: +However, since DLite v0.5.23, plugins are evaluated in separate scopes (which are available in `dlite._plugindict). @@ -198,6 +236,7 @@ An example is available in [ex4]. [strategy design pattern]: https://en.wikipedia.org/wiki/Strategy_pattern [C reference manual]: https://sintef.github.io/dlite/dlite/storage.html +[protocol plugin]: https://sintef.github.io/dlite/user_guide/storage_plugins.html [Python storage plugin template]: https://github.com/SINTEF/dlite/blob/master/doc/user_guide/storage_plugin.py [Python storage plugin example]: https://github.com/SINTEF/dlite/tree/master/examples/storage_plugin [ex1]: https://github.com/SINTEF/dlite/tree/master/examples/ex1 diff --git a/python/DLite-Python/__init__.py b/python/DLite-Python/__init__.py index 2f84958ea..e0b40ce55 100644 --- a/python/DLite-Python/__init__.py +++ b/python/DLite-Python/__init__.py @@ -1,4 +1,4 @@ # This empty package exists for setup.py to not fail on setup(...) because # setup cannot find a package "dlite" in the root directory. We are using -# setup(packages=[dlite"], ...) to start setting up an empty package which is -# then populated by the CMake build. +# setup(packages=[dlite"], ...) to start setting up an empty package which +# is then populated by the CMake build. diff --git a/python/setup.py b/python/setup.py index aa1402af0..00a3f6cbd 100644 --- a/python/setup.py +++ b/python/setup.py @@ -226,7 +226,9 @@ def run(self): str(share / "mapping-plugins" / dlite_compiled_dll_suffix), str(share / "python-storage-plugins" / "*.py"), str(share / "python-mapping-plugins" / "*.py"), + str(share / "python-protocol-plugins" / "*.py"), str(share / "storages" / "*.json"), + str(share / "storages" / "*.yaml"), ] }, ext_modules=[ diff --git a/requirements_full.txt b/requirements_full.txt index e7e7ec5ed..7899f2803 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -13,6 +13,7 @@ pyarrow>=14.0,<18.0 tables>=3.8,<5.0 openpyxl>=3.0.9,<3.2 jinja2>=3.0,<4 +paramiko>=3.0.0,<3.4.1 requests>=2.10,<3 redis>=5.0,<6 minio>=6.0,<8 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 95ea44342..b066df801 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,9 +7,10 @@ if(MSVC AND ${CMAKE_VERSION} VERSION_LESS "3.12.10") endif() -# Generate config.h -# ----------------- +# Generate config headers +# ----------------------- configure_file(config.h.in config.h) +configure_file(config-paths.h.in config-paths.h) include(dliteGenEnv) dlite_genenv(${CMAKE_CURRENT_BINARY_DIR}/config-paths.h NATIVE) diff --git a/src/config-paths.h.in b/src/config-paths.h.in index c0a0de348..211a407ae 100644 --- a/src/config-paths.h.in +++ b/src/config-paths.h.in @@ -3,22 +3,25 @@ #ifndef _DLITE_CONFIG_PATHS_H #define _DLITE_CONFIG_PATHS_H -#define dlite_PATH "@dlite_PATH@" -#define dlite_PATH_EXTRA "@dlite_PATH_EXTRA@" +#define dlite_PATH "@dlite_PATH@" +#define dlite_PATH_EXTRA "@dlite_PATH_EXTRA@" /* FIXME: This has no meaning for Windows but ./tools/dlite-env.c expects this and does not like backslashes */ -#define dlite_LD_LIBRARY_PATH "@dlite_LD_LIBRARY_PATH@" -#define dlite_PYTHONPATH "@dlite_PYTHONPATH@" +#define dlite_LD_LIBRARY_PATH "@dlite_LD_LIBRARY_PATH@" +#define dlite_PYTHONPATH "@dlite_PYTHONPATH@" +#define dlite_PYTHONPATH_NATIVE "@dlite_PYTHONPATH_NATIVE@" -#define dlite_BUILD_ROOT "@dlite_BUILD_ROOT_NATIVE@" -#define dlite_INSTALL_ROOT "@CMAKE_INSTALL_PREFIX@" +//#define dlite_BUILD_ROOT "@dlite_BUILD_ROOT_NATIVE@" +#define dlite_BUILD_ROOT "@dlite_BINARY_DIR@" +#define dlite_INSTALL_ROOT "@CMAKE_INSTALL_PREFIX@" /* hardcoded paths in build tree */ -#define dlite_STORAGE_PLUGINS "@dlite_STORAGE_PLUGINS@" -#define dlite_MAPPING_PLUGINS "@dlite_MAPPING_PLUGINS@" -#define dlite_PYTHON_STORAGE_PLUGINS "@dlite_PYTHON_STORAGE_PLUGINS@" -#define dlite_PYTHON_MAPPING_PLUGINS "@dlite_PYTHON_MAPPING_PLUGINS@" -#define dlite_TEMPLATES "@dlite_TEMPLATES@" -#define dlite_STORAGES "@dlite_STORAGES@" +#define dlite_STORAGE_PLUGINS "@dlite_STORAGE_PLUGINS@" +#define dlite_MAPPING_PLUGINS "@dlite_MAPPING_PLUGINS@" +#define dlite_PYTHON_STORAGE_PLUGINS "@dlite_PYTHON_STORAGE_PLUGINS@" +#define dlite_PYTHON_MAPPING_PLUGINS "@dlite_PYTHON_MAPPING_PLUGINS@" +#define dlite_PYTHON_PROTOCOL_PLUGINS "@dlite_PYTHON_PROTOCOL_PLUGINS@" +#define dlite_TEMPLATES "@dlite_TEMPLATES@" +#define dlite_STORAGES "@dlite_STORAGES@" #endif /* _DLITE_CONFIG_PATHS_H */ diff --git a/src/config.h.in b/src/config.h.in index a3eb611d6..b9ddd148a 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -35,16 +35,17 @@ #endif /* Installation directories relative to DLITE_ROOT */ -#define DLITE_INCLUDE_DIRS "@DLITE_INCLUDE_DIRS@" -#define DLITE_LIBRARY_DIR "@DLITE_LIBRARY_DIR@" -#define DLITE_RUNTIME_DIR "@DLITE_RUNTIME_DIR@" -#define DLITE_PYTHONPATH "@DLITE_PYTHONPATH@" -#define DLITE_STORAGE_PLUGIN_DIRS "@DLITE_STORAGE_PLUGIN_DIRS@" -#define DLITE_MAPPING_PLUGIN_DIRS "@DLITE_MAPPING_PLUGIN_DIRS@" -#define DLITE_PYTHON_STORAGE_PLUGIN_DIRS "@DLITE_PYTHON_STORAGE_PLUGIN_DIRS@" -#define DLITE_PYTHON_MAPPING_PLUGIN_DIRS "@DLITE_PYTHON_MAPPING_PLUGIN_DIRS@" -#define DLITE_TEMPLATE_DIRS "@DLITE_TEMPLATE_DIRS@" -#define DLITE_STORAGES "@DLITE_STORAGES@" +#define DLITE_INCLUDE_DIRS "@DLITE_INCLUDE_DIRS@" +#define DLITE_LIBRARY_DIR "@DLITE_LIBRARY_DIR@" +#define DLITE_RUNTIME_DIR "@DLITE_RUNTIME_DIR@" +#define DLITE_PYTHONPATH "@DLITE_PYTHONPATH@" +#define DLITE_STORAGE_PLUGIN_DIRS "@DLITE_STORAGE_PLUGIN_DIRS@" +#define DLITE_MAPPING_PLUGIN_DIRS "@DLITE_MAPPING_PLUGIN_DIRS@" +#define DLITE_PYTHON_STORAGE_PLUGIN_DIRS "@DLITE_PYTHON_STORAGE_PLUGIN_DIRS@" +#define DLITE_PYTHON_MAPPING_PLUGIN_DIRS "@DLITE_PYTHON_MAPPING_PLUGIN_DIRS@" +#define DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS "@DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS@" +#define DLITE_TEMPLATE_DIRS "@DLITE_TEMPLATE_DIRS@" +#define DLITE_STORAGES "@DLITE_STORAGES@" /* built-in storage plugins */ diff --git a/src/dlite-behavior.c b/src/dlite-behavior.c index 462c29d53..b6f8d0c1b 100644 --- a/src/dlite-behavior.c +++ b/src/dlite-behavior.c @@ -128,10 +128,10 @@ int dlite_behavior_get(const char *name) const char *ver = dlite_get_version(); // current version b->value = (strcmp_semver(ver, b->version_new) >= 0) ? 1 : 0; - dlite_warn("Behavior `%s` is not configured. " - "It will be enabled by default from v%s. " - "See https://sintef.github.io/dlite/user_guide/configure_behavior_changes.html for more info.", - b->name, b->version_new); + dlite_warnx("Behavior `%s` is not configured. " + "It will be enabled by default from v%s. " + "See https://sintef.github.io/dlite/user_guide/configure_behavior_changes.html for more info.", + b->name, b->version_new); } assert(b->value >= 0); diff --git a/src/dlite-entity.c b/src/dlite-entity.c index 52cd535bb..7f9009b24 100644 --- a/src/dlite-entity.c +++ b/src/dlite-entity.c @@ -1144,14 +1144,22 @@ int dlite_instance_save_url(const char *url, const DLiteInstance *inst) */ DLiteInstance *dlite_instance_memload(const char *driver, const unsigned char *buf, size_t size, - const char *id, const char *options) + const char *id, const char *options, + const char *metaid) { const DLiteStoragePlugin *api; + DLiteInstance *inst; if (!(api = dlite_storage_plugin_get(driver))) return NULL; if (!api->memLoadInstance) return err(dliteUnsupportedError, "driver does not support memload: %s", api->name), NULL; - return api->memLoadInstance(api, buf, size, id, options); + + if (!(inst = api->memLoadInstance(api, buf, size, id, options))) + return NULL; + if (metaid) + return dlite_mapping(metaid, (const DLiteInstance **)&inst, 1); + else + return inst; } /* diff --git a/src/dlite-entity.h b/src/dlite-entity.h index 9d52b82d2..bae0254d4 100644 --- a/src/dlite-entity.h +++ b/src/dlite-entity.h @@ -628,7 +628,8 @@ int dlite_instance_save_url(const char *url, const DLiteInstance *inst); */ DLiteInstance *dlite_instance_memload(const char *driver, const unsigned char *buf, size_t size, - const char *id, const char *options); + const char *id, const char *options, + const char *metaid); /** Stores instance `inst` to memory buffer `buf` of size `size`. diff --git a/src/dlite-errors.c b/src/dlite-errors.c index 0c73090ea..d15a9a906 100644 --- a/src/dlite-errors.c +++ b/src/dlite-errors.c @@ -45,6 +45,7 @@ const char *dlite_errname(DLiteErrCode code) case dliteMissingMetadataError: return "DLiteMissingMetadata"; case dliteMetadataExistError: return "DLiteMetadataExist"; case dliteMappingError: return "DLiteMapping"; + case dliteProtocolError: return "DLiteProtocol"; case dlitePythonError: return "DLitePython"; case dliteLastError: return "DLiteUndefined"; @@ -94,6 +95,7 @@ const char *dlite_errdescr(DLiteErrCode code) case dliteMissingMetadataError: return "No metadata with given id"; case dliteMetadataExistError: return "Metadata with given id already exists"; case dliteMappingError: return "Error in instance mappings"; + case dliteProtocolError: return "Error in a protocol plugin"; case dlitePythonError: return "Error calling Python API"; case dliteLastError: return NULL; } diff --git a/src/dlite-errors.h b/src/dlite-errors.h index 816743ec4..e3d8d4f19 100644 --- a/src/dlite-errors.h +++ b/src/dlite-errors.h @@ -40,10 +40,11 @@ typedef enum { dliteMissingMetadataError=-30, /*!< No metadata with given id can be found */ dliteMetadataExistError=-31, /*!< Metadata with given id already exists */ dliteMappingError=-32, /*!< Error in instance mappings */ - dlitePythonError=-33, /*!< Error calling Python API */ + dliteProtocolError=-33, /*!< Error in a protocol plugin */ + dlitePythonError=-34, /*!< Error calling Python API */ /* Should always be the last error */ - dliteLastError=-34 + dliteLastError=-35 } DLiteErrCode; diff --git a/src/dlite-misc.c b/src/dlite-misc.c index 963dd4cdc..0c2554a4b 100644 --- a/src/dlite-misc.c +++ b/src/dlite-misc.c @@ -196,7 +196,7 @@ int dlite_split_meta_uri(const char *uri, char **name, char **version, Returns non-zero on error. */ -int dlite_option_parse(char *options, DLiteOpt *opts) +int dlite_option_parse(char *options, DLiteOpt *opts, DLiteOptFlag flags) { char *p = options; char *buf = NULL; @@ -231,9 +231,14 @@ int dlite_option_parse(char *options, DLiteOpt *opts) } } if (!opts[i].key) { - int len = strcspn(p, "=;&#"); - status = errx(dliteValueError, "unknown option key: '%.*s'", len, p); - goto fail; + if (flags & dliteOptStrict) { + int len = strcspn(p, "=;&#"); + status = errx(dliteValueError, "unknown option key: '%.*s'", len, p); + goto fail; + } else { + p += strcspn(p, ";&#"); + if (*p && strchr(";&", *p)) p++; + } } } fail: @@ -575,6 +580,8 @@ static void dlite_err_handler(const ErrRecord *record) #endif } +/* dlite_errname() with correct call signature */ +static const char *_errname(int eval) { return dlite_errname(eval); } /* Initialises dlite. This function may be called several times. @@ -599,7 +606,7 @@ void dlite_init(void) /* Set up error handling */ err_set_handler(dlite_err_handler); - err_set_nameconv(dlite_errname); + err_set_nameconv(_errname); } } @@ -733,7 +740,7 @@ int dlite_err_ignored_get(DLiteErrCode code) DLiteErrMask *mask = _dlite_err_mask_get(); if (!mask) return 0; if (code > 0 && (*mask & DLITE_ERRBIT(dliteUnknownError))) return 1; - return *mask & DLITE_ERRBIT(code); + return (int)(*mask) & DLITE_ERRBIT(code); } diff --git a/src/dlite-misc.h b/src/dlite-misc.h index 94b7e1cd6..945614c1d 100644 --- a/src/dlite-misc.h +++ b/src/dlite-misc.h @@ -92,6 +92,11 @@ typedef struct _DLiteOpt { const char *descr; /*!< Description of this option */ } DLiteOpt; +typedef enum { + dliteOptDefault=0, /*!< Default option flags */ + dliteOptStrict=1 /*!< Stict mode. Its an error if option is unknown */ +} DLiteOptFlag; + /** Parses the options string `options` and assign corresponding values of the array `opts`. The options string should be a valid url query @@ -110,6 +115,8 @@ typedef struct _DLiteOpt { default values. At return, the values of the provided options are updated. + `flags` should be zero or `dliteOptStrict`. + Returns non-zero on error. Example @@ -138,7 +145,7 @@ typedef struct _DLiteOpt { } ``` */ -int dlite_option_parse(char *options, DLiteOpt *opts); +int dlite_option_parse(char *options, DLiteOpt *opts, DLiteOptFlag flags); /** @} */ @@ -324,7 +331,7 @@ DLiteErrMask *_dlite_err_mask_get(void); void _dlite_err_mask_set(DLiteErrMask mask); #define DLITE_ERRBIT(code) \ - (1<<((code >= 0) ? 0 : (code <= dliteLastError) ? -dliteLastError : -code)) + ((int)1<<(int)((code >= 0) ? 0 : (code <= dliteLastError) ? -dliteLastError : -code)) #define DLITE_NOERR(mask) \ do { \ diff --git a/src/dlite-storage.c b/src/dlite-storage.c index 8576bdc04..897fab741 100644 --- a/src/dlite-storage.c +++ b/src/dlite-storage.c @@ -378,11 +378,14 @@ FUPaths *dlite_storage_paths(void) fu_paths_init_sep(g->storage_paths, "DLITE_STORAGES", "|"); fu_paths_set_platform(g->storage_paths, dlite_get_platform()); - if (dlite_use_build_root()) + if (dlite_use_build_root()) { + fu_paths_append(g->storage_paths, dlite_BUILD_ROOT + "/bindings/python/dlite/share/dlite/storages"); fu_paths_extend(g->storage_paths, dlite_STORAGES, "|"); - else + } else { fu_paths_extend_prefix(g->storage_paths, dlite_root_get(), DLITE_ROOT "/" DLITE_STORAGES, "|"); + } } return g->storage_paths; } diff --git a/src/pyembed/CMakeLists.txt b/src/pyembed/CMakeLists.txt index 9a568eb1a..e8c561e90 100644 --- a/src/pyembed/CMakeLists.txt +++ b/src/pyembed/CMakeLists.txt @@ -1,10 +1,12 @@ # -*- Mode: cmake -*- + set(pyembed_sources pyembed/dlite-pyembed.c pyembed/dlite-pyembed-utils.c pyembed/dlite-python-singletons.c pyembed/dlite-python-storage.c pyembed/dlite-python-mapping.c + pyembed/dlite-python-protocol.c PARENT_SCOPE - ) +) diff --git a/src/pyembed/dlite-pyembed.c b/src/pyembed/dlite-pyembed.c index 554183625..50f9fd37f 100644 --- a/src/pyembed/dlite-pyembed.c +++ b/src/pyembed/dlite-pyembed.c @@ -94,6 +94,7 @@ PyObject *dlite_pyembed_exception(DLiteErrCode code) case dliteMissingMetadataError: return PyExc_LookupError; // dup case dliteMetadataExistError: break; case dliteMappingError: break; + case dliteProtocolError: break; case dlitePythonError: break; case dliteLastError: break; } @@ -270,7 +271,7 @@ DLiteErrCode dlite_pyembed_errcode(PyObject *type) Writes Python error message to `errmsg` (of length `len`) if an Python error has occured. - On return the The Python error indicator is reset. + Resets the Python error indicator. Returns 0 if no error has occured. Otherwise return the number of bytes written to, or would have been written to `errmsg` if it had @@ -435,14 +436,14 @@ void *dlite_pyembed_get_address(const char *symbol) /* Import dlite */ if (!(dlite_name = PyUnicode_FromString("dlite")) || !(dlite_module = PyImport_Import(dlite_name))) - FAIL("cannot import Python package: dlite"); + PYFAILCODE(dlitePythonError, "cannot import Python package: dlite"); /* Get path to _dlite */ if (!(dlite_dict = PyModule_GetDict(dlite_module)) || !(_dlite_module = PyDict_GetItemString(dlite_dict, "_dlite")) || !(_dlite_dict = PyModule_GetDict(_dlite_module)) || !(_dlite_file = PyDict_GetItemString(_dlite_dict, "__file__"))) - FAIL("cannot get path to dlite extension module"); + PYFAILCODE(dlitePythonError, "cannot get path to dlite extension module"); /* Get C path to _dlite */ if (!PyUnicode_Check(_dlite_file) || diff --git a/src/pyembed/dlite-pyembed.h b/src/pyembed/dlite-pyembed.h index 9571c101d..94166bd27 100644 --- a/src/pyembed/dlite-pyembed.h +++ b/src/pyembed/dlite-pyembed.h @@ -24,6 +24,19 @@ #endif +/* Convenient macros */ +#define PYFAILCODE(code, msg) do { \ + dlite_pyembed_err(code, msg); goto fail; } while (0) +#define PYFAILCODE1(code, msg, a1) do { \ + dlite_pyembed_err(code, msg, a1); goto fail; } while (0) +#define PYFAILCODE2(code, msg, a1, a2) do { \ + dlite_pyembed_err(code, msg, a1, a2); goto fail; } while (0) +#define PYFAILCODE3(code, msg, a1, a2, a3) do { \ + dlite_pyembed_err(code, msg, a1, a2, a3); goto fail; } while (0) +#define PYFAILCODE4(code, msg, a1, a2, a3, a4) do { \ + dlite_pyembed_err(code, msg, a1, a2, a3, a4); goto fail; } while (0) + + /** Initialises the embedded Python environment. */ diff --git a/src/pyembed/dlite-python-protocol.c b/src/pyembed/dlite-python-protocol.c new file mode 100644 index 000000000..47ee71804 --- /dev/null +++ b/src/pyembed/dlite-python-protocol.c @@ -0,0 +1,102 @@ +/* + A generic protocol that looks up and loads Python protocol plugins + */ +#include +#include +#include + +/* Python pulls in a lot of defines that conflicts with utils/config.h */ +#define SKIP_UTILS_CONFIG_H + +#include "config-paths.h" + +#include "utils/fileutils.h" +#include "dlite.h" +#include "config.h" +#include "dlite-macros.h" +#include "dlite-misc.h" +#include "dlite-python-protocol.h" +//#include "dlite-pyembed.h" +//#include "dlite-python-singletons.h" + +#define GLOBALS_ID "dlite-python-protocol-id" + + +/* Global variables for dlite-storage-plugins */ +typedef struct { + FUPaths protocol_paths; + int protocol_paths_initialised; +} Globals; + + +/* Frees global state for this module - called by atexit() */ +static void free_globals(void *globals) +{ + Globals *g = globals; + dlite_python_protocol_paths_clear(); + free(g); +} + +/* Return a pointer to global state for this module */ +static Globals *get_globals(void) +{ + Globals *g = dlite_globals_get_state(GLOBALS_ID); + if (!g) { + if (!(g = calloc(1, sizeof(Globals)))) + return dlite_err(dliteMemoryError, "allocation failure"), NULL; + dlite_globals_add_state(GLOBALS_ID, g, free_globals); + } + return g; +} + + +/* + Returns a pointer to Python protocol paths +*/ +FUPaths *dlite_python_protocol_paths(void) +{ + Globals *g; + if (!(g = get_globals())) return NULL; + + if (!g->protocol_paths_initialised) { + int s; + if (fu_paths_init(&g->protocol_paths, + "DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS") < 0) + return dlite_err(1, "cannot initialise " + "DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS"), NULL; + + fu_paths_set_platform(&g->protocol_paths, dlite_get_platform()); + + if (dlite_use_build_root()) + s = fu_paths_extend(&g->protocol_paths, + dlite_PYTHON_PROTOCOL_PLUGINS, NULL); + else + s = fu_paths_extend_prefix(&g->protocol_paths, dlite_root_get(), + DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS, NULL); + if (s < 0) return dlite_err(1, "error initialising dlite python protocol " + "plugin dirs"), NULL; + g->protocol_paths_initialised = 1; + + /* Make sure that dlite DLLs are added to the library search path */ + dlite_add_dll_path(); + + /* Be kind with memory leak software and free memory at exit... */ + //atexit(dlite_python_protocol_paths_clear); + } + + return &g->protocol_paths; +} + +/* + Clears Python protocol search path. +*/ +void dlite_python_protocol_paths_clear(void) +{ + Globals *g; + if (!(g = get_globals())) return; + + if (g->protocol_paths_initialised) { + fu_paths_deinit(&g->protocol_paths); + g->protocol_paths_initialised = 0; + } +} diff --git a/src/pyembed/dlite-python-protocol.h b/src/pyembed/dlite-python-protocol.h new file mode 100644 index 000000000..cb9b04b3e --- /dev/null +++ b/src/pyembed/dlite-python-protocol.h @@ -0,0 +1,25 @@ +#ifndef _DLITE_PYTHON_PROTOCOL_H +#define _DLITE_PYTHON_PROTOCOL_H + +/** + @file + @brief Python protocol +*/ + +#ifdef HAVE_CONFIG_H +#undef HAVE_CONFIG_H +#endif + + +/** + Returns a pointer to Python protocol paths +*/ +FUPaths *dlite_python_protocol_paths(void); + + +/** + Clears Python protocol search path. +*/ +void dlite_python_protocol_paths_clear(void); + +#endif /* _DLITE_PYTHON_PROTOCOL_H */ diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 115de8511..94d2a4932 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -1,7 +1,6 @@ # -*- Mode: cmake -*- project(dlite-src-tests C) - set(tests test_misc test_errors @@ -22,6 +21,10 @@ set(tests test_getuuid ) +if(MSVC) # Unset extra_link_libraries on MSVC to avoid linking against None.lib + unset(extra_link_libraries) +endif() + list(APPEND tests test_json_entity) list(APPEND tests test_storage_lookup) list(APPEND tests test_mapping) diff --git a/src/tests/python/CMakeLists.txt b/src/tests/python/CMakeLists.txt index 953d9a3a4..9899f49c9 100644 --- a/src/tests/python/CMakeLists.txt +++ b/src/tests/python/CMakeLists.txt @@ -6,6 +6,10 @@ set(tests test_python_mapping ) +if(MSVC) # Unset extra_link_libraries on MSVC to avoid linking against None.lib + unset(extra_link_libraries) +endif() + add_definitions( -DTESTDIR=${CMAKE_CURRENT_SOURCE_DIR} -DDLITE_ROOT=${dlite_SOURCE_DIR} @@ -41,9 +45,8 @@ foreach(test ${tests}) set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT "PYTHONHOME=${Python3_RUNTIME_LIBRARY_DIRS}") endif() - # No need to set PYTHONPATH since we are not importing the package in any test here - #set_property(TEST ${test} APPEND PROPERTY - # ENVIRONMENT "PYTHONPATH=${dlite_PYTHONPATH_NATIVE}") + set_property(TEST ${test} APPEND PROPERTY + ENVIRONMENT "PYTHONPATH=${dlite_PYTHONPATH_NATIVE}") if (NOT WIN32) set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${dlite_LD_LIBRARY_PATH_NATIVE}") diff --git a/src/tests/test_misc.c b/src/tests/test_misc.c index a254d5afe..f44ed4fc9 100644 --- a/src/tests/test_misc.c +++ b/src/tests/test_misc.c @@ -98,7 +98,7 @@ MU_TEST(test_option_parse) {0, NULL, NULL, NULL} }; - mu_assert_int_eq(0, dlite_option_parse(options, opts)); + mu_assert_int_eq(0, dlite_option_parse(options, opts, dliteOptStrict)); for (i=0; opts[i].key; i++) { switch (opts[i].c) { case 'N': @@ -120,7 +120,7 @@ MU_TEST(test_option_parse) char options2[] = "name=C;mode=append"; old = err_set_stream(NULL); mu_assert_int_eq(dliteValueError, - dlite_option_parse(options2, opts)); + dlite_option_parse(options2, opts, dliteOptStrict)); err_set_stream(old); } diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index efc1a2737..454c70740 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -22,7 +22,8 @@ check_include_file(byteswap.h HAVE_BYTESWAP_H) check_include_file(locale.h HAVE_LOCALE_H) check_include_file(time.h HAVE_TIME_H) check_include_file(unistd.h HAVE_UNISTD_H) -check_include_file(windows.h HAVE_WINDOWS_H) +check_include_file(windows.h HAVE_WINDOWS_H) # Windows-specific +check_include_file(io.h HAVE_IO_H) # Windows-specific # -- check for symbols set(CMAKE_C_FLAGS_saved ${CMAKE_C_FLAGS}) @@ -97,6 +98,15 @@ check_symbol_exists(localeconv locale.h HAVE__LOCALECONV) check_symbol_exists(va_copy stdarg.h HAVE_VA_COPY) check_symbol_exists(__va_copy stdarg.h HAVE___VA_COPY) +# C11 mutex +check_symbol_exists(mtx_init threads.h HAVE_MTX_INIT) +check_symbol_exists(mtx_lock threads.h HAVE_MTX_LOCK) +check_symbol_exists(mtx_timedlock threads.h HAVE_MTX_TIMEDLOCK) +check_symbol_exists(mtx_trylock threads.h HAVE_MTX_TRYLOCK) +check_symbol_exists(mtx_unlock threads.h HAVE_MTX_UNLOCK) +check_symbol_exists(mtx_destroy threads.h HAVE_MTX_DESTROY) + + # -- typedefs include(CheckTypeSize) @@ -124,7 +134,7 @@ if(WITH_THREADS) # On some 32-bit systems (like manylinux2014_i686) we have to # explicitly link with the threads library - if(CMAKE_SIZEOF_VOID_P EQUAL 4) + if(CMAKE_SIZEOF_VOID_P EQUAL 4 AND CMAKE_THREAD_LIBS_INIT) list(APPEND extra_link_libraries ${CMAKE_THREAD_LIBS_INIT}) endif() diff --git a/src/utils/config.h.in b/src/utils/config.h.in index ebc8ce3ad..abed1046f 100644 --- a/src/utils/config.h.in +++ b/src/utils/config.h.in @@ -37,6 +37,7 @@ #cmakedefine HAVE_TIME_H #cmakedefine HAVE_UNISTD_H #cmakedefine HAVE_WINDOWS_H +#cmakedefine HAVE_IO_H /* Whether symbols exists. If not, they are defined in compat.c or compat/ */ #cmakedefine HAVE_STRDUP @@ -92,6 +93,13 @@ // #cmakedefine HAVE_VA_COPY // #cmakedefine HAVE___VA_COPY +#cmakedefine HAVE_MTX_INIT +#cmakedefine HAVE_MTX_LOCK +#cmakedefine HAVE_MTX_TIMEDLOCK +#cmakedefine HAVE_MTX_TRYLOCK +#cmakedefine HAVE_MTX_UNLOCK +#cmakedefine HAVE_MTX_DESTROY + /* Whether types exists */ #cmakedefine HAVE_CLOCK_T @@ -136,7 +144,9 @@ #if defined WINDOWS && !defined CROSS_TARGET /* Get rid of warnings from MSVS about insecure standard library functions */ +#ifndef _CRT_SECURE_NO_WARNINGS # define _CRT_SECURE_NO_WARNINGS +#endif /* Get rid of warnings about possible loss of data in conversions between integers or floats of different size */ diff --git a/src/utils/dsl.h b/src/utils/dsl.h index 9a37dd0a1..0471ba8f4 100644 --- a/src/utils/dsl.h +++ b/src/utils/dsl.h @@ -129,7 +129,10 @@ typedef void *dsl_handle; #include /* Get rid of warnings about strerror() being deprecated on VS */ +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif + #ifndef CROSS_TARGET #pragma warning(disable: 4996) #endif diff --git a/src/utils/err.c b/src/utils/err.c index e57c0694c..6a3afd8b7 100644 --- a/src/utils/err.c +++ b/src/utils/err.c @@ -20,6 +20,11 @@ #include #endif +#if defined(HAVE_IO_H) && defined(HAVE__ISATTY) +#include +#endif + + /* Thread local storage * https://stackoverflow.com/questions/18298280/how-to-declare-a-variable-as-thread-local-portab */ diff --git a/src/utils/fileutils.c b/src/utils/fileutils.c index 267283a10..00dbde851 100644 --- a/src/utils/fileutils.c +++ b/src/utils/fileutils.c @@ -858,6 +858,7 @@ char *fu_paths_string(const FUPaths *paths) { size_t i, seplen, size=0; char *s, *string; + if (!paths->n) return strdup(""); const char *pathsep = (paths->pathsep) ? paths->pathsep : fu_pathsep(paths->platform); seplen = strlen(pathsep); diff --git a/src/utils/rng.c b/src/utils/rng.c index 397bbfcd4..a5e98c760 100644 --- a/src/utils/rng.c +++ b/src/utils/rng.c @@ -182,18 +182,18 @@ static uint64_t rand_digits(uint64_t n) /* odd random number for low order digit */ u = (rand_msws32_r(&s) % 8) * 2 + 1; - v = (1L<0;) { j = rand_msws32_r(&s); /* get 8 digit 32-bit random word */ for (i=0;i<32;i+=4) { k = (j>>i) & 0xf; /* get a digit */ - if (k!=0 && (c & (1L<", 3, buf); + int n = (int)uri_encode("<\">", 3, buf); mu_assert_string_eq("%3C%22%3E", buf); mu_assert_int_eq(9, n); } MU_TEST(test_encode_middle_null) { - int n = uri_encode("ABC\0DEF", 3, buf); + int n = (int)uri_encode("ABC\0DEF", 3, buf); mu_assert_string_eq("ABC", buf); mu_assert_int_eq(3, n); } MU_TEST(test_encode_middle_null_len) { - int n = uri_encode("ABC\0DEF", 7, buf); + int n = (int)uri_encode("ABC\0DEF", 7, buf); mu_assert_string_eq("ABC%00DEF", buf); mu_assert_int_eq(9, n); } MU_TEST(test_encode_latin1_utf8) { - int n = uri_encode("åäö", strlen("åäö"), buf); + int n = (int)uri_encode("åäö", strlen("åäö"), buf); mu_assert_string_eq("%C3%A5%C3%A4%C3%B6", buf); mu_assert_int_eq(18, n); } MU_TEST(test_encode_utf8) { - int n = uri_encode("❤", strlen("❤"), buf); + int n = (int)uri_encode("❤", strlen("❤"), buf); mu_assert_string_eq("%E2%9D%A4", buf); mu_assert_int_eq(9, n); } /* tests for decode_uri */ MU_TEST(test_decode_empty) { - int n = uri_decode("", 0, buf); + int n = (int)uri_decode("", 0, buf); mu_assert_string_eq("", buf); mu_assert_int_eq(0, n); } MU_TEST(test_decode_something) { - int n = uri_decode("something", 9, buf); + int n = (int)uri_decode("something", 9, buf); mu_assert_string_eq("something", buf); mu_assert_int_eq(9, n); } MU_TEST(test_decode_something_percent) { - int n = uri_decode("something%", 10, buf); + int n = (int)uri_decode("something%", 10, buf); mu_assert_string_eq("something%", buf); mu_assert_int_eq(10, n); } MU_TEST(test_decode_something_percenta) { - int n = uri_decode("something%a", 11, buf); + int n = (int)uri_decode("something%a", 11, buf); mu_assert_string_eq("something%a", buf); mu_assert_int_eq(11, n); } MU_TEST(test_decode_something_zslash) { - int n = uri_decode("something%Z/", 12, buf); + int n = (int)uri_decode("something%Z/", 12, buf); mu_assert_string_eq("something%Z/", buf); mu_assert_int_eq(12, n); } MU_TEST(test_decode_space) { - int n = uri_decode("%20", 3, buf); + int n = (int)uri_decode("%20", 3, buf); mu_assert_string_eq(" ", buf); mu_assert_int_eq(1, n); } MU_TEST(test_decode_percents) { - int n = uri_decode("%25%2520", 8, buf); + int n = (int)uri_decode("%25%2520", 8, buf); mu_assert_string_eq("%%20", buf); mu_assert_int_eq(4, n); } MU_TEST(test_decode_latin1) { - int n = uri_decode("%7Cabc%C3%A5", 12, buf); + int n = (int)uri_decode("%7Cabc%C3%A5", 12, buf); mu_assert_string_eq("|abcå", buf); mu_assert_int_eq(6, n); } MU_TEST(test_decode_symbols) { - int n = uri_decode("~%2A%27%28%29", 13, buf); + int n = (int)uri_decode("~%2A%27%28%29", 13, buf); mu_assert_string_eq("~*'()", buf); mu_assert_int_eq(5, n); } MU_TEST(test_decode_angles) { - int n = uri_decode("%3C%22%3E", 9, buf); + int n = (int)uri_decode("%3C%22%3E", 9, buf); mu_assert_string_eq("<\">", buf); mu_assert_int_eq(3, n); } MU_TEST(test_decode_middle_null) { - int n = uri_decode("ABC%00DEF", 6, buf); + int n = (int)uri_decode("ABC%00DEF", 6, buf); mu_assert_string_eq("ABC\0", buf); mu_assert_int_eq(4, n); } MU_TEST(test_decode_middle_null2) { - int n = uri_decode("ABC%00DEF", 5, buf); + int n = (int)uri_decode("ABC%00DEF", 5, buf); mu_assert_string_eq("ABC%0", buf); mu_assert_int_eq(5, n); } MU_TEST(test_decode_middle_full) { - int n = uri_decode("ABC%00DEF", 9, buf); + int n = (int)uri_decode("ABC%00DEF", 9, buf); mu_assert_string_eq("ABC\0DEF", buf); mu_assert_int_eq(7, n); } diff --git a/src/utils/tests/test_uuid.c b/src/utils/tests/test_uuid.c index a53ac6d6c..2828d1d1e 100644 --- a/src/utils/tests/test_uuid.c +++ b/src/utils/tests/test_uuid.c @@ -17,9 +17,9 @@ #include "sha1.h" #include "uuid.h" -/* Get rid of MSVS warnings */ -#if defined WIN32 || defined _WIN32 || defined __WIN32__ -# pragma warning(disable: 4996) +/* Get rid of MSVC warnings */ +#ifdef _MSC_VER +# pragma warning(disable: 4217 4996) #endif diff --git a/src/utils/tests/test_uuid2.c b/src/utils/tests/test_uuid2.c index 706cd528f..11203aaf0 100644 --- a/src/utils/tests/test_uuid2.c +++ b/src/utils/tests/test_uuid2.c @@ -11,9 +11,9 @@ #include #include "uuid.h" -/* Get rid of MSVS warnings */ -#if defined WIN32 || defined _WIN32 || defined __WIN32__ -# pragma warning(disable: 4217) +/* Get rid of MSVC warnings */ +#ifdef _MSC_VER +# pragma warning( disable : 4217 ) #endif diff --git a/src/utils/urlsplit.c b/src/utils/urlsplit.c index fe33a7da6..c3b8e686d 100644 --- a/src/utils/urlsplit.c +++ b/src/utils/urlsplit.c @@ -367,12 +367,6 @@ int pct_decode(char *buf, long size, const char *encoded) return pct_ndecode(buf, size, encoded, -1); } - -/* Turn off warning on Windows about sscanf() being unsecure. - * Our use here is completely safe and legitimate. */ -#define _CRT_SECURE_NO_WARNINGS - - /* Like pct_decode(), but at most `len` bytes are read from `encoded`. If `len` is negative, all of `encoded` is read. diff --git a/storages/hdf5/dh5lite.c b/storages/hdf5/dh5lite.c index 468f0d8a1..608c0f415 100644 --- a/storages/hdf5/dh5lite.c +++ b/storages/hdf5/dh5lite.c @@ -430,7 +430,7 @@ dh5_open(const DLiteStoragePlugin *api, const char *uri, const char *options) const char **mode = &opts[0].value; UNUSED(api); - if (dlite_option_parse(optcopy, opts)) goto fail; + if (dlite_option_parse(optcopy, opts, 0)) goto fail; H5open(); /* Opens hdf5 library */ diff --git a/storages/json/dlite-json-storage.c b/storages/json/dlite-json-storage.c index 395bce7e7..e7bc8e0a8 100644 --- a/storages/json/dlite-json-storage.c +++ b/storages/json/dlite-json-storage.c @@ -84,7 +84,7 @@ DLiteStorage *json_loader(const DLiteStoragePlugin *api, const char *uri, /* parse options */ char *optcopy = (options) ? strdup(options) : NULL; - if (dlite_option_parse(optcopy, opts)) goto fail; + if (dlite_option_parse(optcopy, opts, 0)) goto fail; char mode = *opts[0].value; int single = (*opts[1].value) ? atob(opts[1].value) : -2; @@ -354,7 +354,7 @@ int json_memsave(const DLiteStoragePlugin *api, }; char *optcopy = (options) ? strdup(options) : NULL; UNUSED(api); - if (dlite_option_parse(optcopy, opts)) goto fail; + if (dlite_option_parse(optcopy, opts, 0)) goto fail; indent = atoi(opts[0].value); if ((*opts[1].value) ? atob(opts[1].value) : dlite_instance_is_meta(inst)) flags |= dliteJsonSingle; @@ -368,7 +368,6 @@ int json_memsave(const DLiteStoragePlugin *api, fail: if (optcopy) free(optcopy); return retval; - } diff --git a/storages/json/tests/CMakeLists.txt b/storages/json/tests/CMakeLists.txt index 70c479093..446e55117 100644 --- a/storages/json/tests/CMakeLists.txt +++ b/storages/json/tests/CMakeLists.txt @@ -7,6 +7,10 @@ set(tests add_definitions(-DDLITE_ROOT=${dlite_SOURCE_DIR}) +if(MSVC) # Unset extra_link_libraries on MSVC to avoid linking against None.lib + unset(extra_link_libraries) +endif() + # We are linking to dlite-plugins-json DLL - this require that this # DLL is in the PATH on Windows. Copying the DLL to the current # BINARY_DIR is a simple way to ensure this. diff --git a/storages/python/CMakeLists.txt b/storages/python/CMakeLists.txt index 17a8a6f94..dde9efddf 100644 --- a/storages/python/CMakeLists.txt +++ b/storages/python/CMakeLists.txt @@ -24,25 +24,11 @@ set_target_properties(dlite-plugins-python PROPERTIES INSTALL_RPATH "$ORIGIN/../../../" ) -# Simplify plugin search path for testing in build tree, copy target -# to ${dlite_BINARY_DIR}/plugins -add_custom_command( - TARGET dlite-plugins-python - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - $ - ${dlite_BINARY_DIR}/plugins - ) - install( TARGETS dlite-plugins-python DESTINATION ${DLITE_STORAGE_PLUGIN_DIRS} -) - -install( - TARGETS dlite-plugins-python - LIBRARY DESTINATION lib + #LIBRARY DESTINATION lib ) install( diff --git a/storages/python/dlite-plugins-python.c b/storages/python/dlite-plugins-python.c index 8bbe52164..cf940c355 100644 --- a/storages/python/dlite-plugins-python.c +++ b/storages/python/dlite-plugins-python.c @@ -75,6 +75,7 @@ opener(const DLiteStoragePlugin *api, const char *location, classname, failmsg())) goto fail; + /* Check if the open() method has set attribute `writable` */ if (PyObject_HasAttrString(obj, "readable")) readable = PyObject_GetAttrString(obj, "readable"); @@ -553,10 +554,11 @@ get_dlite_storage_plugin_api(void *state, int *iter) FAIL1("attribute 'name' (or '__name__') of '%s' is not a string", (char *)PyUnicode_AsUTF8(name)); - if (!(open = PyObject_GetAttrString(cls, "open"))) - FAIL1("'%s' has no method: 'open'", classname); - if (!PyCallable_Check(open)) - FAIL1("attribute 'open' of '%s' is not callable", classname); + if (PyObject_HasAttrString(cls, "open")) { + open = PyObject_GetAttrString(cls, "open"); + if (!PyCallable_Check(open)) + FAIL1("attribute 'open' of '%s' is not callable", classname); + } if (PyObject_HasAttrString(cls, "close")) { close = PyObject_GetAttrString(cls, "close"); @@ -582,10 +584,6 @@ get_dlite_storage_plugin_api(void *state, int *iter) FAIL1("attribute 'save' of '%s' is not callable", classname); } - if (!load && !save) - FAIL1("expect either method 'load()' or 'save()' to be defined in '%s'", - classname); - if (PyObject_HasAttrString(cls, "delete")) { delete = PyObject_GetAttrString(cls, "delete"); if (!PyCallable_Check(delete)) diff --git a/storages/python/python-storage-plugins/mongodb.py b/storages/python/python-storage-plugins/mongodb.py index 4fd13902f..5bd1b283a 100644 --- a/storages/python/python-storage-plugins/mongodb.py +++ b/storages/python/python-storage-plugins/mongodb.py @@ -1,5 +1,7 @@ +"""DLite storage plugin for MongoDB. -"""DLite storage plugin for MongoDB.""" +Note: Continous testing is not run for this plugin. +""" import fnmatch import json diff --git a/storages/python/python-storage-plugins/postgresql.py b/storages/python/python-storage-plugins/postgresql.py index c219c58c5..74aa2a8f4 100644 --- a/storages/python/python-storage-plugins/postgresql.py +++ b/storages/python/python-storage-plugins/postgresql.py @@ -1,3 +1,7 @@ +"""DLite plugin for PostgreSQL. + +Note: Continous testing is not run for this plugin. +""" import os import re import sys diff --git a/storages/python/python-storage-plugins/pyrdf.py b/storages/python/python-storage-plugins/pyrdf.py index 61d2a93de..9d395ac88 100644 --- a/storages/python/python-storage-plugins/pyrdf.py +++ b/storages/python/python-storage-plugins/pyrdf.py @@ -109,18 +109,21 @@ def query(self, pattern: "Optional[str]" = None) -> "Generator[str]": yield str(o) @classmethod - def from_bytes(cls, buffer, id=None): + def from_bytes(cls, buffer, id=None, options=None): """Load instance with given `id` from `buffer`. Arguments: buffer: Bytes or bytearray object to load the instance from. id: ID of instance to load. May be omitted if `buffer` only holds one instance. + options: Options: + - `format`: File format to read from. Returns: New instance. """ - return from_rdf(data=buffer, id=id, format=self.options.get("format")) + opts = Options(options) + return from_rdf(data=buffer, id=id, format=opts.get("format")) @classmethod def to_bytes(cls, inst): diff --git a/storages/python/python-storage-plugins/redis.py b/storages/python/python-storage-plugins/redis.py index c7d49c980..0df0bd267 100644 --- a/storages/python/python-storage-plugins/redis.py +++ b/storages/python/python-storage-plugins/redis.py @@ -1,4 +1,7 @@ -"""DLite storage plugin for Redis written in Python.""" +"""DLite storage plugin for Redis written in Python. + +Note: Continous testing is not run for this plugin. +""" import re from redis import Redis diff --git a/storages/python/tests-c/CMakeLists.txt b/storages/python/tests-c/CMakeLists.txt index c52b4c23f..dd8f06a3e 100644 --- a/storages/python/tests-c/CMakeLists.txt +++ b/storages/python/tests-c/CMakeLists.txt @@ -12,6 +12,7 @@ # Depending on how the server is set up, or if you have a ~/.pgpass # file, PASSWORD can be left undefined. + set(tests test_yaml_storage test_blob_storage @@ -26,6 +27,10 @@ add_definitions( -DCURRENT_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR} ) +if(MSVC) # Unset extra_link_libraries on MSVC to avoid linking against None.lib + unset(extra_link_libraries) +endif() + include(FindPythonModule) # Disable yaml test if PyYAML is not in installed diff --git a/storages/rdf/dlite-rdf.c b/storages/rdf/dlite-rdf.c index c22fec430..d8310edfc 100644 --- a/storages/rdf/dlite-rdf.c +++ b/storages/rdf/dlite-rdf.c @@ -171,7 +171,7 @@ DLiteStorage *rdf_open(const DLiteStoragePlugin *api, const char *uri, if (!(s = calloc(1, sizeof(RdfStorage)))) FAILCODE(dliteMemoryError, "allocation failure"); /* parse options */ - if (dlite_option_parse(optcopy, opts)) goto fail; + if (dlite_option_parse(optcopy, opts, 0)) goto fail; mode = opts[0].value; s->store = (opts[1].value) ? strdup(opts[1].value) : NULL; s->base_uri = (opts[2].value) ? strdup(opts[2].value) : NULL; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f337c5489..fffb103a3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -4,6 +4,10 @@ project(dlite-tools C) # TODO - It would be preferable to link with the shared libraries. # How to do that without breaking the docker build? +if(MSVC) # Unset extra_link_libraries on MSVC to avoid linking against None.lib + unset(extra_link_libraries) +endif() + # dlite-getuuid add_executable(dlite-getuuid dlite-getuuid.c) target_link_libraries(dlite-getuuid @@ -60,6 +64,6 @@ install( ) install( DIRECTORY templates - DESTINATION share/dlite + DESTINATION ${Python3_PREFIX}share/dlite PATTERN "*~" EXCLUDE )