Add shim module and support for GDAL 3 (#804)

* Add shim module and support for GDAL 3

* Add GDAL 2.4 and 3.0 to build matrix

* Don't upgrade deps

* Don't use setup.py directly

* Add PROJ to the build matrix

* Add proj build script

* Update GDAL build script

* Allow travis to wait for GDAL build

* Wait 30 on GDAL build

* Increase wait to 40 and make jobs to 4

* Remove wait so we can see what the problem is

* Unsilence make

* Remove proj apt package

* set PROJ_LIB correctly

* Call OSRFixup for GDAL 1 and 2

* Cast open options for GDALOpenEx

* Add missing OSRFixup declarations

.travis.yml

@@ -1,5 +1,11 @@
+dist: trusty
+
 language: python
-sudo: false
+
+python:
+  - "2.7"
+  - "3.6"
+
 cache:
   directories:
     - $GDALINST
@@ -11,50 +17,48 @@ env:
     - PIP_FIND_LINKS=file://$HOME/.cache/pip/wheels
     - GDALINST=$HOME/gdalinstall
     - GDALBUILD=$HOME/gdalbuild
+    - PROJINST=$HOME/gdalinstall
+    - PROJBUILD=$HOME/projbuild
   matrix:
-    - GDALVERSION="1.11.5"
-    - GDALVERSION="2.0.3"
-    - GDALVERSION="2.1.4"
-    - GDALVERSION="2.2.4"
-    - GDALVERSION="2.3.2"
-    - GDALVERSION="trunk"
+    - GDALVERSION="1.11.5" PROJVERSION="4.8.0"
+    - GDALVERSION="2.0.3" PROJVERSION="4.9.3"
+    - GDALVERSION="2.1.4" PROJVERSION="4.9.3"
+    - GDALVERSION="2.2.4" PROJVERSION="4.9.3"
+    - GDALVERSION="2.3.3" PROJVERSION="4.9.3"
+    - GDALVERSION="2.4.2" PROJVERSION="4.9.3"
+    - GDALVERSION="3.0.1" PROJVERSION="6.1.1"
+    - GDALVERSION="trunk" PROJVERSION="6.1.1"
 
 matrix:
   allow_failures:
-    - env: GDALVERSION="trunk"
+    - env: GDALVERSION="trunk" PROJVERSION="6.1.1"
 
 addons:
   apt:
     packages:
-    - gdal-bin
-    - libproj-dev
-    - libhdf5-serial-dev
     - libgdal-dev
     - libatlas-dev
     - libatlas-base-dev
     - gfortran
 
-python:
-  - "2.7"
-  - "3.6"
-
 before_install:
   - pip install -U pip
   - pip install wheel coveralls>=1.1 --upgrade
   - pip install setuptools==36.0.1
   - pip install wheel
+  - . ./scripts/travis_proj_install.sh
   - . ./scripts/travis_gdal_install.sh
   - export PATH=$GDALINST/gdal-$GDALVERSION/bin:$PATH
   - export LD_LIBRARY_PATH=$GDALINST/gdal-$GDALVERSION/lib:$LD_LIBRARY_PATH
   - export GDAL_DATA=$GDALINST/gdal-$GDALVERSION/share/gdal
-  - export PROJ_LIB=/usr/share/proj
+  - export PROJ_LIB=$GDALINST/gdal-$GDALVERSION/share/proj
   - gdal-config --version
 
 install:
-  - pip install -r requirements-dev.txt
   - if [ "$GDALVERSION" = "trunk" ]; then echo "Using gdal trunk"; elif [ $(gdal-config --version) == "$GDALVERSION" ]; then echo "Using gdal $GDALVERSION"; else echo "NOT using gdal $GDALVERSION as expected; aborting"; exit 1; fi
-  - pip install --upgrade --force-reinstall --global-option=build_ext --global-option='-I$GDALINST/gdal-$GDALVERSION/include' --global-option='-L$GDALINST/gdal-$GDALVERSION/lib' --global-option='-R$GDALINST/gdal-$GDALVERSION/lib' -e .
-  - pip install -e .[test]
+  - "python -m pip wheel -r requirements-dev.txt"
+  - "python -m pip install -r requirements-dev.txt"
+  - "GDAL_CONFIG=$GDALINST/gdal-$GDALVERSION/bin/gdal-config python -m pip install --upgrade --force-reinstall --no-use-pep517 -e .[test]"
   - fio --version
   - gdal-config --version
   - fio --gdal-version

CHANGES.txt

@@ -3,6 +3,11 @@ Changes
 
 All issue numbers are relative to https://github.com/Toblerity/Fiona/issues.
 
+1.8.9 (TBD)
+-----------
+
+- A shim module and support for GDAL 3.0 has been added.
+
 1.8.8 (2019-09-25)
 ------------------
 

fiona/__init__.py

@@ -101,7 +101,7 @@ class Path:
 
 
 __all__ = ['bounds', 'listlayers', 'open', 'prop_type', 'prop_width']
-__version__ = "1.8.8"
+__version__ = "1.8.9dev"
 __gdal_version__ = get_gdal_release_name()
 
 gdal_version = get_gdal_version_tuple()

fiona/_crs.pxd

@@ -12,7 +12,6 @@ cdef extern from "ogr_srs_api.h":
     int     OSRImportFromProj4 (OGRSpatialReferenceH srs, char *proj)
     int     OSRSetFromUserInput (OGRSpatialReferenceH srs, char *input)
     int     OSRAutoIdentifyEPSG (OGRSpatialReferenceH srs)
-    int     OSRFixup(OGRSpatialReferenceH srs)
     const char * OSRGetAuthorityName (OGRSpatialReferenceH srs, const char *key)
     const char * OSRGetAuthorityCode (OGRSpatialReferenceH srs, const char *key)
     OGRSpatialReferenceH  OSRNewSpatialReference (char *wkt)

fiona/_crs.pyx

@@ -10,16 +10,20 @@ import logging
 from six import string_types
 
 from fiona cimport _cpl
+from fiona._shim cimport osr_get_name, osr_set_traditional_axis_mapping_strategy
+
 from fiona.errors import CRSError
 
 
 logger = logging.getLogger(__name__)
 
+cdef int OAMS_TRADITIONAL_GIS_ORDER = 0
+
 
 # Export a WKT string from input crs.
 def crs_to_wkt(crs):
     """Convert a Fiona CRS object to WKT format"""
-    cdef void *cogr_srs = NULL
+    cdef OGRSpatialReferenceH cogr_srs = NULL
     cdef char *proj_c = NULL
 
     cogr_srs = OSRNewSpatialReference(NULL)
@@ -31,6 +35,7 @@ def crs_to_wkt(crs):
         proj_b = crs.encode('utf-8')
         proj_c = proj_b
         OSRSetFromUserInput(cogr_srs, proj_c)
+
     elif isinstance(crs, dict):
         # EPSG is a special case.
         init = crs.get('init')
@@ -53,12 +58,11 @@ def crs_to_wkt(crs):
             proj_b = proj.encode('utf-8')
             proj_c = proj_b
             OSRImportFromProj4(cogr_srs, proj_c)
+
     else:
         raise ValueError("Invalid CRS")
 
-    # Fixup, export to WKT, and set the GDAL dataset's projection.
-    OSRFixup(cogr_srs)
-
+    osr_set_traditional_axis_mapping_strategy(cogr_srs)
     OSRExportToWkt(cogr_srs, &proj_c)
 
     if proj_c == NULL:

fiona/_shim1.pxd

@@ -19,6 +19,8 @@ cdef OGRFieldSubType get_field_subtype(void *fielddefn)
 cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype)
 cdef bint check_capability_create_layer(void *cogr_ds)
 cdef void *get_linear_geometry(void *geom)
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs)
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs)
 
 from fiona._shim cimport OGR_F_GetFieldAsInteger as OGR_F_GetFieldAsInteger64
 from fiona._shim cimport OGR_F_SetFieldInteger as OGR_F_SetFieldInteger64

fiona/_shim1.pyx

@@ -123,3 +123,11 @@ cdef bint check_capability_create_layer(void *cogr_ds):
 
 cdef void *get_linear_geometry(void *geom):
     return geom
+
+
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
+    return ''
+
+
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs):
+    OSRFixup(hSrs)

fiona/_shim2.pxd

@@ -12,3 +12,5 @@ cdef OGRFieldSubType get_field_subtype(void *fielddefn)
 cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype)
 cdef bint check_capability_create_layer(void *cogr_ds)
 cdef void *get_linear_geometry(void *geom)
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs)
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs)

fiona/_shim2.pyx

@@ -125,3 +125,8 @@ cdef bint check_capability_create_layer(void *cogr_ds):
 cdef void *get_linear_geometry(void *geom):
     return OGR_G_GetLinearGeometry(geom, 0.0, NULL)
 
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
+    return ''
+
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs):
+    OSRFixup(hSrs)

fiona/_shim22.pxd

@@ -12,3 +12,5 @@ cdef OGRFieldSubType get_field_subtype(void *fielddefn)
 cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype)
 cdef bint check_capability_create_layer(void *cogr_ds)
 cdef void *get_linear_geometry(void *geom)
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs)
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs)

fiona/_shim22.pyx

@@ -132,3 +132,9 @@ cdef bint check_capability_create_layer(void *cogr_ds):
 
 cdef void *get_linear_geometry(void *geom):
     return OGR_G_GetLinearGeometry(geom, 0.0, NULL)
+
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
+    return ''
+
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs):
+    OSRFixup(hSrs)

fiona/_shim3.pxd

@@ -0,0 +1,16 @@
+include "ogrext3.pxd"
+
+cdef bint is_field_null(void *feature, int n)
+cdef void set_field_null(void *feature, int n)
+cdef void gdal_flush_cache(void *cogr_ds)
+cdef void* gdal_open_vector(const char *path_c, int mode, drivers, options) except NULL
+cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NULL
+cdef OGRErr gdal_start_transaction(void *cogr_ds, int force)
+cdef OGRErr gdal_commit_transaction(void *cogr_ds)
+cdef OGRErr gdal_rollback_transaction(void *cogr_ds)
+cdef OGRFieldSubType get_field_subtype(void *fielddefn)
+cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype)
+cdef bint check_capability_create_layer(void *cogr_ds)
+cdef void *get_linear_geometry(void *geom)
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs)
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs)

fiona/_shim3.pyx

@@ -0,0 +1,149 @@
+"""Shims on top of ogrext for GDAL versions >= 3.0"""
+
+cdef extern from "ogr_api.h":
+
+    int OGR_F_IsFieldNull(void *feature, int n)
+
+
+cdef extern from "ogr_srs_api.h" nogil:
+
+    ctypedef enum OSRAxisMappingStrategy:
+        OAMS_TRADITIONAL_GIS_ORDER
+
+    const char* OSRGetName(OGRSpatialReferenceH hSRS)
+    void OSRSetAxisMappingStrategy(OGRSpatialReferenceH hSRS, OSRAxisMappingStrategy)
+
+
+from fiona.ogrext2 cimport *
+from fiona._err cimport exc_wrap_pointer
+from fiona._err import cpl_errs, CPLE_BaseError, FionaNullPointerError
+from fiona.errors import DriverError
+
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+cdef bint is_field_null(void *feature, int n):
+    if OGR_F_IsFieldNull(feature, n):
+        return True
+    elif not OGR_F_IsFieldSet(feature, n):
+        return True
+    else:
+        return False
+
+
+cdef void set_field_null(void *feature, int n):
+    OGR_F_SetFieldNull(feature, n)
+
+
+cdef void gdal_flush_cache(void *cogr_ds):
+    with cpl_errs:
+        GDALFlushCache(cogr_ds)
+
+
+cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL:
+    cdef void* cogr_ds = NULL
+    cdef char **drvs = NULL
+    cdef void* drv = NULL
+    cdef char **open_opts = NULL
+
+    flags = GDAL_OF_VECTOR | GDAL_OF_VERBOSE_ERROR
+    if mode == 1:
+        flags |= GDAL_OF_UPDATE
+    else:
+        flags |= GDAL_OF_READONLY
+
+    if drivers:
+        for name in drivers:
+            name_b = name.encode()
+            name_c = name_b
+            drv = GDALGetDriverByName(name_c)
+            if drv != NULL:
+                drvs = CSLAddString(drvs, name_c)
+
+    for k, v in options.items():
+
+        if v is None:
+            continue
+
+        k = k.upper().encode('utf-8')
+        if isinstance(v, bool):
+            v = ('ON' if v else 'OFF').encode('utf-8')
+        else:
+            v = str(v).encode('utf-8')
+        log.debug("Set option %r: %r", k, v)
+        open_opts = CSLAddNameValue(open_opts, <const char *>k, <const char *>v)
+
+    open_opts = CSLAddNameValue(open_opts, "VALIDATE_OPEN_OPTIONS", "NO")
+
+    try:
+        cogr_ds = exc_wrap_pointer(
+            GDALOpenEx(path_c, flags, <const char *const *>drvs, <const char *const *>open_opts, NULL)
+        )
+        return cogr_ds
+    except FionaNullPointerError:
+        raise DriverError("Failed to open dataset (mode={}): {}".format(mode, path_c.decode("utf-8")))
+    except CPLE_BaseError as exc:
+        raise DriverError(str(exc))
+    finally:
+        CSLDestroy(drvs)
+        CSLDestroy(open_opts)
+
+
+cdef void* gdal_create(void* cogr_driver, const char *path_c, options) except NULL:
+    cdef char **creation_opts = NULL
+    cdef void *cogr_ds = NULL
+
+    for k, v in options.items():
+        k = k.upper().encode('utf-8')
+        if isinstance(v, bool):
+            v = ('ON' if v else 'OFF').encode('utf-8')
+        else:
+            v = str(v).encode('utf-8')
+        log.debug("Set option %r: %r", k, v)
+        creation_opts = CSLAddNameValue(creation_opts, <const char *>k, <const char *>v)
+
+    try:
+        return exc_wrap_pointer(GDALCreate(cogr_driver, path_c, 0, 0, 0, GDT_Unknown, creation_opts))
+    except FionaNullPointerError:
+        raise DriverError("Failed to create dataset: {}".format(path_c.decode("utf-8")))
+    except CPLE_BaseError as exc:
+        raise DriverError(str(exc))
+    finally:
+        CSLDestroy(creation_opts)
+
+
+cdef OGRErr gdal_start_transaction(void* cogr_ds, int force):
+    return GDALDatasetStartTransaction(cogr_ds, force)
+
+
+cdef OGRErr gdal_commit_transaction(void* cogr_ds):
+    return GDALDatasetCommitTransaction(cogr_ds)
+
+
+cdef OGRErr gdal_rollback_transaction(void* cogr_ds):
+    return GDALDatasetRollbackTransaction(cogr_ds)
+
+
+cdef OGRFieldSubType get_field_subtype(void *fielddefn):
+    return OGR_Fld_GetSubType(fielddefn)
+
+
+cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype):
+    OGR_Fld_SetSubType(fielddefn, subtype)
+
+
+cdef bint check_capability_create_layer(void *cogr_ds):
+    return GDALDatasetTestCapability(cogr_ds, ODsCCreateLayer)
+
+
+cdef void *get_linear_geometry(void *geom):
+    return OGR_G_GetLinearGeometry(geom, 0.0, NULL)
+
+cdef const char* osr_get_name(OGRSpatialReferenceH hSrs):
+        return OSRGetName(hSrs)
+
+cdef void osr_set_traditional_axis_mapping_strategy(OGRSpatialReferenceH hSrs):
+    OSRSetAxisMappingStrategy(hSrs, OAMS_TRADITIONAL_GIS_ORDER)

fiona/_transform.pyx

@@ -8,6 +8,7 @@ import logging
 
 from fiona cimport _cpl, _crs, _csl, _geometry
 from fiona._crs cimport OGRSpatialReferenceH
+from fiona._shim cimport osr_set_traditional_axis_mapping_strategy
 
 from fiona.compat import UserDict
 
@@ -52,6 +53,7 @@ cdef void *_crs_from_crs(object crs):
             auth, val = init.split(':')
             if auth.upper() == 'EPSG':
                 _crs.OSRImportFromEPSG(osr, int(val))
+                osr_set_traditional_axis_mapping_strategy(osr)
         else:
             crs['wktext'] = True
             for k, v in crs.items():
@@ -64,6 +66,7 @@ cdef void *_crs_from_crs(object crs):
             proj_b = proj.encode('utf-8')
             proj_c = proj_b
             _crs.OSRImportFromProj4(osr, proj_c)
+            osr_set_traditional_axis_mapping_strategy(osr)
     # Fall back for CRS strings like "EPSG:3857."
     else:
         proj_b = crs.encode('utf-8')