diff --git a/.github/workflows/alpine/Dockerfile.ci b/.github/workflows/alpine/Dockerfile.ci index 2506ff2739da..05d1c09c09d7 100644 --- a/.github/workflows/alpine/Dockerfile.ci +++ b/.github/workflows/alpine/Dockerfile.ci @@ -22,6 +22,7 @@ RUN apk add \ kealib-dev \ libaec-dev \ libarchive-dev \ + libavif-dev \ libdeflate-dev \ libgeotiff-dev \ libheif-dev \ diff --git a/.github/workflows/alpine_32bit/Dockerfile.ci b/.github/workflows/alpine_32bit/Dockerfile.ci index bd85ef1320cb..7080b05992bb 100644 --- a/.github/workflows/alpine_32bit/Dockerfile.ci +++ b/.github/workflows/alpine_32bit/Dockerfile.ci @@ -24,6 +24,7 @@ RUN apk add \ kealib-dev \ libaec-dev \ libarchive-dev \ + libavif-dev \ libdeflate-dev \ libgeotiff-dev \ libheif-dev \ diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index c9eb17aaaca9..2eee98263c9d 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -325,7 +325,7 @@ jobs: mingw-w64-x86_64-geos mingw-w64-x86_64-libspatialite mingw-w64-x86_64-proj mingw-w64-x86_64-cgal mingw-w64-x86_64-libfreexl mingw-w64-x86_64-hdf5 mingw-w64-x86_64-netcdf mingw-w64-x86_64-poppler mingw-w64-x86_64-podofo mingw-w64-x86_64-postgresql mingw-w64-x86_64-libgeotiff mingw-w64-x86_64-libpng mingw-w64-x86_64-libtiff mingw-w64-x86_64-openjpeg2 - mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-pytest mingw-w64-x86_64-python-setuptools mingw-w64-x86_64-python-lxml mingw-w64-x86_64-swig mingw-w64-x86_64-python-psutil mingw-w64-x86_64-blosc + mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-pytest mingw-w64-x86_64-python-setuptools mingw-w64-x86_64-python-lxml mingw-w64-x86_64-swig mingw-w64-x86_64-python-psutil mingw-w64-x86_64-blosc mingw-w64-x86_64-libavif - name: Setup cache uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 id: cache @@ -429,7 +429,7 @@ jobs: cfitsio freexl geotiff libjpeg-turbo libpq libspatialite libwebp-base pcre pcre2 postgresql \ sqlite tiledb zstd cryptopp cgal doxygen librttopo libkml openssl xz \ openjdk ant qhull armadillo blas blas-devel libblas libcblas liblapack liblapacke blosc libarchive \ - arrow-cpp pyarrow libaec cmake + arrow-cpp pyarrow libaec libavif cmake - name: Check CMake version shell: bash -l {0} run: | diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 8e09edd06b88..905cbdb2a0ab 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -101,6 +101,20 @@ jobs: # SC2129: (style): Consider using { cmd1; cmd2; } >> file instead of individual redirects run: shellcheck -e SC2086,SC2046,SC2164,SC2054,SC2129 $(find . -name '*.sh' -a -not -name ltmain.sh -a -not -wholename "./autotest/*" -a -not -wholename "./.github/*") + binary_files: + runs-on: ubuntu-latest + steps: + + - name: Install Requirements + run: | + sudo apt-get install -y python3 coreutils + + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Detect binary files + run: python3 ./scripts/check_binaries.py + linting: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/fedora_rawhide/Dockerfile.ci b/.github/workflows/fedora_rawhide/Dockerfile.ci index cb764b75ae00..ca6bb876267b 100644 --- a/.github/workflows/fedora_rawhide/Dockerfile.ci +++ b/.github/workflows/fedora_rawhide/Dockerfile.ci @@ -20,6 +20,7 @@ RUN dnf install -y clang make diffutils ccache cmake \ armadillo-devel qhull-devel \ hdf-devel hdf5-devel netcdf-devel \ libpq-devel \ + libavif-devel \ python3-setuptools python3-pip python3-devel python3-lxml swig \ glibc-gconv-extra diff --git a/.github/workflows/ubuntu_22.04/Dockerfile.ci b/.github/workflows/ubuntu_22.04/Dockerfile.ci index d6db8850de46..db60a6e92941 100644 --- a/.github/workflows/ubuntu_22.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_22.04/Dockerfile.ci @@ -16,6 +16,7 @@ RUN apt-get update && \ g++ \ git \ gpsbabel \ + libavif-dev \ libblosc-dev \ libboost-dev \ libcairo2-dev \ diff --git a/.github/workflows/ubuntu_22.04/services.sh b/.github/workflows/ubuntu_22.04/services.sh index d4d9c002aa2c..19fe9c62404a 100755 --- a/.github/workflows/ubuntu_22.04/services.sh +++ b/.github/workflows/ubuntu_22.04/services.sh @@ -7,9 +7,9 @@ set -ex ################## # MSSQL: server side -docker rm -f gdal-sql1 -docker pull mcr.microsoft.com/mssql/server:2017-latest -docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=DummyPassw0rd' -p 1433:1433 --name gdal-sql1 -d mcr.microsoft.com/mssql/server:2017-latest +#docker rm -f gdal-sql1 +#docker pull mcr.microsoft.com/mssql/server:2017-latest +#docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=DummyPassw0rd' -p 1433:1433 --name gdal-sql1 -d mcr.microsoft.com/mssql/server:2017-latest # MySQL 8 docker rm -f gdal-mysql1 @@ -38,7 +38,7 @@ docker run --name gdal-mongo -p 27018:27017 -d mongo:4.4 sleep 10 # MSSQL -docker exec -t gdal-sql1 /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -U SA -P DummyPassw0rd -Q "CREATE DATABASE TestDB" +#docker exec -t gdal-sql1 /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -U SA -P DummyPassw0rd -Q "CREATE DATABASE TestDB" # MySQL docker exec gdal-mysql1 sh -c "echo 'CREATE DATABASE test; SELECT Version()' | mysql -uroot -ppasswd" diff --git a/.github/workflows/ubuntu_22.04/test.sh b/.github/workflows/ubuntu_22.04/test.sh index 9af1dd53f7b7..8bcc8b27ba80 100755 --- a/.github/workflows/ubuntu_22.04/test.sh +++ b/.github/workflows/ubuntu_22.04/test.sh @@ -31,6 +31,6 @@ AZURE_STORAGE_CONNECTION_STRING=${AZURITE_STORAGE_CONNECTION_STRING} python3 -c # MongoDB v3 (cd autotest && MONGODBV3_TEST_PORT=27018 MONGODBV3_TEST_HOST=$IP $PYTEST ogr/ogr_mongodbv3.py) -(cd autotest && OGR_MSSQL_CONNECTION_STRING="MSSQL:server=$IP;database=TestDB;driver=ODBC Driver 17 for SQL Server;UID=SA;PWD=DummyPassw0rd" $PYTEST ogr/ogr_mssqlspatial.py) +#(cd autotest && OGR_MSSQL_CONNECTION_STRING="MSSQL:server=$IP;database=TestDB;driver=ODBC Driver 17 for SQL Server;UID=SA;PWD=DummyPassw0rd" $PYTEST ogr/ogr_mssqlspatial.py) (cd autotest && $PYTEST) diff --git a/.github/workflows/ubuntu_24.04/Dockerfile.ci b/.github/workflows/ubuntu_24.04/Dockerfile.ci index 6839a077a1b1..213d1a36069a 100644 --- a/.github/workflows/ubuntu_24.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_24.04/Dockerfile.ci @@ -16,6 +16,7 @@ RUN apt-get update && \ g++ \ git \ gpsbabel \ + libavif-dev \ libblosc-dev \ libboost-dev \ libcairo2-dev \ diff --git a/.github/workflows/ubuntu_24.04/services.sh b/.github/workflows/ubuntu_24.04/services.sh index d4d9c002aa2c..19fe9c62404a 100755 --- a/.github/workflows/ubuntu_24.04/services.sh +++ b/.github/workflows/ubuntu_24.04/services.sh @@ -7,9 +7,9 @@ set -ex ################## # MSSQL: server side -docker rm -f gdal-sql1 -docker pull mcr.microsoft.com/mssql/server:2017-latest -docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=DummyPassw0rd' -p 1433:1433 --name gdal-sql1 -d mcr.microsoft.com/mssql/server:2017-latest +#docker rm -f gdal-sql1 +#docker pull mcr.microsoft.com/mssql/server:2017-latest +#docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=DummyPassw0rd' -p 1433:1433 --name gdal-sql1 -d mcr.microsoft.com/mssql/server:2017-latest # MySQL 8 docker rm -f gdal-mysql1 @@ -38,7 +38,7 @@ docker run --name gdal-mongo -p 27018:27017 -d mongo:4.4 sleep 10 # MSSQL -docker exec -t gdal-sql1 /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -U SA -P DummyPassw0rd -Q "CREATE DATABASE TestDB" +#docker exec -t gdal-sql1 /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -U SA -P DummyPassw0rd -Q "CREATE DATABASE TestDB" # MySQL docker exec gdal-mysql1 sh -c "echo 'CREATE DATABASE test; SELECT Version()' | mysql -uroot -ppasswd" diff --git a/.github/workflows/ubuntu_24.04/test.sh b/.github/workflows/ubuntu_24.04/test.sh index 5ce09486593d..f653a1fdd752 100755 --- a/.github/workflows/ubuntu_24.04/test.sh +++ b/.github/workflows/ubuntu_24.04/test.sh @@ -34,6 +34,6 @@ AZURE_STORAGE_CONNECTION_STRING=${AZURITE_STORAGE_CONNECTION_STRING} python3 -c # MongoDB v3 (cd autotest && MONGODBV3_TEST_PORT=27018 MONGODBV3_TEST_HOST=$IP $PYTEST ogr/ogr_mongodbv3.py) -(cd autotest && OGR_MSSQL_CONNECTION_STRING="MSSQL:server=$IP;database=TestDB;driver=ODBC Driver 17 for SQL Server;UID=SA;PWD=DummyPassw0rd" $PYTEST ogr/ogr_mssqlspatial.py) +# (cd autotest && OGR_MSSQL_CONNECTION_STRING="MSSQL:server=$IP;database=TestDB;driver=ODBC Driver 17 for SQL Server;UID=SA;PWD=DummyPassw0rd" $PYTEST ogr/ogr_mssqlspatial.py) (cd autotest && $PYTEST) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 56a9e0a98db4..58b2285412a8 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -12,7 +12,7 @@ build: jobs: post_checkout: - - (git --no-pager log --pretty="tformat:%s -- %b" -1 | grep -viqP "skip ci|ci skip") || exit 183 + - (git --no-pager log --pretty="tformat:%s -- %b" -1 | paste -s -d " " | grep -viqP "skip ci|ci skip") || exit 183 pre_build: - ./doc/rtd/pre_build.sh - cd doc && make doxygen generated_rst_files diff --git a/Doxyfile b/Doxyfile index 87534d9a9456..4bf006b3802a 100644 --- a/Doxyfile +++ b/Doxyfile @@ -382,6 +382,7 @@ INPUT = port \ gcore \ frmts/gdalallregister.cpp \ alg \ + alg/viewshed \ frmts/vrt \ apps \ ogr \ diff --git a/MIGRATION_GUIDE.TXT b/MIGRATION_GUIDE.TXT index f1155642d493..146ed58ef8ff 100644 --- a/MIGRATION_GUIDE.TXT +++ b/MIGRATION_GUIDE.TXT @@ -18,6 +18,9 @@ MIGRATION GUIDE FROM GDAL 3.9 to GDAL 3.10 - Python bindings: Band.GetStatistics() and Band.ComputeStatistics() now return a None value in case of error (when exceptions are not enabled) +- New color interpretation (GCI_xxxx) items have been added to the GDALColorInterp + enumeration. Code testing color interpretation may need to be adapted. + MIGRATION GUIDE FROM GDAL 3.8 to GDAL 3.9 ----------------------------------------- diff --git a/alg/CMakeLists.txt b/alg/CMakeLists.txt index ca06ca2fafcc..c4f71ee521a1 100644 --- a/alg/CMakeLists.txt +++ b/alg/CMakeLists.txt @@ -36,7 +36,12 @@ add_library( rasterfill.cpp thinplatespline.cpp gdal_simplesurf.cpp - viewshed.cpp + viewshed/combiner.cpp + viewshed/cumulative.cpp + viewshed/progress.cpp + viewshed/util.cpp + viewshed/viewshed.cpp + viewshed/viewshed_executor.cpp gdalgenericinverse.cpp gdal_interpolateatpoint.cpp ) diff --git a/alg/gdaltransformer.cpp b/alg/gdaltransformer.cpp index 256a79ed6b8b..3eda4c75b31c 100644 --- a/alg/gdaltransformer.cpp +++ b/alg/gdaltransformer.cpp @@ -3828,7 +3828,8 @@ static CPLXMLNode *GDALSerializeApproxTransformer(void *pTransformArg) * @param pBaseTransformArg the callback argument for the high precision * transformer. * @param dfMaxError the maximum cartesian error in the "output" space that - * is to be accepted in the linear approximation. + * is to be accepted in the linear approximation, evaluated as a Manhattan + * distance. * * @return callback pointer suitable for use with GDALApproxTransform(). It * should be deallocated with GDALDestroyApproxTransformer(). diff --git a/alg/gdalwarper.cpp b/alg/gdalwarper.cpp index 079211ae4750..3f09b4b2409a 100644 --- a/alg/gdalwarper.cpp +++ b/alg/gdalwarper.cpp @@ -1167,6 +1167,20 @@ CPLErr GDALWarpDstAlphaMasker(void *pMaskFuncArg, int nBandCount, * will be selected, not just those whose center point falls within the * polygon. * + *
  • XSCALE: Ratio expressing the resampling factor (number of destination + * pixels per source pixel) along the target horizontal axis. + * The scale is used to determine the number of source pixels along the x-axis + * that are considered by the resampling algorithm. + * Equals to one for no resampling, below one for downsampling + * and above one for upsampling. This is automatically computed, for each + * processing chunk, and may thus vary among them, depending on the + * shape of output regions vs input regions. Such variations can be undesired + * in some situations. If the resampling factor can be considered as constant + * over the warped area, setting a constant value can lead to more reproducible + * pixel output.
    • + * + *
  • YSCALE: Same as XSCALE, but along the horizontal axis.
    • + * *
  • OPTIMIZE_SIZE: This defaults to FALSE, but may be set to TRUE * typically when writing to a compressed dataset (GeoTIFF with * COMPRESS creation option set for example) for achieving a smaller @@ -1176,7 +1190,11 @@ CPLErr GDALWarpDstAlphaMasker(void *pMaskFuncArg, int nBandCount, * of the file. However sticking to target block size may cause major * processing slowdown for some particular reprojections. Starting * with GDAL 3.8, OPTIMIZE_SIZE mode is automatically enabled when it is safe - * to do so.
    • + * to do so. + * As this parameter influences the shape of warping chunk, and by default the + * XSCALE and YSCALE parameters are computed per warping chunk, this parameter may + * influence the pixel output. + * * *
  • NUM_THREADS: (GDAL >= 1.10) Can be set to a numeric value or ALL_CPUS to * set the number of threads to use to parallelize the computation part of the @@ -1536,6 +1554,32 @@ void CPL_STDCALL GDALWarpResolveWorkingDataType(GDALWarpOptions *psOptions) psOptions->eWorkingDataType = GDT_Byte; + // If none of the provided input nodata values can be represented in the + // data type of the corresponding source band, ignore them. + if (psOptions->hSrcDS && psOptions->padfSrcNoDataReal) + { + int nCountInvalidSrcNoDataReal = 0; + for (int iBand = 0; iBand < psOptions->nBandCount; iBand++) + { + GDALRasterBandH hSrcBand = GDALGetRasterBand( + psOptions->hSrcDS, psOptions->panSrcBands[iBand]); + + if (hSrcBand && + !GDALIsValueExactAs(psOptions->padfSrcNoDataReal[iBand], + GDALGetRasterDataType(hSrcBand))) + { + nCountInvalidSrcNoDataReal++; + } + } + if (nCountInvalidSrcNoDataReal == psOptions->nBandCount) + { + CPLFree(psOptions->padfSrcNoDataReal); + psOptions->padfSrcNoDataReal = nullptr; + CPLFree(psOptions->padfSrcNoDataImag); + psOptions->padfSrcNoDataImag = nullptr; + } + } + for (int iBand = 0; iBand < psOptions->nBandCount; iBand++) { if (psOptions->hDstDS != nullptr) diff --git a/alg/gdalwarpoperation.cpp b/alg/gdalwarpoperation.cpp index 1448de37f95b..d84d2c08b943 100644 --- a/alg/gdalwarpoperation.cpp +++ b/alg/gdalwarpoperation.cpp @@ -832,22 +832,6 @@ void GDALDestroyWarpOperation(GDALWarpOperationH hOperation) /* CollectChunkList() */ /************************************************************************/ -static int OrderWarpChunk(const void *_a, const void *_b) -{ - const GDALWarpChunk *a = static_cast(_a); - const GDALWarpChunk *b = static_cast(_b); - if (a->dy < b->dy) - return -1; - else if (a->dy > b->dy) - return 1; - else if (a->dx < b->dx) - return -1; - else if (a->dx > b->dx) - return 1; - else - return 0; -} - void GDALWarpOperation::CollectChunkList(int nDstXOff, int nDstYOff, int nDstXSize, int nDstYSize) @@ -859,10 +843,18 @@ void GDALWarpOperation::CollectChunkList(int nDstXOff, int nDstYOff, CollectChunkListInternal(nDstXOff, nDstYOff, nDstXSize, nDstYSize); // Sort chunks from top to bottom, and for equal y, from left to right. - // TODO(schwehr): Use std::sort. - if (pasChunkList) - qsort(pasChunkList, nChunkListCount, sizeof(GDALWarpChunk), - OrderWarpChunk); + if (nChunkListCount > 1) + { + std::sort(pasChunkList, pasChunkList + nChunkListCount, + [](const GDALWarpChunk &a, const GDALWarpChunk &b) + { + if (a.dy < b.dy) + return true; + if (a.dy > b.dy) + return false; + return a.dx < b.dx; + }); + } /* -------------------------------------------------------------------- */ /* Find the global source window. */ diff --git a/alg/viewshed.cpp b/alg/viewshed.cpp deleted file mode 100644 index 6fb84ad6f736..000000000000 --- a/alg/viewshed.cpp +++ /dev/null @@ -1,1064 +0,0 @@ -/****************************************************************************** - * - * Project: Viewshed Generation - * Purpose: Core algorithm implementation for viewshed generation. - * Author: Tamas Szekeres, szekerest@gmail.com - * - ****************************************************************************** - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - ****************************************************************************/ - -#include -#include -#include -#include -#include - -#include "gdal_alg.h" -#include "gdal_priv_templates.hpp" - -#include "viewshed.h" - -/************************************************************************/ -/* GDALViewshedGenerate() */ -/************************************************************************/ - -/** - * Create viewshed from raster DEM. - * - * This algorithm will generate a viewshed raster from an input DEM raster - * by using a modified algorithm of "Generating Viewsheds without Using - * Sightlines" published at - * https://www.asprs.org/wp-content/uploads/pers/2000journal/january/2000_jan_87-90.pdf - * This appoach provides a relatively fast calculation, since the output raster - * is generated in a single scan. The gdal/apps/gdal_viewshed.cpp mainline can - * be used as an example of how to use this function. The output raster will be - * of type Byte or Float64. - * - * \note The algorithm as implemented currently will only output meaningful - * results if the georeferencing is in a projected coordinate reference system. - * - * @param hBand The band to read the DEM data from. Only the part of the raster - * within the specified maxdistance around the observer point is processed. - * - * @param pszDriverName Driver name (GTiff if set to NULL) - * - * @param pszTargetRasterName The name of the target raster to be generated. - * Must not be NULL - * - * @param papszCreationOptions creation options. - * - * @param dfObserverX observer X value (in SRS units) - * - * @param dfObserverY observer Y value (in SRS units) - * - * @param dfObserverHeight The height of the observer above the DEM surface. - * - * @param dfTargetHeight The height of the target above the DEM surface. - * (default 0) - * - * @param dfVisibleVal pixel value for visibility (default 255) - * - * @param dfInvisibleVal pixel value for invisibility (default 0) - * - * @param dfOutOfRangeVal The value to be set for the cells that fall outside of - * the range specified by dfMaxDistance. - * - * @param dfNoDataVal The value to be set for the cells that have no data. - * If set to a negative value, nodata is not set. - * Note: currently, no special processing of input cells at a - * nodata value is done (which may result in erroneous results). - * - * @param dfCurvCoeff Coefficient to consider the effect of the curvature and - * refraction. The height of the DEM is corrected according to the following - * formula: [Height] -= dfCurvCoeff * [Target Distance]^2 / [Earth Diameter] For - * the effect of the atmospheric refraction we can use 0.85714. - * - * @param eMode The mode of the viewshed calculation. - * Possible values GVM_Diagonal = 1, GVM_Edge = 2 (default), GVM_Max = 3, - * GVM_Min = 4. - * - * @param dfMaxDistance maximum distance range to compute viewshed. - * It is also used to clamp the extent of the output - * raster. If set to 0, then unlimited range is assumed, that is to say the - * computation is performed on the extent of the whole - * raster. - * - * @param pfnProgress A GDALProgressFunc that may be used to report progress - * to the user, or to interrupt the algorithm. May be NULL if not required. - * - * @param pProgressArg The callback data for the pfnProgress function. - * - * @param heightMode Type of information contained in output raster. Possible - * values GVOT_NORMAL = 1 (default), GVOT_MIN_TARGET_HEIGHT_FROM_DEM = 2, - * GVOT_MIN_TARGET_HEIGHT_FROM_GROUND = 3 - * - * GVOT_NORMAL returns a raster of type Byte containing - * visible locations. - * - * GVOT_MIN_TARGET_HEIGHT_FROM_DEM and - * GVOT_MIN_TARGET_HEIGHT_FROM_GROUND will return a raster of type Float64 - * containing the minimum target height for target to be visible from the DEM - * surface or ground level respectively. Parameters dfTargetHeight, dfVisibleVal - * and dfInvisibleVal will be ignored. - * - * - * @param papszExtraOptions Future extra options. Must be set to NULL currently. - * - * @return not NULL output dataset on success (to be closed with GDALClose()) or - * NULL if an error occurs. - * - * @since GDAL 3.1 - */ -GDALDatasetH GDALViewshedGenerate( - GDALRasterBandH hBand, const char *pszDriverName, - const char *pszTargetRasterName, CSLConstList papszCreationOptions, - double dfObserverX, double dfObserverY, double dfObserverHeight, - double dfTargetHeight, double dfVisibleVal, double dfInvisibleVal, - double dfOutOfRangeVal, double dfNoDataVal, double dfCurvCoeff, - GDALViewshedMode eMode, double dfMaxDistance, GDALProgressFunc pfnProgress, - void *pProgressArg, GDALViewshedOutputType heightMode, - [[maybe_unused]] CSLConstList papszExtraOptions) -{ - using namespace gdal; - - Viewshed::Options oOpts; - oOpts.outputFormat = pszDriverName; - oOpts.outputFilename = pszTargetRasterName; - oOpts.creationOpts = papszCreationOptions; - oOpts.observer.x = dfObserverX; - oOpts.observer.y = dfObserverY; - oOpts.observer.z = dfObserverHeight; - oOpts.targetHeight = dfTargetHeight; - oOpts.curveCoeff = dfCurvCoeff; - oOpts.maxDistance = dfMaxDistance; - oOpts.nodataVal = dfNoDataVal; - - switch (eMode) - { - case GVM_Edge: - oOpts.cellMode = Viewshed::CellMode::Edge; - break; - case GVM_Diagonal: - oOpts.cellMode = Viewshed::CellMode::Diagonal; - break; - case GVM_Min: - oOpts.cellMode = Viewshed::CellMode::Min; - break; - case GVM_Max: - oOpts.cellMode = Viewshed::CellMode::Max; - break; - } - - switch (heightMode) - { - case GVOT_MIN_TARGET_HEIGHT_FROM_DEM: - oOpts.outputMode = Viewshed::OutputMode::DEM; - break; - case GVOT_MIN_TARGET_HEIGHT_FROM_GROUND: - oOpts.outputMode = Viewshed::OutputMode::Ground; - break; - case GVOT_NORMAL: - oOpts.outputMode = Viewshed::OutputMode::Normal; - break; - } - - if (!GDALIsValueInRange(dfVisibleVal)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "dfVisibleVal out of range. Must be [0, 255]."); - return nullptr; - } - if (!GDALIsValueInRange(dfInvisibleVal)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "dfInvisibleVal out of range. Must be [0, 255]."); - return nullptr; - } - if (!GDALIsValueInRange(dfOutOfRangeVal)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "dfOutOfRangeVal out of range. Must be [0, 255]."); - return nullptr; - } - oOpts.visibleVal = dfVisibleVal; - oOpts.invisibleVal = dfInvisibleVal; - oOpts.outOfRangeVal = dfOutOfRangeVal; - - gdal::Viewshed v(oOpts); - - if (!pfnProgress) - pfnProgress = GDALDummyProgress; - v.run(hBand, pfnProgress, pProgressArg); - - return GDALDataset::FromHandle(v.output().release()); -} - -namespace gdal -{ - -namespace -{ - -// Calculate the height adjustment factor. -double CalcHeightAdjFactor(const GDALDataset *poDataset, double dfCurveCoeff) -{ - const OGRSpatialReference *poDstSRS = poDataset->GetSpatialRef(); - - if (poDstSRS) - { - OGRErr eSRSerr; - - // If we can't get a SemiMajor axis from the SRS, it will be SRS_WGS84_SEMIMAJOR - double dfSemiMajor = poDstSRS->GetSemiMajor(&eSRSerr); - - /* If we fetched the axis from the SRS, use it */ - if (eSRSerr != OGRERR_FAILURE) - return dfCurveCoeff / (dfSemiMajor * 2.0); - - CPLDebug("GDALViewshedGenerate", - "Unable to fetch SemiMajor axis from spatial reference"); - } - return 0; -} - -/// Calculate the height at nDistance units along a line through the origin given the height -/// at nDistance - 1 units along the line. -/// \param nDistance Distance along the line for the target point. -/// \param Za Height at the line one unit previous to the target point. -double CalcHeightLine(int nDistance, double Za) -{ - nDistance = std::abs(nDistance); - assert(nDistance != 1); - return Za * nDistance / (nDistance - 1); -} - -// Calculate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) -// and passing through the line connecting (i - 1, j, Za) and (i, j - 1, Zb). -// In other words, the origin and the two points form a plane and we're calculating Zc -// of the point (i, j, Zc), also on the plane. -double CalcHeightDiagonal(int i, int j, double Za, double Zb) -{ - return (Za * i + Zb * j) / (i + j - 1); -} - -// Calculate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) -// and through the line connecting (i -1, j - 1, Za) and (i - 1, j, Zb). In other words, -// the origin and the other two points form a plane and we're calculating Zc of the -// point (i, j, Zc), also on the plane. -double CalcHeightEdge(int i, int j, double Za, double Zb) -{ - assert(i != j); - return (Za * i + Zb * (j - i)) / (j - 1); -} - -} // unnamed namespace - -/// Calculate the extent of the output raster in terms of the input raster. -/// -/// @param nX observer X position in the input raster -/// @param nY observer Y position in the input raster -/// @return false on error, true otherwise -bool Viewshed::calcOutputExtent(int nX, int nY) -{ - // We start with the assumption that the output size matches the input. - oOutExtent.xStop = GDALGetRasterBandXSize(pSrcBand); - oOutExtent.yStop = GDALGetRasterBandYSize(pSrcBand); - - if (!oOutExtent.contains(nX, nY)) - CPLError(CE_Warning, CPLE_AppDefined, - "NOTE: The observer location falls outside of the DEM area"); - - constexpr double EPSILON = 1e-8; - if (oOpts.maxDistance > 0) - { - //ABELL - This assumes that the transformation is only a scaling. Should be fixed. - // Find the distance in the direction of the transformed unit vector in the X and Y - // directions and use those factors to determine the limiting values in the raster space. - int nXStart = static_cast( - std::floor(nX - adfInvTransform[1] * oOpts.maxDistance + EPSILON)); - int nXStop = static_cast( - std::ceil(nX + adfInvTransform[1] * oOpts.maxDistance - EPSILON) + - 1); - int nYStart = - static_cast(std::floor( - nY - std::fabs(adfInvTransform[5]) * oOpts.maxDistance + - EPSILON)) - - (adfInvTransform[5] > 0 ? 1 : 0); - int nYStop = static_cast( - std::ceil(nY + std::fabs(adfInvTransform[5]) * oOpts.maxDistance - - EPSILON) + - (adfInvTransform[5] < 0 ? 1 : 0)); - - // If the limits are invalid, set the window size to zero to trigger the error below. - if (nXStart >= oOutExtent.xStop || nXStop < 0 || - nYStart >= oOutExtent.yStop || nYStop < 0) - { - oOutExtent = Window(); - } - else - { - oOutExtent.xStart = std::max(nXStart, 0); - oOutExtent.xStop = std::min(nXStop, oOutExtent.xStop); - - oOutExtent.yStart = std::max(nYStart, 0); - oOutExtent.yStop = std::min(nYStop, oOutExtent.yStop); - } - } - - if (oOutExtent.xSize() == 0 || oOutExtent.ySize() == 0) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid target raster size due to transform " - "and/or distance limitation."); - return false; - } - return true; -} - -/// Read a line of raster data. -/// -/// @param nLine Line number to read. -/// @param data Pointer to location in which to store data. -/// @return Success or failure. -bool Viewshed::readLine(int nLine, double *data) -{ - std::lock_guard g(iMutex); - - if (GDALRasterIO(pSrcBand, GF_Read, oOutExtent.xStart, nLine, - oOutExtent.xSize(), 1, data, oOutExtent.xSize(), 1, - GDT_Float64, 0, 0)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "RasterIO error when reading DEM at position (%d,%d), " - "size (%d,%d)", - oOutExtent.xStart, nLine, oOutExtent.xSize(), 1); - return false; - } - return true; -} - -/// Write an output line of either visibility or height data. -/// -/// @param nLine Line number being written. -/// @param vResult Result line to write. -/// @return True on success, false otherwise. -bool Viewshed::writeLine(int nLine, std::vector &vResult) -{ - // GDALRasterIO isn't thread-safe. - std::lock_guard g(oMutex); - - if (GDALRasterIO(pDstBand, GF_Write, 0, nLine - oOutExtent.yStart, - oOutExtent.xSize(), 1, vResult.data(), oOutExtent.xSize(), - 1, GDT_Float64, 0, 0)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "RasterIO error when writing target raster at position " - "(%d,%d), size (%d,%d)", - 0, nLine - oOutExtent.yStart, oOutExtent.xSize(), 1); - return false; - } - return true; -} - -/// Emit progress information saying that a line has been written to output. -/// -/// @return True on success, false otherwise. -bool Viewshed::lineProgress() -{ - if (nLineCount < oCurExtent.ySize()) - nLineCount++; - return emitProgress(nLineCount / static_cast(oCurExtent.ySize())); -} - -/// Emit progress information saying that a fraction of work has been completed. -/// -/// @return True on success, false otherwise. -bool Viewshed::emitProgress(double fraction) -{ - // Call the progress function. - if (!oProgress(fraction, "")) - { - CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); - return false; - } - return true; -} - -/// Adjust the height of the line of data by the observer height and the curvature of the -/// earth. -/// -/// @param nYOffset Y offset of the line being adjusted. -/// @param nX X location of the observer. -/// @param vThisLineVal Line height data. -/// @return [left, right) Leftmost and one past the rightmost cell in the line within -/// the max distance. Indices are limited to the raster extent (right may be just -/// outside the raster). -std::pair Viewshed::adjustHeight(int nYOffset, int nX, - std::vector &vThisLineVal) -{ - int nLeft = 0; - int nRight = oCurExtent.xSize(); - - // Find the starting point in the raster (nX may be outside) - int nXStart = oCurExtent.clampX(nX); - - // If there is a height adjustment factor other than zero or a max distance, - // calculate the adjusted height of the cell, stopping if we've exceeded the max - // distance. - if (static_cast(dfHeightAdjFactor) || dfMaxDistance2 > 0) - { - // Hoist invariants from the loops. - const double dfLineX = adfTransform[2] * nYOffset; - const double dfLineY = adfTransform[5] * nYOffset; - - // Go left - double *pdfHeight = vThisLineVal.data() + nXStart; - for (int nXOffset = nXStart - nX; nXOffset >= -nX; - nXOffset--, pdfHeight--) - { - double dfX = adfTransform[1] * nXOffset + dfLineX; - double dfY = adfTransform[4] * nXOffset + dfLineY; - double dfR2 = dfX * dfX + dfY * dfY; - if (dfR2 > dfMaxDistance2) - { - nLeft = nXOffset + nX + 1; - break; - } - *pdfHeight -= dfHeightAdjFactor * dfR2 + dfZObserver; - } - - // Go right. - pdfHeight = vThisLineVal.data() + nXStart + 1; - for (int nXOffset = nXStart - nX + 1; - nXOffset < oCurExtent.xSize() - nX; nXOffset++, pdfHeight++) - { - double dfX = adfTransform[1] * nXOffset + dfLineX; - double dfY = adfTransform[4] * nXOffset + dfLineY; - double dfR2 = dfX * dfX + dfY * dfY; - if (dfR2 > dfMaxDistance2) - { - nRight = nXOffset + nX; - break; - } - *pdfHeight -= dfHeightAdjFactor * dfR2 + dfZObserver; - } - } - else - { - // No curvature adjustment. Just normalize for the observer height. - double *pdfHeight = vThisLineVal.data(); - for (int i = 0; i < oCurExtent.xSize(); ++i) - { - *pdfHeight -= dfZObserver; - pdfHeight++; - } - } - return {nLeft, nRight}; -} - -/// Create the output dataset. -/// -/// @return True on success, false otherwise. -bool Viewshed::createOutputDataset() -{ - GDALDriverManager *hMgr = GetGDALDriverManager(); - GDALDriver *hDriver = hMgr->GetDriverByName(oOpts.outputFormat.c_str()); - if (!hDriver) - { - CPLError(CE_Failure, CPLE_AppDefined, "Cannot get driver"); - return false; - } - - /* create output raster */ - poDstDS.reset(hDriver->Create( - oOpts.outputFilename.c_str(), oOutExtent.xSize(), oOutExtent.ySize(), 1, - oOpts.outputMode == OutputMode::Normal ? GDT_Byte : GDT_Float64, - const_cast(oOpts.creationOpts.List()))); - if (!poDstDS) - { - CPLError(CE_Failure, CPLE_AppDefined, "Cannot create dataset for %s", - oOpts.outputFilename.c_str()); - return false; - } - - /* copy srs */ - GDALDatasetH hSrcDS = GDALGetBandDataset(pSrcBand); - if (hSrcDS) - poDstDS->SetSpatialRef( - GDALDataset::FromHandle(hSrcDS)->GetSpatialRef()); - - std::array adfDstTransform; - adfDstTransform[0] = adfTransform[0] + adfTransform[1] * oOutExtent.xStart + - adfTransform[2] * oOutExtent.yStart; - adfDstTransform[1] = adfTransform[1]; - adfDstTransform[2] = adfTransform[2]; - adfDstTransform[3] = adfTransform[3] + adfTransform[4] * oOutExtent.xStart + - adfTransform[5] * oOutExtent.yStart; - adfDstTransform[4] = adfTransform[4]; - adfDstTransform[5] = adfTransform[5]; - poDstDS->SetGeoTransform(adfDstTransform.data()); - - pDstBand = poDstDS->GetRasterBand(1); - if (!pDstBand) - { - CPLError(CE_Failure, CPLE_AppDefined, "Cannot get band for %s", - oOpts.outputFilename.c_str()); - return false; - } - - if (oOpts.nodataVal >= 0) - GDALSetRasterNoDataValue(pDstBand, oOpts.nodataVal); - return true; -} - -namespace -{ - -double doDiagonal(int nXOffset, [[maybe_unused]] int nYOffset, - double dfThisPrev, double dfLast, - [[maybe_unused]] double dfLastPrev) -{ - return CalcHeightDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast); -} - -double doEdge(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, - double dfLastPrev) -{ - if (nXOffset >= nYOffset) - return CalcHeightEdge(nYOffset, nXOffset, dfLastPrev, dfThisPrev); - else - return CalcHeightEdge(nXOffset, nYOffset, dfLastPrev, dfLast); -} - -double doMin(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, - double dfLastPrev) -{ - double dfEdge = doEdge(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); - double dfDiagonal = - doDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); - return std::min(dfEdge, dfDiagonal); -} - -double doMax(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, - double dfLastPrev) -{ - double dfEdge = doEdge(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); - double dfDiagonal = - doDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); - return std::max(dfEdge, dfDiagonal); -} - -} // unnamed namespace - -/// Process the part of the first line to the left of the observer. -/// -/// @param nX X coordinate of the observer. -/// @param iStart X coordinate of the first cell to the left of the observer to be procssed. -/// @param iEnd X coordinate one past the last cell to be processed. -/// @param vResult Vector in which to store the visibility/height results. -/// @param vThisLineVal Height of each cell in the line being processed. -void Viewshed::processFirstLineLeft(int nX, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal) -{ - // If end is to the right of start, everything is taken care of by right processing. - if (iEnd >= iStart) - return; - - iStart = oCurExtent.clampX(iStart); - - double *pThis = vThisLineVal.data() + iStart; - - // If the start cell is next to the observer, just mark it visible. - if (iStart + 1 == nX || iStart + 1 == oCurExtent.xStop) - { - if (oOpts.outputMode == OutputMode::Normal) - vResult[iStart] = oOpts.visibleVal; - else - setOutput(vResult[iStart], *pThis, *pThis); - iStart--; - pThis--; - } - - // Go from the observer to the left, calculating Z as we go. - for (int iPixel = iStart; iPixel > iEnd; iPixel--, pThis--) - { - int nXOffset = std::abs(iPixel - nX); - double dfZ = CalcHeightLine(nXOffset, *(pThis + 1)); - setOutput(vResult[iPixel], *pThis, dfZ); - } - // For cells outside of the [start, end) range, set the outOfRange value. - std::fill(vResult.begin(), vResult.begin() + iEnd + 1, oOpts.outOfRangeVal); -} - -/// Process the part of the first line to the right of the observer. -/// -/// @param nX X coordinate of the observer. -/// @param iStart X coordinate of the first cell to the right of the observer to be processed. -/// @param iEnd X coordinate one past the last cell to be processed. -/// @param vResult Vector in which to store the visibility/height results. -/// @param vThisLineVal Height of each cell in the line being processed. -void Viewshed::processFirstLineRight(int nX, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal) -{ - // If start is to the right of end, everything is taken care of by left processing. - if (iStart >= iEnd) - return; - - iStart = oCurExtent.clampX(iStart); - - double *pThis = vThisLineVal.data() + iStart; - - // If the start cell is next to the observer, just mark it visible. - if (iStart - 1 == nX || iStart == oCurExtent.xStart) - { - if (oOpts.outputMode == OutputMode::Normal) - vResult[iStart] = oOpts.visibleVal; - else - setOutput(vResult[iStart], *pThis, *pThis); - iStart++; - pThis++; - } - - // Go from the observer to the right, calculating Z as we go. - for (int iPixel = iStart; iPixel < iEnd; iPixel++, pThis++) - { - int nXOffset = std::abs(iPixel - nX); - double dfZ = CalcHeightLine(nXOffset, *(pThis - 1)); - setOutput(vResult[iPixel], *pThis, dfZ); - } - // For cells outside of the [start, end) range, set the outOfRange value. - std::fill(vResult.begin() + iEnd, vResult.end(), oOpts.outOfRangeVal); -} - -/// Process a line to the left of the observer. -/// -/// @param nX X coordinate of the observer. -/// @param nYOffset Offset of the line being processed from the observer -/// @param iStart X coordinate of the first cell to the left of the observer to be processed. -/// @param iEnd X coordinate one past the last cell to be processed. -/// @param vResult Vector in which to store the visibility/height results. -/// @param vThisLineVal Height of each cell in the line being processed. -/// @param vLastLineVal Observable height of each cell in the previous line processed. -void Viewshed::processLineLeft(int nX, int nYOffset, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal, - std::vector &vLastLineVal) -{ - // If start to the left of end, everything is taken care of by processing right. - if (iStart <= iEnd) - return; - iStart = oCurExtent.clampX(iStart); - - nYOffset = std::abs(nYOffset); - double *pThis = vThisLineVal.data() + iStart; - double *pLast = vLastLineVal.data() + iStart; - - // If the observer is to the right of the raster, mark the first cell to the left as - // visible. This may mark an out-of-range cell with a value, but this will be fixed - // with the out of range assignment at the end. - if (iStart == oCurExtent.xStop - 1) - { - if (oOpts.outputMode == OutputMode::Normal) - vResult[iStart] = oOpts.visibleVal; - else - setOutput(vResult[iStart], *pThis, *pThis); - iStart--; - pThis--; - pLast--; - } - - // Go from the observer to the left, calculating Z as we go. - for (int iPixel = iStart; iPixel > iEnd; iPixel--, pThis--, pLast--) - { - int nXOffset = std::abs(iPixel - nX); - double dfZ; - if (nXOffset == nYOffset) - { - if (nXOffset == 1) - dfZ = *pThis; - else - dfZ = CalcHeightLine(nXOffset, *(pLast + 1)); - } - else - dfZ = - oZcalc(nXOffset, nYOffset, *(pThis + 1), *pLast, *(pLast + 1)); - setOutput(vResult[iPixel], *pThis, dfZ); - } - - // For cells outside of the [start, end) range, set the outOfRange value. - std::fill(vResult.begin(), vResult.begin() + iEnd + 1, oOpts.outOfRangeVal); -} - -/// Process a line to the right of the observer. -/// -/// @param nX X coordinate of the observer. -/// @param nYOffset Offset of the line being processed from the observer -/// @param iStart X coordinate of the first cell to the right of the observer to be processed. -/// @param iEnd X coordinate one past the last cell to be processed. -/// @param vResult Vector in which to store the visibility/height results. -/// @param vThisLineVal Height of each cell in the line being processed. -/// @param vLastLineVal Observable height of each cell in the previous line processed. -void Viewshed::processLineRight(int nX, int nYOffset, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal, - std::vector &vLastLineVal) -{ - // If start is to the right of end, everything is taken care of by processing left. - if (iStart >= iEnd) - return; - iStart = oCurExtent.clampX(iStart); - - nYOffset = std::abs(nYOffset); - double *pThis = vThisLineVal.data() + iStart; - double *pLast = vLastLineVal.data() + iStart; - - // If the observer is to the left of the raster, mark the first cell to the right as - // visible. This may mark an out-of-range cell with a value, but this will be fixed - // with the out of range assignment at the end. - if (iStart == 0) - { - if (oOpts.outputMode == OutputMode::Normal) - vResult[iStart] = oOpts.visibleVal; - else - setOutput(vResult[0], *pThis, *pThis); - iStart++; - pThis++; - pLast++; - } - - // Go from the observer to the right, calculating Z as we go. - for (int iPixel = iStart; iPixel < iEnd; iPixel++, pThis++, pLast++) - { - int nXOffset = std::abs(iPixel - nX); - double dfZ; - if (nXOffset == nYOffset) - { - if (nXOffset == 1) - dfZ = *pThis; - else - dfZ = CalcHeightLine(nXOffset, *(pLast - 1)); - } - else - dfZ = - oZcalc(nXOffset, nYOffset, *(pThis - 1), *pLast, *(pLast - 1)); - setOutput(vResult[iPixel], *pThis, dfZ); - } - - // For cells outside of the [start, end) range, set the outOfRange value. - std::fill(vResult.begin() + iEnd, vResult.end(), oOpts.outOfRangeVal); -} - -/// Set the output Z value depending on the observable height and computation mode. -/// -/// dfResult Reference to the result cell -/// dfCellVal Reference to the current cell height. Replace with observable height. -/// dfZ Minimum observable height at cell. -void Viewshed::setOutput(double &dfResult, double &dfCellVal, double dfZ) -{ - if (oOpts.outputMode != OutputMode::Normal) - { - dfResult += (dfZ - dfCellVal); - dfResult = std::max(0.0, dfResult); - } - else - dfResult = (dfCellVal + oOpts.targetHeight < dfZ) ? oOpts.invisibleVal - : oOpts.visibleVal; - dfCellVal = std::max(dfCellVal, dfZ); -} - -/// Process the first line (the one with the Y coordinate the same as the observer). -/// -/// @param nX X location of the observer -/// @param nY Y location of the observer -/// @param vLastLineVal Vector in which to store the read line. Becomes the last line -/// in further processing. -/// @return True on success, false otherwise. -bool Viewshed::processFirstLine(int nX, int nY, - std::vector &vLastLineVal) -{ - int nLine = oOutExtent.clampY(nY); - int nYOffset = nLine - nY; - - std::vector vResult(oOutExtent.xSize()); - std::vector vThisLineVal(oOutExtent.xSize()); - - if (!readLine(nLine, vThisLineVal.data())) - return false; - - // If the observer is outside of the raster, take the specified value as the Z height, - // otherwise, take it as an offset from the raster height at that location. - dfZObserver = oOpts.observer.z; - if (oCurExtent.containsX(nX)) - { - dfZObserver += vThisLineVal[nX]; - if (oOpts.outputMode == OutputMode::Normal) - vResult[nX] = oOpts.visibleVal; - } - dfHeightAdjFactor = CalcHeightAdjFactor(poDstDS.get(), oOpts.curveCoeff); - - // In DEM mode the base is the pre-adjustment value. In ground mode the base is zero. - if (oOpts.outputMode == OutputMode::DEM) - vResult = vThisLineVal; - - // iLeft and iRight are the processing limits for the line. - const auto [iLeft, iRight] = adjustHeight(nYOffset, nX, vThisLineVal); - - if (!oCurExtent.containsY(nY)) - processFirstLineTopOrBottom(iLeft, iRight, vResult, vThisLineVal); - else - { - auto t1 = std::async(std::launch::async, - [&, left = iLeft]() { - processFirstLineLeft(nX, nX - 1, left - 1, - vResult, vThisLineVal); - }); - - auto t2 = std::async(std::launch::async, - [&, right = iRight]() { - processFirstLineRight(nX, nX + 1, right, - vResult, vThisLineVal); - }); - t1.wait(); - t2.wait(); - } - - // Make the current line the last line. - vLastLineVal = std::move(vThisLineVal); - - // Create the output writer. - if (!writeLine(nLine, vResult)) - return false; - - if (!lineProgress()) - return false; - return true; -} - -// If the observer is above or below the raster, set all cells in the first line near the -// observer as observable provided they're in range. Mark cells out of range as such. -/// @param iLeft Leftmost observable raster position in range of the target line. -/// @param iRight One past the rightmost observable raster position of the target line. -/// @param vResult Result line. -/// @param vThisLineVal Heights of the cells in the target line -void Viewshed::processFirstLineTopOrBottom(int iLeft, int iRight, - std::vector &vResult, - std::vector &vThisLineVal) -{ - double *pResult = vResult.data() + iLeft; - double *pThis = vThisLineVal.data() + iLeft; - for (int iPixel = iLeft; iPixel < iRight; ++iPixel, ++pResult, pThis++) - { - if (oOpts.outputMode == OutputMode::Normal) - *pResult = oOpts.visibleVal; - else - setOutput(*pResult, *pThis, *pThis); - } - std::fill(vResult.begin(), vResult.begin() + iLeft, oOpts.outOfRangeVal); - std::fill(vResult.begin() + iRight, vResult.begin() + oCurExtent.xStop, - oOpts.outOfRangeVal); -} - -/// Process a line above or below the observer. -/// -/// @param nX X location of the observer -/// @param nY Y location of the observer -/// @param nLine Line number being processed. -/// @param vLastLineVal Vector in which to store the read line. Becomes the last line -/// in further processing. -/// @return True on success, false otherwise. -bool Viewshed::processLine(int nX, int nY, int nLine, - std::vector &vLastLineVal) -{ - int nYOffset = nLine - nY; - std::vector vResult(oOutExtent.xSize()); - std::vector vThisLineVal(oOutExtent.xSize()); - - if (!readLine(nLine, vThisLineVal.data())) - return false; - - // In DEM mode the base is the input DEM value. - // In ground mode the base is zero. - if (oOpts.outputMode == OutputMode::DEM) - vResult = vThisLineVal; - - // Adjust height of the read line. - const auto [iLeft, iRight] = adjustHeight(nYOffset, nX, vThisLineVal); - - // Handle the initial position on the line. - if (oCurExtent.containsX(nX)) - { - if (iLeft < iRight) - { - double dfZ; - if (std::abs(nYOffset) == 1) - dfZ = vThisLineVal[nX]; - else - dfZ = CalcHeightLine(nYOffset, vLastLineVal[nX]); - setOutput(vResult[nX], vThisLineVal[nX], dfZ); - } - else - vResult[nX] = oOpts.outOfRangeVal; - } - - // process left half then right half of line - auto t1 = - std::async(std::launch::async, - [&, left = iLeft]() - { - processLineLeft(nX, nYOffset, nX - 1, left - 1, vResult, - vThisLineVal, vLastLineVal); - }); - - auto t2 = - std::async(std::launch::async, - [&, right = iRight]() - { - processLineRight(nX, nYOffset, nX + 1, right, vResult, - vThisLineVal, vLastLineVal); - }); - t1.wait(); - t2.wait(); - - // Make the current line the last line. - vLastLineVal = std::move(vThisLineVal); - - if (!writeLine(nLine, vResult)) - return false; - - if (!lineProgress()) - return false; - return true; -} - -/// Compute the viewshed of a raster band. -/// -/// @param band Pointer to the raster band to be processed. -/// @param pfnProgress Pointer to the progress function. Can be null. -/// @param pProgressArg Argument passed to the progress function -/// @return True on success, false otherwise. -bool Viewshed::run(GDALRasterBandH band, GDALProgressFunc pfnProgress, - void *pProgressArg) -{ - using namespace std::placeholders; - - nLineCount = 0; - pSrcBand = static_cast(band); - - oProgress = std::bind(pfnProgress, _1, _2, pProgressArg); - - if (!emitProgress(0)) - return false; - - // set up geotransformation - GDALDatasetH hSrcDS = GDALGetBandDataset(pSrcBand); - if (hSrcDS != nullptr) - GDALGetGeoTransform(hSrcDS, adfTransform.data()); - - if (!GDALInvGeoTransform(adfTransform.data(), adfInvTransform.data())) - { - CPLError(CE_Failure, CPLE_AppDefined, "Cannot invert geotransform"); - return false; - } - - // calculate observer position - double dfX, dfY; - GDALApplyGeoTransform(adfInvTransform.data(), oOpts.observer.x, - oOpts.observer.y, &dfX, &dfY); - if (!GDALIsValueInRange(dfX)) - { - CPLError(CE_Failure, CPLE_AppDefined, "Observer X value out of range"); - return false; - } - if (!GDALIsValueInRange(dfY)) - { - CPLError(CE_Failure, CPLE_AppDefined, "Observer Y value out of range"); - return false; - } - int nX = static_cast(dfX); - int nY = static_cast(dfY); - - // calculate the area of interest - if (!calcOutputExtent(nX, nY)) - return false; - - // normalize horizontal index to [ 0, oOutExtent.xSize() ) - //ABELL - verify this won't underflow. - oCurExtent = oOutExtent; - oCurExtent.shiftX(-oOutExtent.xStart); - nX -= oOutExtent.xStart; - - // create the output dataset - if (!createOutputDataset()) - return false; - - std::vector vFirstLineVal(oCurExtent.xSize()); - - if (!processFirstLine(nX, nY, vFirstLineVal)) - return false; - - if (oOpts.cellMode == CellMode::Edge) - oZcalc = doEdge; - else if (oOpts.cellMode == CellMode::Diagonal) - oZcalc = doDiagonal; - else if (oOpts.cellMode == CellMode::Min) - oZcalc = doMin; - else if (oOpts.cellMode == CellMode::Max) - oZcalc = doMax; - - // scan upwards - int yStart = oCurExtent.clampY(nY); - std::atomic err(false); - auto tUp = std::async(std::launch::async, - [&]() - { - std::vector vLastLineVal = vFirstLineVal; - - for (int nLine = yStart - 1; - nLine >= oCurExtent.yStart && !err; nLine--) - if (!processLine(nX, nY, nLine, vLastLineVal)) - err = true; - }); - - // scan downwards - auto tDown = - std::async(std::launch::async, - [&]() - { - std::vector vLastLineVal = vFirstLineVal; - - for (int nLine = yStart + 1; - nLine < oCurExtent.yStop && !err; nLine++) - if (!processLine(nX, nY, nLine, vLastLineVal)) - err = true; - }); - - tUp.wait(); - tDown.wait(); - - if (!emitProgress(1)) - return false; - - return true; -} - -} // namespace gdal diff --git a/alg/viewshed.h b/alg/viewshed.h deleted file mode 100644 index 13a1ab558091..000000000000 --- a/alg/viewshed.h +++ /dev/null @@ -1,262 +0,0 @@ -/****************************************************************************** - * - * Project: Viewshed Generation - * Purpose: Core algorithm implementation for viewshed generation. - * Author: Tamas Szekeres, szekerest@gmail.com - * - ****************************************************************************** - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - ****************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cpl_progress.h" -#include "gdal_priv.h" - -namespace gdal -{ - -/** - * Class to support viewshed raster generation. - */ -class Viewshed -{ - public: - /** - * Raster output mode. - */ - enum class OutputMode - { - Normal, //!< Normal output mode (visibility only) - DEM, //!< Output height from DEM - Ground //!< Output height from ground - }; - - /** - * Cell height calculation mode. - */ - enum class CellMode - { - Diagonal, //!< Diagonal Mode - Edge, //!< Edge Mode - Max, //!< Maximum value produced by Diagonal and Edge mode - Min //!< Minimum value produced by Diagonal and Edge mode - }; - - /** - * A point. - */ - struct Point - { - double x; //!< X value - double y; //!< Y value - double z; //!< Z value - }; - - /** - * A window in a raster including pixels in [xStart, xStop) and [yStart, yStop). - */ - struct Window - { - int xStart{}; //!< X start position - int xStop{}; //!< X end position - int yStart{}; //!< Y start position - int yStop{}; //!< Y end position - - /// \brief Window size in the X direction. - int xSize() const - { - return xStop - xStart; - } - - /// \brief Window size in the Y direction. - int ySize() const - { - return yStop - yStart; - } - - /// \brief Determine if the X window contains the index. - /// \param nX Index to check - /// \return True if the index is contained, false otherwise. - bool containsX(int nX) const - { - return nX >= xStart && nX < xStop; - } - - /// \brief Determine if the Y window contains the index. - /// \param nY Index to check - /// \return True if the index is contained, false otherwise. - bool containsY(int nY) const - { - return nY >= xStart && nY < yStop; - } - - /// \brief Determine if the window contains the index. - /// \param nX X coordinate of the index to check - /// \param nY Y coordinate of the index to check - /// \return True if the index is contained, false otherwise. - bool contains(int nX, int nY) const - { - return containsX(nX) && containsY(nY); - } - - /// \brief Clamp the argument to be in the window in the X dimension. - /// \param nX Value to clamp. - /// \return Clamped value. - int clampX(int nX) const - { - return xSize() ? std::clamp(nX, xStart, xStop - 1) : xStart; - } - - /// \brief Clamp the argument to be in the window in the Y dimension. - /// \param nY Value to clamp. - /// \return Clamped value. - int clampY(int nY) const - { - return ySize() ? std::clamp(nY, yStart, yStop - 1) : yStart; - } - - /// \brief Shift the X dimension by nShift. - /// \param nShift Amount to shift - void shiftX(int nShift) - { - xStart += nShift; - xStop += nShift; - } - }; - - /** - * Options for viewshed generation. - */ - struct Options - { - Point observer{0, 0, 0}; //!< x, y, and z of the observer - double visibleVal{255}; //!< raster output value for visible pixels. - double invisibleVal{ - 0}; //!< raster output value for non-visible pixels. - double outOfRangeVal{ - 0}; //!< raster output value for pixels outside of max distance. - double nodataVal{-1}; //!< raster output value for pixels with no data - double targetHeight{0.0}; //!< target height above the DEM surface - double maxDistance{ - 0.0}; //!< maximum distance from observer to compute value - double curveCoeff{.85714}; //!< coefficient for atmospheric refraction - OutputMode outputMode{OutputMode::Normal}; //!< Output information. - //!< Normal, Height from DEM or Height from ground - std::string outputFormat{}; //!< output raster format - std::string outputFilename{}; //!< output raster filename - CPLStringList creationOpts{}; //!< options for output raster creation - CellMode cellMode{ - CellMode::Edge}; //!< Mode of cell height calculation. - }; - - /** - * Constructor. - * - * @param opts Options to use when calculating viewshed. - */ - CPL_DLL explicit Viewshed(const Options &opts) - : oOpts{opts}, oOutExtent{}, oCurExtent{}, - dfMaxDistance2{opts.maxDistance * opts.maxDistance}, - dfZObserver{0}, poDstDS{}, pSrcBand{}, pDstBand{}, - dfHeightAdjFactor{0}, nLineCount{0}, adfTransform{0, 1, 0, 0, 0, 1}, - adfInvTransform{}, oProgress{}, oZcalc{}, oMutex{}, iMutex{} - { - if (dfMaxDistance2 == 0) - dfMaxDistance2 = std::numeric_limits::max(); - } - - Viewshed(const Viewshed &) = delete; - Viewshed &operator=(const Viewshed &) = delete; - - CPL_DLL bool run(GDALRasterBandH hBand, - GDALProgressFunc pfnProgress = GDALDummyProgress, - void *pProgressArg = nullptr); - - /** - * Fetch a pointer to the created raster band. - * - * @return Unique pointer to the viewshed dataset. - */ - CPL_DLL std::unique_ptr output() - { - return std::move(poDstDS); - } - - private: - Options oOpts; - Window oOutExtent; - Window oCurExtent; - double dfMaxDistance2; - double dfZObserver; - std::unique_ptr poDstDS; - GDALRasterBand *pSrcBand; - GDALRasterBand *pDstBand; - double dfHeightAdjFactor; - int nLineCount; - std::array adfTransform; - std::array adfInvTransform; - using ProgressFunc = std::function; - ProgressFunc oProgress; - using ZCalc = std::function; - ZCalc oZcalc; - std::mutex oMutex; - std::mutex iMutex; - - void setOutput(double &dfResult, double &dfCellVal, double dfZ); - double calcHeight(double dfZ, double dfZ2); - bool readLine(int nLine, double *data); - bool writeLine(int nLine, std::vector &vResult); - bool processLine(int nX, int nY, int nLine, - std::vector &vLastLineVal); - bool processFirstLine(int nX, int nY, std::vector &vLastLineVal); - void processFirstLineLeft(int nX, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal); - void processFirstLineRight(int nX, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal); - void processFirstLineTopOrBottom(int iLeft, int iRight, - std::vector &vResult, - std::vector &vThisLineVal); - void processLineLeft(int nX, int nYOffset, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal, - std::vector &vLastLineVal); - void processLineRight(int nX, int nYOffset, int iStart, int iEnd, - std::vector &vResult, - std::vector &vThisLineVal, - std::vector &vLastLineVal); - std::pair adjustHeight(int iLine, int nX, - std::vector &thisLineVal); - bool calcOutputExtent(int nX, int nY); - bool createOutputDataset(); - bool lineProgress(); - bool emitProgress(double fraction); -}; - -} // namespace gdal diff --git a/alg/viewshed/combiner.cpp b/alg/viewshed/combiner.cpp new file mode 100644 index 000000000000..74f8b3f6c7bb --- /dev/null +++ b/alg/viewshed/combiner.cpp @@ -0,0 +1,78 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "combiner.h" +#include "util.h" + +namespace gdal +{ +namespace viewshed +{ + +/// Read viewshed executor output and sum it up in our owned memory raster. +void Combiner::run() +{ + DatasetPtr pTempDataset; + + while (m_inputQueue.pop(pTempDataset)) + { + if (!m_dataset) + m_dataset = std::move(pTempDataset); + else + sum(std::move(pTempDataset)); + } + // Queue remaining summed rasters. + queueOutputBuffer(); +} + +/// Add the values of the source dataset to those of the owned dataset. +/// @param src Source dataset. +void Combiner::sum(DatasetPtr src) +{ + if (!m_dataset) + { + m_dataset = std::move(src); + return; + } + size_t size = bandSize(*m_dataset->GetRasterBand(1)); + + uint8_t *dstP = + static_cast(m_dataset->GetInternalHandle("MEMORY1")); + uint8_t *srcP = static_cast(src->GetInternalHandle("MEMORY1")); + for (size_t i = 0; i < size; ++i) + *dstP++ += *srcP++; + // If we've seen 255 inputs, queue our raster for output and rollup since we might overflow + // otherwise. + if (++m_count == 255) + queueOutputBuffer(); +} + +/// Queue the owned buffer as for output. +void Combiner::queueOutputBuffer() +{ + if (m_dataset) + m_outputQueue.push(std::move(m_dataset)); + m_count = 0; +} + +} // namespace viewshed +} // namespace gdal diff --git a/alg/viewshed/combiner.h b/alg/viewshed/combiner.h new file mode 100644 index 000000000000..0e6ce9787a48 --- /dev/null +++ b/alg/viewshed/combiner.h @@ -0,0 +1,72 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef VIEWSHED_COMBINER_H_INCLUDED +#define VIEWSHED_COMBINER_H_INCLUDED + +#include "cumulative.h" +#include "viewshed_types.h" + +namespace gdal +{ +namespace viewshed +{ + +/// Reads completed viewshed rasters and sums them together. When the +/// summed values may exceed the 8-bit limit, push it on the output +/// queue. +class Combiner +{ + public: + /// Constructor + /// @param inputQueue Reference to input queue of datasets + /// @param outputQueue Reference to output queue of datasets + Combiner(Cumulative::DatasetQueue &inputQueue, + Cumulative::DatasetQueue &outputQueue) + : m_inputQueue(inputQueue), m_outputQueue(outputQueue) + { + } + + /// Copy ctor. Allows initialization in a vector of Combiners. + /// @param src Source Combiner. + // cppcheck-suppress missingMemberCopy + Combiner(const Combiner &src) + : m_inputQueue(src.m_inputQueue), m_outputQueue(src.m_outputQueue) + { + } + + void queueOutputBuffer(); + void run(); + + private: + Cumulative::DatasetQueue &m_inputQueue; + Cumulative::DatasetQueue &m_outputQueue; + DatasetPtr m_dataset{}; + size_t m_count{0}; + + void sum(DatasetPtr srcDs); +}; + +} // namespace viewshed +} // namespace gdal + +#endif diff --git a/alg/viewshed/cumulative.cpp b/alg/viewshed/cumulative.cpp new file mode 100644 index 000000000000..606ca23c8215 --- /dev/null +++ b/alg/viewshed/cumulative.cpp @@ -0,0 +1,233 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include +#include +#include + +#include "cpl_worker_thread_pool.h" + +#include "combiner.h" +#include "cumulative.h" +#include "notifyqueue.h" +#include "util.h" +#include "viewshed_executor.h" + +namespace gdal +{ +namespace viewshed +{ + +/// Constructor +/// +/// @param opts Options for viewshed generation. +Cumulative::Cumulative(const Options &opts) : m_opts(opts) +{ +} + +/// Destructor +/// +Cumulative::~Cumulative() = default; + +/// Compute the cumulative viewshed of a raster band. +/// +/// @param srcFilename Source filename. +/// @param pfnProgress Pointer to the progress function. Can be null. +/// @param pProgressArg Argument passed to the progress function +/// @return True on success, false otherwise. +bool Cumulative::run(const std::string &srcFilename, + GDALProgressFunc pfnProgress, void *pProgressArg) +{ + // In cumulative mode, we run the executors in normal mode and want "1" where things + // are visible. + m_opts.outputMode = OutputMode::Normal; + m_opts.visibleVal = 1; + + DatasetPtr srcDS( + GDALDataset::FromHandle(GDALOpen(srcFilename.c_str(), GA_ReadOnly))); + if (!srcDS) + { + CPLError(CE_Failure, CPLE_AppDefined, "Unable open source file."); + return false; + } + + GDALRasterBand *pSrcBand = srcDS->GetRasterBand(1); + + // In cumulative mode, the output extent is always the entire source raster. + m_extent.xStop = GDALGetRasterBandXSize(pSrcBand); + m_extent.yStop = GDALGetRasterBandYSize(pSrcBand); + + // Make a bunch of observer locations based on the spacing and stick them on a queue + // to be handled by viewshed executors. + for (int x = 0; x < m_extent.xStop; x += m_opts.observerSpacing) + for (int y = 0; y < m_extent.yStop; y += m_opts.observerSpacing) + m_observerQueue.push({x, y}); + m_observerQueue.done(); + + // Run executors. + const int numThreads = m_opts.numJobs; + std::atomic err = false; + std::atomic running = numThreads; + Progress progress(pfnProgress, pProgressArg, + m_observerQueue.size() * m_extent.ySize()); + CPLWorkerThreadPool executorPool(numThreads); + for (int i = 0; i < numThreads; ++i) + executorPool.SubmitJob( + [this, &srcFilename, &progress, &err, &running] + { runExecutor(srcFilename, progress, err, running); }); + + // Run combiners that create 8-bit sums of executor jobs. + CPLWorkerThreadPool combinerPool(numThreads); + std::vector combiners(numThreads, + Combiner(m_datasetQueue, m_rollupQueue)); + for (Combiner &c : combiners) + combinerPool.SubmitJob([&c] { c.run(); }); + + // Run 32-bit rollup job that combines the 8-bit results from the combiners. + std::thread sum([this] { rollupRasters(); }); + + // When the combiner jobs are done, all the data is in the rollup queue. + combinerPool.WaitCompletion(); + if (m_datasetQueue.isStopped()) + return false; + m_rollupQueue.done(); + + // Wait for finalBuf to be fully filled. + sum.join(); + // The executors should exit naturally, but we wait here so that we don't outrun their + // completion and exit with outstanding threads. + executorPool.WaitCompletion(); + + // Scale the data so that we can write an 8-bit raster output. + scaleOutput(); + if (!writeOutput(createOutputDataset(*pSrcBand, m_opts, m_extent))) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Unable to write to output file."); + return false; + } + progress.emit(1); + + return true; +} + +/// Run an executor (single viewshed) +/// @param srcFilename Source filename +/// @param progress Progress supporting support. +/// @param err Shared error flag. +/// @param running Shared count of number of executors running. +void Cumulative::runExecutor(const std::string &srcFilename, Progress &progress, + std::atomic &err, std::atomic &running) +{ + DatasetPtr srcDs(GDALDataset::Open(srcFilename.c_str(), GA_ReadOnly)); + if (!srcDs) + { + err = true; + } + else + { + Location loc; + while (!err && m_observerQueue.pop(loc)) + { + GDALDriver *memDriver = + GetGDALDriverManager()->GetDriverByName("MEM"); + DatasetPtr dstDs(memDriver ? memDriver->Create("", m_extent.xSize(), + m_extent.ySize(), 1, + GDT_Byte, nullptr) + : nullptr); + if (!dstDs) + { + err = true; + } + else + { + ViewshedExecutor executor( + *srcDs->GetRasterBand(1), *dstDs->GetRasterBand(1), loc.x, + loc.y, m_extent, m_extent, m_opts, progress); + err = !executor.run(); + if (!err) + m_datasetQueue.push(std::move(dstDs)); + } + } + } + + // Job done. Set the output queue state. If all the executor jobs have completed, + // set the dataset output queue done. + if (err) + m_datasetQueue.stop(); + else + { + running--; + if (!running) + m_datasetQueue.done(); + } +} + +// Add 8-bit rasters into the 32-bit raster buffer. +void Cumulative::rollupRasters() +{ + DatasetPtr pDS; + + m_finalBuf.resize(m_extent.size()); + while (m_rollupQueue.pop(pDS)) + { + uint8_t *srcP = + static_cast(pDS->GetInternalHandle("MEMORY1")); + for (size_t i = 0; i < m_extent.size(); ++i) + m_finalBuf[i] += srcP[i]; + } +} + +/// Scale the output so that it's fully spread in 8 bits. Perhaps this shouldn't happen if +/// the max is less than 255? +void Cumulative::scaleOutput() +{ + uint32_t m = 0; // This gathers all the bits set. + for (uint32_t &val : m_finalBuf) + m = std::max(val, m); + + if (m == 0) + return; + + double factor = + std::numeric_limits::max() / static_cast(m); + for (uint32_t &val : m_finalBuf) + val = static_cast(std::floor(factor * val)); +} + +/// Write the output dataset. +/// @param pDstDS Pointer to the destination dataset. +/// @return True if the write was successful, false otherwise. +bool Cumulative::writeOutput(DatasetPtr pDstDS) +{ + if (!pDstDS) + return false; + + GDALRasterBand *pDstBand = pDstDS->GetRasterBand(1); + return (pDstBand->RasterIO(GF_Write, 0, 0, m_extent.xSize(), + m_extent.ySize(), m_finalBuf.data(), + m_extent.xSize(), m_extent.ySize(), GDT_UInt32, + 0, 0, nullptr) == 0); +} + +} // namespace viewshed +} // namespace gdal diff --git a/alg/viewshed/cumulative.h b/alg/viewshed/cumulative.h new file mode 100644 index 000000000000..c9d2f8a3f958 --- /dev/null +++ b/alg/viewshed/cumulative.h @@ -0,0 +1,87 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef VIEWSHED_CUMULATIVE_H_INCLUDED +#define VIEWSHED_CUMULATIVE_H_INCLUDED + +#include +#include + +#include "notifyqueue.h" +#include "progress.h" +#include "viewshed_types.h" + +namespace gdal +{ +namespace viewshed +{ + +class Progress; + +/// Generates a cumulative viewshed from a matrix of observers. +class Cumulative +{ + public: + CPL_DLL explicit Cumulative(const Options &opts); + // We define an explicit destructor, whose implementation is in libgdal, + // otherwise with gcc 9.4 of Ubuntu 20.04 in debug mode, this would need to + // redefinition of the NotifyQueue class in both libgdal and gdal_viewshed, + // leading to weird things related to mutex. + CPL_DLL ~Cumulative(); + CPL_DLL bool run(const std::string &srcFilename, + GDALProgressFunc pfnProgress = GDALDummyProgress, + void *pProgressArg = nullptr); + + private: + friend class Combiner; // Provides access to the queue types. + + struct Location + { + int x; + int y; + }; + + using Buf32 = std::vector; + using ObserverQueue = NotifyQueue; + using DatasetQueue = NotifyQueue; + + Window m_extent{}; + Options m_opts; + ObserverQueue m_observerQueue{}; + DatasetQueue m_datasetQueue{}; + DatasetQueue m_rollupQueue{}; + Buf32 m_finalBuf{}; + + void runExecutor(const std::string &srcFilename, Progress &progress, + std::atomic &err, std::atomic &running); + void rollupRasters(); + void scaleOutput(); + bool writeOutput(DatasetPtr pDstDS); + + Cumulative(const Cumulative &) = delete; + Cumulative &operator=(const Cumulative &) = delete; +}; + +} // namespace viewshed +} // namespace gdal + +#endif diff --git a/alg/viewshed/notifyqueue.h b/alg/viewshed/notifyqueue.h new file mode 100644 index 000000000000..e0e3f9da3725 --- /dev/null +++ b/alg/viewshed/notifyqueue.h @@ -0,0 +1,142 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef VIEWSHED_NOTIFYQUEUE_H_INCLUDED +#define VIEWSHED_NOTIFYQUEUE_H_INCLUDED + +#include "cpl_port.h" + +#include +#include +#include + +namespace gdal +{ +namespace viewshed +{ + +/// This is a thread-safe queue. Things placed in the queue must be move-constructible. +/// Readers will wait until there is something in the queue or the queue is empty or stopped. +/// If the queue is stopped (error), it will never be in the done state. If in the +/// done state (all writers have finished), it will never be in the error state. +template class NotifyQueue +{ + public: + /// Destructor + ~NotifyQueue() + { + done(); + } + + /// Push an object on the queue and notify readers. + /// \param t Object to be moved onto the queue. + void push(T &&t) + { + { + std::lock_guard lock(m_mutex); + m_queue.push(std::move(t)); + } + m_cv.notify_all(); + } + + /// Get an item from the queue. + /// \param t Reference to an item to to which a queued item will be moved. + /// \return True if an item was popped. False otherwise. Use isStopped() or isDone() + /// to determine the state if you care when false is returned. + bool pop(T &t) + { + std::unique_lock lock(m_mutex); + m_cv.wait(lock, + [this] { return !m_queue.empty() || m_done || m_stop; }); + + if (m_stop) + return false; + + if (m_queue.size()) + { + t = std::move(m_queue.front()); + m_queue.pop(); + return true; + } + + // m_done must be true and the queue is empty. + return false; + } + + /// When we're done putting things in the queue, set the end condition. + void done() + { + { + std::lock_guard lock(m_mutex); + m_done = !m_stop; // If we're already stopped, we can't be done. + } + m_cv.notify_all(); + } + + /// Unblock all readers regardless of queue state. + void stop() + { + { + std::lock_guard lock(m_mutex); + m_stop = !m_done; // If we're already done, we can't be stopped. + } + m_cv.notify_all(); + } + + /// Determine if the queue was emptied completely. Call after pop() returns false + /// to check queue state. + /// \return Whether the queue was emptied completely. + bool isDone() + { + std::lock_guard lock(m_mutex); + return m_done; + } + + /// Determine if the queue was stopped. Call after pop() returns false + /// to check queue state. + /// \return Whether the queue was stopped. + bool isStopped() + { + std::lock_guard lock(m_mutex); + return m_stop; + } + + /// Get the current size of the queue. + /// \return Current queue size. + size_t size() const + { + std::lock_guard lock(m_mutex); + return m_queue.size(); + } + + private: + std::queue m_queue{}; + mutable std::mutex m_mutex{}; + std::condition_variable m_cv{}; + bool m_done{false}; + bool m_stop{false}; +}; + +} // namespace viewshed +} // namespace gdal + +#endif diff --git a/alg/viewshed/progress.cpp b/alg/viewshed/progress.cpp new file mode 100644 index 000000000000..3f02b324d9f8 --- /dev/null +++ b/alg/viewshed/progress.cpp @@ -0,0 +1,81 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include + +#include "progress.h" + +#include "cpl_error.h" + +namespace gdal +{ +namespace viewshed +{ + +/// Constructor +/// @param pfnProgress Pointer to progress function. +/// @param pProgressArg Pointer to progress function data. +/// @param expectedLines Number of lines expected to be processed. +Progress::Progress(GDALProgressFunc pfnProgress, void *pProgressArg, + size_t expectedLines) + : m_expectedLines(std::max(expectedLines, static_cast(1))) +{ + using namespace std::placeholders; + + // cppcheck-suppress useInitializationList + m_cb = std::bind(pfnProgress, _1, _2, pProgressArg); +} + +/// Emit progress information saying that a line has been written to output. +/// +/// @return True on success, false otherwise. +bool Progress::lineComplete() +{ + double fraction; + { + std::lock_guard lock(m_mutex); + + if (m_lines < m_expectedLines) + m_lines++; + fraction = m_lines / static_cast(m_expectedLines); + } + return emit(fraction); +} + +/// Emit progress information saying that a fraction of work has been completed. +/// +/// @return True on success, false otherwise. +bool Progress::emit(double fraction) +{ + std::lock_guard lock(m_mutex); + + // Call the progress function. + if (!m_cb(fraction, "")) + { + CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); + return false; + } + return true; +} + +} // namespace viewshed +} // namespace gdal diff --git a/alg/viewshed/progress.h b/alg/viewshed/progress.h new file mode 100644 index 000000000000..0eaee7d449c8 --- /dev/null +++ b/alg/viewshed/progress.h @@ -0,0 +1,59 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef VIEWSHED_PROGRESS_H_INCLUDED +#define VIEWSHED_PROGRESS_H_INCLUDED + +#include +#include + +#include "cpl_progress.h" + +namespace gdal +{ +namespace viewshed +{ + +/// Support for progress reporting in viewshed construction. Determines the faction of +/// progress made based on the number of raster lines completed. +class Progress +{ + public: + Progress(GDALProgressFunc pfnProgress, void *pProgressArg, + size_t expectedLines); + + bool lineComplete(); + bool emit(double fraction); + + private: + using ProgressFunc = std::function; + + size_t m_lines{0}; ///< Number of lines completed. + size_t m_expectedLines; ///< Number of lines expected. + std::mutex m_mutex{}; ///< Progress function might not be thread-safe. + ProgressFunc m_cb{}; ///< Progress callback function. +}; + +} // namespace viewshed +} // namespace gdal + +#endif diff --git a/alg/viewshed/util.cpp b/alg/viewshed/util.cpp new file mode 100644 index 000000000000..cf935ed3b3c9 --- /dev/null +++ b/alg/viewshed/util.cpp @@ -0,0 +1,104 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include + +#include "gdal_priv.h" +#include "util.h" +#include "viewshed_types.h" + +namespace gdal +{ +namespace viewshed +{ + +/// Get the band size +/// +/// @param band Raster band +/// @return The raster band size. +size_t bandSize(GDALRasterBand &band) +{ + return static_cast(band.GetXSize()) * band.GetYSize(); +} + +/// Create the output dataset. +/// +/// @param srcBand Source raster band. +/// @param opts Options. +/// @param extent Output dataset extent. +/// @return The output dataset to be filled with data. +DatasetPtr createOutputDataset(GDALRasterBand &srcBand, const Options &opts, + const Window &extent) +{ + GDALDriverManager *hMgr = GetGDALDriverManager(); + GDALDriver *hDriver = hMgr->GetDriverByName(opts.outputFormat.c_str()); + if (!hDriver) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot get driver"); + return nullptr; + } + + /* create output raster */ + DatasetPtr dataset(hDriver->Create( + opts.outputFilename.c_str(), extent.xSize(), extent.ySize(), 1, + opts.outputMode == OutputMode::Normal ? GDT_Byte : GDT_Float64, + const_cast(opts.creationOpts.List()))); + if (!dataset) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot create dataset for %s", + opts.outputFilename.c_str()); + return nullptr; + } + + /* copy srs */ + dataset->SetSpatialRef(srcBand.GetDataset()->GetSpatialRef()); + + std::array adfSrcTransform; + std::array adfDstTransform; + srcBand.GetDataset()->GetGeoTransform(adfSrcTransform.data()); + adfDstTransform[0] = adfSrcTransform[0] + + adfSrcTransform[1] * extent.xStart + + adfSrcTransform[2] * extent.yStart; + adfDstTransform[1] = adfSrcTransform[1]; + adfDstTransform[2] = adfSrcTransform[2]; + adfDstTransform[3] = adfSrcTransform[3] + + adfSrcTransform[4] * extent.xStart + + adfSrcTransform[5] * extent.yStart; + adfDstTransform[4] = adfSrcTransform[4]; + adfDstTransform[5] = adfSrcTransform[5]; + dataset->SetGeoTransform(adfDstTransform.data()); + + GDALRasterBand *pBand = dataset->GetRasterBand(1); + if (!pBand) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot get band for %s", + opts.outputFilename.c_str()); + return nullptr; + } + + if (opts.nodataVal >= 0) + GDALSetRasterNoDataValue(pBand, opts.nodataVal); + return dataset; +} + +} // namespace viewshed +} // namespace gdal diff --git a/alg/viewshed/util.h b/alg/viewshed/util.h new file mode 100644 index 000000000000..3ebf6a9203dc --- /dev/null +++ b/alg/viewshed/util.h @@ -0,0 +1,41 @@ +/****************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef VIEWSHED_UTIL_H_INCLUDED +#define VIEWSHED_UTIL_H_INCLUDED + +#include "viewshed_types.h" + +namespace gdal +{ +namespace viewshed +{ + +size_t bandSize(GDALRasterBand &band); + +DatasetPtr createOutputDataset(GDALRasterBand &srcBand, const Options &opts, + const Window &extent); + +} // namespace viewshed +} // namespace gdal + +#endif diff --git a/alg/viewshed/viewshed.cpp b/alg/viewshed/viewshed.cpp new file mode 100644 index 000000000000..df5638bb70ac --- /dev/null +++ b/alg/viewshed/viewshed.cpp @@ -0,0 +1,365 @@ +/****************************************************************************** + * + * Project: Viewshed Generation + * Purpose: Core algorithm implementation for viewshed generation. + * Author: Tamas Szekeres, szekerest@gmail.com + * + * (c) 2024 info@hobu.co + * + ****************************************************************************** + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include + +#include "gdal_alg.h" +#include "gdal_priv_templates.hpp" + +#include "progress.h" +#include "util.h" +#include "viewshed.h" +#include "viewshed_executor.h" + +/************************************************************************/ +/* GDALViewshedGenerate() */ +/************************************************************************/ + +/** + * Create viewshed from raster DEM. + * + * This algorithm will generate a viewshed raster from an input DEM raster + * by using a modified algorithm of "Generating Viewsheds without Using + * Sightlines" published at + * https://www.asprs.org/wp-content/uploads/pers/2000journal/january/2000_jan_87-90.pdf + * This approach provides a relatively fast calculation, since the output raster + * is generated in a single scan. The gdal/apps/gdal_viewshed.cpp mainline can + * be used as an example of how to use this function. The output raster will be + * of type Byte or Float64. + * + * \note The algorithm as implemented currently will only output meaningful + * results if the georeferencing is in a projected coordinate reference system. + * + * @param hBand The band to read the DEM data from. Only the part of the raster + * within the specified maxdistance around the observer point is processed. + * + * @param pszDriverName Driver name (GTiff if set to NULL) + * + * @param pszTargetRasterName The name of the target raster to be generated. + * Must not be NULL + * + * @param papszCreationOptions creation options. + * + * @param dfObserverX observer X value (in SRS units) + * + * @param dfObserverY observer Y value (in SRS units) + * + * @param dfObserverHeight The height of the observer above the DEM surface. + * + * @param dfTargetHeight The height of the target above the DEM surface. + * (default 0) + * + * @param dfVisibleVal pixel value for visibility (default 255) + * + * @param dfInvisibleVal pixel value for invisibility (default 0) + * + * @param dfOutOfRangeVal The value to be set for the cells that fall outside of + * the range specified by dfMaxDistance. + * + * @param dfNoDataVal The value to be set for the cells that have no data. + * If set to a negative value, nodata is not set. + * Note: currently, no special processing of input cells at a + * nodata value is done (which may result in erroneous results). + * + * @param dfCurvCoeff Coefficient to consider the effect of the curvature and + * refraction. The height of the DEM is corrected according to the following + * formula: [Height] -= dfCurvCoeff * [Target Distance]^2 / [Earth Diameter] For + * the effect of the atmospheric refraction we can use 0.85714. + * + * @param eMode The mode of the viewshed calculation. + * Possible values GVM_Diagonal = 1, GVM_Edge = 2 (default), GVM_Max = 3, + * GVM_Min = 4. + * + * @param dfMaxDistance maximum distance range to compute viewshed. + * It is also used to clamp the extent of the output + * raster. If set to 0, then unlimited range is assumed, that is to say the + * computation is performed on the extent of the whole + * raster. + * + * @param pfnProgress A GDALProgressFunc that may be used to report progress + * to the user, or to interrupt the algorithm. May be NULL if not required. + * + * @param pProgressArg The callback data for the pfnProgress function. + * + * @param heightMode Type of information contained in output raster. Possible + * values GVOT_NORMAL = 1 (default), GVOT_MIN_TARGET_HEIGHT_FROM_DEM = 2, + * GVOT_MIN_TARGET_HEIGHT_FROM_GROUND = 3 + * + * GVOT_NORMAL returns a raster of type Byte containing + * visible locations. + * + * GVOT_MIN_TARGET_HEIGHT_FROM_DEM and + * GVOT_MIN_TARGET_HEIGHT_FROM_GROUND will return a raster of type Float64 + * containing the minimum target height for target to be visible from the DEM + * surface or ground level respectively. Parameters dfTargetHeight, dfVisibleVal + * and dfInvisibleVal will be ignored. + * + * + * @param papszExtraOptions Future extra options. Must be set to NULL currently. + * + * @return not NULL output dataset on success (to be closed with GDALClose()) or + * NULL if an error occurs. + * + * @since GDAL 3.1 + */ +GDALDatasetH GDALViewshedGenerate( + GDALRasterBandH hBand, const char *pszDriverName, + const char *pszTargetRasterName, CSLConstList papszCreationOptions, + double dfObserverX, double dfObserverY, double dfObserverHeight, + double dfTargetHeight, double dfVisibleVal, double dfInvisibleVal, + double dfOutOfRangeVal, double dfNoDataVal, double dfCurvCoeff, + GDALViewshedMode eMode, double dfMaxDistance, GDALProgressFunc pfnProgress, + void *pProgressArg, GDALViewshedOutputType heightMode, + [[maybe_unused]] CSLConstList papszExtraOptions) +{ + using namespace gdal; + + viewshed::Options oOpts; + oOpts.outputFormat = pszDriverName; + oOpts.outputFilename = pszTargetRasterName; + oOpts.creationOpts = papszCreationOptions; + oOpts.observer.x = dfObserverX; + oOpts.observer.y = dfObserverY; + oOpts.observer.z = dfObserverHeight; + oOpts.targetHeight = dfTargetHeight; + oOpts.curveCoeff = dfCurvCoeff; + oOpts.maxDistance = dfMaxDistance; + oOpts.nodataVal = dfNoDataVal; + + switch (eMode) + { + case GVM_Edge: + oOpts.cellMode = viewshed::CellMode::Edge; + break; + case GVM_Diagonal: + oOpts.cellMode = viewshed::CellMode::Diagonal; + break; + case GVM_Min: + oOpts.cellMode = viewshed::CellMode::Min; + break; + case GVM_Max: + oOpts.cellMode = viewshed::CellMode::Max; + break; + } + + switch (heightMode) + { + case GVOT_MIN_TARGET_HEIGHT_FROM_DEM: + oOpts.outputMode = viewshed::OutputMode::DEM; + break; + case GVOT_MIN_TARGET_HEIGHT_FROM_GROUND: + oOpts.outputMode = viewshed::OutputMode::Ground; + break; + case GVOT_NORMAL: + oOpts.outputMode = viewshed::OutputMode::Normal; + break; + } + + if (!GDALIsValueInRange(dfVisibleVal)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "dfVisibleVal out of range. Must be [0, 255]."); + return nullptr; + } + if (!GDALIsValueInRange(dfInvisibleVal)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "dfInvisibleVal out of range. Must be [0, 255]."); + return nullptr; + } + if (!GDALIsValueInRange(dfOutOfRangeVal)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "dfOutOfRangeVal out of range. Must be [0, 255]."); + return nullptr; + } + oOpts.visibleVal = dfVisibleVal; + oOpts.invisibleVal = dfInvisibleVal; + oOpts.outOfRangeVal = dfOutOfRangeVal; + + gdal::viewshed::Viewshed v(oOpts); + + if (!pfnProgress) + pfnProgress = GDALDummyProgress; + v.run(hBand, pfnProgress, pProgressArg); + + return GDALDataset::FromHandle(v.output().release()); +} + +namespace gdal +{ +namespace viewshed +{ + +namespace +{ + +bool getTransforms(GDALRasterBand &band, double *pFwdTransform, + double *pRevTransform) +{ + band.GetDataset()->GetGeoTransform(pFwdTransform); + if (!GDALInvGeoTransform(pFwdTransform, pRevTransform)) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot invert geotransform"); + return false; + } + return true; +} + +} // unnamed namespace + +Viewshed::Viewshed(const Options &opts) : oOpts{opts} +{ +} + +Viewshed::~Viewshed() = default; + +/// Calculate the extent of the output raster in terms of the input raster and +/// save the input raster extent. +/// +/// @return false on error, true otherwise +bool Viewshed::calcExtents(int nX, int nY, + const std::array &adfInvTransform) +{ + // We start with the assumption that the output size matches the input. + oOutExtent.xStop = GDALGetRasterBandXSize(pSrcBand); + oOutExtent.yStop = GDALGetRasterBandYSize(pSrcBand); + + if (!oOutExtent.contains(nX, nY)) + CPLError(CE_Warning, CPLE_AppDefined, + "NOTE: The observer location falls outside of the DEM area"); + + constexpr double EPSILON = 1e-8; + if (oOpts.maxDistance > 0) + { + //ABELL - This assumes that the transformation is only a scaling. Should be fixed. + // Find the distance in the direction of the transformed unit vector in the X and Y + // directions and use those factors to determine the limiting values in the raster space. + int nXStart = static_cast( + std::floor(nX - adfInvTransform[1] * oOpts.maxDistance + EPSILON)); + int nXStop = static_cast( + std::ceil(nX + adfInvTransform[1] * oOpts.maxDistance - EPSILON) + + 1); + int nYStart = + static_cast(std::floor( + nY - std::fabs(adfInvTransform[5]) * oOpts.maxDistance + + EPSILON)) - + (adfInvTransform[5] > 0 ? 1 : 0); + int nYStop = static_cast( + std::ceil(nY + std::fabs(adfInvTransform[5]) * oOpts.maxDistance - + EPSILON) + + (adfInvTransform[5] < 0 ? 1 : 0)); + + // If the limits are invalid, set the window size to zero to trigger the error below. + if (nXStart >= oOutExtent.xStop || nXStop < 0 || + nYStart >= oOutExtent.yStop || nYStop < 0) + { + oOutExtent = Window(); + } + else + { + oOutExtent.xStart = std::max(nXStart, 0); + oOutExtent.xStop = std::min(nXStop, oOutExtent.xStop); + + oOutExtent.yStart = std::max(nYStart, 0); + oOutExtent.yStop = std::min(nYStop, oOutExtent.yStop); + } + } + + if (oOutExtent.xSize() == 0 || oOutExtent.ySize() == 0) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid target raster size due to transform " + "and/or distance limitation."); + return false; + } + + // normalize horizontal index to [ 0, oOutExtent.xSize() ) + oCurExtent = oOutExtent; + oCurExtent.shiftX(-oOutExtent.xStart); + + return true; +} + +/// Compute the viewshed of a raster band. +/// +/// @param band Pointer to the raster band to be processed. +/// @param pfnProgress Pointer to the progress function. Can be null. +/// @param pProgressArg Argument passed to the progress function +/// @return True on success, false otherwise. +bool Viewshed::run(GDALRasterBandH band, GDALProgressFunc pfnProgress, + void *pProgressArg) +{ + pSrcBand = static_cast(band); + + std::array adfFwdTransform; + std::array adfInvTransform; + if (!getTransforms(*pSrcBand, adfFwdTransform.data(), + adfInvTransform.data())) + return false; + + // calculate observer position + double dfX, dfY; + GDALApplyGeoTransform(adfInvTransform.data(), oOpts.observer.x, + oOpts.observer.y, &dfX, &dfY); + if (!GDALIsValueInRange(dfX)) + { + CPLError(CE_Failure, CPLE_AppDefined, "Observer X value out of range"); + return false; + } + if (!GDALIsValueInRange(dfY)) + { + CPLError(CE_Failure, CPLE_AppDefined, "Observer Y value out of range"); + return false; + } + int nX = static_cast(dfX); + int nY = static_cast(dfY); + + // Must calculate extents in order to make the output dataset. + if (!calcExtents(nX, nY, adfInvTransform)) + return false; + + poDstDS = createOutputDataset(*pSrcBand, oOpts, oOutExtent); + if (!poDstDS) + return false; + + // Create the progress reporter. + Progress oProgress(pfnProgress, pProgressArg, oOutExtent.ySize()); + + // Execute the viewshed algorithm. + GDALRasterBand *pDstBand = poDstDS->GetRasterBand(1); + ViewshedExecutor executor(*pSrcBand, *pDstBand, nX, nY, oOutExtent, + oCurExtent, oOpts, oProgress); + executor.run(); + oProgress.emit(1); + return static_cast(poDstDS); +} + +} // namespace viewshed +} // namespace gdal diff --git a/alg/viewshed/viewshed.h b/alg/viewshed/viewshed.h new file mode 100644 index 000000000000..996276596dd9 --- /dev/null +++ b/alg/viewshed/viewshed.h @@ -0,0 +1,105 @@ +/****************************************************************************** + * + * Project: Viewshed Generation + * Purpose: Core algorithm implementation for viewshed generation. + * Author: Tamas Szekeres, szekerest@gmail.com + * + * (c) 2024 info@hobu.co + * + ****************************************************************************** + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef VIEWSHED_H_INCLUDED +#define VIEWSHED_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpl_progress.h" +#include "gdal_priv.h" +#include "viewshed_types.h" + +namespace gdal +{ +namespace viewshed +{ + +/** + * Class to support viewshed raster generation. + */ +class Viewshed +{ + public: + /** + * Constructor. + * + * @param opts Options to use when calculating viewshed. + */ + CPL_DLL explicit Viewshed(const Options &opts); + + /** Destructor */ + CPL_DLL ~Viewshed(); + + CPL_DLL bool run(GDALRasterBandH hBand, + GDALProgressFunc pfnProgress = GDALDummyProgress, + void *pProgressArg = nullptr); + + /** + * Fetch a pointer to the created raster band. + * + * @return Unique pointer to the viewshed dataset. + */ + CPL_DLL DatasetPtr output() + { + return std::move(poDstDS); + } + + private: + Options oOpts; + Window oOutExtent{}; + Window oCurExtent{}; + DatasetPtr poDstDS{}; + GDALRasterBand *pSrcBand = nullptr; + + DatasetPtr execute(int nX, int nY, const std::string &outFilename); + void setOutput(double &dfResult, double &dfCellVal, double dfZ); + double calcHeight(double dfZ, double dfZ2); + bool readLine(int nLine, double *data); + std::pair adjustHeight(int iLine, int nX, + std::vector &thisLineVal); + bool calcExtents(int nX, int nY, + const std::array &adfInvTransform); + + Viewshed(const Viewshed &) = delete; + Viewshed &operator=(const Viewshed &) = delete; +}; + +} // namespace viewshed +} // namespace gdal + +#endif diff --git a/alg/viewshed/viewshed_executor.cpp b/alg/viewshed/viewshed_executor.cpp new file mode 100644 index 000000000000..83c9669b9346 --- /dev/null +++ b/alg/viewshed/viewshed_executor.cpp @@ -0,0 +1,687 @@ +/****************************************************************************** + * + * Project: Viewshed Generation + * Purpose: Core algorithm implementation for viewshed generation. + * Author: Tamas Szekeres, szekerest@gmail.com + * + * (c) 2024 info@hobu.co + * + ****************************************************************************** + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include +#include +#include +#include + +#include "viewshed_executor.h" +#include "progress.h" + +namespace gdal +{ +namespace viewshed +{ + +namespace +{ + +/// Calculate the height at nDistance units along a line through the origin given the height +/// at nDistance - 1 units along the line. +/// \param nDistance Distance along the line for the target point. +/// \param Za Height at the line one unit previous to the target point. +double CalcHeightLine(int nDistance, double Za) +{ + nDistance = std::abs(nDistance); + assert(nDistance != 1); + return Za * nDistance / (nDistance - 1); +} + +// Calculate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) +// and passing through the line connecting (i - 1, j, Za) and (i, j - 1, Zb). +// In other words, the origin and the two points form a plane and we're calculating Zc +// of the point (i, j, Zc), also on the plane. +double CalcHeightDiagonal(int i, int j, double Za, double Zb) +{ + return (Za * i + Zb * j) / (i + j - 1); +} + +// Calculate the height Zc of a point (i, j, Zc) given a line through the origin (0, 0, 0) +// and through the line connecting (i -1, j - 1, Za) and (i - 1, j, Zb). In other words, +// the origin and the other two points form a plane and we're calculating Zc of the +// point (i, j, Zc), also on the plane. +double CalcHeightEdge(int i, int j, double Za, double Zb) +{ + assert(i != j); + return (Za * i + Zb * (j - i)) / (j - 1); +} + +double doDiagonal(int nXOffset, [[maybe_unused]] int nYOffset, + double dfThisPrev, double dfLast, + [[maybe_unused]] double dfLastPrev) +{ + return CalcHeightDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast); +} + +double doEdge(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, + double dfLastPrev) +{ + if (nXOffset >= nYOffset) + return CalcHeightEdge(nYOffset, nXOffset, dfLastPrev, dfThisPrev); + else + return CalcHeightEdge(nXOffset, nYOffset, dfLastPrev, dfLast); +} + +double doMin(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, + double dfLastPrev) +{ + double dfEdge = doEdge(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + double dfDiagonal = + doDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + return std::min(dfEdge, dfDiagonal); +} + +double doMax(int nXOffset, int nYOffset, double dfThisPrev, double dfLast, + double dfLastPrev) +{ + double dfEdge = doEdge(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + double dfDiagonal = + doDiagonal(nXOffset, nYOffset, dfThisPrev, dfLast, dfLastPrev); + return std::max(dfEdge, dfDiagonal); +} + +} // unnamed namespace + +/// Constructor -- the viewshed algorithm executor +/// @param srcBand Source raster band +/// @param dstBand Destination raster band +/// @param nX X position of observer +/// @param nY Y position of observer +/// @param outExtent Extent of output raster (relative to input) +/// @param curExtent Extent of active raster. +/// @param opts Configuration options. +/// @param progress Reference to the progress tracker. +ViewshedExecutor::ViewshedExecutor(GDALRasterBand &srcBand, + GDALRasterBand &dstBand, int nX, int nY, + const Window &outExtent, + const Window &curExtent, const Options &opts, + Progress &progress) + : m_pool(4), m_srcBand(srcBand), m_dstBand(dstBand), oOutExtent(outExtent), + oCurExtent(curExtent), m_nX(nX - oOutExtent.xStart), m_nY(nY), + oOpts(opts), oProgress(progress), + m_dfMaxDistance2(opts.maxDistance * opts.maxDistance) +{ + if (m_dfMaxDistance2 == 0) + m_dfMaxDistance2 = std::numeric_limits::max(); + m_srcBand.GetDataset()->GetGeoTransform(m_adfTransform.data()); +} + +// calculate the height adjustment factor. +double ViewshedExecutor::calcHeightAdjFactor() +{ + std::lock_guard g(oMutex); + + const OGRSpatialReference *poDstSRS = + m_dstBand.GetDataset()->GetSpatialRef(); + + if (poDstSRS) + { + OGRErr eSRSerr; + + // If we can't get a SemiMajor axis from the SRS, it will be SRS_WGS84_SEMIMAJOR + double dfSemiMajor = poDstSRS->GetSemiMajor(&eSRSerr); + + /* If we fetched the axis from the SRS, use it */ + if (eSRSerr != OGRERR_FAILURE) + return oOpts.curveCoeff / (dfSemiMajor * 2.0); + + CPLDebug("GDALViewshedGenerate", + "Unable to fetch SemiMajor axis from spatial reference"); + } + return 0; +} + +/// Set the output Z value depending on the observable height and computation mode. +/// +/// dfResult Reference to the result cell +/// dfCellVal Reference to the current cell height. Replace with observable height. +/// dfZ Minimum observable height at cell. +void ViewshedExecutor::setOutput(double &dfResult, double &dfCellVal, + double dfZ) +{ + if (oOpts.outputMode != OutputMode::Normal) + { + dfResult += (dfZ - dfCellVal); + dfResult = std::max(0.0, dfResult); + } + else + dfResult = (dfCellVal + oOpts.targetHeight < dfZ) ? oOpts.invisibleVal + : oOpts.visibleVal; + dfCellVal = std::max(dfCellVal, dfZ); +} + +/// Read a line of raster data. +/// +/// @param nLine Line number to read. +/// @param data Pointer to location in which to store data. +/// @return Success or failure. +bool ViewshedExecutor::readLine(int nLine, double *data) +{ + std::lock_guard g(iMutex); + + if (GDALRasterIO(&m_srcBand, GF_Read, oOutExtent.xStart, nLine, + oOutExtent.xSize(), 1, data, oOutExtent.xSize(), 1, + GDT_Float64, 0, 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "RasterIO error when reading DEM at position (%d,%d), " + "size (%d,%d)", + oOutExtent.xStart, nLine, oOutExtent.xSize(), 1); + return false; + } + return true; +} + +/// Write an output line of either visibility or height data. +/// +/// @param nLine Line number being written. +/// @param vResult Result line to write. +/// @return True on success, false otherwise. +bool ViewshedExecutor::writeLine(int nLine, std::vector &vResult) +{ + // GDALRasterIO isn't thread-safe. + std::lock_guard g(oMutex); + + if (GDALRasterIO(&m_dstBand, GF_Write, 0, nLine - oOutExtent.yStart, + oOutExtent.xSize(), 1, vResult.data(), oOutExtent.xSize(), + 1, GDT_Float64, 0, 0)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "RasterIO error when writing target raster at position " + "(%d,%d), size (%d,%d)", + 0, nLine - oOutExtent.yStart, oOutExtent.xSize(), 1); + return false; + } + return true; +} + +/// Adjust the height of the line of data by the observer height and the curvature of the +/// earth. +/// +/// @param nYOffset Y offset of the line being adjusted. +/// @param vThisLineVal Line height data. +/// @return [left, right) Leftmost and one past the rightmost cell in the line within +/// the max distance. Indices are limited to the raster extent (right may be just +/// outside the raster). +std::pair +ViewshedExecutor::adjustHeight(int nYOffset, std::vector &vThisLineVal) +{ + int nLeft = 0; + int nRight = oCurExtent.xSize(); + + // Find the starting point in the raster (m_nX may be outside) + int nXStart = oCurExtent.clampX(m_nX); + + // If there is a height adjustment factor other than zero or a max distance, + // calculate the adjusted height of the cell, stopping if we've exceeded the max + // distance. + if (static_cast(m_dfHeightAdjFactor) || m_dfMaxDistance2 > 0) + { + // Hoist invariants from the loops. + const double dfLineX = m_adfTransform[2] * nYOffset; + const double dfLineY = m_adfTransform[5] * nYOffset; + + // Go left + double *pdfHeight = vThisLineVal.data() + nXStart; + for (int nXOffset = nXStart - m_nX; nXOffset >= -m_nX; + nXOffset--, pdfHeight--) + { + double dfX = m_adfTransform[1] * nXOffset + dfLineX; + double dfY = m_adfTransform[4] * nXOffset + dfLineY; + double dfR2 = dfX * dfX + dfY * dfY; + if (dfR2 > m_dfMaxDistance2) + { + nLeft = nXOffset + m_nX + 1; + break; + } + *pdfHeight -= m_dfHeightAdjFactor * dfR2 + m_dfZObserver; + } + + // Go right. + pdfHeight = vThisLineVal.data() + nXStart + 1; + for (int nXOffset = nXStart - m_nX + 1; + nXOffset < oCurExtent.xSize() - m_nX; nXOffset++, pdfHeight++) + { + double dfX = m_adfTransform[1] * nXOffset + dfLineX; + double dfY = m_adfTransform[4] * nXOffset + dfLineY; + double dfR2 = dfX * dfX + dfY * dfY; + if (dfR2 > m_dfMaxDistance2) + { + nRight = nXOffset + m_nX; + break; + } + *pdfHeight -= m_dfHeightAdjFactor * dfR2 + m_dfZObserver; + } + } + else + { + // No curvature adjustment. Just normalize for the observer height. + double *pdfHeight = vThisLineVal.data(); + for (int i = 0; i < oCurExtent.xSize(); ++i) + { + *pdfHeight -= m_dfZObserver; + pdfHeight++; + } + } + return {nLeft, nRight}; +} + +/// Process the first line (the one with the Y coordinate the same as the observer). +/// +/// @param vLastLineVal Vector in which to store the read line. Becomes the last line +/// in further processing. +/// @return True on success, false otherwise. +bool ViewshedExecutor::processFirstLine(std::vector &vLastLineVal) +{ + int nLine = oOutExtent.clampY(m_nY); + int nYOffset = nLine - m_nY; + + std::vector vResult(oOutExtent.xSize()); + std::vector vThisLineVal(oOutExtent.xSize()); + + if (!readLine(nLine, vThisLineVal.data())) + return false; + + // If the observer is outside of the raster, take the specified value as the Z height, + // otherwise, take it as an offset from the raster height at that location. + m_dfZObserver = oOpts.observer.z; + if (oCurExtent.containsX(m_nX)) + { + m_dfZObserver += vThisLineVal[m_nX]; + if (oOpts.outputMode == OutputMode::Normal) + vResult[m_nX] = oOpts.visibleVal; + } + m_dfHeightAdjFactor = calcHeightAdjFactor(); + + // In DEM mode the base is the pre-adjustment value. In ground mode the base is zero. + if (oOpts.outputMode == OutputMode::DEM) + vResult = vThisLineVal; + + // iLeft and iRight are the processing limits for the line. + const auto [iLeft, iRight] = adjustHeight(nYOffset, vThisLineVal); + + if (!oCurExtent.containsY(m_nY)) + processFirstLineTopOrBottom(iLeft, iRight, vResult, vThisLineVal); + else + { + CPLJobQueuePtr pQueue = m_pool.CreateJobQueue(); + pQueue->SubmitJob( + [&, left = iLeft]() { + processFirstLineLeft(m_nX - 1, left - 1, vResult, vThisLineVal); + }); + + pQueue->SubmitJob( + [&, right = iRight]() + { processFirstLineRight(m_nX + 1, right, vResult, vThisLineVal); }); + pQueue->WaitCompletion(); + } + + // Make the current line the last line. + vLastLineVal = std::move(vThisLineVal); + + // Create the output writer. + if (!writeLine(nLine, vResult)) + return false; + + return oProgress.lineComplete(); +} + +// If the observer is above or below the raster, set all cells in the first line near the +// observer as observable provided they're in range. Mark cells out of range as such. +/// @param iLeft Leftmost observable raster position in range of the target line. +/// @param iRight One past the rightmost observable raster position of the target line. +/// @param vResult Result line. +/// @param vThisLineVal Heights of the cells in the target line +void ViewshedExecutor::processFirstLineTopOrBottom( + int iLeft, int iRight, std::vector &vResult, + std::vector &vThisLineVal) +{ + double *pResult = vResult.data() + iLeft; + double *pThis = vThisLineVal.data() + iLeft; + for (int iPixel = iLeft; iPixel < iRight; ++iPixel, ++pResult, pThis++) + { + if (oOpts.outputMode == OutputMode::Normal) + *pResult = oOpts.visibleVal; + else + setOutput(*pResult, *pThis, *pThis); + } + std::fill(vResult.begin(), vResult.begin() + iLeft, oOpts.outOfRangeVal); + std::fill(vResult.begin() + iRight, vResult.begin() + oCurExtent.xStop, + oOpts.outOfRangeVal); +} + +/// Process the part of the first line to the left of the observer. +/// +/// @param iStart X coordinate of the first cell to the left of the observer to be procssed. +/// @param iEnd X coordinate one past the last cell to be processed. +/// @param vResult Vector in which to store the visibility/height results. +/// @param vThisLineVal Height of each cell in the line being processed. +void ViewshedExecutor::processFirstLineLeft(int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal) +{ + // If end is to the right of start, everything is taken care of by right processing. + if (iEnd >= iStart) + return; + + iStart = oCurExtent.clampX(iStart); + + double *pThis = vThisLineVal.data() + iStart; + + // If the start cell is next to the observer, just mark it visible. + if (iStart + 1 == m_nX || iStart + 1 == oCurExtent.xStop) + { + if (oOpts.outputMode == OutputMode::Normal) + vResult[iStart] = oOpts.visibleVal; + else + setOutput(vResult[iStart], *pThis, *pThis); + iStart--; + pThis--; + } + + // Go from the observer to the left, calculating Z as we go. + for (int iPixel = iStart; iPixel > iEnd; iPixel--, pThis--) + { + int nXOffset = std::abs(iPixel - m_nX); + double dfZ = CalcHeightLine(nXOffset, *(pThis + 1)); + setOutput(vResult[iPixel], *pThis, dfZ); + } + // For cells outside of the [start, end) range, set the outOfRange value. + std::fill(vResult.begin(), vResult.begin() + iEnd + 1, oOpts.outOfRangeVal); +} + +/// Process the part of the first line to the right of the observer. +/// +/// @param iStart X coordinate of the first cell to the right of the observer to be processed. +/// @param iEnd X coordinate one past the last cell to be processed. +/// @param vResult Vector in which to store the visibility/height results. +/// @param vThisLineVal Height of each cell in the line being processed. +void ViewshedExecutor::processFirstLineRight(int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal) +{ + // If start is to the right of end, everything is taken care of by left processing. + if (iStart >= iEnd) + return; + + iStart = oCurExtent.clampX(iStart); + + double *pThis = vThisLineVal.data() + iStart; + + // If the start cell is next to the observer, just mark it visible. + if (iStart - 1 == m_nX || iStart == oCurExtent.xStart) + { + if (oOpts.outputMode == OutputMode::Normal) + vResult[iStart] = oOpts.visibleVal; + else + setOutput(vResult[iStart], *pThis, *pThis); + iStart++; + pThis++; + } + + // Go from the observer to the right, calculating Z as we go. + for (int iPixel = iStart; iPixel < iEnd; iPixel++, pThis++) + { + int nXOffset = std::abs(iPixel - m_nX); + double dfZ = CalcHeightLine(nXOffset, *(pThis - 1)); + setOutput(vResult[iPixel], *pThis, dfZ); + } + // For cells outside of the [start, end) range, set the outOfRange value. + std::fill(vResult.begin() + iEnd, vResult.end(), oOpts.outOfRangeVal); +} + +/// Process a line to the left of the observer. +/// +/// @param nYOffset Offset of the line being processed from the observer +/// @param iStart X coordinate of the first cell to the left of the observer to be processed. +/// @param iEnd X coordinate one past the last cell to be processed. +/// @param vResult Vector in which to store the visibility/height results. +/// @param vThisLineVal Height of each cell in the line being processed. +/// @param vLastLineVal Observable height of each cell in the previous line processed. +void ViewshedExecutor::processLineLeft(int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal) +{ + // If start to the left of end, everything is taken care of by processing right. + if (iStart <= iEnd) + return; + iStart = oCurExtent.clampX(iStart); + + nYOffset = std::abs(nYOffset); + double *pThis = vThisLineVal.data() + iStart; + double *pLast = vLastLineVal.data() + iStart; + + // If the observer is to the right of the raster, mark the first cell to the left as + // visible. This may mark an out-of-range cell with a value, but this will be fixed + // with the out of range assignment at the end. + if (iStart == oCurExtent.xStop - 1) + { + if (oOpts.outputMode == OutputMode::Normal) + vResult[iStart] = oOpts.visibleVal; + else + setOutput(vResult[iStart], *pThis, *pThis); + iStart--; + pThis--; + pLast--; + } + + // Go from the observer to the left, calculating Z as we go. + for (int iPixel = iStart; iPixel > iEnd; iPixel--, pThis--, pLast--) + { + int nXOffset = std::abs(iPixel - m_nX); + double dfZ; + if (nXOffset == nYOffset) + { + if (nXOffset == 1) + dfZ = *pThis; + else + dfZ = CalcHeightLine(nXOffset, *(pLast + 1)); + } + else + dfZ = + oZcalc(nXOffset, nYOffset, *(pThis + 1), *pLast, *(pLast + 1)); + setOutput(vResult[iPixel], *pThis, dfZ); + } + + // For cells outside of the [start, end) range, set the outOfRange value. + std::fill(vResult.begin(), vResult.begin() + iEnd + 1, oOpts.outOfRangeVal); +} + +/// Process a line to the right of the observer. +/// +/// @param nYOffset Offset of the line being processed from the observer +/// @param iStart X coordinate of the first cell to the right of the observer to be processed. +/// @param iEnd X coordinate one past the last cell to be processed. +/// @param vResult Vector in which to store the visibility/height results. +/// @param vThisLineVal Height of each cell in the line being processed. +/// @param vLastLineVal Observable height of each cell in the previous line processed. +void ViewshedExecutor::processLineRight(int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal) +{ + // If start is to the right of end, everything is taken care of by processing left. + if (iStart >= iEnd) + return; + iStart = oCurExtent.clampX(iStart); + + nYOffset = std::abs(nYOffset); + double *pThis = vThisLineVal.data() + iStart; + double *pLast = vLastLineVal.data() + iStart; + + // If the observer is to the left of the raster, mark the first cell to the right as + // visible. This may mark an out-of-range cell with a value, but this will be fixed + // with the out of range assignment at the end. + if (iStart == 0) + { + if (oOpts.outputMode == OutputMode::Normal) + vResult[iStart] = oOpts.visibleVal; + else + setOutput(vResult[0], *pThis, *pThis); + iStart++; + pThis++; + pLast++; + } + + // Go from the observer to the right, calculating Z as we go. + for (int iPixel = iStart; iPixel < iEnd; iPixel++, pThis++, pLast++) + { + int nXOffset = std::abs(iPixel - m_nX); + double dfZ; + if (nXOffset == nYOffset) + { + if (nXOffset == 1) + dfZ = *pThis; + else + dfZ = CalcHeightLine(nXOffset, *(pLast - 1)); + } + else + dfZ = + oZcalc(nXOffset, nYOffset, *(pThis - 1), *pLast, *(pLast - 1)); + setOutput(vResult[iPixel], *pThis, dfZ); + } + + // For cells outside of the [start, end) range, set the outOfRange value. + std::fill(vResult.begin() + iEnd, vResult.end(), oOpts.outOfRangeVal); +} + +/// Process a line above or below the observer. +/// +/// @param nLine Line number being processed. +/// @param vLastLineVal Vector in which to store the read line. Becomes the last line +/// in further processing. +/// @return True on success, false otherwise. +bool ViewshedExecutor::processLine(int nLine, std::vector &vLastLineVal) +{ + int nYOffset = nLine - m_nY; + std::vector vResult(oOutExtent.xSize()); + std::vector vThisLineVal(oOutExtent.xSize()); + + if (!readLine(nLine, vThisLineVal.data())) + return false; + + // In DEM mode the base is the input DEM value. + // In ground mode the base is zero. + if (oOpts.outputMode == OutputMode::DEM) + vResult = vThisLineVal; + + // Adjust height of the read line. + const auto [iLeft, iRight] = adjustHeight(nYOffset, vThisLineVal); + + // Handle the initial position on the line. + if (oCurExtent.containsX(m_nX)) + { + if (iLeft < iRight) + { + double dfZ; + if (std::abs(nYOffset) == 1) + dfZ = vThisLineVal[m_nX]; + else + dfZ = CalcHeightLine(nYOffset, vLastLineVal[m_nX]); + setOutput(vResult[m_nX], vThisLineVal[m_nX], dfZ); + } + else + vResult[m_nX] = oOpts.outOfRangeVal; + } + + // process left half then right half of line + CPLJobQueuePtr pQueue = m_pool.CreateJobQueue(); + pQueue->SubmitJob( + [&, left = iLeft]() + { + processLineLeft(nYOffset, m_nX - 1, left - 1, vResult, vThisLineVal, + vLastLineVal); + }); + pQueue->SubmitJob( + [&, right = iRight]() + { + processLineRight(nYOffset, m_nX + 1, right, vResult, vThisLineVal, + vLastLineVal); + }); + pQueue->WaitCompletion(); + + // Make the current line the last line. + vLastLineVal = std::move(vThisLineVal); + + if (!writeLine(nLine, vResult)) + return false; + + return oProgress.lineComplete(); +} + +/// Run the viewshed computation +/// @return Success as true or false. +bool ViewshedExecutor::run() +{ + std::vector vFirstLineVal(oCurExtent.xSize()); + if (!processFirstLine(vFirstLineVal)) + return false; + + if (oOpts.cellMode == CellMode::Edge) + oZcalc = doEdge; + else if (oOpts.cellMode == CellMode::Diagonal) + oZcalc = doDiagonal; + else if (oOpts.cellMode == CellMode::Min) + oZcalc = doMin; + else if (oOpts.cellMode == CellMode::Max) + oZcalc = doMax; + + // scan upwards + int yStart = oCurExtent.clampY(m_nY); + std::atomic err(false); + CPLJobQueuePtr pQueue = m_pool.CreateJobQueue(); + pQueue->SubmitJob( + [&]() + { + std::vector vLastLineVal = vFirstLineVal; + + for (int nLine = yStart - 1; nLine >= oCurExtent.yStart && !err; + nLine--) + if (!processLine(nLine, vLastLineVal)) + err = true; + }); + + // scan downwards + pQueue->SubmitJob( + [&]() + { + std::vector vLastLineVal = vFirstLineVal; + + for (int nLine = yStart + 1; nLine < oCurExtent.yStop && !err; + nLine++) + if (!processLine(nLine, vLastLineVal)) + err = true; + }); + return true; +} + +} // namespace viewshed +} // namespace gdal diff --git a/alg/viewshed/viewshed_executor.h b/alg/viewshed/viewshed_executor.h new file mode 100644 index 000000000000..291581e79b07 --- /dev/null +++ b/alg/viewshed/viewshed_executor.h @@ -0,0 +1,103 @@ +/****************************************************************************** + * + * Project: Viewshed Generation + * Purpose: Core algorithm implementation for viewshed generation. + * Author: Tamas Szekeres, szekerest@gmail.com + * + * (c) 2024 info@hobu.co + * + ****************************************************************************** + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#pragma once + +#include +#include + +#include "gdal_priv.h" +#include "cpl_worker_thread_pool.h" + +#include "viewshed_types.h" + +namespace gdal +{ +namespace viewshed +{ + +class Progress; + +/// Executes a viewshed computation on a source band, placing the result +/// in the destination band. +class ViewshedExecutor +{ + public: + ViewshedExecutor(GDALRasterBand &srcBand, GDALRasterBand &dstBand, int nX, + int nY, const Window &oOutExtent, const Window &oCurExtent, + const Options &opts, Progress &oProgress); + bool run(); + + private: + CPLWorkerThreadPool m_pool; + GDALRasterBand &m_srcBand; + GDALRasterBand &m_dstBand; + const Window oOutExtent; + const Window oCurExtent; + const int m_nX; + const int m_nY; + const Options oOpts; + Progress &oProgress; + double m_dfHeightAdjFactor{0}; + double m_dfMaxDistance2; + double m_dfZObserver{0}; + std::mutex iMutex{}; + std::mutex oMutex{}; + std::array m_adfTransform{0, 1, 0, 0, 0, 1}; + double (*oZcalc)(int, int, double, double, double){}; + + double calcHeightAdjFactor(); + void setOutput(double &dfResult, double &dfCellVal, double dfZ); + bool readLine(int nLine, double *data); + bool writeLine(int nLine, std::vector &vResult); + bool processLine(int nLine, std::vector &vLastLineVal); + bool processFirstLine(std::vector &vLastLineVal); + void processFirstLineLeft(int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal); + void processFirstLineRight(int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal); + void processFirstLineTopOrBottom(int iLeft, int iRight, + std::vector &vResult, + std::vector &vThisLineVal); + void processLineLeft(int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal); + void processLineRight(int nYOffset, int iStart, int iEnd, + std::vector &vResult, + std::vector &vThisLineVal, + std::vector &vLastLineVal); + std::pair adjustHeight(int iLine, + std::vector &thisLineVal); +}; + +} // namespace viewshed +} // namespace gdal diff --git a/alg/viewshed/viewshed_types.h b/alg/viewshed/viewshed_types.h new file mode 100644 index 000000000000..fa1f0b2b9ead --- /dev/null +++ b/alg/viewshed/viewshed_types.h @@ -0,0 +1,176 @@ +/**************************************************************************** + * (c) 2024 info@hobu.co + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ +#ifndef VIEWSHED_TYPES_H_INCLUDED +#define VIEWSHED_TYPES_H_INCLUDED + +#include +#include + +#include "gdal_priv.h" + +namespace gdal +{ +namespace viewshed +{ + +/// Unique pointer to GDAL dataset. +using DatasetPtr = std::unique_ptr; + +/** + * Raster output mode. + */ +enum class OutputMode +{ + Normal, //!< Normal output mode (visibility only) + DEM, //!< Output height from DEM + Ground, //!< Output height from ground + Cumulative //!< Output observability heat map +}; + +/** + * Cell height calculation mode. + */ +enum class CellMode +{ + Diagonal, //!< Diagonal Mode + Edge, //!< Edge Mode + Max, //!< Maximum value produced by Diagonal and Edge mode + Min //!< Minimum value produced by Diagonal and Edge mode +}; + +/** + * A point. + */ +struct Point +{ + double x; //!< X value + double y; //!< Y value + double z; //!< Z value +}; + +/** + * Options for viewshed generation. + */ +struct Options +{ + Point observer{0, 0, 0}; //!< x, y, and z of the observer + double visibleVal{255}; //!< raster output value for visible pixels. + double invisibleVal{0}; //!< raster output value for non-visible pixels. + double outOfRangeVal{ + 0}; //!< raster output value for pixels outside of max distance. + double nodataVal{-1}; //!< raster output value for pixels with no data + double targetHeight{0.0}; //!< target height above the DEM surface + double maxDistance{ + 0.0}; //!< maximum distance from observer to compute value + double curveCoeff{.85714}; //!< coefficient for atmospheric refraction + OutputMode outputMode{OutputMode::Normal}; //!< Output information. + //!< Normal, Height from DEM or Height from ground + std::string outputFormat{}; //!< output raster format + std::string outputFilename{}; //!< output raster filename + CPLStringList creationOpts{}; //!< options for output raster creation + CellMode cellMode{CellMode::Edge}; //!< Mode of cell height calculation. + int observerSpacing{10}; //!< Observer spacing in cumulative mode. + uint8_t numJobs{3}; //!< Relative number of jobs in cumulative mode. +}; + +/** + * A window in a raster including pixels in [xStart, xStop) and [yStart, yStop). + */ +struct Window +{ + int xStart{}; //!< X start position + int xStop{}; //!< X end position + int yStart{}; //!< Y start position + int yStop{}; //!< Y end position + + /// \brief Window size in the X direction. + int xSize() const + { + return xStop - xStart; + } + + /// \brief Window size in the Y direction. + int ySize() const + { + return yStop - yStart; + } + + /// \brief Number of cells. + size_t size() const + { + return static_cast(xSize()) * ySize(); + } + + /// \brief Determine if the X window contains the index. + /// \param nX Index to check + /// \return True if the index is contained, false otherwise. + bool containsX(int nX) const + { + return nX >= xStart && nX < xStop; + } + + /// \brief Determine if the Y window contains the index. + /// \param nY Index to check + /// \return True if the index is contained, false otherwise. + bool containsY(int nY) const + { + return nY >= xStart && nY < yStop; + } + + /// \brief Determine if the window contains the index. + /// \param nX X coordinate of the index to check + /// \param nY Y coordinate of the index to check + /// \return True if the index is contained, false otherwise. + bool contains(int nX, int nY) const + { + return containsX(nX) && containsY(nY); + } + + /// \brief Clamp the argument to be in the window in the X dimension. + /// \param nX Value to clamp. + /// \return Clamped value. + int clampX(int nX) const + { + return xSize() ? std::clamp(nX, xStart, xStop - 1) : xStart; + } + + /// \brief Clamp the argument to be in the window in the Y dimension. + /// \param nY Value to clamp. + /// \return Clamped value. + int clampY(int nY) const + { + return ySize() ? std::clamp(nY, yStart, yStop - 1) : yStart; + } + + /// \brief Shift the X dimension by nShift. + /// \param nShift Amount to shift + void shiftX(int nShift) + { + xStart += nShift; + xStop += nShift; + } +}; + +} // namespace viewshed +} // namespace gdal + +#endif diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 554ca13f1955..4ce116ae1b01 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -158,6 +158,9 @@ if (BUILD_APPS) add_executable(gdalasyncread EXCLUDE_FROM_ALL gdalasyncread.cpp) add_executable(gdalwarpsimple EXCLUDE_FROM_ALL gdalwarpsimple.c) add_executable(multireadtest EXCLUDE_FROM_ALL multireadtest.cpp) + if(NOT MSVC AND CMAKE_THREAD_LIBS_INIT) + target_link_libraries(multireadtest PRIVATE ${CMAKE_THREAD_LIBS_INIT}) + endif() add_executable(test_ogrsf test_ogrsf.cpp) add_executable(testreprojmulti EXCLUDE_FROM_ALL testreprojmulti.cpp) diff --git a/apps/gdal_contour.cpp b/apps/gdal_contour.cpp index 7dd5f9f9bddb..1248f7b2c190 100644 --- a/apps/gdal_contour.cpp +++ b/apps/gdal_contour.cpp @@ -155,13 +155,9 @@ GDALContourAppOptionsGetParser(GDALContourOptions *psOptions) .help(_("Generate levels on an exponential scale: base ^ k, for k an " "integer.")); - argParser->add_argument("-fl") - .metavar("") - .nargs(argparse::nargs_pattern::at_least_one) - .scan<'g', double>() - .action([psOptions](const std::string &s) - { psOptions->adfFixedLevels.push_back(CPLAtof(s.c_str())); }) - .help(_("Name one or more \"fixed levels\" to extract.")); + // Dealt manually as argparse::nargs_pattern::at_least_one is problematic + argParser->add_argument("-fl").scan<'g', double>().metavar("").help( + _("Name one or more \"fixed levels\" to extract.")); argParser->add_argument("-off") .metavar("") @@ -246,6 +242,7 @@ MAIN_START(argc, argv) if (argc < 2) { + try { GDALContourOptions sOptions; @@ -261,11 +258,54 @@ MAIN_START(argc, argv) } GDALContourOptions sOptions; + CPLStringList aosArgv; try { + /* -------------------------------------------------------------------- */ + /* Pre-processing for custom syntax that ArgumentParser does not */ + /* support. */ + /* -------------------------------------------------------------------- */ + for (int i = 1; i < argc && argv != nullptr && argv[i] != nullptr; i++) + { + // argparser is confused by arguments that have at_least_one + // cardinality, if they immediately precede positional arguments. + if (EQUAL(argv[i], "-fl") && argv[i + 1]) + { + if (strchr(argv[i + 1], ' ')) + { + const CPLStringList aosTokens( + CSLTokenizeString(argv[i + 1])); + for (const char *pszToken : aosTokens) + { + sOptions.adfFixedLevels.push_back(CPLAtof(pszToken)); + } + i += 1; + } + else + { + auto isNumeric = [](const char *pszArg) -> bool + { + char *pszEnd = nullptr; + CPLStrtod(pszArg, &pszEnd); + return pszEnd != nullptr && pszEnd[0] == '\0'; + }; + + while (i < argc - 1 && isNumeric(argv[i + 1])) + { + sOptions.adfFixedLevels.push_back(CPLAtof(argv[i + 1])); + i += 1; + } + } + } + else + { + aosArgv.AddString(argv[i]); + } + } + auto argParser = GDALContourAppOptionsGetParser(&sOptions); - argParser->parse_args_without_binary_name(argv + 1); + argParser->parse_args_without_binary_name(aosArgv.List()); if (sOptions.dfInterval == 0.0 && sOptions.adfFixedLevels.empty() && sOptions.dfExpBase == 0.0) diff --git a/apps/gdal_rasterize_bin.cpp b/apps/gdal_rasterize_bin.cpp index 772a5cc9d766..5121cfb3e27d 100644 --- a/apps/gdal_rasterize_bin.cpp +++ b/apps/gdal_rasterize_bin.cpp @@ -36,33 +36,11 @@ /* Usage() */ /************************************************************************/ -static void Usage(bool bIsError, const char *pszErrorMsg = nullptr) +static void Usage() { - fprintf( - bIsError ? stderr : stdout, - "Usage: gdal_rasterize [--help] [--help-general]\n" - " [-b ]... [-i] [-at]\n" - " [-oo =]...\n" - " {[-burn ]... | [-a ] | [-3d]} [-add]\n" - " [-l ]... [-where ] " - "[-sql |@]\n" - " [-dialect ] [-of ] [-a_srs ] [-to " - "=]...\n" - " [-co =]... [-a_nodata ] [-init " - "]...\n" - " [-te ] [-tr ] [-tap] " - "[-ts " - "]\n" - " [-ot " - "{Byte/Int8/Int16/UInt16/UInt32/Int32/UInt64/Int64/Float32/Float64/\n" - " CInt16/CInt32/CFloat32/CFloat64}] [-optim " - "{AUTO|VECTOR|RASTER}] [-q]\n" - " \n"); - - if (pszErrorMsg != nullptr) - fprintf(stderr, "\nFAILURE: %s\n", pszErrorMsg); - exit(bIsError ? 1 : 0); + fprintf(stderr, "%s\n", GDALRasterizeAppGetParserUsage().c_str()); + exit(1); } /************************************************************************/ @@ -71,6 +49,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg = nullptr) MAIN_START(argc, argv) { + /* Check strict compilation and runtime library version as we use C++ API */ if (!GDAL_CHECK_VERSION(argv[0])) exit(1); @@ -78,51 +57,36 @@ MAIN_START(argc, argv) EarlySetConfigOptions(argc, argv); /* -------------------------------------------------------------------- */ - /* Generic arg processing. */ + /* Register standard GDAL drivers, and process generic GDAL */ + /* command options. */ /* -------------------------------------------------------------------- */ GDALAllRegister(); argc = GDALGeneralCmdLineProcessor(argc, &argv, 0); if (argc < 1) exit(-argc); - for (int i = 0; i < argc; i++) - { - if (EQUAL(argv[i], "--utility_version")) - { - printf("%s was compiled against GDAL %s and " - "is running against GDAL %s\n", - argv[0], GDAL_RELEASE_NAME, GDALVersionInfo("RELEASE_NAME")); - CSLDestroy(argv); - return 0; - } - else if (EQUAL(argv[i], "--help")) - { - Usage(false); - } - } + /* -------------------------------------------------------------------- */ + /* Generic arg processing. */ + /* -------------------------------------------------------------------- */ GDALRasterizeOptionsForBinary sOptionsForBinary; - // coverity[tainted_data] - GDALRasterizeOptions *psOptions = - GDALRasterizeOptionsNew(argv + 1, &sOptionsForBinary); + std::unique_ptr + psOptions{GDALRasterizeOptionsNew(argv + 1, &sOptionsForBinary), + GDALRasterizeOptionsFree}; + CSLDestroy(argv); - if (psOptions == nullptr) + if (!psOptions) { - Usage(true); + Usage(); } if (!(sOptionsForBinary.bQuiet)) { - GDALRasterizeOptionsSetProgress(psOptions, GDALTermProgress, nullptr); + GDALRasterizeOptionsSetProgress(psOptions.get(), GDALTermProgress, + nullptr); } - if (sOptionsForBinary.osSource.empty()) - Usage(true, "No input file specified."); - - if (!sOptionsForBinary.bDestSpecified) - Usage(true, "No output file specified."); - /* -------------------------------------------------------------------- */ /* Open input file. */ /* -------------------------------------------------------------------- */ @@ -186,16 +150,19 @@ MAIN_START(argc, argv) } int bUsageError = FALSE; - GDALDatasetH hRetDS = GDALRasterize(sOptionsForBinary.osDest.c_str(), - hDstDS, hInDS, psOptions, &bUsageError); + GDALDatasetH hRetDS = + GDALRasterize(sOptionsForBinary.osDest.c_str(), hDstDS, hInDS, + psOptions.get(), &bUsageError); + if (bUsageError == TRUE) - Usage(true); + Usage(); + int nRetCode = hRetDS ? 0 : 1; GDALClose(hInDS); + if (GDALClose(hRetDS) != CE_None) nRetCode = 1; - GDALRasterizeOptionsFree(psOptions); GDALDestroyDriverManager(); diff --git a/apps/gdal_rasterize_lib.cpp b/apps/gdal_rasterize_lib.cpp index 493922ee1068..e3b7d346bb0a 100644 --- a/apps/gdal_rasterize_lib.cpp +++ b/apps/gdal_rasterize_lib.cpp @@ -50,17 +50,314 @@ #include "ogr_api.h" #include "ogr_core.h" #include "ogr_srs_api.h" +#include "gdalargumentparser.h" /************************************************************************/ -/* ArgIsNumericRasterize() */ +/* GDALRasterizeOptions() */ /************************************************************************/ -static bool ArgIsNumericRasterize(const char *pszArg) +struct GDALRasterizeOptions +{ + std::vector anBandList{}; + std::vector adfBurnValues{}; + bool bInverse = false; + std::string osFormat{}; + bool b3D = false; + GDALProgressFunc pfnProgress = GDALDummyProgress; + void *pProgressData = nullptr; + std::vector aosLayers{}; + std::string osSQL{}; + std::string osDialect{}; + std::string osBurnAttribute{}; + std::string osWHERE{}; + CPLStringList aosRasterizeOptions{}; + CPLStringList aosTO{}; + double dfXRes = 0; + double dfYRes = 0; + CPLStringList aosCreationOptions{}; + GDALDataType eOutputType = GDT_Unknown; + std::vector adfInitVals{}; + std::string osNoData{}; + OGREnvelope sEnvelop{}; + int nXSize = 0; + int nYSize = 0; + OGRSpatialReference oOutputSRS{}; + + bool bTargetAlignedPixels = false; + bool bCreateOutput = false; +}; + +/************************************************************************/ +/* GDALRasterizeOptionsGetParser() */ +/************************************************************************/ +static std::unique_ptr +GDALRasterizeOptionsGetParser(GDALRasterizeOptions *psOptions, + GDALRasterizeOptionsForBinary *psOptionsForBinary) { - char *pszEnd = nullptr; - CPLStrtod(pszArg, &pszEnd); - return pszEnd != nullptr && pszEnd[0] == '\0'; + auto argParser = std::make_unique( + "gdal_rasterize", /* bForBinary=*/psOptionsForBinary != nullptr); + + argParser->add_description(_("Burns vector geometries into a raster.")); + + argParser->add_epilog( + _("This program burns vector geometries (points, lines, and polygons) " + "into the raster band(s) of a raster image.")); + + // Dealt manually as argparse::nargs_pattern::at_least_one is problematic + argParser->add_argument("-b") + .metavar("") + .append() + .scan<'i', int>() + //.nargs(argparse::nargs_pattern::at_least_one) + .help(_("The band(s) to burn values into.")); + + argParser->add_argument("-i") + .flag() + .store_into(psOptions->bInverse) + .help(_("Invert rasterization.")); + + argParser->add_argument("-at") + .flag() + .action( + [psOptions](const std::string &) { + psOptions->aosRasterizeOptions.SetNameValue("ALL_TOUCHED", + "TRUE"); + }) + .help(_("Enables the ALL_TOUCHED rasterization option.")); + + // Mutually exclusive options: -burn, -3d, -a + { + // Required if options for binary + auto &group = argParser->add_mutually_exclusive_group( + psOptionsForBinary != nullptr); + + // Dealt manually as argparse::nargs_pattern::at_least_one is problematic + group.add_argument("-burn") + .metavar("") + .scan<'g', double>() + .append() + //.nargs(argparse::nargs_pattern::at_least_one) + .help(_("A fixed value to burn into the raster band(s).")); + + group.add_argument("-a") + .metavar("") + .store_into(psOptions->osBurnAttribute) + .help(_("Name of the field in the input layer to get the burn " + "values from.")); + + group.add_argument("-3d") + .flag() + .store_into(psOptions->b3D) + .action( + [psOptions](const std::string &) { + psOptions->aosRasterizeOptions.SetNameValue( + "BURN_VALUE_FROM", "Z"); + }) + .help(_("Indicates that a burn value should be extracted from the " + "\"Z\" values of the feature.")); + } + + argParser->add_argument("-add") + .flag() + .action( + [psOptions](const std::string &) { + psOptions->aosRasterizeOptions.SetNameValue("MERGE_ALG", "ADD"); + }) + .help(_("Instead of burning a new value, this adds the new value to " + "the existing raster.")); + + // Undocumented + argParser->add_argument("-chunkysize") + .flag() + .hidden() + .action( + [psOptions](const std::string &s) { + psOptions->aosRasterizeOptions.SetNameValue("CHUNKYSIZE", + s.c_str()); + }); + + // Mutually exclusive -l, -sql + { + auto &group = argParser->add_mutually_exclusive_group(false); + + group.add_argument("-l") + .metavar("") + .append() + .store_into(psOptions->aosLayers) + .help(_("Name of the layer(s) to process.")); + + group.add_argument("-sql") + .metavar("") + .store_into(psOptions->osSQL) + .action( + [psOptions](const std::string &sql) + { + GByte *pabyRet = nullptr; + if (!sql.empty() && sql.at(0) == '@' && + VSIIngestFile(nullptr, sql.substr(1).c_str(), &pabyRet, + nullptr, 1024 * 1024)) + { + GDALRemoveBOM(pabyRet); + char *pszSQLStatement = + reinterpret_cast(pabyRet); + psOptions->osSQL = CPLStrdup( + GDALRemoveSQLComments(pszSQLStatement).c_str()); + VSIFree(pszSQLStatement); + } + }) + .help( + _("An SQL statement to be evaluated against the datasource to " + "produce a virtual layer of features to be burned in.")); + } + + argParser->add_argument("-where") + .metavar("") + .store_into(psOptions->osWHERE) + .help(_("An optional SQL WHERE style query expression to be applied to " + "select features " + "to burn in from the input layer(s).")); + + argParser->add_argument("-dialect") + .metavar("") + .store_into(psOptions->osDialect) + .help(_("The SQL dialect to use for the SQL expression.")); + + // Store later + argParser->add_argument("-a_nodata") + .metavar("") + .help(_("Assign a specified nodata value to output bands.")); + + // Dealt manually as argparse::nargs_pattern::at_least_one is problematic + argParser->add_argument("-init") + .metavar("") + .append() + //.nargs(argparse::nargs_pattern::at_least_one) + .scan<'g', double>() + .help(_("Initialize the output bands to the specified value.")); + + argParser->add_argument("-a_srs") + .metavar("") + .action( + [psOptions](const std::string &osOutputSRSDef) + { + if (psOptions->oOutputSRS.SetFromUserInput( + osOutputSRSDef.c_str()) != OGRERR_NONE) + { + throw std::invalid_argument( + std::string("Failed to process SRS definition: ") + .append(osOutputSRSDef)); + } + psOptions->bCreateOutput = true; + }) + .help(_("The spatial reference system to use for the output raster.")); + + argParser->add_argument("-to") + .metavar("=") + .append() + .action([psOptions](const std::string &s) + { psOptions->aosTO.AddString(s.c_str()); }) + .help(_("Set a transformer option.")); + + // Store later + argParser->add_argument("-te") + .metavar(" ") + .nargs(4) + .scan<'g', double>() + .help(_("Set georeferenced extents of output file to be created.")); + + // Mutex with tr + { + auto &group = argParser->add_mutually_exclusive_group(false); + + // Store later + group.add_argument("-tr") + .metavar(" ") + .nargs(2) + .scan<'g', double>() + .help( + _("Set output file resolution in target georeferenced units.")); + + // Store later + auto &arg = group.add_argument("-ts") + .metavar(" ") + .nargs(2) + .scan<'i', int>() + .help(_("Set output file size in pixels and lines.")); + + argParser->add_hidden_alias_for(arg, "-outsize"); + } + + argParser->add_argument("-tap") + .flag() + .store_into(psOptions->bTargetAlignedPixels) + .action([psOptions](const std::string &) + { psOptions->bCreateOutput = true; }) + .help(_("Align the coordinates of the extent to the values of the " + "output raster.")); + + argParser->add_argument("-optim") + .metavar("AUTO|VECTOR|RASTER") + .action( + [psOptions](const std::string &s) { + psOptions->aosRasterizeOptions.SetNameValue("OPTIM", s.c_str()); + }) + .help(_("Force the algorithm used.")); + + argParser->add_creation_options_argument(psOptions->aosCreationOptions) + .action([psOptions](const std::string &) + { psOptions->bCreateOutput = true; }); + + argParser->add_output_type_argument(psOptions->eOutputType) + .action([psOptions](const std::string &) + { psOptions->bCreateOutput = true; }); + + argParser->add_output_format_argument(psOptions->osFormat) + .action([psOptions](const std::string &) + { psOptions->bCreateOutput = true; }); + + if (psOptionsForBinary) + { + + argParser->add_open_options_argument( + psOptionsForBinary->aosOpenOptions); + + argParser->add_quiet_argument(&psOptionsForBinary->bQuiet); + + argParser->add_argument("src_datasource") + .metavar("") + .store_into(psOptionsForBinary->osSource) + .help(_("Any vector supported readable datasource.")); + + argParser->add_argument("dst_filename") + .metavar("") + .store_into(psOptionsForBinary->osDest) + .help(_("The GDAL raster supported output file.")); + } + + return argParser; +} + +/************************************************************************/ +/* GDALRasterizeAppGetParserUsage() */ +/************************************************************************/ + +std::string GDALRasterizeAppGetParserUsage() +{ + try + { + GDALRasterizeOptions sOptions; + GDALRasterizeOptionsForBinary sOptionsForBinary; + auto argParser = + GDALRasterizeOptionsGetParser(&sOptions, &sOptionsForBinary); + return argParser->usage(); + } + catch (const std::exception &err) + { + CPLError(CE_Failure, CPLE_AppDefined, "Unexpected exception: %s", + err.what()); + return std::string(); + } } /************************************************************************/ @@ -221,9 +518,10 @@ static CPLErr ProcessLayer(OGRLayerH hSrcLayer, bool bSRSIsSet, GDALDatasetH hDstDS, const std::vector &anBandList, const std::vector &adfBurnValues, bool b3D, - bool bInverse, const char *pszBurnAttribute, - CSLConstList papszRasterizeOptions, char **papszTO, - GDALProgressFunc pfnProgress, void *pProgressData) + bool bInverse, const std::string &osBurnAttribute, + CSLConstList papszRasterizeOptions, + CSLConstList papszTO, GDALProgressFunc pfnProgress, + void *pProgressData) { /* -------------------------------------------------------------------- */ @@ -289,14 +587,15 @@ static CPLErr ProcessLayer(OGRLayerH hSrcLayer, bool bSRSIsSet, /* -------------------------------------------------------------------- */ int iBurnField = -1; bool bUseInt64 = false; - if (pszBurnAttribute) + if (!osBurnAttribute.empty()) { OGRFeatureDefnH hLayerDefn = OGR_L_GetLayerDefn(hSrcLayer); - iBurnField = OGR_FD_GetFieldIndex(hLayerDefn, pszBurnAttribute); + iBurnField = OGR_FD_GetFieldIndex(hLayerDefn, osBurnAttribute.c_str()); if (iBurnField == -1) { CPLError(CE_Failure, CPLE_AppDefined, - "Failed to find field %s on layer %s.", pszBurnAttribute, + "Failed to find field %s on layer %s.", + osBurnAttribute.c_str(), OGR_FD_GetName(OGR_L_GetLayerDefn(hSrcLayer))); if (hCT != nullptr) OCTDestroyCoordinateTransformation(hCT); @@ -350,7 +649,7 @@ static CPLErr ProcessLayer(OGRLayerH hSrcLayer, bool bSRSIsSet, adfFullBurnValues.push_back(adfBurnValues[std::min( iBand, static_cast(adfBurnValues.size()) - 1)]); - else if (pszBurnAttribute) + else if (!osBurnAttribute.empty()) { if (bUseInt64) anFullBurnValues.push_back( @@ -481,7 +780,7 @@ static GDALDatasetH CreateOutputDataset( const std::vector &ahLayers, OGRSpatialReferenceH hSRS, OGREnvelope sEnvelop, GDALDriverH hDriver, const char *pszDest, int nXSize, int nYSize, double dfXRes, double dfYRes, bool bTargetAlignedPixels, - int nBandCount, GDALDataType eOutputType, char **papszCreationOptions, + int nBandCount, GDALDataType eOutputType, CSLConstList papszCreationOptions, const std::vector &adfInitVals, const char *pszNoData) { bool bFirstLayer = true; @@ -625,41 +924,6 @@ static GDALDatasetH CreateOutputDataset( return hDstDS; } -struct GDALRasterizeOptions -{ - /*! output format. Use the short format name. */ - char *pszFormat; - - /*! the progress function to use */ - GDALProgressFunc pfnProgress; - - /*! pointer to the progress data variable */ - void *pProgressData; - - bool bCreateOutput; - bool b3D; - bool bInverse; - char **papszLayers; - char *pszSQL; - char *pszDialect; - char *pszBurnAttribute; - char *pszWHERE; - std::vector anBandList; - std::vector adfBurnValues; - char **papszRasterizeOptions; - char **papszTO; - double dfXRes; - double dfYRes; - char **papszCreationOptions; - GDALDataType eOutputType; - std::vector adfInitVals; - char *pszNoData; - OGREnvelope sEnvelop; - int nXSize, nYSize; - OGRSpatialReferenceH hSRS; - bool bTargetAlignedPixels; -}; - /************************************************************************/ /* GDALRasterize() */ /************************************************************************/ @@ -734,7 +998,7 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, if (pszDest == nullptr) pszDest = GDALGetDescription(hDstDS); - if (psOptions->pszSQL == nullptr && psOptions->papszLayers == nullptr && + if (psOptions->osSQL.empty() && psOptions->aosLayers.empty() && GDALDatasetGetLayerCount(hSrcDataset) != 1) { CPLError(CE_Failure, CPLE_NotSupported, @@ -756,7 +1020,7 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, if (bCreateOutput) { CPLString osFormat; - if (psOptions->pszFormat == nullptr) + if (psOptions->osFormat.empty()) { osFormat = GetOutputDriverForRaster(pszDest); if (osFormat.empty()) @@ -767,7 +1031,7 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, } else { - osFormat = psOptions->pszFormat; + osFormat = psOptions->osFormat; } /* -------------------------------------------------------------------- @@ -798,12 +1062,11 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, CPLAssert(bCreateOutput); CPLAssert(hDriver); GDALDataType eOutputType = psOptions->eOutputType; - if (eOutputType == GDT_Unknown && - psOptions->pszBurnAttribute != nullptr) + if (eOutputType == GDT_Unknown && !psOptions->osBurnAttribute.empty()) { OGRFeatureDefnH hLayerDefn = OGR_L_GetLayerDefn(hLayer); - const int iBurnField = - OGR_FD_GetFieldIndex(hLayerDefn, psOptions->pszBurnAttribute); + const int iBurnField = OGR_FD_GetFieldIndex( + hLayerDefn, psOptions->osBurnAttribute.c_str()); if (iBurnField >= 0 && OGR_Fld_GetType(OGR_FD_GetFieldDefn( hLayerDefn, iBurnField)) == OFTInteger64) { @@ -823,15 +1086,23 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, return eOutputType; }; + // Store SRS handle + OGRSpatialReferenceH hSRS = + psOptions->oOutputSRS.IsEmpty() + ? nullptr + : OGRSpatialReference::ToHandle( + const_cast(&psOptions->oOutputSRS)); + /* -------------------------------------------------------------------- */ /* Process SQL request. */ /* -------------------------------------------------------------------- */ CPLErr eErr = CE_Failure; - if (psOptions->pszSQL != nullptr) + if (!psOptions->osSQL.empty()) { - OGRLayerH hLayer = GDALDatasetExecuteSQL( - hSrcDataset, psOptions->pszSQL, nullptr, psOptions->pszDialect); + OGRLayerH hLayer = + GDALDatasetExecuteSQL(hSrcDataset, psOptions->osSQL.c_str(), + nullptr, psOptions->osDialect.c_str()); if (hLayer != nullptr) { if (bCreateOutput) @@ -841,13 +1112,12 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, const GDALDataType eOutputType = GetOutputDataType(hLayer); hDstDS = CreateOutputDataset( - ahLayers, psOptions->hSRS, psOptions->sEnvelop, hDriver, - pszDest, psOptions->nXSize, psOptions->nYSize, - psOptions->dfXRes, psOptions->dfYRes, - psOptions->bTargetAlignedPixels, + ahLayers, hSRS, psOptions->sEnvelop, hDriver, pszDest, + psOptions->nXSize, psOptions->nYSize, psOptions->dfXRes, + psOptions->dfYRes, psOptions->bTargetAlignedPixels, static_cast(psOptions->anBandList.size()), eOutputType, - psOptions->papszCreationOptions, psOptions->adfInitVals, - psOptions->pszNoData); + psOptions->aosCreationOptions, psOptions->adfInitVals, + psOptions->osNoData.c_str()); if (hDstDS == nullptr) { GDALDatasetReleaseResultSet(hSrcDataset, hLayer); @@ -857,10 +1127,10 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, } eErr = ProcessLayer( - hLayer, psOptions->hSRS != nullptr, hDstDS, - psOptions->anBandList, psOptions->adfBurnValues, psOptions->b3D, - psOptions->bInverse, psOptions->pszBurnAttribute, - psOptions->papszRasterizeOptions, psOptions->papszTO, + hLayer, hSRS != nullptr, hDstDS, psOptions->anBandList, + psOptions->adfBurnValues, psOptions->b3D, psOptions->bInverse, + psOptions->osBurnAttribute.c_str(), + psOptions->aosRasterizeOptions, psOptions->aosTO, psOptions->pfnProgress, psOptions->pProgressData); GDALDatasetReleaseResultSet(hSrcDataset, hLayer); @@ -871,9 +1141,9 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, /* Create output file if necessary. */ /* -------------------------------------------------------------------- */ const int nLayerCount = - (psOptions->pszSQL == nullptr && psOptions->papszLayers == nullptr) + (psOptions->osSQL.empty() && psOptions->aosLayers.empty()) ? 1 - : CSLCount(psOptions->papszLayers); + : static_cast(psOptions->aosLayers.size()); if (bCreateOutput && hDstDS == nullptr) { @@ -884,16 +1154,18 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, for (int i = 0; i < nLayerCount; i++) { OGRLayerH hLayer; - if (psOptions->papszLayers) - hLayer = GDALDatasetGetLayerByName(hSrcDataset, - psOptions->papszLayers[i]); + if (psOptions->aosLayers.size() > static_cast(i)) + hLayer = GDALDatasetGetLayerByName( + hSrcDataset, psOptions->aosLayers[i].c_str()); else hLayer = GDALDatasetGetLayer(hSrcDataset, 0); if (hLayer == nullptr) { - CPLError( - CE_Failure, CPLE_AppDefined, "Unable to find layer \"%s\".", - psOptions->papszLayers ? psOptions->papszLayers[i] : "0"); + CPLError(CE_Failure, CPLE_AppDefined, + "Unable to find layer \"%s\".", + psOptions->aosLayers.size() > static_cast(i) + ? psOptions->aosLayers[i].c_str() + : "0"); GDALRasterizeOptionsFree(psOptionsToFree); return nullptr; } @@ -912,12 +1184,12 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, } hDstDS = CreateOutputDataset( - ahLayers, psOptions->hSRS, psOptions->sEnvelop, hDriver, pszDest, + ahLayers, hSRS, psOptions->sEnvelop, hDriver, pszDest, psOptions->nXSize, psOptions->nYSize, psOptions->dfXRes, psOptions->dfYRes, psOptions->bTargetAlignedPixels, static_cast(psOptions->anBandList.size()), eOutputType, - psOptions->papszCreationOptions, psOptions->adfInitVals, - psOptions->pszNoData); + psOptions->aosCreationOptions, psOptions->adfInitVals, + psOptions->osNoData.c_str()); if (hDstDS == nullptr) { GDALRasterizeOptionsFree(psOptionsToFree); @@ -932,23 +1204,25 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, for (int i = 0; i < nLayerCount; i++) { OGRLayerH hLayer; - if (psOptions->papszLayers) + if (psOptions->aosLayers.size() > static_cast(i)) hLayer = GDALDatasetGetLayerByName(hSrcDataset, - psOptions->papszLayers[i]); + psOptions->aosLayers[i].c_str()); else hLayer = GDALDatasetGetLayer(hSrcDataset, 0); if (hLayer == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, "Unable to find layer \"%s\".", - psOptions->papszLayers ? psOptions->papszLayers[i] : "0"); + psOptions->aosLayers.size() > static_cast(i) + ? psOptions->aosLayers[i].c_str() + : "0"); eErr = CE_Failure; break; } - if (psOptions->pszWHERE) + if (!psOptions->osWHERE.empty()) { - if (OGR_L_SetAttributeFilter(hLayer, psOptions->pszWHERE) != + if (OGR_L_SetAttributeFilter(hLayer, psOptions->osWHERE.c_str()) != OGRERR_NONE) { eErr = CE_Failure; @@ -960,11 +1234,12 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, 0.0, 1.0 * (i + 1) / nLayerCount, psOptions->pfnProgress, psOptions->pProgressData); - eErr = ProcessLayer( - hLayer, psOptions->hSRS != nullptr, hDstDS, psOptions->anBandList, - psOptions->adfBurnValues, psOptions->b3D, psOptions->bInverse, - psOptions->pszBurnAttribute, psOptions->papszRasterizeOptions, - psOptions->papszTO, GDALScaledProgress, pScaledProgress); + eErr = ProcessLayer(hLayer, !psOptions->oOutputSRS.IsEmpty(), hDstDS, + psOptions->anBandList, psOptions->adfBurnValues, + psOptions->b3D, psOptions->bInverse, + psOptions->osBurnAttribute.c_str(), + psOptions->aosRasterizeOptions, psOptions->aosTO, + GDALScaledProgress, pScaledProgress); GDALDestroyScaledProgress(pScaledProgress); if (eErr != CE_None) @@ -983,6 +1258,18 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, return hDstDS; } +/************************************************************************/ +/* ArgIsNumericRasterize() */ +/************************************************************************/ + +static bool ArgIsNumericRasterize(const char *pszArg) + +{ + char *pszEnd = nullptr; + CPLStrtod(pszArg, &pszEnd); + return pszEnd != nullptr && pszEnd[0] == '\0'; +} + /************************************************************************/ /* GDALRasterizeOptionsNew() */ /************************************************************************/ @@ -1007,69 +1294,36 @@ GDALRasterizeOptions * GDALRasterizeOptionsNew(char **papszArgv, GDALRasterizeOptionsForBinary *psOptionsForBinary) { - GDALRasterizeOptions *psOptions = new GDALRasterizeOptions; - - psOptions->pszFormat = nullptr; - psOptions->pfnProgress = GDALDummyProgress; - psOptions->pProgressData = nullptr; - psOptions->bCreateOutput = false; - psOptions->b3D = false; - psOptions->bInverse = false; - // sEnvelop implicitly initialized - psOptions->papszCreationOptions = nullptr; - psOptions->papszLayers = nullptr; - psOptions->pszSQL = nullptr; - psOptions->pszDialect = nullptr; - psOptions->pszBurnAttribute = nullptr; - psOptions->pszWHERE = nullptr; - psOptions->papszRasterizeOptions = nullptr; - psOptions->papszTO = nullptr; - psOptions->dfXRes = 0; - psOptions->dfYRes = 0; - psOptions->eOutputType = GDT_Unknown; - psOptions->pszNoData = nullptr; - psOptions->nXSize = 0; - psOptions->nYSize = 0; - psOptions->hSRS = nullptr; - psOptions->bTargetAlignedPixels = false; - - bool bGotSourceFilename = false; - bool bGotDestFilename = false; + + auto psOptions = std::make_unique(); + + /*-------------------------------------------------------------------- */ + /* Parse arguments. */ + /*-------------------------------------------------------------------- */ + + CPLStringList aosArgv; + /* -------------------------------------------------------------------- */ - /* Handle command line arguments. */ + /* Pre-processing for custom syntax that ArgumentParser does not */ + /* support. */ /* -------------------------------------------------------------------- */ const int argc = CSLCount(papszArgv); - for (int i = 0; papszArgv != nullptr && i < argc; i++) + for (int i = 0; i < argc && papszArgv != nullptr && papszArgv[i] != nullptr; + i++) { - if (i < argc - 1 && - (EQUAL(papszArgv[i], "-of") || EQUAL(papszArgv[i], "-f"))) + // argparser will be confused if the value of a string argument + // starts with a negative sign. + if (EQUAL(papszArgv[i], "-a_nodata") && papszArgv[i + 1]) { ++i; - CPLFree(psOptions->pszFormat); - psOptions->pszFormat = CPLStrdup(papszArgv[i]); + const std::string s = papszArgv[i]; + psOptions->osNoData = s; psOptions->bCreateOutput = true; } - else if (EQUAL(papszArgv[i], "-q") || EQUAL(papszArgv[i], "-quiet")) - { - if (psOptionsForBinary) - { - psOptionsForBinary->bQuiet = true; - } - else - { - CPLError(CE_Failure, CPLE_NotSupported, - "%s switch only supported from gdal_rasterize binary.", - papszArgv[i]); - } - } - - else if (i < argc - 1 && EQUAL(papszArgv[i], "-a")) - { - CPLFree(psOptions->pszBurnAttribute); - psOptions->pszBurnAttribute = CPLStrdup(papszArgv[++i]); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-b")) + // argparser is confused by arguments that have at_least_one + // cardinality, if they immediately precede positional arguments. + else if (EQUAL(papszArgv[i], "-burn") && papszArgv[i + 1]) { if (strchr(papszArgv[i + 1], ' ')) { @@ -1077,7 +1331,7 @@ GDALRasterizeOptionsNew(char **papszArgv, CSLTokenizeString(papszArgv[i + 1])); for (const char *pszToken : aosTokens) { - psOptions->anBandList.push_back(atoi(pszToken)); + psOptions->adfBurnValues.push_back(CPLAtof(pszToken)); } i += 1; } @@ -1085,42 +1339,18 @@ GDALRasterizeOptionsNew(char **papszArgv, { while (i < argc - 1 && ArgIsNumericRasterize(papszArgv[i + 1])) { - psOptions->anBandList.push_back(atoi(papszArgv[i + 1])); + psOptions->adfBurnValues.push_back( + CPLAtof(papszArgv[i + 1])); i += 1; } } + + // Dummy value to make argparse happy, as at least one of + // -burn, -a or -3d is required + aosArgv.AddString("-burn"); + aosArgv.AddString("0"); } - else if (EQUAL(papszArgv[i], "-3d")) - { - psOptions->b3D = true; - psOptions->papszRasterizeOptions = CSLSetNameValue( - psOptions->papszRasterizeOptions, "BURN_VALUE_FROM", "Z"); - } - else if (EQUAL(papszArgv[i], "-add")) - { - psOptions->papszRasterizeOptions = CSLSetNameValue( - psOptions->papszRasterizeOptions, "MERGE_ALG", "ADD"); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-chunkysize")) - { - psOptions->papszRasterizeOptions = CSLSetNameValue( - psOptions->papszRasterizeOptions, "CHUNKYSIZE", papszArgv[++i]); - } - else if (EQUAL(papszArgv[i], "-i")) - { - psOptions->bInverse = true; - } - else if (EQUAL(papszArgv[i], "-at")) - { - psOptions->papszRasterizeOptions = CSLSetNameValue( - psOptions->papszRasterizeOptions, "ALL_TOUCHED", "TRUE"); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-optim")) - { - psOptions->papszRasterizeOptions = CSLSetNameValue( - psOptions->papszRasterizeOptions, "OPTIM", papszArgv[++i]); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-burn")) + else if (EQUAL(papszArgv[i], "-init") && papszArgv[i + 1]) { if (strchr(papszArgv[i + 1], ' ')) { @@ -1128,7 +1358,7 @@ GDALRasterizeOptionsNew(char **papszArgv, CSLTokenizeString(papszArgv[i + 1])); for (const char *pszToken : aosTokens) { - psOptions->adfBurnValues.push_back(CPLAtof(pszToken)); + psOptions->adfInitVals.push_back(CPLAtof(pszToken)); } i += 1; } @@ -1136,48 +1366,13 @@ GDALRasterizeOptionsNew(char **papszArgv, { while (i < argc - 1 && ArgIsNumericRasterize(papszArgv[i + 1])) { - psOptions->adfBurnValues.push_back( - CPLAtof(papszArgv[i + 1])); + psOptions->adfInitVals.push_back(CPLAtof(papszArgv[i + 1])); i += 1; } } + psOptions->bCreateOutput = true; } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-where")) - { - CPLFree(psOptions->pszWHERE); - psOptions->pszWHERE = CPLStrdup(papszArgv[++i]); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-l")) - { - psOptions->papszLayers = - CSLAddString(psOptions->papszLayers, papszArgv[++i]); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-sql")) - { - ++i; - CPLFree(psOptions->pszSQL); - GByte *pabyRet = nullptr; - if (papszArgv[i][0] == '@' && - VSIIngestFile(nullptr, papszArgv[i] + 1, &pabyRet, nullptr, - 1024 * 1024)) - { - GDALRemoveBOM(pabyRet); - char *pszSQLStatement = reinterpret_cast(pabyRet); - psOptions->pszSQL = - CPLStrdup(GDALRemoveSQLComments(pszSQLStatement).c_str()); - VSIFree(pszSQLStatement); - } - else - { - psOptions->pszSQL = CPLStrdup(papszArgv[i]); - } - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-dialect")) - { - CPLFree(psOptions->pszDialect); - psOptions->pszDialect = CPLStrdup(papszArgv[++i]); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-init")) + else if (EQUAL(papszArgv[i], "-b") && papszArgv[i + 1]) { if (strchr(papszArgv[i + 1], ' ')) { @@ -1185,7 +1380,7 @@ GDALRasterizeOptionsNew(char **papszArgv, CSLTokenizeString(papszArgv[i + 1])); for (const char *pszToken : aosTokens) { - psOptions->adfInitVals.push_back(CPLAtof(pszToken)); + psOptions->anBandList.push_back(atoi(pszToken)); } i += 1; } @@ -1193,263 +1388,138 @@ GDALRasterizeOptionsNew(char **papszArgv, { while (i < argc - 1 && ArgIsNumericRasterize(papszArgv[i + 1])) { - psOptions->adfInitVals.push_back(CPLAtof(papszArgv[i + 1])); + psOptions->anBandList.push_back(atoi(papszArgv[i + 1])); i += 1; } } - psOptions->bCreateOutput = true; } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-a_nodata")) + else { - CPLFree(psOptions->pszNoData); - psOptions->pszNoData = CPLStrdup(papszArgv[i + 1]); - i += 1; - psOptions->bCreateOutput = true; + aosArgv.AddString(papszArgv[i]); } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-a_srs")) - { - OSRDestroySpatialReference(psOptions->hSRS); - psOptions->hSRS = OSRNewSpatialReference(nullptr); - - if (OSRSetFromUserInput(psOptions->hSRS, papszArgv[i + 1]) != - OGRERR_NONE) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Failed to process SRS definition: %s", - papszArgv[i + 1]); - GDALRasterizeOptionsFree(psOptions); - return nullptr; - } + } - i++; - psOptions->bCreateOutput = true; - } + try + { + auto argParser = + GDALRasterizeOptionsGetParser(psOptions.get(), psOptionsForBinary); + argParser->parse_args_without_binary_name(aosArgv.List()); - else if (i < argc - 4 && EQUAL(papszArgv[i], "-te")) - { - psOptions->sEnvelop.MinX = CPLAtof(papszArgv[++i]); - psOptions->sEnvelop.MinY = CPLAtof(papszArgv[++i]); - psOptions->sEnvelop.MaxX = CPLAtof(papszArgv[++i]); - psOptions->sEnvelop.MaxY = CPLAtof(papszArgv[++i]); - psOptions->bCreateOutput = true; - } - else if (i < argc - 4 && EQUAL(papszArgv[i], "-a_ullr")) - { - psOptions->sEnvelop.MinX = CPLAtof(papszArgv[++i]); - psOptions->sEnvelop.MaxY = CPLAtof(papszArgv[++i]); - psOptions->sEnvelop.MaxX = CPLAtof(papszArgv[++i]); - psOptions->sEnvelop.MinY = CPLAtof(papszArgv[++i]); - psOptions->bCreateOutput = true; - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-co")) + // Check all no store_into args + if (auto oTe = argParser->present>("-te")) { - psOptions->papszCreationOptions = - CSLAddString(psOptions->papszCreationOptions, papszArgv[++i]); + psOptions->sEnvelop.MinX = oTe.value()[0]; + psOptions->sEnvelop.MinY = oTe.value()[1]; + psOptions->sEnvelop.MaxX = oTe.value()[2]; + psOptions->sEnvelop.MaxY = oTe.value()[3]; psOptions->bCreateOutput = true; } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-ot")) + + if (auto oTr = argParser->present>("-tr")) { - for (int iType = 1; iType < GDT_TypeCount; iType++) - { - const GDALDataType eType = static_cast(iType); - if (GDALGetDataTypeName(eType) != nullptr && - EQUAL(GDALGetDataTypeName(eType), papszArgv[i + 1])) - { - psOptions->eOutputType = eType; - } - } + psOptions->dfXRes = oTr.value()[0]; + psOptions->dfYRes = oTr.value()[1]; - if (psOptions->eOutputType == GDT_Unknown) + if (psOptions->dfXRes <= 0 || psOptions->dfYRes <= 0) { CPLError(CE_Failure, CPLE_AppDefined, - "Unknown output pixel type: %s", papszArgv[i + 1]); - GDALRasterizeOptionsFree(psOptions); + "Wrong value for -tr parameter."); return nullptr; } - i++; + psOptions->bCreateOutput = true; } - else if (i < argc - 2 && (EQUAL(papszArgv[i], "-ts") || - EQUAL(papszArgv[i], "-outsize"))) + + if (auto oTs = argParser->present("-ts")) { - psOptions->nXSize = atoi(papszArgv[++i]); - psOptions->nYSize = atoi(papszArgv[++i]); + psOptions->nXSize = oTs.value(); + psOptions->nYSize = oTs.value(); + if (psOptions->nXSize <= 0 || psOptions->nYSize <= 0) { CPLError(CE_Failure, CPLE_AppDefined, "Wrong value for -ts parameter."); - GDALRasterizeOptionsFree(psOptions); return nullptr; } + psOptions->bCreateOutput = true; } - else if (i < argc - 2 && EQUAL(papszArgv[i], "-tr")) + + if (psOptions->bCreateOutput) { - psOptions->dfXRes = CPLAtof(papszArgv[++i]); - psOptions->dfYRes = fabs(CPLAtof(papszArgv[++i])); - if (psOptions->dfXRes == 0 || psOptions->dfYRes == 0) + if (psOptions->dfXRes == 0 && psOptions->dfYRes == 0 && + psOptions->nXSize == 0 && psOptions->nYSize == 0) { - CPLError(CE_Failure, CPLE_AppDefined, - "Wrong value for -tr parameter."); - GDALRasterizeOptionsFree(psOptions); + CPLError(CE_Failure, CPLE_NotSupported, + "'-tr xres yres' or '-ts xsize ysize' is required."); return nullptr; } - psOptions->bCreateOutput = true; - } - else if (EQUAL(papszArgv[i], "-tap")) - { - psOptions->bTargetAlignedPixels = true; - psOptions->bCreateOutput = true; - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-to")) - { - psOptions->papszTO = - CSLAddString(psOptions->papszTO, papszArgv[++i]); - } - else if (i < argc - 1 && EQUAL(papszArgv[i], "-oo")) - { - i++; - if (psOptionsForBinary) + + if (psOptions->bTargetAlignedPixels && psOptions->dfXRes == 0 && + psOptions->dfYRes == 0) { - psOptionsForBinary->aosOpenOptions.AddString(papszArgv[i]); + CPLError(CE_Failure, CPLE_NotSupported, + "-tap option cannot be used without using -tr."); + return nullptr; } - else + + if (!psOptions->anBandList.empty()) { CPLError( CE_Failure, CPLE_NotSupported, - "-oo switch only supported from gdal_rasterize binary."); - } - } - else if (papszArgv[i][0] == '-') - { - CPLError(CE_Failure, CPLE_NotSupported, "Unknown option name '%s'", - papszArgv[i]); - GDALRasterizeOptionsFree(psOptions); - return nullptr; - } - else if (!bGotSourceFilename) - { - bGotSourceFilename = true; - if (psOptionsForBinary) - { - psOptionsForBinary->osSource = papszArgv[i]; - } - else - { - CPLError(CE_Failure, CPLE_NotSupported, - "{source_filename} only supported from gdal_rasterize " - "binary."); - } - } - else if (!bGotDestFilename) - { - bGotDestFilename = true; - if (psOptionsForBinary) - { - psOptionsForBinary->bDestSpecified = true; - psOptionsForBinary->osDest = papszArgv[i]; + "-b option cannot be used when creating a GDAL dataset."); + return nullptr; } - else + + int nBandCount = 1; + + if (!psOptions->adfBurnValues.empty()) + nBandCount = static_cast(psOptions->adfBurnValues.size()); + + if (static_cast(psOptions->adfInitVals.size()) > nBandCount) + nBandCount = static_cast(psOptions->adfInitVals.size()); + + if (psOptions->adfInitVals.size() == 1) { - CPLError(CE_Failure, CPLE_NotSupported, - "{dest_filename} only supported from gdal_rasterize " - "binary."); + for (int i = 1; i <= nBandCount - 1; i++) + psOptions->adfInitVals.push_back(psOptions->adfInitVals[0]); } - } - else - { - CPLError(CE_Failure, CPLE_NotSupported, - "Too many command options '%s'", papszArgv[i]); - GDALRasterizeOptionsFree(psOptions); - return nullptr; - } - } - int nExclusiveOptionsCount = 0; - nExclusiveOptionsCount += !psOptions->adfBurnValues.empty() ? 1 : 0; - nExclusiveOptionsCount += psOptions->pszBurnAttribute != nullptr ? 1 : 0; - nExclusiveOptionsCount += psOptions->b3D ? 1 : 0; - if (nExclusiveOptionsCount != 1) - { - if (nExclusiveOptionsCount == 0 && psOptionsForBinary == nullptr) - { - psOptions->adfBurnValues.push_back(255); + for (int i = 1; i <= nBandCount; i++) + psOptions->anBandList.push_back(i); } else { - CPLError(CE_Failure, CPLE_NotSupported, - "One and only one of -3d, -burn or -a is required."); - GDALRasterizeOptionsFree(psOptions); - return nullptr; + if (psOptions->anBandList.empty()) + psOptions->anBandList.push_back(1); } - } - if (psOptions->bCreateOutput) - { - if (psOptions->dfXRes == 0 && psOptions->dfYRes == 0 && - psOptions->nXSize == 0 && psOptions->nYSize == 0) + if (!psOptions->osDialect.empty() && !psOptions->osWHERE.empty() && + !psOptions->osSQL.empty()) { - CPLError(CE_Failure, CPLE_NotSupported, - "'-tr xres yres' or '-ts xsize ysize' is required."); - GDALRasterizeOptionsFree(psOptions); - return nullptr; - } - - if (psOptions->bTargetAlignedPixels && psOptions->dfXRes == 0 && - psOptions->dfYRes == 0) - { - CPLError(CE_Failure, CPLE_NotSupported, - "-tap option cannot be used without using -tr."); - GDALRasterizeOptionsFree(psOptions); - return nullptr; + CPLError(CE_Warning, CPLE_AppDefined, + "-dialect is ignored with -where. Use -sql instead"); } - if (!psOptions->anBandList.empty()) + if (psOptionsForBinary) { - CPLError(CE_Failure, CPLE_NotSupported, - "-b option cannot be used when creating a GDAL dataset."); - GDALRasterizeOptionsFree(psOptions); - return nullptr; + psOptionsForBinary->bCreateOutput = psOptions->bCreateOutput; + if (!psOptions->osFormat.empty()) + psOptionsForBinary->osFormat = psOptions->osFormat; } - - int nBandCount = 1; - - if (!psOptions->adfBurnValues.empty()) - nBandCount = static_cast(psOptions->adfBurnValues.size()); - - if (static_cast(psOptions->adfInitVals.size()) > nBandCount) - nBandCount = static_cast(psOptions->adfInitVals.size()); - - if (psOptions->adfInitVals.size() == 1) + else if (psOptions->adfBurnValues.empty() && + psOptions->osBurnAttribute.empty() && !psOptions->b3D) { - for (int i = 1; i <= nBandCount - 1; i++) - psOptions->adfInitVals.push_back(psOptions->adfInitVals[0]); + psOptions->adfBurnValues.push_back(255); } - - for (int i = 1; i <= nBandCount; i++) - psOptions->anBandList.push_back(i); - } - else - { - if (psOptions->anBandList.empty()) - psOptions->anBandList.push_back(1); - } - - if (psOptions->pszDialect != nullptr && psOptions->pszWHERE != nullptr && - psOptions->pszSQL == nullptr) - { - CPLError(CE_Warning, CPLE_AppDefined, - "-dialect is ignored with -where. Use -sql instead"); } - - if (psOptionsForBinary) + catch (const std::exception &e) { - psOptionsForBinary->bCreateOutput = psOptions->bCreateOutput; - if (psOptions->pszFormat) - psOptionsForBinary->osFormat = psOptions->pszFormat; + CPLError(CE_Failure, CPLE_AppDefined, "%s", e.what()); + return nullptr; } - return psOptions; + return psOptions.release(); } /************************************************************************/ @@ -1466,21 +1536,6 @@ GDALRasterizeOptionsNew(char **papszArgv, void GDALRasterizeOptionsFree(GDALRasterizeOptions *psOptions) { - if (psOptions == nullptr) - return; - - CPLFree(psOptions->pszFormat); - CSLDestroy(psOptions->papszCreationOptions); - CSLDestroy(psOptions->papszLayers); - CSLDestroy(psOptions->papszRasterizeOptions); - CSLDestroy(psOptions->papszTO); - CPLFree(psOptions->pszSQL); - CPLFree(psOptions->pszDialect); - CPLFree(psOptions->pszBurnAttribute); - CPLFree(psOptions->pszWHERE); - CPLFree(psOptions->pszNoData); - OSRDestroySpatialReference(psOptions->hSRS); - delete psOptions; } diff --git a/apps/gdal_translate_lib.cpp b/apps/gdal_translate_lib.cpp index 79d903017568..fdd31db6289b 100644 --- a/apps/gdal_translate_lib.cpp +++ b/apps/gdal_translate_lib.cpp @@ -2737,18 +2737,11 @@ static void CopyBandInfo(GDALRasterBand *poSrcBand, GDALRasterBand *poDstBand, static int GetColorInterp(const char *pszStr) { - if (EQUAL(pszStr, "red")) - return GCI_RedBand; - if (EQUAL(pszStr, "green")) - return GCI_GreenBand; - if (EQUAL(pszStr, "blue")) - return GCI_BlueBand; - if (EQUAL(pszStr, "alpha")) - return GCI_AlphaBand; - if (EQUAL(pszStr, "gray") || EQUAL(pszStr, "grey")) - return GCI_GrayIndex; if (EQUAL(pszStr, "undefined")) return GCI_Undefined; + const int eInterp = GDALGetColorInterpretationByName(pszStr); + if (eInterp != GCI_Undefined) + return eInterp; CPLError(CE_Warning, CPLE_NotSupported, "Unsupported color interpretation: %s", pszStr); return -1; @@ -3078,7 +3071,8 @@ GDALTranslateOptionsGetParser(GDALTranslateOptions *psOptions, _("Add the indicated ground control point to the output dataset.")); argParser->add_argument("-colorinterp") - .metavar("{red|green|blue|alpha|gray|undefined},...") + .metavar("{red|green|blue|alpha|gray|undefined|pan|coastal|rededge|nir|" + "swir|mwir|lwir|...},...") .action( [psOptions](const std::string &s) { @@ -3093,7 +3087,8 @@ GDALTranslateOptionsGetParser(GDALTranslateOptions *psOptions, argParser->add_argument("-colorinterp_X") .append() - .metavar("{red|green|blue|alpha|gray|undefined}") + .metavar("{red|green|blue|alpha|gray|undefined|pan|coastal|rededge|nir|" + "swir|mwir|lwir|...}") .help(_("Override the color interpretation of band X.")); { diff --git a/apps/gdal_utils_priv.h b/apps/gdal_utils_priv.h index 2aac09d0dd1f..7814c26babeb 100644 --- a/apps/gdal_utils_priv.h +++ b/apps/gdal_utils_priv.h @@ -252,6 +252,8 @@ std::string CPL_DLL GDALTileIndexAppGetParserUsage(); std::string CPL_DLL GDALFootprintAppGetParserUsage(); +std::string CPL_DLL GDALRasterizeAppGetParserUsage(); + /** * Returns the gdaldem usage help string * @param osProcessingMode Processing mode (subparser name) diff --git a/apps/gdal_viewshed.cpp b/apps/gdal_viewshed.cpp index fa529dc24432..c5025ee04272 100644 --- a/apps/gdal_viewshed.cpp +++ b/apps/gdal_viewshed.cpp @@ -31,46 +31,42 @@ #include "gdal.h" #include "gdalargumentparser.h" -#include "viewshed.h" +#include "viewshed/cumulative.h" +#include "viewshed/viewshed.h" -/************************************************************************/ -/* main() */ -/************************************************************************/ - -MAIN_START(argc, argv) +namespace gdal { - using namespace gdal; - EarlySetConfigOptions(argc, argv); - - GDALAllRegister(); - - argc = GDALGeneralCmdLineProcessor(argc, &argv, 0); - CPLStringList aosArgv; - aosArgv.Assign(argv, /* bTakeOwnership= */ true); - if (argc < 1) - std::exit(-argc); - - GDALArgumentParser argParser(aosArgv[0], /* bForBinary=*/true); - - argParser.add_description( - _("Calculates a viewshed raster from an input raster DEM.")); +namespace +{ - argParser.add_epilog(_("For more details, consult " - "https://gdal.org/programs/gdal_viewshed.html")); +struct Options +{ + viewshed::Options opts; + std::string osSrcFilename; + int nBandIn{1}; + bool bQuiet; +}; + +/// Parse arguments into options structure. +/// +/// \param argParser Argument parser +/// \param aosArgv Command line options as a string list +/// \return Command line parsed as options +Options parseArgs(GDALArgumentParser &argParser, const CPLStringList &aosArgv) +{ + Options localOpts; - Viewshed::Options opts; + viewshed::Options &opts = localOpts.opts; argParser.add_output_format_argument(opts.outputFormat); argParser.add_argument("-ox") .store_into(opts.observer.x) - .required() .metavar("") .help(_("The X position of the observer (in SRS units).")); argParser.add_argument("-oy") .store_into(opts.observer.y) - .required() .metavar("") .help(_("The Y position of the observer (in SRS units).")); @@ -130,6 +126,14 @@ MAIN_START(argc, argv) .nargs(1) .help(_("Maximum distance from observer to compute visibility.")); + argParser.add_argument("-j") + .default_value(3) + .store_into(opts.numJobs) + .metavar("") + .nargs(1) + .help(_( + "Number of relative simultaneous jobs to run in cumulative mode")); + // Value for standard atmospheric refraction. See // doc/source/programs/gdal_viewshed.rst argParser.add_argument("-cc") @@ -140,36 +144,42 @@ MAIN_START(argc, argv) .help(_("Coefficient to consider the effect of the curvature and " "refraction.")); - int nBandIn = 1; argParser.add_argument("-b") - .default_value(nBandIn) - .store_into(nBandIn) + .default_value(localOpts.nBandIn) + .store_into(localOpts.nBandIn) .metavar("") .nargs(1) .help(_("Select an input band band containing the DEM data.")); argParser.add_argument("-om") - .choices("NORMAL", "DEM", "GROUND") - .metavar("NORMAL|DEM|GROUND") + .choices("NORMAL", "DEM", "GROUND", "ACCUM") + .metavar("NORMAL|DEM|GROUND|ACCUM") .action( [&into = opts.outputMode](const std::string &value) { if (EQUAL(value.c_str(), "DEM")) - into = Viewshed::OutputMode::DEM; + into = viewshed::OutputMode::DEM; else if (EQUAL(value.c_str(), "GROUND")) - into = Viewshed::OutputMode::Ground; + into = viewshed::OutputMode::Ground; + else if (EQUAL(value.c_str(), "ACCUM")) + into = viewshed::OutputMode::Cumulative; else - into = Viewshed::OutputMode::Normal; + into = viewshed::OutputMode::Normal; }) .nargs(1) .help(_("Sets what information the output contains.")); - bool bQuiet = false; - argParser.add_quiet_argument(&bQuiet); + argParser.add_argument("-os") + .default_value(10) + .store_into(opts.observerSpacing) + .metavar("") + .nargs(1) + .help(_("Spacing between observer cells when using cumulative mode.")); + + argParser.add_quiet_argument(&localOpts.bQuiet); - std::string osSrcFilename; argParser.add_argument("src_filename") - .store_into(osSrcFilename) + .store_into(localOpts.osSrcFilename) .metavar(""); argParser.add_argument("dst_filename") @@ -185,6 +195,16 @@ MAIN_START(argc, argv) argParser.display_error_and_usage(err); std::exit(1); } + return localOpts; +} + +/// Validate specified options. +/// +/// \param localOpts Options to validate +/// \param argParser Argument parser +void validateArgs(Options &localOpts, const GDALArgumentParser &argParser) +{ + viewshed::Options &opts = localOpts.opts; if (opts.maxDistance < 0) { @@ -203,59 +223,149 @@ MAIN_START(argc, argv) } } + if (opts.outputMode != viewshed::OutputMode::Cumulative) + { + for (const char *opt : {"-os", "-j"}) + if (argParser.is_used(opt)) + { + std::string err = "Option " + std::string(opt) + + " can only be used in cumulative mode."; + CPLError(CE_Failure, CPLE_AppDefined, "%s", err.c_str()); + exit(2); + } + } + + if (opts.outputMode == viewshed::OutputMode::Cumulative) + { + for (const char *opt : {"-ox", "-oy", "-vv", "-iv", "-md"}) + if (argParser.is_used(opt)) + { + std::string err = "Option " + std::string(opt) + + " can't be used in cumulative mode."; + CPLError(CE_Failure, CPLE_AppDefined, "%s", err.c_str()); + exit(2); + } + } + else + { + for (const char *opt : {"-ox", "-oy"}) + if (!argParser.is_used(opt)) + { + std::string err = + "Option " + std::string(opt) + " is required."; + CPLError(CE_Failure, CPLE_AppDefined, "%s", err.c_str()); + exit(2); + } + } + // For double values that are out of range for byte raster output, // set to zero. Values less than zero are sentinel as NULL nodata. - if (opts.outputMode == Viewshed::OutputMode::Normal && + if (opts.outputMode == viewshed::OutputMode::Normal && opts.nodataVal > std::numeric_limits::max()) opts.nodataVal = 0; +} + +// Adjust the coefficient of curvature for non-earth SRS. +/// \param curveCoeff Current curve coefficient +/// \param hSrcDS Source dataset +/// \return Adjusted curve coefficient. +double adjustCurveCoeff(double curveCoeff, GDALDatasetH hSrcDS) +{ + const OGRSpatialReference *poSRS = + GDALDataset::FromHandle(hSrcDS)->GetSpatialRef(); + if (poSRS) + { + OGRErr eSRSerr; + const double dfSemiMajor = poSRS->GetSemiMajor(&eSRSerr); + if (eSRSerr != OGRERR_FAILURE && + fabs(dfSemiMajor - SRS_WGS84_SEMIMAJOR) > + 0.05 * SRS_WGS84_SEMIMAJOR) + { + curveCoeff = 1.0; + CPLDebug("gdal_viewshed", + "Using -cc=1.0 as a non-Earth CRS has been detected"); + } + } + return curveCoeff; +} + +} // unnamed namespace +} // namespace gdal + +/************************************************************************/ +/* main() */ +/************************************************************************/ + +MAIN_START(argc, argv) +{ + using namespace gdal; + + EarlySetConfigOptions(argc, argv); + + GDALAllRegister(); + + argc = GDALGeneralCmdLineProcessor(argc, &argv, 0); + CPLStringList aosArgv; + aosArgv.Assign(argv, /* bTakeOwnership= */ true); + if (argc < 1) + std::exit(-argc); + + GDALArgumentParser argParser(aosArgv[0], /* bForBinary=*/true); + + argParser.add_description( + _("Calculates a viewshed raster from an input raster DEM.")); + + argParser.add_epilog(_("For more details, consult " + "https://gdal.org/programs/gdal_viewshed.html")); + + Options localOpts = parseArgs(argParser, aosArgv); + viewshed::Options &opts = localOpts.opts; + + validateArgs(localOpts, argParser); /* -------------------------------------------------------------------- */ /* Open source raster file. */ /* -------------------------------------------------------------------- */ - GDALDatasetH hSrcDS = GDALOpen(osSrcFilename.c_str(), GA_ReadOnly); + GDALDatasetH hSrcDS = + GDALOpen(localOpts.osSrcFilename.c_str(), GA_ReadOnly); if (hSrcDS == nullptr) exit(2); - GDALRasterBandH hBand = GDALGetRasterBand(hSrcDS, nBandIn); + GDALRasterBandH hBand = GDALGetRasterBand(hSrcDS, localOpts.nBandIn); if (hBand == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, - "Band %d does not exist on dataset.", nBandIn); + "Band %d does not exist on dataset.", localOpts.nBandIn); exit(2); } if (!argParser.is_used("-cc")) - { - const OGRSpatialReference *poSRS = - GDALDataset::FromHandle(hSrcDS)->GetSpatialRef(); - if (poSRS) - { - OGRErr eSRSerr; - const double dfSemiMajor = poSRS->GetSemiMajor(&eSRSerr); - if (eSRSerr != OGRERR_FAILURE && - fabs(dfSemiMajor - SRS_WGS84_SEMIMAJOR) > - 0.05 * SRS_WGS84_SEMIMAJOR) - { - opts.curveCoeff = 1.0; - CPLDebug("gdal_viewshed", - "Using -cc=1.0 as a non-Earth CRS has been detected"); - } - } - } + opts.curveCoeff = adjustCurveCoeff(opts.curveCoeff, hSrcDS); /* -------------------------------------------------------------------- */ /* Invoke. */ /* -------------------------------------------------------------------- */ - Viewshed oViewshed(opts); - - bool bSuccess = - oViewshed.run(hBand, bQuiet ? GDALDummyProgress : GDALTermProgress); - GDALDatasetH hDstDS = GDALDataset::FromHandle(oViewshed.output().release()); + GDALDatasetH hDstDS; - GDALClose(hSrcDS); - if (GDALClose(hDstDS) != CE_None) - bSuccess = false; + bool bSuccess; + if (opts.outputMode == viewshed::OutputMode::Cumulative) + { + viewshed::Cumulative oViewshed(opts); + bSuccess = oViewshed.run(localOpts.osSrcFilename, + localOpts.bQuiet ? GDALDummyProgress + : GDALTermProgress); + } + else + { + viewshed::Viewshed oViewshed(opts); + bSuccess = oViewshed.run(hBand, localOpts.bQuiet ? GDALDummyProgress + : GDALTermProgress); + hDstDS = GDALDataset::FromHandle(oViewshed.output().release()); + GDALClose(hSrcDS); + if (GDALClose(hDstDS) != CE_None) + bSuccess = false; + } GDALDestroyDriverManager(); OGRCleanupAll(); diff --git a/apps/gdaldem_lib.cpp b/apps/gdaldem_lib.cpp index 6d9ef904c02a..c42bd152ec5f 100644 --- a/apps/gdaldem_lib.cpp +++ b/apps/gdaldem_lib.cpp @@ -100,6 +100,7 @@ #include "commonutils.h" #include "gdalargumentparser.h" +#include #include #include #include @@ -1666,13 +1667,10 @@ GDALColorReliefGetRGBA(const std::vector &asColorAssociation, if (eColorSelectionMode == COLOR_SELECTION_NEAREST_ENTRY && asColorAssociation[i - 1].dfVal != dfVal) { - size_t index = i; - if (dfVal - asColorAssociation[i - 1].dfVal < - asColorAssociation[i].dfVal - dfVal) - { - --index; - } - + const size_t index = (dfVal - asColorAssociation[i - 1].dfVal < + asColorAssociation[i].dfVal - dfVal) + ? i - 1 + : i; *pnR = asColorAssociation[index].nR; *pnG = asColorAssociation[index].nG; *pnB = asColorAssociation[index].nB; @@ -1692,34 +1690,21 @@ GDALColorReliefGetRGBA(const std::vector &asColorAssociation, const double dfRatio = (dfVal - asColorAssociation[i - 1].dfVal) / (asColorAssociation[i].dfVal - asColorAssociation[i - 1].dfVal); - *pnR = static_cast(0.45 + asColorAssociation[i - 1].nR + - dfRatio * (asColorAssociation[i].nR - - asColorAssociation[i - 1].nR)); - if (*pnR < 0) - *pnR = 0; - else if (*pnR > 255) - *pnR = 255; - *pnG = static_cast(0.45 + asColorAssociation[i - 1].nG + - dfRatio * (asColorAssociation[i].nG - - asColorAssociation[i - 1].nG)); - if (*pnG < 0) - *pnG = 0; - else if (*pnG > 255) - *pnG = 255; - *pnB = static_cast(0.45 + asColorAssociation[i - 1].nB + - dfRatio * (asColorAssociation[i].nB - - asColorAssociation[i - 1].nB)); - if (*pnB < 0) - *pnB = 0; - else if (*pnB > 255) - *pnB = 255; - *pnA = static_cast(0.45 + asColorAssociation[i - 1].nA + - dfRatio * (asColorAssociation[i].nA - - asColorAssociation[i - 1].nA)); - if (*pnA < 0) - *pnA = 0; - else if (*pnA > 255) - *pnA = 255; + const auto LinearInterpolation = [dfRatio](int nValBefore, int nVal) + { + return std::clamp(static_cast(0.45 + nValBefore + + dfRatio * (nVal - nValBefore)), + 0, 255); + }; + + *pnR = LinearInterpolation(asColorAssociation[i - 1].nR, + asColorAssociation[i].nR); + *pnG = LinearInterpolation(asColorAssociation[i - 1].nG, + asColorAssociation[i].nG); + *pnB = LinearInterpolation(asColorAssociation[i - 1].nB, + asColorAssociation[i].nB); + *pnA = LinearInterpolation(asColorAssociation[i - 1].nA, + asColorAssociation[i].nA); return true; } @@ -1767,14 +1752,13 @@ static bool GDALColorReliefFindNamedColor(const char *pszColorName, int *pnR, *pnR = 0; *pnG = 0; *pnB = 0; - for (unsigned int i = 0; i < sizeof(namedColors) / sizeof(namedColors[0]); - i++) + for (const auto &namedColor : namedColors) { - if (EQUAL(pszColorName, namedColors[i].name)) + if (EQUAL(pszColorName, namedColor.name)) { - *pnR = static_cast(255.0 * namedColors[i].r); - *pnG = static_cast(255.0 * namedColors[i].g); - *pnB = static_cast(255.0 * namedColors[i].b); + *pnR = static_cast(255.0 * namedColor.r); + *pnG = static_cast(255.0 * namedColor.g); + *pnB = static_cast(255.0 * namedColor.b); return true; } } @@ -2178,8 +2162,9 @@ GDALColorRelief(GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand1, /* for GDT_Byte, GDT_Int16 or GDT_UInt16 */ /* -------------------------------------------------------------------- */ int nIndexOffset = 0; - GByte *pabyPrecomputed = GDALColorReliefPrecompute( - hSrcBand, asColorAssociation, eColorSelectionMode, &nIndexOffset); + std::unique_ptr pabyPrecomputed( + GDALColorReliefPrecompute(hSrcBand, asColorAssociation, + eColorSelectionMode, &nIndexOffset)); /* -------------------------------------------------------------------- */ /* Initialize progress counter. */ @@ -2188,15 +2173,17 @@ GDALColorRelief(GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand1, const int nXSize = GDALGetRasterBandXSize(hSrcBand); const int nYSize = GDALGetRasterBandYSize(hSrcBand); - float *pafSourceBuf = nullptr; - int *panSourceBuf = nullptr; + std::unique_ptr pafSourceBuf; + std::unique_ptr panSourceBuf; if (pabyPrecomputed) - panSourceBuf = - static_cast(VSI_MALLOC2_VERBOSE(sizeof(int), nXSize)); + panSourceBuf.reset( + static_cast(VSI_MALLOC2_VERBOSE(sizeof(int), nXSize))); else - pafSourceBuf = - static_cast(VSI_MALLOC2_VERBOSE(sizeof(float), nXSize)); - GByte *pabyDestBuf1 = static_cast(VSI_MALLOC2_VERBOSE(4, nXSize)); + pafSourceBuf.reset( + static_cast(VSI_MALLOC2_VERBOSE(sizeof(float), nXSize))); + std::unique_ptr pabyDestBuf( + static_cast(VSI_MALLOC2_VERBOSE(4, nXSize))); + GByte *pabyDestBuf1 = pabyDestBuf.get(); GByte *pabyDestBuf2 = pabyDestBuf1 ? pabyDestBuf1 + nXSize : nullptr; GByte *pabyDestBuf3 = pabyDestBuf2 ? pabyDestBuf2 + nXSize : nullptr; GByte *pabyDestBuf4 = pabyDestBuf3 ? pabyDestBuf3 + nXSize : nullptr; @@ -2205,22 +2192,12 @@ GDALColorRelief(GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand1, (pabyPrecomputed == nullptr && pafSourceBuf == nullptr) || pabyDestBuf1 == nullptr) { - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - return CE_Failure; } if (!pfnProgress(0.0, nullptr, pProgressData)) { CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - return CE_Failure; } @@ -2234,34 +2211,33 @@ GDALColorRelief(GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand1, /* Read source buffer */ CPLErr eErr = GDALRasterIO( hSrcBand, GF_Read, 0, i, nXSize, 1, - panSourceBuf ? static_cast(panSourceBuf) - : static_cast(pafSourceBuf), + panSourceBuf ? static_cast(panSourceBuf.get()) + : static_cast(pafSourceBuf.get()), nXSize, 1, panSourceBuf ? GDT_Int32 : GDT_Float32, 0, 0); if (eErr != CE_None) { - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); return eErr; } if (pabyPrecomputed) { + const auto pabyPrecomputedRaw = pabyPrecomputed.get(); + const auto panSourceBufRaw = panSourceBuf.get(); for (int j = 0; j < nXSize; j++) { - int nIndex = panSourceBuf[j] + nIndexOffset; - pabyDestBuf1[j] = pabyPrecomputed[4 * nIndex]; - pabyDestBuf2[j] = pabyPrecomputed[4 * nIndex + 1]; - pabyDestBuf3[j] = pabyPrecomputed[4 * nIndex + 2]; - pabyDestBuf4[j] = pabyPrecomputed[4 * nIndex + 3]; + int nIndex = panSourceBufRaw[j] + nIndexOffset; + pabyDestBuf1[j] = pabyPrecomputedRaw[4 * nIndex]; + pabyDestBuf2[j] = pabyPrecomputedRaw[4 * nIndex + 1]; + pabyDestBuf3[j] = pabyPrecomputedRaw[4 * nIndex + 2]; + pabyDestBuf4[j] = pabyPrecomputedRaw[4 * nIndex + 3]; } } else { + const auto pafSourceBufRaw = pafSourceBuf.get(); for (int j = 0; j < nXSize; j++) { - GDALColorReliefGetRGBA(asColorAssociation, pafSourceBuf[j], + GDALColorReliefGetRGBA(asColorAssociation, pafSourceBufRaw[j], eColorSelectionMode, &nR, &nG, &nB, &nA); pabyDestBuf1[j] = static_cast(nR); pabyDestBuf2[j] = static_cast(nG); @@ -2275,75 +2251,36 @@ GDALColorRelief(GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand1, */ eErr = GDALRasterIO(hDstBand1, GF_Write, 0, i, nXSize, 1, pabyDestBuf1, nXSize, 1, GDT_Byte, 0, 0); - if (eErr != CE_None) + if (eErr == CE_None) { - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - - return eErr; + eErr = GDALRasterIO(hDstBand2, GF_Write, 0, i, nXSize, 1, + pabyDestBuf2, nXSize, 1, GDT_Byte, 0, 0); } - - eErr = GDALRasterIO(hDstBand2, GF_Write, 0, i, nXSize, 1, pabyDestBuf2, - nXSize, 1, GDT_Byte, 0, 0); - if (eErr != CE_None) + if (eErr == CE_None) { - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - - return eErr; + eErr = GDALRasterIO(hDstBand3, GF_Write, 0, i, nXSize, 1, + pabyDestBuf3, nXSize, 1, GDT_Byte, 0, 0); } - - eErr = GDALRasterIO(hDstBand3, GF_Write, 0, i, nXSize, 1, pabyDestBuf3, - nXSize, 1, GDT_Byte, 0, 0); - if (eErr != CE_None) - { - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - - return eErr; - } - - if (hDstBand4) + if (eErr == CE_None && hDstBand4) { eErr = GDALRasterIO(hDstBand4, GF_Write, 0, i, nXSize, 1, pabyDestBuf4, nXSize, 1, GDT_Byte, 0, 0); - if (eErr != CE_None) - { - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - - return eErr; - } } - if (!pfnProgress(1.0 * (i + 1) / nYSize, nullptr, pProgressData)) + if (eErr == CE_None && + !pfnProgress(1.0 * (i + 1) / nYSize, nullptr, pProgressData)) { CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated"); - - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - - return CE_Failure; + eErr = CE_Failure; + } + if (eErr != CE_None) + { + return eErr; } } pfnProgress(1.0, nullptr, pProgressData); - VSIFree(pabyPrecomputed); - CPLFree(pafSourceBuf); - CPLFree(panSourceBuf); - CPLFree(pabyDestBuf1); - return CE_None; } @@ -2587,24 +2524,23 @@ template static float GDALRoughnessAlg(const T *afWin, float /*fDstNoDataValue*/, void * /*pData*/) { - // Roughness is the largest difference - // between any two cells + // Roughness is the largest difference between any two cells - T pafRoughnessMin = afWin[0]; - T pafRoughnessMax = afWin[0]; + T fRoughnessMin = afWin[0]; + T fRoughnessMax = afWin[0]; for (int k = 1; k < 9; k++) { - if (afWin[k] > pafRoughnessMax) + if (afWin[k] > fRoughnessMax) { - pafRoughnessMax = afWin[k]; + fRoughnessMax = afWin[k]; } - if (afWin[k] < pafRoughnessMin) + if (afWin[k] < fRoughnessMin) { - pafRoughnessMin = afWin[k]; + fRoughnessMin = afWin[k]; } } - return static_cast(pafRoughnessMax - pafRoughnessMin); + return static_cast(fRoughnessMax - fRoughnessMin); } /************************************************************************/ diff --git a/apps/gdalinfo_lib.cpp b/apps/gdalinfo_lib.cpp index 293bdbbf66d2..e61ef39ff8e5 100644 --- a/apps/gdalinfo_lib.cpp +++ b/apps/gdalinfo_lib.cpp @@ -1128,25 +1128,21 @@ char *GDALInfo(GDALDatasetH hDataset, const GDALInfoOptions *psOptions) json_object_object_add(poStacEOBand, "name", poBandName); } - if (GDALGetDescription(hBand) != nullptr && - strlen(GDALGetDescription(hBand)) > 0) + const char *pszBandDesc = GDALGetDescription(hBand); + if (pszBandDesc != nullptr && strlen(pszBandDesc) > 0) { if (bJson) { - json_object *poBandDescription = - json_object_new_string(GDALGetDescription(hBand)); json_object_object_add(poBand, "description", - poBandDescription); + json_object_new_string(pszBandDesc)); - json_object *poStacBandDescription = - json_object_new_string(GDALGetDescription(hBand)); json_object_object_add(poStacEOBand, "description", - poStacBandDescription); + json_object_new_string(pszBandDesc)); } else { Concat(osStr, psOptions->bStdoutOutput, " Description = %s\n", - GDALGetDescription(hBand)); + pszBandDesc); } } else @@ -1161,6 +1157,17 @@ char *GDALInfo(GDALDatasetH hDataset, const GDALInfoOptions *psOptions) } } + if (bJson) + { + const char *pszCommonName = GDALGetSTACCommonNameFromColorInterp( + GDALGetRasterColorInterpretation(hBand)); + if (pszCommonName) + { + json_object_object_add(poStacEOBand, "common_name", + json_object_new_string(pszCommonName)); + } + } + { int bGotMin = FALSE; int bGotMax = FALSE; @@ -2269,6 +2276,9 @@ static void GDALInfoReportMetadata(const GDALInfoOptions *psOptions, GDALInfoPrintMetadata(psOptions, hObject, "RPC", "RPC Metadata", pszIndent, bJson, poMetadata, osStr); } + + GDALInfoPrintMetadata(psOptions, hObject, "IMAGERY", "Imagery", pszIndent, + bJson, poMetadata, osStr); } /************************************************************************/ diff --git a/apps/gdalmdiminfo_lib.cpp b/apps/gdalmdiminfo_lib.cpp index c6d1372164ca..0502cc4d090f 100644 --- a/apps/gdalmdiminfo_lib.cpp +++ b/apps/gdalmdiminfo_lib.cpp @@ -1222,7 +1222,7 @@ char *GDALMultiDimInfo(GDALDatasetH hDataset, } curGroup = std::move(curGroupNew); } - const char *pszArrayName = aosTokens[aosTokens.size() - 1]; + const char *pszArrayName = aosTokens.back(); auto array(curGroup->OpenMDArray(pszArrayName)); if (!array) { diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 7e43acdce3ad..53fa14271b03 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -1175,8 +1175,8 @@ static bool DealWithCOGOptions(CPLStringList &aosCreateOptions, int nSrcCount, GDALWarpAppOptions oClonedOptions(*psOptions); oClonedOptions.bQuiet = true; - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/gdalwarp/%p.tif", &oClonedOptions); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("gdalwarp_tmp.tif")); CPLStringList aosTmpGTiffCreateOptions; aosTmpGTiffCreateOptions.SetNameValue("SPARSE_OK", "YES"); aosTmpGTiffCreateOptions.SetNameValue("TILED", "YES"); @@ -4428,10 +4428,14 @@ static GDALDatasetH GDALWarpCreateOutput( nPixels = static_cast((psOptions->dfMaxX - psOptions->dfMinX + (psOptions->dfXRes / 2.0)) / psOptions->dfXRes); + if (nPixels == 0) + nPixels = 1; nLines = static_cast( (std::fabs(psOptions->dfMaxY - psOptions->dfMinY) + (psOptions->dfYRes / 2.0)) / psOptions->dfYRes); + if (nLines == 0) + nLines = 1; adfDstGeoTransform[0] = psOptions->dfMinX; adfDstGeoTransform[3] = psOptions->dfMaxY; adfDstGeoTransform[1] = psOptions->dfXRes; @@ -4445,7 +4449,7 @@ static GDALDatasetH GDALWarpCreateOutput( { // Try to detect if the edge of the raster would be blank // Cf https://github.com/OSGeo/gdal/issues/7905 - while (true) + while (nPixels > 1 || nLines > 1) { UpdateGeoTransformandAndPixelLines(); @@ -4547,18 +4551,30 @@ static GDALDatasetH GDALWarpCreateOutput( if (bTopBlankLine) { + if (psOptions->dfMaxY - psOptions->dfMinY <= + 2 * psOptions->dfYRes) + break; psOptions->dfMaxY -= psOptions->dfYRes; } if (bBottomBlankLine) { + if (psOptions->dfMaxY - psOptions->dfMinY <= + 2 * psOptions->dfYRes) + break; psOptions->dfMinY += psOptions->dfYRes; } if (bLeftBlankCol) { + if (psOptions->dfMaxX - psOptions->dfMinX <= + 2 * psOptions->dfXRes) + break; psOptions->dfMinX += psOptions->dfXRes; } if (bRightBlankCol) { + if (psOptions->dfMaxX - psOptions->dfMinX <= + 2 * psOptions->dfXRes) + break; psOptions->dfMaxX -= psOptions->dfXRes; } } diff --git a/apps/multireadtest.cpp b/apps/multireadtest.cpp index d8a4e1cc24b8..e0c737bcf521 100644 --- a/apps/multireadtest.cpp +++ b/apps/multireadtest.cpp @@ -30,27 +30,32 @@ #include "gdal_alg.h" #include "cpl_multiproc.h" #include "cpl_string.h" + +#include +#include +#include #include static int nIterations = 1; static bool bLockOnOpen = false; static int nOpenIterations = 1; static volatile int nPendingThreads = 0; +static bool bThreadCanFinish = false; +static std::mutex oMutex; +static std::condition_variable oCond; static const char *pszFilename = nullptr; static int nChecksum = 0; static int nWidth = 0; static int nHeight = 0; -static CPLMutex *pGlobalMutex = nullptr; - /************************************************************************/ /* Usage() */ /************************************************************************/ static void Usage() { - printf("multireadtest [-lock_on_open] [-open_in_main] [-t ]\n" - " [-i ] [-oi ]\n" + printf("multireadtest [[-thread_safe] | [[-lock_on_open] [-open_in_main]]\n" + " [-t ] [-i ] [-oi ]\n" " [-width ] [-height ]\n" " filename\n"); exit(1); @@ -74,12 +79,12 @@ static void WorkerFunc(void *arg) else { if (bLockOnOpen) - CPLAcquireMutex(pGlobalMutex, 100.0); + oMutex.lock(); hDS = GDALOpen(pszFilename, GA_ReadOnly); if (bLockOnOpen) - CPLReleaseMutex(pGlobalMutex); + oMutex.unlock(); } for (int iIter = 0; iIter < nIterations && hDS != nullptr; iIter++) @@ -99,10 +104,10 @@ static void WorkerFunc(void *arg) if (hDS && hDSIn == nullptr) { if (bLockOnOpen) - CPLAcquireMutex(pGlobalMutex, 100.0); + oMutex.lock(); GDALClose(hDS); if (bLockOnOpen) - CPLReleaseMutex(pGlobalMutex); + oMutex.unlock(); } else if (hDSIn != nullptr) { @@ -110,9 +115,13 @@ static void WorkerFunc(void *arg) } } - CPLAcquireMutex(pGlobalMutex, 100.0); - nPendingThreads--; - CPLReleaseMutex(pGlobalMutex); + { + std::unique_lock oLock(oMutex); + nPendingThreads--; + oCond.notify_all(); + while (!bThreadCanFinish) + oCond.wait(oLock); + } } /************************************************************************/ @@ -131,6 +140,10 @@ int main(int argc, char **argv) int nThreadCount = 4; bool bOpenInThreads = true; + bool bThreadSafe = false; + bool bJoinAfterClosing = false; + bool bDetach = false; + bool bClose = true; for (int iArg = 1; iArg < argc; iArg++) { @@ -154,6 +167,10 @@ int main(int argc, char **argv) { nHeight = atoi(argv[++iArg]); } + else if (EQUAL(argv[iArg], "-thread_safe")) + { + bThreadSafe = true; + } else if (EQUAL(argv[iArg], "-lock_on_open")) { bLockOnOpen = true; @@ -162,6 +179,18 @@ int main(int argc, char **argv) { bOpenInThreads = false; } + else if (EQUAL(argv[iArg], "-join_after_closing")) + { + bJoinAfterClosing = true; + } + else if (EQUAL(argv[iArg], "-detach")) + { + bDetach = true; + } + else if (EQUAL(argv[iArg], "-do_not_close")) + { + bClose = false; + } else if (pszFilename == nullptr) { pszFilename = argv[iArg]; @@ -186,12 +215,10 @@ int main(int argc, char **argv) /* -------------------------------------------------------------------- */ /* Get the checksum of band1. */ /* -------------------------------------------------------------------- */ - GDALDatasetH hDS = nullptr; - GDALAllRegister(); for (int i = 0; i < 2; i++) { - hDS = GDALOpen(pszFilename, GA_ReadOnly); + GDALDatasetH hDS = GDALOpen(pszFilename, GA_ReadOnly); if (hDS == nullptr) exit(1); @@ -210,39 +237,83 @@ int main(int argc, char **argv) /* -------------------------------------------------------------------- */ /* Fire off worker threads. */ /* -------------------------------------------------------------------- */ - pGlobalMutex = CPLCreateMutex(); - CPLReleaseMutex(pGlobalMutex); nPendingThreads = nThreadCount; + GDALDatasetH hThreadSafeDS = nullptr; + if (bThreadSafe) + { + hThreadSafeDS = + GDALOpenEx(pszFilename, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE, + nullptr, nullptr, nullptr); + if (!hThreadSafeDS) + exit(1); + } + std::vector aoThreads; std::vector aoDS; for (int iThread = 0; iThread < nThreadCount; iThread++) { - hDS = nullptr; - if (!bOpenInThreads) + GDALDatasetH hDS = nullptr; + if (bThreadSafe) { - hDS = GDALOpen(pszFilename, GA_ReadOnly); - if (!hDS) + hDS = hThreadSafeDS; + } + else + { + if (!bOpenInThreads) { - printf("GDALOpen() failed.\n"); - exit(1); + hDS = GDALOpen(pszFilename, GA_ReadOnly); + if (!hDS) + { + printf("GDALOpen() failed.\n"); + exit(1); + } + aoDS.push_back(hDS); } - aoDS.push_back(hDS); } - if (CPLCreateThread(WorkerFunc, hDS) == -1) + aoThreads.push_back(std::thread([hDS]() { WorkerFunc(hDS); })); + } + + { + std::unique_lock oLock(oMutex); + while (nPendingThreads > 0) { - printf("CPLCreateThread() failed.\n"); - exit(1); + // printf("nPendingThreads = %d\n", nPendingThreads); + oCond.wait(oLock); } } - while (nPendingThreads > 0) - CPLSleep(0.5); - - CPLDestroyMutex(pGlobalMutex); + if (!bJoinAfterClosing && !bDetach) + { + { + std::lock_guard oLock(oMutex); + bThreadCanFinish = true; + oCond.notify_all(); + } + for (auto &oThread : aoThreads) + oThread.join(); + } for (size_t i = 0; i < aoDS.size(); ++i) GDALClose(aoDS[i]); + if (bClose) + GDALClose(hThreadSafeDS); + + if (bDetach) + { + for (auto &oThread : aoThreads) + oThread.detach(); + } + else if (bJoinAfterClosing) + { + { + std::lock_guard oLock(oMutex); + bThreadCanFinish = true; + oCond.notify_all(); + } + for (auto &oThread : aoThreads) + oThread.join(); + } printf("All threads complete.\n"); @@ -250,5 +321,13 @@ int main(int argc, char **argv) GDALDestroyDriverManager(); + { + std::lock_guard oLock(oMutex); + bThreadCanFinish = true; + oCond.notify_all(); + } + + printf("End of main.\n"); + return 0; } diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index e42155fdfa49..28866b20c7d5 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -40,9 +40,12 @@ #include #include +#include +#include #include #include #include +#include #include #include #include @@ -68,8 +71,10 @@ #include "ogr_p.h" #include "ogr_recordbatch.h" #include "ogr_spatialref.h" +#include "ogrlayerarrow.h" #include "ogrlayerdecorator.h" #include "ogrsf_frmts.h" +#include "ogr_wkb.h" typedef enum { @@ -519,38 +524,39 @@ class SetupTargetLayer bool &bError); public: - GDALDataset *m_poSrcDS; - GDALDataset *m_poDstDS; - char **m_papszLCO; - OGRSpatialReference *m_poOutputSRS; + GDALDataset *m_poSrcDS = nullptr; + GDALDataset *m_poDstDS = nullptr; + CSLConstList m_papszLCO = nullptr; + const OGRSpatialReference *m_poUserSourceSRS = nullptr; + const OGRSpatialReference *m_poOutputSRS = nullptr; bool m_bTransform = false; - bool m_bNullifyOutputSRS; + bool m_bNullifyOutputSRS = false; bool m_bSelFieldsSet = false; - char **m_papszSelFields; - bool m_bAppend; - bool m_bAddMissingFields; - int m_eGType; - GeomTypeConversion m_eGeomTypeConversion; - int m_nCoordDim; - bool m_bOverwrite; - char **m_papszFieldTypesToString; - char **m_papszMapFieldType; - bool m_bUnsetFieldWidth; - bool m_bExplodeCollections; - const char *m_pszZField; - char **m_papszFieldMap; - const char *m_pszWHERE; - bool m_bExactFieldNameMatch; - bool m_bQuiet; - bool m_bForceNullable; - bool m_bResolveDomains; - bool m_bUnsetDefault; - bool m_bUnsetFid; - bool m_bPreserveFID; - bool m_bCopyMD; - bool m_bNativeData; - bool m_bNewDataSource; - const char *m_pszCTPipeline; + CSLConstList m_papszSelFields = nullptr; + bool m_bAppend = false; + bool m_bAddMissingFields = false; + int m_eGType = 0; + GeomTypeConversion m_eGeomTypeConversion = GTC_DEFAULT; + int m_nCoordDim = 0; + bool m_bOverwrite = false; + CSLConstList m_papszFieldTypesToString = nullptr; + CSLConstList m_papszMapFieldType = nullptr; + bool m_bUnsetFieldWidth = false; + bool m_bExplodeCollections = false; + const char *m_pszZField = nullptr; + CSLConstList m_papszFieldMap = nullptr; + const char *m_pszWHERE = nullptr; + bool m_bExactFieldNameMatch = false; + bool m_bQuiet = false; + bool m_bForceNullable = false; + bool m_bResolveDomains = false; + bool m_bUnsetDefault = false; + bool m_bUnsetFid = false; + bool m_bPreserveFID = false; + bool m_bCopyMD = false; + bool m_bNativeData = false; + bool m_bNewDataSource = false; + const char *m_pszCTPipeline = nullptr; std::unique_ptr Setup(OGRLayer *poSrcLayer, const char *pszNewLayerName, @@ -559,11 +565,10 @@ class SetupTargetLayer class LayerTranslator { - static bool TranslateArrow(const TargetLayerInfo *psInfo, - GIntBig nCountLayerFeatures, - GIntBig *pnReadFeatureCount, - GDALProgressFunc pfnProgress, void *pProgressArg, - const GDALVectorTranslateOptions *psOptions); + bool TranslateArrow(TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, + GIntBig *pnReadFeatureCount, + GDALProgressFunc pfnProgress, void *pProgressArg, + const GDALVectorTranslateOptions *psOptions); public: GDALDataset *m_poSrcDS = nullptr; @@ -571,9 +576,9 @@ class LayerTranslator bool m_bTransform = false; bool m_bWrapDateline = false; CPLString m_osDateLineOffset{}; - OGRSpatialReference *m_poOutputSRS = nullptr; + const OGRSpatialReference *m_poOutputSRS = nullptr; bool m_bNullifyOutputSRS = false; - OGRSpatialReference *m_poUserSourceSRS = nullptr; + const OGRSpatialReference *m_poUserSourceSRS = nullptr; OGRCoordinateTransformation *m_poGCPCoordTrans = nullptr; int m_eGType = -1; GeomTypeConversion m_eGeomTypeConversion = GTC_DEFAULT; @@ -585,20 +590,20 @@ class LayerTranslator OGRGeometry *m_poClipSrcOri = nullptr; bool m_bWarnedClipSrcSRS = false; - std::unique_ptr m_poClipSrcReprojectedToSrcSRS; + std::unique_ptr m_poClipSrcReprojectedToSrcSRS{}; const OGRSpatialReference *m_poClipSrcReprojectedToSrcSRS_SRS = nullptr; OGREnvelope m_oClipSrcEnv{}; OGRGeometry *m_poClipDstOri = nullptr; bool m_bWarnedClipDstSRS = false; - std::unique_ptr m_poClipDstReprojectedToDstSRS; + std::unique_ptr m_poClipDstReprojectedToDstSRS{}; const OGRSpatialReference *m_poClipDstReprojectedToDstSRS_SRS = nullptr; OGREnvelope m_oClipDstEnv{}; bool m_bExplodeCollections = false; bool m_bNativeData = false; GIntBig m_nLimit = -1; - OGRGeometryFactory::TransformWithOptionsCache m_transformWithOptionsCache; + OGRGeometryFactory::TransformWithOptionsCache m_transformWithOptionsCache{}; bool Translate(OGRFeature *poFeatureIn, TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, GIntBig *pnReadFeatureCount, @@ -1205,6 +1210,14 @@ class GCPCoordTransformation : public OGRCoordinateTransformation virtual OGRCoordinateTransformation *GetInverse() const override { + static std::once_flag flag; + std::call_once(flag, + []() + { + CPLDebug("OGR2OGR", + "GCPCoordTransformation::GetInverse() " + "called, but not implemented"); + }); return nullptr; } }; @@ -1215,20 +1228,24 @@ class GCPCoordTransformation : public OGRCoordinateTransformation class CompositeCT : public OGRCoordinateTransformation { + OGRCoordinateTransformation *const poCT1; + const bool bOwnCT1; + OGRCoordinateTransformation *const poCT2; + const bool bOwnCT2; + + // Working buffer + std::vector m_anErrorCode{}; + CompositeCT(const CompositeCT &other) : poCT1(other.poCT1 ? other.poCT1->Clone() : nullptr), bOwnCT1(true), - poCT2(other.poCT2 ? other.poCT2->Clone() : nullptr), bOwnCT2(true) + poCT2(other.poCT2 ? other.poCT2->Clone() : nullptr), bOwnCT2(true), + m_anErrorCode({}) { } CompositeCT &operator=(const CompositeCT &) = delete; public: - OGRCoordinateTransformation *poCT1; - bool bOwnCT1; - OGRCoordinateTransformation *poCT2; - bool bOwnCT2; - CompositeCT(OGRCoordinateTransformation *poCT1In, bool bOwnCT1In, OGRCoordinateTransformation *poCT2In, bool bOwnCT2In) : poCT1(poCT1In), bOwnCT1(bOwnCT1In), poCT2(poCT2In), bOwnCT2(bOwnCT2In) @@ -1290,9 +1307,52 @@ class CompositeCT : public OGRCoordinateTransformation return nResult; } + virtual int TransformWithErrorCodes(size_t nCount, double *x, double *y, + double *z, double *t, + int *panErrorCodes) override + { + if (poCT1 && poCT2 && panErrorCodes) + { + m_anErrorCode.resize(nCount); + int nResult = poCT1->TransformWithErrorCodes(nCount, x, y, z, t, + m_anErrorCode.data()); + if (nResult) + nResult = poCT2->TransformWithErrorCodes(nCount, x, y, z, t, + panErrorCodes); + for (size_t i = 0; i < nCount; ++i) + { + if (m_anErrorCode[i]) + panErrorCodes[i] = m_anErrorCode[i]; + } + return nResult; + } + int nResult = TRUE; + if (poCT1) + nResult = poCT1->TransformWithErrorCodes(nCount, x, y, z, t, + panErrorCodes); + if (nResult && poCT2) + nResult = poCT2->TransformWithErrorCodes(nCount, x, y, z, t, + panErrorCodes); + return nResult; + } + virtual OGRCoordinateTransformation *GetInverse() const override { - return nullptr; + if (!poCT1 && !poCT2) + return nullptr; + if (!poCT2) + return poCT1->GetInverse(); + if (!poCT1) + return poCT2->GetInverse(); + auto poInvCT1 = + std::unique_ptr(poCT1->GetInverse()); + auto poInvCT2 = + std::unique_ptr(poCT2->GetInverse()); + if (!poInvCT1 || !poInvCT2) + return nullptr; + return std::make_unique(poInvCT2.release(), true, + poInvCT1.release(), true) + .release(); } }; @@ -1302,9 +1362,14 @@ class CompositeCT : public OGRCoordinateTransformation class AxisMappingCoordinateTransformation : public OGRCoordinateTransformation { - public: bool bSwapXY = false; + explicit AxisMappingCoordinateTransformation(bool bSwapXYIn) + : bSwapXY(bSwapXYIn) + { + } + + public: AxisMappingCoordinateTransformation(const std::vector &mappingIn, const std::vector &mappingOut) { @@ -1358,9 +1423,23 @@ class AxisMappingCoordinateTransformation : public OGRCoordinateTransformation return true; } + virtual int TransformWithErrorCodes(size_t nCount, double *x, double *y, + double * /*z*/, double * /*t*/, + int *panErrorCodes) override + { + for (size_t i = 0; i < nCount; i++) + { + if (panErrorCodes) + panErrorCodes[i] = 0; + if (bSwapXY) + std::swap(x[i], y[i]); + } + return true; + } + virtual OGRCoordinateTransformation *GetInverse() const override { - return nullptr; + return new AxisMappingCoordinateTransformation(bSwapXY); } }; @@ -1455,7 +1534,7 @@ static int GetFieldType(const char *pszArg, int *pnSubFieldType) *pnSubFieldType = -1; CPLString osArgSubType = pszOpenParenthesis + 1; if (!osArgSubType.empty() && osArgSubType.back() == ')') - osArgSubType.resize(osArgSubType.size() - 1); + osArgSubType.pop_back(); for (int iSubType = 0; iSubType <= static_cast(OFSTMaxSubType); iSubType++) { @@ -2835,6 +2914,7 @@ GDALDatasetH GDALVectorTranslate(const char *pszDest, GDALDatasetH hDstDS, oSetup.m_poOutputSRS = oOutputSRSHolder.get(); oSetup.m_bTransform = psOptions->bTransform; oSetup.m_bNullifyOutputSRS = psOptions->bNullifyOutputSRS; + oSetup.m_poUserSourceSRS = poSourceSRS; oSetup.m_bSelFieldsSet = psOptions->bSelFieldsSet; oSetup.m_papszSelFields = psOptions->aosSelFields.List(); oSetup.m_bAppend = bAppend; @@ -3732,8 +3812,8 @@ static OGRwkbGeometryType ConvertType(GeomTypeConversion eGeomTypeConversion, static void DoFieldTypeConversion(GDALDataset *poDstDS, OGRFieldDefn &oFieldDefn, - char **papszFieldTypesToString, - char **papszMapFieldType, + CSLConstList papszFieldTypesToString, + CSLConstList papszMapFieldType, bool bUnsetFieldWidth, bool bQuiet, bool bForceNullable, bool bUnsetDefault) { @@ -3878,6 +3958,47 @@ static void DoFieldTypeConversion(GDALDataset *poDstDS, } } +/************************************************************************/ +/* GetArrowGeomFieldIndex() */ +/************************************************************************/ + +static int GetArrowGeomFieldIndex(const struct ArrowSchema *psLayerSchema, + const char *pszFieldName) +{ + if (strcmp(psLayerSchema->format, "+s") == 0) // struct + { + for (int i = 0; i < psLayerSchema->n_children; ++i) + { + const auto psSchema = psLayerSchema->children[i]; + if (strcmp(psSchema->format, "z") == 0) // binary + { + if (strcmp(psSchema->name, pszFieldName) == 0) + { + return i; + } + else + { + // Check if ARROW:extension:name = ogc.wkb or geoarrow.wkb + const char *pabyMetadata = psSchema->metadata; + if (pabyMetadata) + { + const auto oMetadata = + OGRParseArrowMetadata(pabyMetadata); + auto oIter = oMetadata.find(ARROW_EXTENSION_NAME_KEY); + if (oIter != oMetadata.end() && + (oIter->second == EXTENSION_NAME_OGC_WKB || + oIter->second == EXTENSION_NAME_GEOARROW_WKB)) + { + return i; + } + } + } + } + } + } + return -1; +} + /************************************************************************/ /* SetupTargetLayer::CanUseWriteArrowBatch() */ /************************************************************************/ @@ -3905,11 +4026,11 @@ bool SetupTargetLayer::CanUseWriteArrowBatch( !psOptions->aosLCO.FetchNameValue("BATCH_SIZE") && CPLTestBool(CPLGetConfigOption("OGR2OGR_USE_ARROW_API", "YES"))) || CPLTestBool(CPLGetConfigOption("OGR2OGR_USE_ARROW_API", "NO"))) && - !psOptions->bSkipFailures && !psOptions->bTransform && - !psOptions->poClipSrc && !psOptions->poClipDst && - psOptions->oGCPs.nGCPCount == 0 && !psOptions->bWrapDateline && - !m_papszSelFields && !m_bAddMissingFields && - m_eGType == GEOMTYPE_UNCHANGED && psOptions->eGeomOp == GEOMOP_NONE && + !psOptions->bSkipFailures && !psOptions->poClipSrc && + !psOptions->poClipDst && psOptions->oGCPs.nGCPCount == 0 && + !psOptions->bWrapDateline && !m_papszSelFields && + !m_bAddMissingFields && m_eGType == GEOMTYPE_UNCHANGED && + psOptions->eGeomOp == GEOMOP_NONE && m_eGeomTypeConversion == GTC_DEFAULT && m_nCoordDim < 0 && !m_papszFieldTypesToString && !m_papszMapFieldType && !m_bUnsetFieldWidth && !m_bExplodeCollections && !m_pszZField && @@ -3918,6 +4039,26 @@ bool SetupTargetLayer::CanUseWriteArrowBatch( psOptions->dfXYRes == OGRGeomCoordinatePrecision::UNKNOWN && !psOptions->bMakeValid && !psOptions->bSkipInvalidGeom) { + if (psOptions->bTransform) + { + // To simplify implementation for now + if (poSrcLayer->GetLayerDefn()->GetGeomFieldCount() != 1 || + poDstLayer->GetLayerDefn()->GetGeomFieldCount() != 1) + { + return false; + } + auto poSrcSRS = m_poUserSourceSRS ? m_poUserSourceSRS + : poSrcLayer->GetLayerDefn() + ->GetGeomFieldDefn(0) + ->GetSpatialRef(); + if (!poSrcSRS || + !OGRGeometryFactory::isTransformWithOptionsRegularTransform( + poSrcSRS, m_poOutputSRS, nullptr)) + { + return false; + } + } + struct ArrowArrayStream streamSrc; const char *const apszOptions[] = {"SILENCE_GET_SCHEMA_ERROR=YES", nullptr}; @@ -3926,6 +4067,15 @@ bool SetupTargetLayer::CanUseWriteArrowBatch( struct ArrowSchema schemaSrc; if (streamSrc.get_schema(&streamSrc, &schemaSrc) == 0) { + if (psOptions->bTransform && + GetArrowGeomFieldIndex(&schemaSrc, + poSrcLayer->GetGeometryColumn()) < 0) + { + schemaSrc.release(&schemaSrc); + streamSrc.release(&streamSrc); + return false; + } + std::string osErrorMsg; if (poDstLayer->IsArrowSchemaSupported(&schemaSrc, nullptr, osErrorMsg)) @@ -5624,7 +5774,7 @@ SetupCT(TargetLayerInfo *psInfo, OGRLayer *poSrcLayer, bool bTransform, /************************************************************************/ bool LayerTranslator::TranslateArrow( - const TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, + TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, GIntBig *pnReadFeatureCount, GDALProgressFunc pfnProgress, void *pProgressArg, const GDALVectorTranslateOptions *psOptions) { @@ -5674,10 +5824,52 @@ bool LayerTranslator::TranslateArrow( return false; } + int iArrowGeomFieldIndex = -1; + if (m_bTransform) + { + iArrowGeomFieldIndex = GetArrowGeomFieldIndex( + &schema, psInfo->m_poSrcLayer->GetGeometryColumn()); + if (!SetupCT(psInfo, psInfo->m_poSrcLayer, m_bTransform, + m_bWrapDateline, m_osDateLineOffset, m_poUserSourceSRS, + nullptr, m_poOutputSRS, m_poGCPCoordTrans, false)) + { + return false; + } + } + bool bRet = true; GIntBig nCount = 0; bool bGoOn = true; + std::vector abyModifiedWKB; + const int nNumReprojectionThreads = []() + { + const int nNumCPUs = CPLGetNumCPUs(); + if (nNumCPUs <= 1) + { + return 1; + } + else + { + const char *pszNumThreads = + CPLGetConfigOption("GDAL_NUM_THREADS", nullptr); + if (pszNumThreads) + { + if (EQUAL(pszNumThreads, "ALL_CPUS")) + return CPLGetNumCPUs(); + return std::min(atoi(pszNumThreads), 1024); + } + else + { + return std::max(2, nNumCPUs / 2); + } + } + }(); + + // Somewhat arbitrary threshold (config option only/mostly for autotest purposes) + const int MIN_FEATURES_FOR_THREADED_REPROJ = atoi(CPLGetConfigOption( + "OGR2OGR_MIN_FEATURES_FOR_THREADED_REPROJ", "10000")); + while (bGoOn) { struct ArrowArray array; @@ -5714,9 +5906,122 @@ bool LayerTranslator::TranslateArrow( nCount += array.length; } + const auto nArrayLength = array.length; + + // Coordinate reprojection + const void *backupGeomArrayBuffers2 = nullptr; + if (m_bTransform) + { + auto *psGeomArray = array.children[iArrowGeomFieldIndex]; + GByte *pabyWKB = static_cast( + const_cast(psGeomArray->buffers[2])); + const uint32_t *panOffsets = + static_cast(psGeomArray->buffers[1]); + auto poCT = psInfo->m_aoReprojectionInfo[0].m_poCT.get(); + + try + { + abyModifiedWKB.resize(panOffsets[nArrayLength]); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, "Out of memory"); + bRet = false; + if (array.release) + array.release(&array); + break; + } + memcpy(abyModifiedWKB.data(), pabyWKB, panOffsets[nArrayLength]); + backupGeomArrayBuffers2 = psGeomArray->buffers[2]; + psGeomArray->buffers[2] = abyModifiedWKB.data(); + + std::atomic atomicRet{true}; + const auto oReprojectionLambda = + [psGeomArray, nArrayLength, panOffsets, &atomicRet, + &abyModifiedWKB, &poCT](int iThread, int nThreads) + { + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv3D; + auto poThisCT = + std::unique_ptr(poCT->Clone()); + if (!poThisCT) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot clone OGRCoordinateTransformation"); + atomicRet = false; + return; + } + + const GByte *pabyValidity = + static_cast(psGeomArray->buffers[0]); + const size_t iStart = + static_cast(iThread * nArrayLength / nThreads); + const size_t iMax = static_cast( + (iThread + 1) * nArrayLength / nThreads); + for (size_t i = iStart; i < iMax; ++i) + { + const size_t iShifted = + static_cast(i + psGeomArray->offset); + if (!pabyValidity || (pabyValidity[iShifted >> 8] & + (1 << (iShifted % 8))) != 0) + { + const auto nWKBSize = + panOffsets[iShifted + 1] - panOffsets[iShifted]; + if (!OGRWKBTransform( + abyModifiedWKB.data() + panOffsets[iShifted], + nWKBSize, poThisCT.get(), oCache, sEnv3D)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Reprojection failed"); + atomicRet = false; + break; + } + } + } + }; + + if (nArrayLength >= MIN_FEATURES_FOR_THREADED_REPROJ && + nNumReprojectionThreads >= 2) + { + std::vector> oTasks; + for (int iThread = 0; iThread < nNumReprojectionThreads; + ++iThread) + { + oTasks.emplace_back(std::async(std::launch::async, + oReprojectionLambda, iThread, + nNumReprojectionThreads)); + } + for (auto &oTask : oTasks) + { + oTask.get(); + } + } + else + { + oReprojectionLambda(0, 1); + } + + bRet = atomicRet; + if (!bRet) + { + psGeomArray->buffers[2] = backupGeomArrayBuffers2; + if (array.release) + array.release(&array); + break; + } + } + // Write batch to target layer - if (!psInfo->m_poDstLayer->WriteArrowBatch( - &schema, &array, aosOptionsWriteArrowBatch.List())) + const bool bWriteOK = psInfo->m_poDstLayer->WriteArrowBatch( + &schema, &array, aosOptionsWriteArrowBatch.List()); + + if (backupGeomArrayBuffers2) + { + auto *psGeomArray = array.children[iArrowGeomFieldIndex]; + psGeomArray->buffers[2] = backupGeomArrayBuffers2; + } + + if (!bWriteOK) { CPLError(CE_Failure, CPLE_AppDefined, "WriteArrowBatch() failed"); if (array.release) @@ -5747,46 +6052,6 @@ bool LayerTranslator::TranslateArrow( schema.release(&schema); - // Ugly hack to work around https://github.com/OSGeo/gdal/issues/9497 - // Deleting a RecordBatchReader obtained from arrow::dataset::Scanner.ToRecordBatchReader() - // is a lengthy operation since all batches are read in its destructors. - // Here we ask to our custom I/O layer to return in error to short circuit - // that lengthy operation. - if (auto poDS = psInfo->m_poSrcLayer->GetDataset()) - { - if (poDS->GetLayerCount() == 1 && poDS->GetDriver() && - EQUAL(poDS->GetDriver()->GetDescription(), "PARQUET")) - { - bool bStopIO = false; - const char *pszArrowStopIO = - CPLGetConfigOption("OGR_ARROW_STOP_IO", nullptr); - if (pszArrowStopIO && CPLTestBool(pszArrowStopIO)) - { - bStopIO = true; - } - else if (!pszArrowStopIO) - { - std::string osExePath; - osExePath.resize(1024); - if (CPLGetExecPath(osExePath.data(), - static_cast(osExePath.size()))) - { - osExePath.resize(strlen(osExePath.data())); - if (strcmp(CPLGetBasename(osExePath.data()), "ogr2ogr") == - 0) - { - bStopIO = true; - } - } - } - if (bStopIO) - { - CPLSetConfigOption("OGR_ARROW_STOP_IO", "YES"); - CPLDebug("OGR2OGR", "Forcing interruption of Parquet I/O"); - } - } - } - stream.release(&stream); return bRet; } @@ -6880,7 +7145,7 @@ static std::unique_ptr GDALVectorTranslateOptionsGetParser( osGeomName.c_str() + osGeomName.size() - 1, "Z")) { bIs3D = true; - osGeomName.resize(osGeomName.size() - 1); + osGeomName.pop_back(); } if (EQUAL(osGeomName.c_str(), "NONE")) { diff --git a/apps/ogrdissolve.cpp b/apps/ogrdissolve.cpp index f92ced90fae5..cf79c595b60c 100644 --- a/apps/ogrdissolve.cpp +++ b/apps/ogrdissolve.cpp @@ -163,7 +163,7 @@ MAIN_START(nArgc, papszArgv) "Z")) { bIs3D = TRUE; - osGeomName.resize(osGeomName.size() - 1); + osGeomName.pop_back(); } if (EQUAL(osGeomName, "NONE")) eGType = wkbNone; diff --git a/autotest/cpp/test_alg.cpp b/autotest/cpp/test_alg.cpp index d8c5d64a4b44..76024d5f88c3 100644 --- a/autotest/cpp/test_alg.cpp +++ b/autotest/cpp/test_alg.cpp @@ -151,6 +151,37 @@ TEST_F(test_alg, GDALWarpResolveWorkingDataType_padfSrcNoDataReal) GDALDestroyWarpOptions(psOptions); } +// GDALWarpResolveWorkingDataType: effect of padfSrcNoDataReal +TEST_F(test_alg, GDALWarpResolveWorkingDataType_padfSrcNoDataReal_with_band) +{ + GDALDatasetUniquePtr poDS(GDALDriver::FromHandle(GDALGetDriverByName("MEM")) + ->Create("", 1, 1, 1, GDT_Byte, nullptr)); + GDALWarpOptions *psOptions = GDALCreateWarpOptions(); + // False-positive: hSrcDS is no longer used after GDALDestroyWarpOptions() + // coverity[escape] + psOptions->hSrcDS = GDALDataset::ToHandle(poDS.get()); + psOptions->nBandCount = 1; + psOptions->panSrcBands = + static_cast(CPLMalloc(psOptions->nBandCount * sizeof(int))); + psOptions->panSrcBands[0] = 1; + psOptions->padfSrcNoDataReal = + static_cast(CPLMalloc(sizeof(double))); + psOptions->padfSrcNoDataReal[0] = 0.0; + GDALWarpResolveWorkingDataType(psOptions); + EXPECT_EQ(psOptions->eWorkingDataType, GDT_Byte); + + psOptions->padfSrcNoDataReal[0] = -1.0; + GDALWarpResolveWorkingDataType(psOptions); + EXPECT_EQ(psOptions->eWorkingDataType, GDT_Byte); + + psOptions->eWorkingDataType = GDT_Unknown; + psOptions->padfSrcNoDataReal[0] = 2.0; + GDALWarpResolveWorkingDataType(psOptions); + EXPECT_EQ(psOptions->eWorkingDataType, GDT_Byte); + + GDALDestroyWarpOptions(psOptions); +} + // GDALWarpResolveWorkingDataType: effect of padfSrcNoDataImag TEST_F(test_alg, GDALWarpResolveWorkingDataType_padfSrcNoDataImag) { diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp index d2ea0ed7da1c..84f182e7e02a 100644 --- a/autotest/cpp/test_cpl.cpp +++ b/autotest/cpp/test_cpl.cpp @@ -52,6 +52,7 @@ #include "cpl_auto_close.h" #include "cpl_minixml.h" #include "cpl_quad_tree.h" +#include "cpl_spawn.h" #include "cpl_worker_thread_pool.h" #include "cpl_vsi_virtual.h" #include "cpl_threadsafe_queue.hpp" @@ -2845,22 +2846,23 @@ TEST_F(test_cpl, CPLJSONDocument) } // Test CPLRecodeIconv() with re-allocation +// (this test also passed on Windows using its native recoding API) TEST_F(test_cpl, CPLRecodeIconv) { -#ifdef CPL_RECODE_ICONV +#if defined(CPL_RECODE_ICONV) || defined(_WIN32) int N = 32800; char *pszIn = static_cast(CPLMalloc(N + 1)); for (int i = 0; i < N; i++) - pszIn[i] = '\xE9'; + pszIn[i] = '\xA1'; pszIn[N] = 0; char *pszExpected = static_cast(CPLMalloc(N * 2 + 1)); for (int i = 0; i < N; i++) { - pszExpected[2 * i] = '\xC3'; - pszExpected[2 * i + 1] = '\xA9'; + pszExpected[2 * i] = '\xD0'; + pszExpected[2 * i + 1] = '\x81'; } pszExpected[N * 2] = 0; - char *pszRet = CPLRecode(pszIn, "ISO-8859-2", CPL_ENC_UTF8); + char *pszRet = CPLRecode(pszIn, "ISO-8859-5", CPL_ENC_UTF8); EXPECT_EQ(memcmp(pszExpected, pszRet, N * 2 + 1), 0); CPLFree(pszIn); CPLFree(pszRet); @@ -2870,6 +2872,50 @@ TEST_F(test_cpl, CPLRecodeIconv) #endif } +// Test CP1252 to UTF-8 +TEST_F(test_cpl, CPLRecodeStubCP1252_to_UTF8_strict_alloc) +{ + CPLClearRecodeWarningFlags(); + CPLErrorReset(); + CPLPushErrorHandler(CPLQuietErrorHandler); + // Euro character expands to 3-bytes + char *pszRet = CPLRecode("\x80", "CP1252", CPL_ENC_UTF8); + CPLPopErrorHandler(); + EXPECT_STREQ(CPLGetLastErrorMsg(), ""); + EXPECT_EQ(memcmp(pszRet, "\xE2\x82\xAC\x00", 4), 0); + CPLFree(pszRet); +} + +// Test CP1252 to UTF-8 +TEST_F(test_cpl, CPLRecodeStubCP1252_to_UTF8_with_ascii) +{ + CPLClearRecodeWarningFlags(); + CPLErrorReset(); + CPLPushErrorHandler(CPLQuietErrorHandler); + char *pszRet = CPLRecode("x\x80y", "CP1252", CPL_ENC_UTF8); + CPLPopErrorHandler(); + EXPECT_STREQ(CPLGetLastErrorMsg(), ""); + EXPECT_EQ(memcmp(pszRet, "x\xE2\x82\xACy\x00", 6), 0); + CPLFree(pszRet); +} + +// Test CP1252 to UTF-8 +TEST_F(test_cpl, CPLRecodeStubCP1252_to_UTF8_with_warning) +{ + CPLClearRecodeWarningFlags(); + CPLErrorReset(); + CPLPushErrorHandler(CPLQuietErrorHandler); + // \x90 is an invalid CP1252 character. Will be skipped + char *pszRet = CPLRecode("\x90\x80", "CP1252", CPL_ENC_UTF8); + CPLPopErrorHandler(); + EXPECT_STREQ( + CPLGetLastErrorMsg(), + "One or several characters couldn't be converted correctly from CP1252 " + "to UTF-8. This warning will not be emitted anymore"); + EXPECT_EQ(memcmp(pszRet, "\xE2\x82\xAC\x00", 4), 0); + CPLFree(pszRet); +} + // Test CPLHTTPParseMultipartMime() TEST_F(test_cpl, CPLHTTPParseMultipartMime) { @@ -5261,4 +5307,218 @@ TEST_F(test_cpl, CPLUTF8ForceToASCII) } } +#ifndef _WIN32 +TEST_F(test_cpl, CPLSpawn) +{ + VSIStatBufL sStatBuf; + if (VSIStatL("/bin/true", &sStatBuf) == 0) + { + const char *const apszArgs[] = {"/bin/true", nullptr}; + EXPECT_EQ(CPLSpawn(apszArgs, nullptr, nullptr, false), 0); + } + if (VSIStatL("/bin/false", &sStatBuf) == 0) + { + const char *const apszArgs[] = {"/bin/false", nullptr}; + EXPECT_EQ(CPLSpawn(apszArgs, nullptr, nullptr, false), 1); + } + + { + const char *const apszArgs[] = {"/i_do/not/exist", nullptr}; + CPLPushErrorHandler(CPLQuietErrorHandler); + EXPECT_EQ(CPLSpawn(apszArgs, nullptr, nullptr, false), -1); + CPLPopErrorHandler(); + } +} +#endif + +static bool ENDS_WITH(const char *pszStr, const char *pszEnd) +{ + return strlen(pszStr) >= strlen(pszEnd) && + strcmp(pszStr + strlen(pszStr) - strlen(pszEnd), pszEnd) == 0; +} + +TEST_F(test_cpl, VSIMemGenerateHiddenFilename) +{ + { + // Initial cleanup + VSIRmdirRecursive("/vsimem/"); + VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."); + + // Generate unlisted filename + const std::string osFilename1 = VSIMemGenerateHiddenFilename(nullptr); + const char *pszFilename1 = osFilename1.c_str(); + EXPECT_TRUE(STARTS_WITH(pszFilename1, "/vsimem/.#!HIDDEN!#./")); + EXPECT_TRUE(ENDS_WITH(pszFilename1, "/unnamed")); + + { + // Check the file doesn't exist yet + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(pszFilename1, &sStat), -1); + } + + // Create the file with some content + GByte abyDummyData[1] = {0}; + VSIFCloseL(VSIFileFromMemBuffer(pszFilename1, abyDummyData, + sizeof(abyDummyData), false)); + + { + // Check the file exists now + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(pszFilename1, &sStat), 0); + } + + // Get's back content + EXPECT_EQ(VSIGetMemFileBuffer(pszFilename1, nullptr, false), + abyDummyData); + + { + // Check the hidden file doesn't popup + const CPLStringList aosFiles(VSIReadDir("/vsimem/")); + EXPECT_EQ(aosFiles.size(), 0); + } + + { + // Check that we can list the below directory if we know it exists + // and there's just one subdir + const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_EQ(aosFiles.size(), 1); + } + + { + // but that it is not an explicit directory + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL("/vsimem/.#!HIDDEN!#.", &sStat), -1); + } + + // Creates second file + const std::string osFilename2 = VSIMemGenerateHiddenFilename(nullptr); + const char *pszFilename2 = osFilename2.c_str(); + EXPECT_TRUE(strcmp(pszFilename1, pszFilename2) != 0); + + // Create it + VSIFCloseL(VSIFileFromMemBuffer(pszFilename2, abyDummyData, + sizeof(abyDummyData), false)); + + { + // Check that we can list the root hidden dir if we know it exists + const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_EQ(aosFiles.size(), 2); + } + + { + // Create an explicit subdirectory in a hidden directory + const std::string osBaseName = + VSIMemGenerateHiddenFilename(nullptr); + const std::string osSubDir = + CPLFormFilename(osBaseName.c_str(), "mysubdir", nullptr); + EXPECT_EQ(VSIMkdir(osSubDir.c_str(), 0), 0); + + // Check the subdirectory exists + { + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osSubDir.c_str(), &sStat), 0); + } + + // but not its hidden parent + { + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osBaseName.c_str(), &sStat), -1); + } + + // Create file within the subdirectory + VSIFCloseL(VSIFileFromMemBuffer( + CPLFormFilename(osSubDir.c_str(), "my.bin", nullptr), + abyDummyData, sizeof(abyDummyData), false)); + + { + // Check that we can list the subdirectory + const CPLStringList aosFiles(VSIReadDir(osSubDir.c_str())); + EXPECT_EQ(aosFiles.size(), 1); + } + + { + // Check that we can list the root hidden dir if we know it exists + const CPLStringList aosFiles( + VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_EQ(aosFiles.size(), 3); + } + } + + // Directly create a directory with the return of VSIMemGenerateHiddenFilename() + { + const std::string osDirname = VSIMemGenerateHiddenFilename(nullptr); + EXPECT_EQ(VSIMkdir(osDirname.c_str(), 0), 0); + + // Check the subdirectory exists + { + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), 0); + } + + // Create file within the subdirectory + VSIFCloseL(VSIFileFromMemBuffer( + CPLFormFilename(osDirname.c_str(), "my.bin", nullptr), + abyDummyData, sizeof(abyDummyData), false)); + + { + // Check there's a file in this subdirectory + const CPLStringList aosFiles(VSIReadDir(osDirname.c_str())); + EXPECT_EQ(aosFiles.size(), 1); + } + + EXPECT_EQ(VSIRmdirRecursive(osDirname.c_str()), 0); + + { + // Check there's no longer any file in this subdirectory + const CPLStringList aosFiles(VSIReadDir(osDirname.c_str())); + EXPECT_EQ(aosFiles.size(), 0); + } + + { + // Check that it no longer exists + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), -1); + } + } + + // Check that operations on "/vsimem/" do not interfere with hidden files + { + // Create regular file + VSIFCloseL(VSIFileFromMemBuffer("/vsimem/regular_file", + abyDummyData, sizeof(abyDummyData), + false)); + + // Check it is visible + EXPECT_EQ(CPLStringList(VSIReadDir("/vsimem/")).size(), 1); + + // Clean root /vsimem/ + VSIRmdirRecursive("/vsimem/"); + + // No more user files + EXPECT_TRUE(CPLStringList(VSIReadDir("/vsimem/")).empty()); + + // But still hidden files + EXPECT_TRUE( + !CPLStringList(VSIReadDir("/vsimem/.#!HIDDEN!#.")).empty()); + } + + // Clean-up hidden files + EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0); + + { + // Check the root hidden dir is empty + const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_TRUE(aosFiles.empty()); + } + + EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0); + } + + { + const std::string osFilename = VSIMemGenerateHiddenFilename("foo.bar"); + const char *pszFilename = osFilename.c_str(); + EXPECT_TRUE(STARTS_WITH(pszFilename, "/vsimem/.#!HIDDEN!#./")); + EXPECT_TRUE(ENDS_WITH(pszFilename, "/foo.bar")); + } +} } // namespace diff --git a/autotest/cpp/test_gdal.cpp b/autotest/cpp/test_gdal.cpp index be310432af49..c5046797e65f 100644 --- a/autotest/cpp/test_gdal.cpp +++ b/autotest/cpp/test_gdal.cpp @@ -260,12 +260,74 @@ TEST_F(test_gdal, GDALDataTypeUnion_special_cases) false /* complex */), GDT_Int64); - EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Byte, -128, 0), GDT_Int16); - EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Byte, -32768, 0), GDT_Int16); - EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Byte, -32769, 0), GDT_Int32); - EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Float32, -99999, 0), GDT_Float32); - EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Float32, -99999.9876, 0), + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Byte, -128, false), GDT_Int16); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Byte, -32768, false), GDT_Int16); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Byte, -32769, false), GDT_Int32); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Int8, 127, false), GDT_Int8); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Int8, 128, false), GDT_Int16); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Int16, 32767, false), GDT_Int16); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Int16, 32768, false), GDT_Int32); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_UInt16, 65535, false), GDT_UInt16); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_UInt16, 65536, false), GDT_UInt32); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Int32, INT32_MAX, false), + GDT_Int32); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Int32, INT32_MAX + 1.0, false), + GDT_Int64); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_UInt32, UINT32_MAX, false), + GDT_UInt32); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_UInt32, UINT32_MAX + 1.0, false), + GDT_UInt64); + + // (1 << 63) - 1024 + EXPECT_EQ( + GDALDataTypeUnionWithValue(GDT_Int64, 9223372036854774784.0, false), + GDT_Int64); + // (1 << 63) - 512 + EXPECT_EQ( + GDALDataTypeUnionWithValue(GDT_Int64, 9223372036854775296.0, false), + GDT_Float64); + + // (1 << 64) - 2048 + EXPECT_EQ( + GDALDataTypeUnionWithValue(GDT_UInt64, 18446744073709549568.0, false), + GDT_UInt64); + // (1 << 64) + 4096 + EXPECT_EQ( + GDALDataTypeUnionWithValue(GDT_UInt64, 18446744073709555712.0, false), + GDT_Float64); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Float32, -99999, false), + GDT_Float32); + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Float32, -99999.9876, false), + GDT_Float64); + EXPECT_EQ(GDALDataTypeUnionWithValue( + GDT_Float32, std::numeric_limits::quiet_NaN(), false), + GDT_Float32); + EXPECT_EQ(GDALDataTypeUnionWithValue( + GDT_Float32, -std::numeric_limits::infinity(), false), + GDT_Float32); + EXPECT_EQ(GDALDataTypeUnionWithValue( + GDT_Float32, -std::numeric_limits::infinity(), false), + GDT_Float32); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Float64, -99999.9876, false), + GDT_Float64); + EXPECT_EQ(GDALDataTypeUnionWithValue( + GDT_Float64, std::numeric_limits::quiet_NaN(), false), GDT_Float64); + EXPECT_EQ(GDALDataTypeUnionWithValue( + GDT_Float64, -std::numeric_limits::infinity(), false), + GDT_Float64); + EXPECT_EQ(GDALDataTypeUnionWithValue( + GDT_Float64, -std::numeric_limits::infinity(), false), + GDT_Float64); + + EXPECT_EQ(GDALDataTypeUnionWithValue(GDT_Unknown, 0, false), GDT_Byte); } // Test GDALAdjustValueToDataType() @@ -807,6 +869,106 @@ TEST_F(test_gdal, GDALIsValueExactAs) GDALIsValueExactAs(std::numeric_limits::quiet_NaN())); } +// Test GDALIsValueExactAs() +TEST_F(test_gdal, GDALIsValueExactAs_C_func) +{ + EXPECT_TRUE(GDALIsValueExactAs(0, GDT_Byte)); + EXPECT_TRUE(GDALIsValueExactAs(255, GDT_Byte)); + EXPECT_FALSE(GDALIsValueExactAs(-1, GDT_Byte)); + EXPECT_FALSE(GDALIsValueExactAs(256, GDT_Byte)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_Byte)); + + EXPECT_TRUE(GDALIsValueExactAs(-128, GDT_Int8)); + EXPECT_TRUE(GDALIsValueExactAs(127, GDT_Int8)); + EXPECT_FALSE(GDALIsValueExactAs(-129, GDT_Int8)); + EXPECT_FALSE(GDALIsValueExactAs(128, GDT_Int8)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_Int8)); + + EXPECT_TRUE(GDALIsValueExactAs(0, GDT_UInt16)); + EXPECT_TRUE(GDALIsValueExactAs(65535, GDT_UInt16)); + EXPECT_FALSE(GDALIsValueExactAs(-1, GDT_UInt16)); + EXPECT_FALSE(GDALIsValueExactAs(65536, GDT_UInt16)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_UInt16)); + + EXPECT_TRUE(GDALIsValueExactAs(-32768, GDT_Int16)); + EXPECT_TRUE(GDALIsValueExactAs(32767, GDT_Int16)); + EXPECT_FALSE(GDALIsValueExactAs(-32769, GDT_Int16)); + EXPECT_FALSE(GDALIsValueExactAs(32768, GDT_Int16)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_Int16)); + + EXPECT_TRUE(GDALIsValueExactAs(std::numeric_limits::lowest(), + GDT_UInt32)); + EXPECT_TRUE( + GDALIsValueExactAs(std::numeric_limits::max(), GDT_UInt32)); + EXPECT_FALSE(GDALIsValueExactAs( + std::numeric_limits::lowest() - 1.0, GDT_UInt32)); + EXPECT_FALSE(GDALIsValueExactAs(std::numeric_limits::max() + 1.0, + GDT_UInt32)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_UInt32)); + + EXPECT_TRUE( + GDALIsValueExactAs(std::numeric_limits::lowest(), GDT_Int32)); + EXPECT_TRUE( + GDALIsValueExactAs(std::numeric_limits::max(), GDT_Int32)); + EXPECT_FALSE(GDALIsValueExactAs( + std::numeric_limits::lowest() - 1.0, GDT_Int32)); + EXPECT_FALSE(GDALIsValueExactAs(std::numeric_limits::max() + 1.0, + GDT_Int32)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_Int32)); + + EXPECT_TRUE(GDALIsValueExactAs( + static_cast(std::numeric_limits::lowest()), + GDT_UInt64)); + // (1 << 64) - 2048 + EXPECT_TRUE(GDALIsValueExactAs(18446744073709549568.0, GDT_UInt64)); + EXPECT_FALSE(GDALIsValueExactAs( + static_cast(std::numeric_limits::lowest()) - 1.0, + GDT_UInt64)); + // (1 << 64) + EXPECT_FALSE(GDALIsValueExactAs(18446744073709551616.0, GDT_UInt64)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_UInt64)); + + EXPECT_TRUE(GDALIsValueExactAs( + static_cast(std::numeric_limits::lowest()), + GDT_Int64)); + // (1 << 63) - 1024 + EXPECT_TRUE(GDALIsValueExactAs(9223372036854774784.0, GDT_Int64)); + EXPECT_FALSE(GDALIsValueExactAs( + static_cast(std::numeric_limits::lowest()) - 2048.0, + GDT_Int64)); + // (1 << 63) - 512 + EXPECT_FALSE(GDALIsValueExactAs(9223372036854775296.0, GDT_Int64)); + EXPECT_FALSE(GDALIsValueExactAs(0.5, GDT_Int64)); + + EXPECT_TRUE( + GDALIsValueExactAs(-std::numeric_limits::max(), GDT_Float32)); + EXPECT_TRUE( + GDALIsValueExactAs(std::numeric_limits::max(), GDT_Float32)); + EXPECT_TRUE(GDALIsValueExactAs(-std::numeric_limits::infinity(), + GDT_Float32)); + EXPECT_TRUE(GDALIsValueExactAs(std::numeric_limits::infinity(), + GDT_Float32)); + EXPECT_TRUE(GDALIsValueExactAs(std::numeric_limits::quiet_NaN(), + GDT_Float32)); + EXPECT_TRUE( + !GDALIsValueExactAs(-std::numeric_limits::max(), GDT_Float32)); + EXPECT_TRUE( + !GDALIsValueExactAs(std::numeric_limits::max(), GDT_Float32)); + + EXPECT_TRUE(GDALIsValueExactAs(-std::numeric_limits::infinity(), + GDT_Float64)); + EXPECT_TRUE(GDALIsValueExactAs(std::numeric_limits::infinity(), + GDT_Float64)); + EXPECT_TRUE( + GDALIsValueExactAs(-std::numeric_limits::max(), GDT_Float64)); + EXPECT_TRUE( + GDALIsValueExactAs(std::numeric_limits::max(), GDT_Float64)); + EXPECT_TRUE(GDALIsValueExactAs(std::numeric_limits::quiet_NaN(), + GDT_Float64)); + + EXPECT_TRUE(GDALIsValueExactAs(0, GDT_CInt16)); +} + #ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp index e75d0b6253b9..8fe4e7469977 100644 --- a/autotest/cpp/test_ogr.cpp +++ b/autotest/cpp/test_ogr.cpp @@ -4201,4 +4201,73 @@ TEST_F(test_ogr, OGRCurve_reversePoints) } } +// Test OGRGeometryFactory::transformWithOptions() +TEST_F(test_ogr, transformWithOptions) +{ + // Projected CRS to national geographic CRS (not including poles or antimeridian) + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "LINESTRING(700000 6600000, 700001 6600001)", nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + + OGRSpatialReference oEPSG_2154; + oEPSG_2154.importFromEPSG(2154); // "RGF93 v1 / Lambert-93" + OGRSpatialReference oEPSG_4171; + oEPSG_4171.importFromEPSG(4171); // "RGF93 v1" + oEPSG_4171.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + auto poCT = std::unique_ptr( + OGRCreateCoordinateTransformation(&oEPSG_2154, &oEPSG_4171)); + OGRGeometryFactory::TransformWithOptionsCache oCache; + poGeom = OGRGeometryFactory::transformWithOptions(poGeom, poCT.get(), + nullptr, oCache); + EXPECT_NEAR(poGeom->toLineString()->getX(0), 3, 1e-8); + EXPECT_NEAR(poGeom->toLineString()->getY(0), 46.5, 1e-8); + + delete poGeom; + } + +#ifdef HAVE_GEOS + // Projected CRS to national geographic CRS including antimeridian + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "LINESTRING(657630.64 4984896.17,815261.43 4990738.26)", nullptr, + &poGeom); + ASSERT_NE(poGeom, nullptr); + + OGRSpatialReference oEPSG_6329; + oEPSG_6329.importFromEPSG(6329); // "NAD83(2011) / UTM zone 60N" + OGRSpatialReference oEPSG_6318; + oEPSG_6318.importFromEPSG(6318); // "NAD83(2011)" + oEPSG_6318.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + auto poCT = std::unique_ptr( + OGRCreateCoordinateTransformation(&oEPSG_6329, &oEPSG_6318)); + OGRGeometryFactory::TransformWithOptionsCache oCache; + poGeom = OGRGeometryFactory::transformWithOptions(poGeom, poCT.get(), + nullptr, oCache); + EXPECT_EQ(poGeom->getGeometryType(), wkbMultiLineString); + if (poGeom->getGeometryType() == wkbMultiLineString) + { + const auto poMLS = poGeom->toMultiLineString(); + EXPECT_EQ(poMLS->getNumGeometries(), 2); + if (poMLS->getNumGeometries() == 2) + { + const auto poLS = poMLS->getGeometryRef(0); + EXPECT_EQ(poLS->getNumPoints(), 2); + if (poLS->getNumPoints() == 2) + { + EXPECT_NEAR(poLS->getX(0), 179, 1e-6); + EXPECT_NEAR(poLS->getY(0), 45, 1e-6); + EXPECT_NEAR(poLS->getX(1), 180, 1e-6); + EXPECT_NEAR(poLS->getY(1), 45.004384301691303, 1e-6); + } + } + } + + delete poGeom; + } +#endif +} + } // namespace diff --git a/autotest/cpp/test_ogr_wkb.cpp b/autotest/cpp/test_ogr_wkb.cpp index ed86e66fd6a6..dba3eedc46a0 100644 --- a/autotest/cpp/test_ogr_wkb.cpp +++ b/autotest/cpp/test_ogr_wkb.cpp @@ -524,4 +524,354 @@ INSTANTIATE_TEST_SUITE_P( OGRWKBIntersectsPessimisticFixture::ParamType> &l_info) { return std::get<6>(l_info.param); }); +class OGRWKBTransformFixture + : public test_ogr_wkb, + public ::testing::WithParamInterface< + std::tuple> +{ + public: + static std::vector< + std::tuple> + GetTupleValues() + { + return { + std::make_tuple("POINT EMPTY", wkbNDR, "POINT EMPTY", + "POINT_EMPTY_NDR"), + std::make_tuple("POINT EMPTY", wkbXDR, "POINT EMPTY", + "POINT_EMPTY_XDR"), + std::make_tuple("POINT (1 2)", wkbNDR, "POINT (2 4)", "POINT_NDR"), + std::make_tuple("POINT (1 2)", wkbXDR, "POINT (2 4)", "POINT_XDR"), + std::make_tuple("POINT Z EMPTY", wkbNDR, "POINT Z EMPTY", + "POINT_Z_EMPTY_NDR"), + std::make_tuple("POINT Z EMPTY", wkbXDR, "POINT Z EMPTY", + "POINT_Z_EMPTY_XDR"), + std::make_tuple("POINT Z (1 2 3)", wkbNDR, "POINT Z (2 4 6)", + "POINT_Z_NDR"), + std::make_tuple("POINT Z (1 2 3)", wkbXDR, "POINT Z (2 4 6)", + "POINT_Z_XDR"), + std::make_tuple("POINT M EMPTY", wkbNDR, "POINT M EMPTY", + "POINT_M_EMPTY_NDR"), + std::make_tuple("POINT M EMPTY", wkbXDR, "POINT M EMPTY", + "POINT_M_EMPTY_XDR"), + std::make_tuple("POINT M (1 2 -10)", wkbNDR, "POINT M (2 4 -10)", + "POINT_M_NDR"), + std::make_tuple("POINT M (1 2 -10)", wkbXDR, "POINT M (2 4 -10)", + "POINT_M_XDR"), + std::make_tuple("POINT ZM EMPTY", wkbNDR, "POINT ZM EMPTY", + "POINT_ZM_EMPTY_NDR"), + std::make_tuple("POINT ZM EMPTY", wkbXDR, "POINT ZM EMPTY", + "POINT_ZM_EMPTY_XDR"), + std::make_tuple("POINT ZM (1 2 3 10)", wkbNDR, + "POINT ZM (2 4 6 10)", "POINT_ZM_NDR"), + std::make_tuple("POINT ZM (1 2 3 10)", wkbXDR, + "POINT ZM (2 4 6 10)", "POINT_ZM_XDR"), + + std::make_tuple("LINESTRING EMPTY", wkbNDR, "LINESTRING EMPTY", + "LINESTRING_EMPTY"), + std::make_tuple("LINESTRING (1 2,11 12)", wkbNDR, + "LINESTRING (2 4,12 14)", "LINESTRING_NDR"), + std::make_tuple("LINESTRING (1 2,11 12)", wkbXDR, + "LINESTRING (2 4,12 14)", "LINESTRING_XDR"), + std::make_tuple("LINESTRING Z EMPTY", wkbNDR, "LINESTRING Z EMPTY", + "LINESTRING_Z_EMPTY"), + std::make_tuple("LINESTRING Z (1 2 3,11 12 13)", wkbNDR, + "LINESTRING Z (2 4 6,12 14 16)", + "LINESTRING_Z_NDR"), + std::make_tuple("LINESTRING Z (1 2 3,11 12 13)", wkbXDR, + "LINESTRING Z (2 4 6,12 14 16)", + "LINESTRING_Z_XDR"), + std::make_tuple("LINESTRING M EMPTY", wkbNDR, "LINESTRING M EMPTY", + "LINESTRING_M_EMPTY"), + std::make_tuple("LINESTRING M (1 2 -10,11 12 -20)", wkbNDR, + "LINESTRING M (2 4 -10,12 14 -20)", + "LINESTRING_M_NDR"), + std::make_tuple("LINESTRING M (1 2 -10,11 12 -20)", wkbXDR, + "LINESTRING M (2 4 -10,12 14 -20)", + "LINESTRING_M_XDR"), + std::make_tuple("LINESTRING ZM EMPTY", wkbNDR, + "LINESTRING ZM EMPTY", "LINESTRING_ZM_EMPTY"), + std::make_tuple("LINESTRING ZM (1 2 3 -10,11 12 13 -20)", wkbNDR, + "LINESTRING ZM (2 4 6 -10,12 14 16 -20)", + "LINESTRING_ZM_NDR"), + std::make_tuple("LINESTRING ZM (1 2 3 -10,11 12 13 -20)", wkbXDR, + "LINESTRING ZM (2 4 6 -10,12 14 16 -20)", + "LINESTRING_ZM_XDR"), + + // I know the polygon is invalid, but this is enough for our purposes + std::make_tuple("POLYGON EMPTY", wkbNDR, "POLYGON EMPTY", + "POLYGON_EMPTY"), + std::make_tuple("POLYGON ((1 2,11 12))", wkbNDR, + "POLYGON ((2 4,12 14))", "POLYGON_NDR"), + std::make_tuple("POLYGON ((1 2,11 12))", wkbXDR, + "POLYGON ((2 4,12 14))", "POLYGON_XDR"), + std::make_tuple("POLYGON ((1 2,11 12),(21 22,31 32))", wkbNDR, + "POLYGON ((2 4,12 14),(22 24,32 34))", + "POLYGON_TWO_RINGS"), + std::make_tuple("POLYGON Z EMPTY", wkbNDR, "POLYGON Z EMPTY", + "POLYGON_Z_EMPTY"), + std::make_tuple("POLYGON Z ((1 2 3,11 12 13))", wkbNDR, + "POLYGON Z ((2 4 6,12 14 16))", "POLYGON_Z_NDR"), + std::make_tuple("POLYGON Z ((1 2 3,11 12 13))", wkbXDR, + "POLYGON Z ((2 4 6,12 14 16))", "POLYGON_Z_XDR"), + std::make_tuple("POLYGON M EMPTY", wkbNDR, "POLYGON M EMPTY", + "POLYGON_M_EMPTY"), + std::make_tuple("POLYGON M ((1 2 -10,11 12 -20))", wkbNDR, + "POLYGON M ((2 4 -10,12 14 -20))", "POLYGON_M_NDR"), + std::make_tuple("POLYGON M ((1 2 -10,11 12 -20))", wkbXDR, + "POLYGON M ((2 4 -10,12 14 -20))", "POLYGON_M_XDR"), + std::make_tuple("POLYGON ZM EMPTY", wkbNDR, "POLYGON ZM EMPTY", + "POLYGON_ZM_EMPTY"), + std::make_tuple("POLYGON ZM ((1 2 3 -10,11 12 13 -20))", wkbNDR, + "POLYGON ZM ((2 4 6 -10,12 14 16 -20))", + "POLYGON_ZM_NDR"), + std::make_tuple("POLYGON ZM ((1 2 3 -10,11 12 13 -20))", wkbXDR, + "POLYGON ZM ((2 4 6 -10,12 14 16 -20))", + "POLYGON_ZM_XDR"), + + std::make_tuple("MULTIPOINT EMPTY", wkbNDR, "MULTIPOINT EMPTY", + "MULTIPOINT_EMPTY_NDR"), + std::make_tuple("MULTIPOINT ((1 2),(11 12))", wkbNDR, + "MULTIPOINT ((2 4),(12 14))", "MULTIPOINT_NDR"), + std::make_tuple("MULTIPOINT Z ((1 2 3),(11 12 13))", wkbXDR, + "MULTIPOINT Z ((2 4 6),(12 14 16))", + "MULTIPOINT_Z_XDR"), + + std::make_tuple("MULTILINESTRING ((1 2,11 12))", wkbNDR, + "MULTILINESTRING ((2 4,12 14))", + "MULTILINESTRING_NDR"), + + std::make_tuple("MULTIPOLYGON (((1 2,11 12)))", wkbNDR, + "MULTIPOLYGON (((2 4,12 14)))", "MULTIPOLYGON_NDR"), + + std::make_tuple("GEOMETRYCOLLECTION (POLYGON ((1 2,11 12)))", + wkbNDR, + "GEOMETRYCOLLECTION (POLYGON ((2 4,12 14)))", + "GEOMETRYCOLLECTION_NDR"), + + std::make_tuple("CIRCULARSTRING (1 2,11 12,21 22)", wkbNDR, + "CIRCULARSTRING (2 4,12 14,22 24)", + "CIRCULARSTRING_NDR"), + + std::make_tuple("COMPOUNDCURVE ((1 2,11 12))", wkbNDR, + "COMPOUNDCURVE ((2 4,12 14))", "COMPOUNDCURVE_NDR"), + + std::make_tuple("CURVEPOLYGON ((1 2,11 12,21 22,1 2))", wkbNDR, + "CURVEPOLYGON ((2 4,12 14,22 24,2 4))", + "CURVEPOLYGON_NDR"), + + std::make_tuple("MULTICURVE ((1 2,11 12))", wkbNDR, + "MULTICURVE ((2 4,12 14))", "MULTICURVE_NDR"), + + std::make_tuple("MULTISURFACE (((1 2,11 12)))", wkbNDR, + "MULTISURFACE (((2 4,12 14)))", "MULTISURFACE_NDR"), + + std::make_tuple("TRIANGLE ((1 2,11 12,21 22,1 2))", wkbNDR, + "TRIANGLE ((2 4,12 14,22 24,2 4))", "TRIANGLE_NDR"), + + std::make_tuple("POLYHEDRALSURFACE (((1 2,11 12,21 22,1 2)))", + wkbNDR, + "POLYHEDRALSURFACE (((2 4,12 14,22 24,2 4)))", + "POLYHEDRALSURFACE_NDR"), + + std::make_tuple("TIN (((1 2,11 12,21 22,1 2)))", wkbNDR, + "TIN (((2 4,12 14,22 24,2 4)))", "TIN_NDR"), + }; + } +}; + +struct MyCT final : public OGRCoordinateTransformation +{ + const bool m_bSuccess; + + explicit MyCT(bool bSuccess = true) : m_bSuccess(bSuccess) + { + } + + const OGRSpatialReference *GetSourceCS() const override + { + return nullptr; + } + + const OGRSpatialReference *GetTargetCS() const override + { + return nullptr; + } + + int Transform(size_t nCount, double *x, double *y, double *z, double *, + int *pabSuccess) override + { + for (size_t i = 0; i < nCount; ++i) + { + x[i] += 1; + y[i] += 2; + if (z) + z[i] += 3; + if (pabSuccess) + pabSuccess[i] = m_bSuccess; + } + return true; + } + + OGRCoordinateTransformation *Clone() const override + { + return new MyCT(); + } + + OGRCoordinateTransformation *GetInverse() const override + { + return nullptr; + } // unused +}; + +TEST_P(OGRWKBTransformFixture, test) +{ + const char *pszInput = std::get<0>(GetParam()); + OGRwkbByteOrder eByteOrder = std::get<1>(GetParam()); + const char *pszOutput = std::get<2>(GetParam()); + + MyCT oCT; + oCT.GetSourceCS(); // just for code coverage purpose + oCT.GetTargetCS(); // just for code coverage purpose + delete oCT.Clone(); // just for code coverage purpose + delete oCT.GetInverse(); // just for code coverage purpose + + OGRGeometry *poGeom = nullptr; + EXPECT_EQ(OGRGeometryFactory::createFromWkt(pszInput, nullptr, &poGeom), + OGRERR_NONE); + ASSERT_TRUE(poGeom != nullptr); + std::vector abyWkb(poGeom->WkbSize()); + poGeom->exportToWkb(eByteOrder, abyWkb.data(), wkbVariantIso); + delete poGeom; + + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv; + EXPECT_TRUE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + const auto abyWkbOri = abyWkb; + + poGeom = nullptr; + OGRGeometryFactory::createFromWkb(abyWkb.data(), nullptr, &poGeom, + abyWkb.size()); + ASSERT_TRUE(poGeom != nullptr); + char *pszWKT = nullptr; + poGeom->exportToWkt(&pszWKT, wkbVariantIso); + delete poGeom; + EXPECT_STREQ(pszWKT, pszOutput); + CPLFree(pszWKT); + + { + CPLErrorHandlerPusher oErrorHandler(CPLQuietErrorHandler); + + // Truncated geometry + for (size_t i = 0; i < abyWkb.size(); ++i) + { + abyWkb = abyWkbOri; + EXPECT_FALSE(OGRWKBTransform(abyWkb.data(), i, &oCT, oCache, sEnv)); + } + + // Check altering all bytes + for (size_t i = 0; i < abyWkb.size(); ++i) + { + abyWkb = abyWkbOri; + abyWkb[i] = 0xff; + CPL_IGNORE_RET_VAL(OGRWKBTransform(abyWkb.data(), abyWkb.size(), + &oCT, oCache, sEnv)); + } + + if (abyWkb.size() > 9) + { + abyWkb = abyWkbOri; + if (!STARTS_WITH(pszInput, "POINT")) + { + // Corrupt number of sub-geometries + abyWkb[5] = 0xff; + abyWkb[6] = 0xff; + abyWkb[7] = 0xff; + abyWkb[8] = 0xff; + EXPECT_FALSE(OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, + oCache, sEnv)); + } + } + } +} + +INSTANTIATE_TEST_SUITE_P( + test_ogr_wkb, OGRWKBTransformFixture, + ::testing::ValuesIn(OGRWKBTransformFixture::GetTupleValues()), + [](const ::testing::TestParamInfo + &l_info) { return std::get<3>(l_info.param); }); + +TEST_F(test_ogr_wkb, OGRWKBTransformFixture_rec_collection) +{ + std::vector abyWkb; + constexpr int BEYOND_ALLOWED_RECURSION_LEVEL = 128; + for (int i = 0; i < BEYOND_ALLOWED_RECURSION_LEVEL; ++i) + { + abyWkb.push_back(wkbNDR); + abyWkb.push_back(wkbGeometryCollection); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(1); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + } + { + abyWkb.push_back(wkbNDR); + abyWkb.push_back(wkbGeometryCollection); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + } + + MyCT oCT; + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv; + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); +} + +TEST_F(test_ogr_wkb, OGRWKBTransformFixture_ct_failure) +{ + MyCT oCT(/* bSuccess = */ false); + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv; + { + OGRPoint p(1, 2); + std::vector abyWkb(p.WkbSize()); + static_cast(p).exportToWkb(wkbNDR, abyWkb.data(), + wkbVariantIso); + + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + } + { + OGRLineString ls; + ls.addPoint(1, 2); + std::vector abyWkb(ls.WkbSize()); + static_cast(ls).exportToWkb(wkbNDR, abyWkb.data(), + wkbVariantIso); + + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + } + { + OGRPolygon p; + auto poLR = std::make_unique(); + poLR->addPoint(1, 2); + p.addRing(std::move(poLR)); + std::vector abyWkb(p.WkbSize()); + static_cast(p).exportToWkb(wkbNDR, abyWkb.data(), + wkbVariantIso); + + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + } +} + } // namespace diff --git a/autotest/cpp/test_viewshed.cpp b/autotest/cpp/test_viewshed.cpp index 95382ba0a814..ec5846501f9d 100644 --- a/autotest/cpp/test_viewshed.cpp +++ b/autotest/cpp/test_viewshed.cpp @@ -33,10 +33,12 @@ #include "gtest_include.h" -#include "viewshed.h" +#include "viewshed/viewshed.h" namespace gdal { +namespace viewshed +{ namespace { @@ -45,9 +47,9 @@ using DatasetPtr = std::unique_ptr; using Transform = std::array; Transform identity{0, 1, 0, 0, 0, 1}; -Viewshed::Options stdOptions(int x, int y) +Options stdOptions(int x, int y) { - Viewshed::Options opts; + Options opts; opts.observer.x = x; opts.observer.y = y; opts.outputFilename = "none"; @@ -57,13 +59,13 @@ Viewshed::Options stdOptions(int x, int y) return opts; } -Viewshed::Options stdOptions(const Coord &observer) +Options stdOptions(const Coord &observer) { return stdOptions(observer.first, observer.second); } DatasetPtr runViewshed(const int8_t *in, int xlen, int ylen, - const Viewshed::Options &opts) + const Options &opts) { Viewshed v(opts); @@ -157,8 +159,8 @@ TEST(Viewshed, simple_height) { std::array dem; SCOPED_TRACE("simple_height:dem"); - Viewshed::Options opts = stdOptions(2, 2); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(2, 2); + opts.outputMode = OutputMode::DEM; DatasetPtr output = runViewshed(in.data(), xlen, ylen, opts); @@ -179,8 +181,8 @@ TEST(Viewshed, simple_height) { std::array ground; SCOPED_TRACE("simple_height:ground"); - Viewshed::Options opts = stdOptions(2, 2); - opts.outputMode = Viewshed::OutputMode::Ground; + Options opts = stdOptions(2, 2); + opts.outputMode = OutputMode::Ground; DatasetPtr output = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = output->GetRasterBand(1); @@ -210,8 +212,8 @@ TEST(Viewshed, dem_vs_ground) const int ylen = 1; std::array out; - Viewshed::Options opts = stdOptions(observer); - opts.outputMode = Viewshed::OutputMode::Ground; + Options opts = stdOptions(observer); + opts.outputMode = OutputMode::Ground; // Verify ground mode. DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); @@ -223,7 +225,7 @@ TEST(Viewshed, dem_vs_ground) EXPECT_DOUBLE_EQ(out[i], ground[i]); // Verify DEM mode. - opts.outputMode = Viewshed::OutputMode::DEM; + opts.outputMode = OutputMode::DEM; ds = runViewshed(in.data(), xlen, ylen, opts); band = ds->GetRasterBand(1); err = band->RasterIO(GF_Read, 0, 0, xlen, ylen, out.data(), xlen, ylen, @@ -262,8 +264,8 @@ TEST(Viewshed, oor_right) // clang-format on { - Viewshed::Options opts = stdOptions(6, 1); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(6, 1); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -285,8 +287,8 @@ TEST(Viewshed, oor_right) } { - Viewshed::Options opts = stdOptions(6, 2); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(6, 2); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -323,8 +325,8 @@ TEST(Viewshed, oor_left) // clang-format on { - Viewshed::Options opts = stdOptions(-2, 1); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(-2, 1); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -346,8 +348,8 @@ TEST(Viewshed, oor_left) } { - Viewshed::Options opts = stdOptions(-2, 2); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(-2, 2); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -384,8 +386,8 @@ TEST(Viewshed, oor_above) // clang-format on { - Viewshed::Options opts = stdOptions(2, -2); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(2, -2); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -408,8 +410,8 @@ TEST(Viewshed, oor_above) } { - Viewshed::Options opts = stdOptions(-2, -2); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(-2, -2); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -446,8 +448,8 @@ TEST(Viewshed, oor_below) // clang-format on { - Viewshed::Options opts = stdOptions(2, 4); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(2, 4); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -470,8 +472,8 @@ TEST(Viewshed, oor_below) } { - Viewshed::Options opts = stdOptions(6, 4); - opts.outputMode = Viewshed::OutputMode::DEM; + Options opts = stdOptions(6, 4); + opts.outputMode = OutputMode::DEM; DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); GDALRasterBand *band = ds->GetRasterBand(1); std::array out; @@ -493,4 +495,5 @@ TEST(Viewshed, oor_below) } } +} // namespace viewshed } // namespace gdal diff --git a/autotest/gcore/basic_test.py b/autotest/gcore/basic_test.py index 2719fc963dcc..7e496fea622b 100755 --- a/autotest/gcore/basic_test.py +++ b/autotest/gcore/basic_test.py @@ -990,3 +990,13 @@ def test_band_getitem(): with pytest.raises(IndexError): ds[5] + + +def test_colorinterp(): + + d = {} + for c in range(gdal.GCI_Max + 1): + name = gdal.GetColorInterpretationName(c) + assert name not in d + d[name] = c + assert gdal.GetColorInterpretationByName(name) == c diff --git a/autotest/gcore/data/gtiff/unknown_colorinterp.tif b/autotest/gcore/data/gtiff/unknown_colorinterp.tif new file mode 100644 index 000000000000..02466ea4db36 Binary files /dev/null and b/autotest/gcore/data/gtiff/unknown_colorinterp.tif differ diff --git a/autotest/gcore/data/tar_of_tar_gzip.tar b/autotest/gcore/data/tar_of_tar_gzip.tar new file mode 100644 index 000000000000..ed18401a2bc5 Binary files /dev/null and b/autotest/gcore/data/tar_of_tar_gzip.tar differ diff --git a/autotest/gcore/rasterio.py b/autotest/gcore/rasterio.py index cee967b15724..777f0841bfca 100755 --- a/autotest/gcore/rasterio.py +++ b/autotest/gcore/rasterio.py @@ -884,29 +884,153 @@ def test_rasterio_12(): # Test cubic resampling with masking -def test_rasterio_13(): +@pytest.mark.parametrize( + "dt", + [ + "Byte", + "Int8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Float32", + "Float64", + ], +) +def test_rasterio_13(dt): numpy = pytest.importorskip("numpy") - for dt in [gdal.GDT_Byte, gdal.GDT_UInt16, gdal.GDT_UInt32]: + dt = gdal.GetDataTypeByName(dt) + mem_ds = gdal.GetDriverByName("MEM").Create("", 4, 3, 1, dt) + mem_ds.GetRasterBand(1).SetNoDataValue(0) + if dt == gdal.GDT_Int8: + x = (1 << 7) - 1 + elif dt == gdal.GDT_Byte: + x = (1 << 8) - 1 + elif dt == gdal.GDT_Int16: + x = (1 << 15) - 1 + elif dt == gdal.GDT_UInt16: + x = (1 << 16) - 1 + elif dt == gdal.GDT_Int32: + x = (1 << 31) - 1 + elif dt == gdal.GDT_UInt32: + x = (1 << 32) - 1 + elif dt == gdal.GDT_Int64: + x = (1 << 63) - 1 + elif dt == gdal.GDT_UInt64: + x = (1 << 64) - 2048 + elif dt == gdal.GDT_Float32: + x = 1.5 + else: + x = 1.23456 + mem_ds.GetRasterBand(1).WriteArray( + numpy.array([[0, 0, 0, 0], [0, x, 0, 0], [0, 0, 0, 0]]) + ) - mem_ds = gdal.GetDriverByName("MEM").Create("", 4, 3, 1, dt) - mem_ds.GetRasterBand(1).SetNoDataValue(0) - mem_ds.GetRasterBand(1).WriteArray( - numpy.array([[0, 0, 0, 0], [0, 255, 0, 0], [0, 0, 0, 0]]) - ) + ar_ds = mem_ds.ReadAsArray( + 0, 0, 4, 3, buf_xsize=8, buf_ysize=3, resample_alg=gdal.GRIORA_Cubic + ) - ar_ds = mem_ds.ReadAsArray( - 0, 0, 4, 3, buf_xsize=8, buf_ysize=3, resample_alg=gdal.GRIORA_Cubic - ) + expected_ar = numpy.array( + [ + [0, 0, 0, 0, 0, 0, 0, 0], + [0, x, x, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0], + ] + ) + assert numpy.array_equal(ar_ds, expected_ar) - expected_ar = numpy.array( - [ - [0, 0, 0, 0, 0, 0, 0, 0], - [0, 255, 255, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0], - ] - ) - assert numpy.array_equal(ar_ds, expected_ar), (ar_ds, dt) + +############################################################################### +# Test nearest and mode resampling + + +@pytest.mark.parametrize( + "dt", + [ + "Byte", + "Int8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Float32", + "Float64", + "CInt16", + "CInt32", + "CFloat32", + "CFloat64", + ], +) +@pytest.mark.parametrize( + "resample_alg", [gdal.GRIORA_NearestNeighbour, gdal.GRIORA_Mode] +) +@pytest.mark.parametrize("use_nan", [True, False]) +def test_rasterio_nearest_or_mode(dt, resample_alg, use_nan): + numpy = pytest.importorskip("numpy") + gdal_array = pytest.importorskip("osgeo.gdal_array") + + dt = gdal.GetDataTypeByName(dt) + mem_ds = gdal.GetDriverByName("MEM").Create("", 4, 4, 1, dt) + if dt == gdal.GDT_Int8: + x = (1 << 7) - 1 + elif dt == gdal.GDT_Byte: + x = (1 << 8) - 1 + elif dt == gdal.GDT_Int16 or dt == gdal.GDT_CInt16: + x = (1 << 15) - 1 + elif dt == gdal.GDT_UInt16: + x = (1 << 16) - 1 + elif dt == gdal.GDT_Int32 or dt == gdal.GDT_CInt32: + x = (1 << 31) - 1 + elif dt == gdal.GDT_UInt32: + x = (1 << 32) - 1 + elif dt == gdal.GDT_Int64: + x = (1 << 63) - 1 + elif dt == gdal.GDT_UInt64: + x = (1 << 64) - 1 + elif dt == gdal.GDT_Float32 or dt == gdal.GDT_CFloat32: + x = float("nan") if use_nan else 1.5 + else: + x = float("nan") if use_nan else 1.234567890123 + + if gdal.DataTypeIsComplex(dt): + val = complex(x, x) + else: + val = x + + dtype = gdal_array.flip_code(dt) + mem_ds.GetRasterBand(1).WriteArray(numpy.full((4, 4), val, dtype=dtype)) + + ar_ds = mem_ds.ReadAsArray( + 0, 0, 4, 4, buf_xsize=1, buf_ysize=1, resample_alg=resample_alg + ) + + expected_ar = numpy.array([[val]]).astype(dtype) + if math.isnan(x): + if gdal.DataTypeIsComplex(dt): + assert math.isnan(ar_ds[0][0].real) and math.isnan(ar_ds[0][0].imag) + else: + assert math.isnan(ar_ds[0][0]) + else: + assert numpy.array_equal(ar_ds, expected_ar) + + resample_alg_mapping = { + gdal.GRIORA_NearestNeighbour: "NEAR", + gdal.GRIORA_Mode: "MODE", + } + mem_ds.BuildOverviews(resample_alg_mapping[resample_alg], [4]) + ar_ds = mem_ds.GetRasterBand(1).GetOverview(0).ReadAsArray() + if math.isnan(x): + if gdal.DataTypeIsComplex(dt): + assert math.isnan(ar_ds[0][0].real) and math.isnan(ar_ds[0][0].imag) + else: + assert math.isnan(ar_ds[0][0]) + else: + assert numpy.array_equal(ar_ds, expected_ar) ############################################################################### diff --git a/autotest/gcore/relationship.py b/autotest/gcore/relationship.py index 60d32f24085e..57569ec3598d 100755 --- a/autotest/gcore/relationship.py +++ b/autotest/gcore/relationship.py @@ -35,7 +35,7 @@ def test_gdal_relationship(): - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.Relationship(None, None, None, gdal.GRC_ONE_TO_ONE) relationship = gdal.Relationship( diff --git a/autotest/gcore/thread_test.py b/autotest/gcore/thread_test.py index 9fe563757aae..91e8d574b202 100755 --- a/autotest/gcore/thread_test.py +++ b/autotest/gcore/thread_test.py @@ -29,6 +29,7 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### +import shutil import threading import gdaltest @@ -75,3 +76,540 @@ def test_thread_test_1(): ret = False assert ret + + +def launch_threads(get_band, expected_cs, on_mask_band=False): + res = [True] + + def verify_checksum(): + for i in range(1000): + got_cs = get_band().Checksum() + if got_cs != expected_cs: + res[0] = False + assert False, (got_cs, expected_cs) + + threads = [threading.Thread(target=verify_checksum) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] + + +def test_thread_safe_open(): + + ds = gdal.OpenEx("data/byte.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE) + assert ds.IsThreadSafe(gdal.OF_RASTER) + assert not ds.IsThreadSafe(gdal.OF_RASTER | gdal.OF_UPDATE) + + def get_band(): + return ds.GetRasterBand(1) + + launch_threads(get_band, 4672) + + # Check that GetThreadSafeDataset() on an already thread-safe dataset + # return itself. + ref_count_before = ds.GetRefCount() + thread_safe_ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + assert ds.GetRefCount() == ref_count_before + 1 + assert thread_safe_ds.GetRefCount() == ref_count_before + 1 + del thread_safe_ds + assert ds.GetRefCount() == ref_count_before + + +def test_thread_safe_create(): + + ds = gdal.OpenEx("data/byte.tif", gdal.OF_RASTER) + assert not ds.IsThreadSafe(gdal.OF_RASTER) + assert ds.GetRefCount() == 1 + thread_safe_ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + assert thread_safe_ds.IsThreadSafe(gdal.OF_RASTER) + assert ds.GetRefCount() == 2 + del ds + + def get_band(): + return thread_safe_ds.GetRasterBand(1) + + launch_threads(get_band, 4672) + + +def test_thread_safe_create_close_src_ds(): + + ds = gdal.OpenEx("data/byte.tif", gdal.OF_RASTER) + thread_safe_ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + ds.Close() + with pytest.raises(Exception): + thread_safe_ds.RasterCount + + +def test_thread_safe_src_cannot_be_reopened(tmp_vsimem): + + tmpfilename = str(tmp_vsimem / "byte.tif") + gdal.Translate(tmpfilename, "data/byte.tif") + + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + gdal.Unlink(tmpfilename) + with pytest.raises(Exception): + ds.GetRasterBand(1).Checksum() + + +@pytest.mark.parametrize( + "flag", [gdal.OF_UPDATE, gdal.OF_VECTOR, gdal.OF_MULTIDIM_RASTER, gdal.OF_GNM] +) +def test_thread_safe_incompatible_open_flags(flag): + with pytest.raises(Exception, match="mutually exclusive"): + gdal.OpenEx("data/byte.tif", gdal.OF_THREAD_SAFE | flag) + + +def test_thread_safe_src_alter_after_opening(tmp_vsimem): + + tmpfilename = str(tmp_vsimem / "byte.tif") + + gdal.Translate(tmpfilename, "data/byte.tif") + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + gdal.GetDriverByName("GTiff").Create( + tmpfilename, ds.RasterXSize + 1, ds.RasterYSize, ds.RasterCount + ) + with pytest.raises(Exception): + ds.GetRasterBand(1).Checksum() + + gdal.Translate(tmpfilename, "data/byte.tif") + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + gdal.GetDriverByName("GTiff").Create( + tmpfilename, ds.RasterXSize, ds.RasterYSize + 1, ds.RasterCount + ) + with pytest.raises(Exception): + ds.GetRasterBand(1).Checksum() + + gdal.Translate(tmpfilename, "data/byte.tif") + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + gdal.GetDriverByName("GTiff").Create( + tmpfilename, ds.RasterXSize, ds.RasterYSize, ds.RasterCount + 1 + ) + with pytest.raises(Exception): + ds.GetRasterBand(1).Checksum() + + gdal.Translate(tmpfilename, "data/byte.tif") + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + gdal.GetDriverByName("GTiff").Create( + tmpfilename, ds.RasterXSize, ds.RasterYSize, ds.RasterCount, gdal.GDT_Int16 + ) + with pytest.raises(Exception): + ds.GetRasterBand(1).Checksum() + + gdal.Translate(tmpfilename, "data/byte.tif") + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + gdal.GetDriverByName("GTiff").Create( + tmpfilename, + ds.RasterXSize, + ds.RasterYSize, + ds.RasterCount, + options=["TILED=YES"], + ) + with pytest.raises(Exception): + ds.GetRasterBand(1).Checksum() + + with gdal.Translate(tmpfilename, "data/byte.tif") as ds: + ds.BuildOverviews("NEAR", [2]) + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + ds.GetRasterBand(1).GetOverviewCount() + gdal.GetDriverByName("GTiff").Create( + tmpfilename, ds.RasterXSize, ds.RasterYSize, ds.RasterCount + ) + with pytest.raises(Exception): + ds.GetRasterBand(1).GetOverview(0).Checksum() + + +def test_thread_safe_mask_band(): + + with gdal.Open("data/stefan_full_rgba.tif") as src_ds: + expected_cs = src_ds.GetRasterBand(1).GetMaskBand().Checksum() + + with gdal.OpenEx( + "data/stefan_full_rgba.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE + ) as ds: + + def get_band(): + return ds.GetRasterBand(1).GetMaskBand() + + launch_threads(get_band, expected_cs) + + with gdal.OpenEx( + "data/stefan_full_rgba.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE + ) as ds: + band = ds.GetRasterBand(1).GetMaskBand() + + def get_band(): + return band + + launch_threads(get_band, expected_cs) + + +def test_thread_safe_mask_of_mask_band(): + + with gdal.Open("data/stefan_full_rgba.tif") as src_ds: + expected_cs = src_ds.GetRasterBand(1).GetMaskBand().GetMaskBand().Checksum() + + with gdal.OpenEx( + "data/stefan_full_rgba.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE + ) as ds: + + def get_band(): + return ds.GetRasterBand(1).GetMaskBand().GetMaskBand() + + launch_threads(get_band, expected_cs) + + +def test_thread_safe_mask_band_implicit_mem_ds(): + + with gdal.Open("data/stefan_full_rgba.tif") as src_ds: + ds = gdal.GetDriverByName("MEM").CreateCopy("", src_ds) + expected_cs = src_ds.GetRasterBand(1).GetMaskBand().Checksum() + assert ds.GetRasterBand(1).GetMaskBand().Checksum() == expected_cs + + ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + + def get_band(): + return ds.GetRasterBand(1).GetMaskBand() + + launch_threads(get_band, expected_cs) + + +def test_thread_safe_mask_band_explicit_mem_ds(): + + with gdal.Open("data/stefan_full_rgba.tif") as src_ds: + ds = gdal.GetDriverByName("MEM").Create( + "", src_ds.RasterXSize, src_ds.RasterYSize, 1 + ) + ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET) + ds.GetRasterBand(1).GetMaskBand().WriteRaster( + 0, + 0, + src_ds.RasterXSize, + src_ds.RasterYSize, + src_ds.GetRasterBand(1).GetMaskBand().ReadRaster(), + ) + + expected_cs = src_ds.GetRasterBand(1).GetMaskBand().Checksum() + assert ds.GetRasterBand(1).GetMaskBand().Checksum() == expected_cs + + ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + + def get_band(): + return ds.GetRasterBand(1).GetMaskBand() + + launch_threads(get_band, expected_cs) + + +def test_thread_safe_overview(tmp_path): + + tmpfilename = str(tmp_path / "byte.tif") + shutil.copy("data/byte.tif", tmpfilename) + with gdal.Open(tmpfilename, gdal.GA_Update) as ds: + ds.BuildOverviews("NEAR", [2]) + expected_cs = ds.GetRasterBand(1).GetOverview(0).Checksum() + + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + assert ds.GetRasterBand(1).GetOverviewCount() == 1 + assert ds.GetRasterBand(1).GetOverview(-1) is None + assert ds.GetRasterBand(1).GetOverview(1) is None + + def get_band(): + return ds.GetRasterBand(1).GetOverview(0) + + launch_threads(get_band, expected_cs) + + with gdal.OpenEx(tmpfilename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + band = ds.GetRasterBand(1).GetOverview(0) + + def get_band(): + return band + + launch_threads(get_band, expected_cs) + + +def test_thread_safe_overview_mem_ds(): + + with gdal.Open("data/byte.tif") as src_ds: + ds = gdal.GetDriverByName("MEM").CreateCopy("", src_ds) + ds.BuildOverviews("NEAR", [2]) + expected_cs = ds.GetRasterBand(1).GetOverview(0).Checksum() + + ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + + assert ds.GetRasterBand(1).GetOverviewCount() == 1 + + def get_band(): + return ds.GetRasterBand(1).GetOverview(0) + + launch_threads(get_band, expected_cs) + + def get_band(): + return ds.GetRasterBand(1).GetSampleOverview(1) + + launch_threads(get_band, expected_cs) + + band = ds.GetRasterBand(1).GetOverview(0) + + def get_band(): + return band + + launch_threads(get_band, expected_cs) + + +def test_thread_safe_open_options(tmp_path): + + tmpfilename = str(tmp_path / "byte.tif") + shutil.copy("data/byte.tif", tmpfilename) + with gdal.Open(tmpfilename, gdal.GA_Update) as ds: + ds.BuildOverviews("NEAR", [2]) + expected_cs = ds.GetRasterBand(1).GetOverview(0).Checksum() + + with gdal.OpenEx( + tmpfilename, + gdal.OF_RASTER | gdal.OF_THREAD_SAFE, + open_options=["OVERVIEW_LEVEL=0"], + ) as ds: + + def get_band(): + return ds.GetRasterBand(1) + + launch_threads(get_band, expected_cs) + + +@pytest.mark.require_driver("HDF5") +@pytest.mark.require_driver("netCDF") +def test_thread_safe_reuse_same_driver_as_prototype(): + """Checks that thread-safe mode honours the opening driver""" + + with gdal.OpenEx( + "../gdrivers/data/netcdf/byte_hdf5_starting_at_offset_1024.nc", + gdal.OF_RASTER | gdal.OF_THREAD_SAFE, + allowed_drivers=["HDF5"], + ) as ds: + + ret = [False] + + def thread_func(): + assert ds.GetMetadataItem("_NCProperties") is not None + ret[0] = True + + t = threading.Thread(target=thread_func) + t.start() + t.join() + assert ret[0] + + +def test_thread_safe_no_rat(): + + ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + assert ds.GetRasterBand(1).GetDefaultRAT() is None + + +def test_thread_safe_rat(): + + ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + ds.GetRasterBand(1).SetDefaultRAT(gdal.RasterAttributeTable()) + ds = ds.GetThreadSafeDataset(gdal.OF_RASTER) + assert ds.GetRasterBand(1).GetDefaultRAT() is not None + + +@pytest.mark.require_driver("HFA") +def test_thread_safe_unsupported_rat(): + + with gdal.OpenEx( + "../gdrivers/data/hfa/87test.img", gdal.OF_RASTER | gdal.OF_THREAD_SAFE + ) as ds: + with pytest.raises( + Exception, + match="not supporting a non-GDALDefaultRasterAttributeTable implementation", + ): + ds.GetRasterBand(1).GetDefaultRAT() + + +def test_thread_safe_many_datasets(): + + tab_ds = [ + gdal.OpenEx( + "data/byte.tif" if (i % 3) < 2 else "data/utmsmall.tif", + gdal.OF_RASTER | gdal.OF_THREAD_SAFE, + ) + for i in range(100) + ] + + res = [True] + + def check(): + for _ in range(10): + for i, ds in enumerate(tab_ds): + if ds.GetRasterBand(1).Checksum() != (4672 if (i % 3) < 2 else 50054): + res[0] = False + + threads = [threading.Thread(target=check) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] + + +def test_thread_safe_BeginAsyncReader(): + + with gdal.OpenEx("data/byte.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + with pytest.raises(Exception, match="not supported"): + ds.BeginAsyncReader(0, 0, ds.RasterXSize, ds.RasterYSize) + + +def test_thread_safe_GetVirtualMem(): + + pytest.importorskip("numpy") + pytest.importorskip("osgeo.gdal_array") + + with gdal.OpenEx("data/byte.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + with pytest.raises(Exception, match="not supported"): + ds.GetRasterBand(1).GetVirtualMemAutoArray(gdal.GF_Read) + + +def test_thread_safe_GetMetadadata(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + with gdal.GetDriverByName("GTiff").Create(filename, 1, 1) as ds: + ds.SetMetadataItem("foo", "bar") + ds.GetRasterBand(1).SetMetadataItem("bar", "baz") + + with gdal.OpenEx(filename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + assert ds.GetMetadataItem("foo") == "bar" + assert ds.GetMetadataItem("not existing") is None + assert ds.GetMetadata() == {"foo": "bar"} + assert ds.GetMetadata("not existing") == {} + assert ds.GetRasterBand(1).GetMetadataItem("bar") == "baz" + assert ds.GetRasterBand(1).GetMetadataItem("not existing") is None + assert ds.GetRasterBand(1).GetMetadata() == {"bar": "baz"} + assert ds.GetRasterBand(1).GetMetadata("not existing") == {} + + +def test_thread_safe_GetUnitType(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + with gdal.GetDriverByName("GTiff").Create(filename, 1, 1) as ds: + ds.GetRasterBand(1).SetUnitType("foo") + + with gdal.OpenEx(filename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + assert ds.GetRasterBand(1).GetUnitType() == "foo" + + +def test_thread_safe_GetColorTable(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + with gdal.GetDriverByName("GTiff").Create(filename, 1, 1) as ds: + ct = gdal.ColorTable() + ct.SetColorEntry(0, (1, 2, 3, 255)) + ds.GetRasterBand(1).SetColorTable(ct) + + with gdal.OpenEx(filename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + res = [None] + + def thread_job(): + res[0] = ds.GetRasterBand(1).GetColorTable() + + t = threading.Thread(target=thread_job) + t.start() + t.join() + assert res[0] + assert res[0].GetColorEntry(0) == (1, 2, 3, 255) + ct = ds.GetRasterBand(1).GetColorTable() + assert ct.GetColorEntry(0) == (1, 2, 3, 255) + + +def test_thread_safe_GetSpatialRef(): + + with gdal.OpenEx("data/byte.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + + res = [True] + + def check(): + for i in range(100): + + if len(ds.GetGCPs()) != 0: + res[0] = False + assert False + + if ds.GetGCPSpatialRef(): + res[0] = False + assert False + + if ds.GetGCPProjection(): + res[0] = False + assert False + + srs = ds.GetSpatialRef() + if not srs: + res[0] = False + assert False + if not srs.IsProjected(): + res[0] = False + assert False + if "NAD27 / UTM zone 11N" not in srs.ExportToWkt(): + res[0] = False + assert False + + if "NAD27 / UTM zone 11N" not in ds.GetProjectionRef(): + res[0] = False + assert False + + threads = [threading.Thread(target=check) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] + + +def test_thread_safe_GetGCPs(): + + with gdal.OpenEx( + "data/byte_gcp_pixelispoint.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE + ) as ds: + + res = [True] + + def check(): + for i in range(100): + + if len(ds.GetGCPs()) != 4: + res[0] = False + assert False + + gcp_srs = ds.GetGCPSpatialRef() + if gcp_srs is None: + res[0] = False + assert False + if not gcp_srs.IsGeographic(): + res[0] = False + assert False + if "unretrievable - using WGS84" not in gcp_srs.ExportToWkt(): + res[0] = False + assert False + + gcp_wkt = ds.GetGCPProjection() + if not gcp_wkt: + res[0] = False + assert False + if "unretrievable - using WGS84" not in gcp_wkt: + res[0] = False + assert False + + if ds.GetSpatialRef(): + res[0] = False + assert False + if ds.GetProjectionRef() != "": + res[0] = False + assert False + + threads = [threading.Thread(target=check) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] diff --git a/autotest/gcore/tiff_ovr.py b/autotest/gcore/tiff_ovr.py index ba49b4b3fb45..d047ea29843e 100755 --- a/autotest/gcore/tiff_ovr.py +++ b/autotest/gcore/tiff_ovr.py @@ -2642,7 +2642,9 @@ def test_tiff_ovr_fallback_to_multiband_overview_generate(): "data/byte.tif", options="-b 1 -b 1 -b 1 -co INTERLEAVE=BAND -co TILED=YES -outsize 1024 1024", ) - with gdaltest.config_option("GDAL_OVR_CHUNK_MAX_SIZE", "1000"): + with gdaltest.config_options( + {"GDAL_OVR_CHUNK_MAX_SIZE": "1000", "GDAL_OVR_TEMP_DRIVER": "MEM"} + ): ds.BuildOverviews("NEAR", overviewlist=[2, 4, 8]) ds = None diff --git a/autotest/gcore/tiff_read.py b/autotest/gcore/tiff_read.py index 152687229ba3..3a18f98579a5 100755 --- a/autotest/gcore/tiff_read.py +++ b/autotest/gcore/tiff_read.py @@ -5315,3 +5315,15 @@ def error_handler(type, code, msg): webserver.server_stop(webserver_process, webserver_port) gdal.VSICurlClearCache() + + +############################################################################### +# Test reading a unrecognized value in the special COLORINTERP item in +# GDAL_METADATA + + +def test_tiff_read_unrecognized_color_interpretation(): + + ds = gdal.Open("data/gtiff/unknown_colorinterp.tif") + assert ds.GetRasterBand(1).GetColorInterpretation() == gdal.GCI_Undefined + assert ds.GetRasterBand(1).GetMetadataItem("COLOR_INTERPRETATION") == "XXXX" diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index 03aae9ca57e0..c7bcdef00130 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -11597,3 +11597,139 @@ def test_tiff_write_colormap_256_mult_factor(tmp_vsimem): and ct.GetColorEntry(1) == (0, 1, 2, 255) and ct.GetColorEntry(2) == (254, 254, 254, 255) ), "Wrong color table entry." + + +############################################################################### +@pytest.mark.require_creation_option("GTiff", "JPEG") +@pytest.mark.parametrize( + "xsize,ysize,options,expected_error_msg", + [ + ( + 65501, + 1, + ["COMPRESS=JPEG"], + "COMPRESS=JPEG is only compatible of un-tiled images whose width is lesser or equal to 65500 pixels", + ), + ( + 1, + 65501, + ["COMPRESS=JPEG", "BLOCKYSIZE=65501"], + "COMPRESS=JPEG is only compatible of images whose BLOCKYSIZE is lesser or equal to 65500 pixels", + ), + ( + 1, + 1, + ["COMPRESS=JPEG", "TILED=YES", "BLOCKXSIZE=65536"], + "COMPRESS=JPEG is only compatible of tiled images whose BLOCKXSIZE is lesser or equal to 65500 pixels", + ), + ( + 1, + 1, + ["COMPRESS=JPEG", "TILED=YES", "BLOCKYSIZE=65536"], + "COMPRESS=JPEG is only compatible of images whose BLOCKYSIZE is lesser or equal to 65500 pixels", + ), + ], +) +@gdaltest.enable_exceptions() +def test_tiff_write_too_large_jpeg( + tmp_vsimem, xsize, ysize, options, expected_error_msg +): + + filename = str(tmp_vsimem / "test.tif") + with pytest.raises(Exception, match=expected_error_msg): + gdal.GetDriverByName("GTiff").Create(filename, xsize, ysize, options=options) + + +############################################################################### +@pytest.mark.require_creation_option("GTiff", "WEBP") +@pytest.mark.parametrize( + "xsize,ysize,options,expected_error_msg", + [ + ( + 16384, + 1, + ["COMPRESS=WEBP"], + "COMPRESS=WEBP is only compatible of un-tiled images whose width is lesser or equal to 16383 pixels", + ), + ( + 1, + 16384, + ["COMPRESS=WEBP", "BLOCKYSIZE=16384"], + "COMPRESS=WEBP is only compatible of images whose BLOCKYSIZE is lesser or equal to 16383 pixels", + ), + ( + 1, + 1, + ["COMPRESS=WEBP", "TILED=YES", "BLOCKXSIZE=16384"], + "COMPRESS=WEBP is only compatible of tiled images whose BLOCKXSIZE is lesser or equal to 16383 pixels", + ), + ( + 1, + 1, + ["COMPRESS=WEBP", "TILED=YES", "BLOCKYSIZE=16384"], + "COMPRESS=WEBP is only compatible of images whose BLOCKYSIZE is lesser or equal to 16383 pixels", + ), + ], +) +@gdaltest.enable_exceptions() +def test_tiff_write_too_large_webp( + tmp_vsimem, xsize, ysize, options, expected_error_msg +): + + filename = str(tmp_vsimem / "test.tif") + with pytest.raises(Exception, match=expected_error_msg): + gdal.GetDriverByName("GTiff").Create(filename, xsize, ysize, options=options) + + +############################################################################### +# Test writing/reading band IMAGERY metadata + + +def test_tiff_write_band_IMAGERY(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + with gdal.GetDriverByName("GTiff").Create(filename, 1, 1) as ds: + ds.GetRasterBand(1).SetMetadataItem("foo", "bar", "IMAGERY") + with gdal.Open(filename) as ds: + assert ds.GetRasterBand(1).GetMetadataDomainList() == ["IMAGERY"] + with gdal.Open(filename) as ds: + assert ds.GetRasterBand(1).GetMetadataItem("foo", "IMAGERY") == "bar" + with gdal.Open(filename) as ds: + assert ds.GetRasterBand(1).GetMetadata_Dict("IMAGERY") == {"foo": "bar"} + + filename2 = str(tmp_vsimem / "test2.tif") + + with gdal.Open(filename) as ds: + gdal.GetDriverByName("GTiff").CreateCopy(filename2, ds) + with gdal.Open(filename2) as ds: + assert ds.GetRasterBand(1).GetMetadata_Dict("IMAGERY") == {"foo": "bar"} + + with gdal.Open(filename) as ds: + gdal.GetDriverByName("GTiff").CreateCopy( + filename2, ds, options=["COPY_SRC_MDD=YES"] + ) + with gdal.Open(filename2) as ds: + assert ds.GetRasterBand(1).GetMetadata_Dict("IMAGERY") == {"foo": "bar"} + + with gdal.Open(filename) as ds: + gdal.GetDriverByName("GTiff").CreateCopy( + filename2, ds, options=["COPY_SRC_MDD=NO"] + ) + with gdal.Open(filename2) as ds: + assert ds.GetRasterBand(1).GetMetadataDomainList() is None + assert ds.GetRasterBand(1).GetMetadata_Dict("IMAGERY") == {} + + with gdal.Open(filename) as ds: + gdal.GetDriverByName("GTiff").CreateCopy( + filename2, ds, options=["SRC_MDD=not_existing"] + ) + with gdal.Open(filename2) as ds: + assert ds.GetRasterBand(1).GetMetadataDomainList() is None + assert ds.GetRasterBand(1).GetMetadata_Dict("IMAGERY") == {} + + with gdal.Open(filename) as ds: + gdal.GetDriverByName("GTiff").CreateCopy( + filename2, ds, options=["SRC_MDD=not_existing", "SRC_MDD=IMAGERY"] + ) + with gdal.Open(filename2) as ds: + assert ds.GetRasterBand(1).GetMetadata_Dict("IMAGERY") == {"foo": "bar"} diff --git a/autotest/gcore/vsiaz.py b/autotest/gcore/vsiaz.py index 60701a3c7742..1bd12627c1a0 100755 --- a/autotest/gcore/vsiaz.py +++ b/autotest/gcore/vsiaz.py @@ -453,6 +453,10 @@ def test_vsiaz_sas_fake(): # Test write +@pytest.mark.skipif( + "CI" in os.environ, + reason="Flaky", +) def test_vsiaz_fake_write(): if gdaltest.webserver_port == 0: diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index cd40d72aa5c7..45746a20f57b 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -1139,15 +1139,38 @@ def test_vsicurl_GDAL_HTTP_HEADERS(server): filename = ( "/vsicurl/http://localhost:%d/test_vsicurl_GDAL_HTTP_HEADERS.bin" % server.port ) - gdal.SetPathSpecificOption( - filename, - "GDAL_HTTP_HEADERS", - r'Foo: Bar,"Baz: escaped backslash \\, escaped double-quote \", end of value",Another: Header', - ) - with webserver.install_http_handler(handler): - statres = gdal.VSIStatL(filename) - gdal.SetPathSpecificOption(filename, "GDAL_HTTP_HEADERS", None) - assert statres.size == 3 + try: + gdal.SetPathSpecificOption( + filename, + "GDAL_HTTP_HEADERS", + r'Foo: Bar,"Baz: escaped backslash \\, escaped double-quote \", end of value",Another: Header', + ) + with webserver.install_http_handler(handler): + statres = gdal.VSIStatL(filename) + assert statres.size == 3 + + gdal.VSICurlClearCache() + handler = webserver.SequentialHandler() + handler.add( + "HEAD", + "/test_vsicurl_GDAL_HTTP_HEADERS.bin", + 200, + {"Content-Length": "3"}, + expected_headers={ + "Foo": "Bar", + "Baz": r'escaped backslash \, escaped double-quote ", end of value', + "Another": "Header", + }, + ) + with webserver.install_http_handler(handler): + statres = gdal.VSIStatL( + "/vsicurl_streaming/http://localhost:%d/test_vsicurl_GDAL_HTTP_HEADERS.bin" + % server.port + ) + assert statres.size == 3 + + finally: + gdal.SetPathSpecificOption(filename, "GDAL_HTTP_HEADERS", None) ############################################################################### diff --git a/autotest/gcore/vsifile.py b/autotest/gcore/vsifile.py index 1aa63cd9082d..bf57128f21f7 100755 --- a/autotest/gcore/vsifile.py +++ b/autotest/gcore/vsifile.py @@ -148,6 +148,28 @@ def vsifile_generic(filename, options=[]): if fp: gdal.VSIFCloseL(fp) + gdal.Unlink(filename) + + if not filename.startswith("/vsicrypt/"): + assert gdal.RmdirRecursive(filename + "/i_dont_exist") == -1 + + subdir = filename + "/subdir" + assert gdal.MkdirRecursive(subdir + "/subsubdir", 0o755) == 0 + + assert gdal.VSIStatL(subdir) is not None + assert gdal.VSIStatL(subdir + "/subsubdir") is not None + + if not filename.startswith("/vsimem/"): + assert gdal.Rmdir(subdir) == -1 + assert gdal.VSIStatL(subdir) is not None + + # Safety belt... + assert filename.startswith("tmp/") or filename.startswith("/vsimem/") + assert gdal.RmdirRecursive(filename) == 0 + + assert gdal.VSIStatL(subdir) is None + assert gdal.VSIStatL(subdir + "/subsubdir") is None + ############################################################################### # Test /vsimem @@ -679,6 +701,15 @@ def test_vsifile_14(): ) +############################################################################### +# Test bugfix for https://github.com/OSGeo/gdal/issues/10821 + + +def test_vsifile_vsitar_of_vsitar(): + + gdal.Open("/vsitar/{/vsitar/data/tar_of_tar_gzip.tar}/byte_tif.tar.gz/byte.tif") + + ############################################################################### # Test issue with Error() not detecting end of corrupted gzip stream (#6944) @@ -774,12 +805,8 @@ def test_vsifile_19(): def test_vsifile_20(): - try: + with pytest.raises(Exception): gdal.VSIFReadL(1, 1, None) - except ValueError: - return - - pytest.fail() ############################################################################### @@ -1708,7 +1735,7 @@ def test_vsifile_MultipartUpload(): with gdal.quiet_errors(): assert gdal.MultipartUploadGetCapabilities("foo") is None with gdal.ExceptionMgr(useExceptions=True): - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadGetCapabilities(None) with pytest.raises( @@ -1717,7 +1744,7 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadGetCapabilities("foo") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadStart(None) with pytest.raises( @@ -1726,9 +1753,9 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadStart("foo") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAddPart(None, "", 1, 0, b"") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAddPart("", None, 1, 0, b"") with pytest.raises( @@ -1737,9 +1764,9 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadAddPart("", "", 1, 0, b"") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadEnd(None, "", [], 0) - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadEnd("", None, [], 0) with pytest.raises( @@ -1748,9 +1775,9 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadEnd("", "", [], 0) - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAbort(None, "") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAbort("", None) with pytest.raises( diff --git a/autotest/gcore/vsigs.py b/autotest/gcore/vsigs.py index 75cbdad3198f..f619f79a779f 100755 --- a/autotest/gcore/vsigs.py +++ b/autotest/gcore/vsigs.py @@ -1188,6 +1188,15 @@ def method(request): assert data == "foo" + handler = webserver.SequentialHandler() + handler.add("GET", "/gs_fake_bucket/resource2", custom_method=method) + with webserver.install_http_handler(handler): + f = open_for_read("/vsigs/gs_fake_bucket/resource2") + assert f is not None + data = gdal.VSIFReadL(1, 4, f).decode("ascii") + gdal.VSIFCloseL(f) + assert data == "foo" + gdal.Unlink("/vsimem/service_account.json") diff --git a/autotest/gcore/vsioss.py b/autotest/gcore/vsioss.py index be75e9506ce5..1b4565452459 100755 --- a/autotest/gcore/vsioss.py +++ b/autotest/gcore/vsioss.py @@ -28,6 +28,7 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### +import os import stat import sys @@ -841,7 +842,8 @@ def test_vsioss_5(server): @pytest.mark.skipif( - gdaltest.is_travis_branch("macos_build"), reason="randomly fails on macos" + "CI" in os.environ, + reason="Flaky", ) @gdaltest.disable_exceptions() def test_vsioss_6(server): diff --git a/autotest/gdrivers/avif.py b/autotest/gdrivers/avif.py new file mode 100644 index 000000000000..0e42ca88e00c --- /dev/null +++ b/autotest/gdrivers/avif.py @@ -0,0 +1,273 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# +# Project: GDAL/OGR Test Suite +# Purpose: Test AVIF driver +# Author: Even Rouault, +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import base64 +import shutil + +import gdaltest +import pytest + +from osgeo import gdal + +pytestmark = pytest.mark.require_driver("AVIF") + + +def has_avif_encoder(): + drv = gdal.GetDriverByName("AVIF") + return drv is not None and drv.GetMetadataItem("DMD_CREATIONOPTIONLIST") is not None + + +def test_avif_subdatasets(tmp_path): + + filename = str(tmp_path / "out.avif") + shutil.copy("data/avif/colors-animated-8bpc-alpha-exif-xmp.avif", filename) + + ds = gdal.Open(filename) + assert ds + assert len(ds.GetSubDatasets()) == 5 + subds1_name = ds.GetSubDatasets()[0][0] + subds2_name = ds.GetSubDatasets()[1][0] + + ds = gdal.Open(subds1_name) + assert ds + assert ds.RasterXSize == 150 + assert ds.GetRasterBand(1).GetMetadataItem("STATISTICS_MINIMUM") is None + assert ds.GetRasterBand(1).ComputeStatistics(False) + assert ds.GetRasterBand(1).GetMetadataItem("STATISTICS_MINIMUM") is not None + ds.Close() + + ds = gdal.Open(subds1_name) + assert ds.GetRasterBand(1).GetMetadataItem("STATISTICS_MINIMUM") is not None + + ds = gdal.Open(subds2_name) + assert ds + assert ds.RasterXSize == 150 + assert ds.GetRasterBand(1).GetMetadataItem("STATISTICS_MINIMUM") is None + + with pytest.raises(Exception): + gdal.Open(f"AVIF:0:{filename}") + with pytest.raises(Exception): + gdal.Open(f"AVIF:6:{filename}") + with pytest.raises(Exception): + gdal.Open("AVIF:1:non_existing.heic") + with pytest.raises(Exception): + gdal.Open("AVIF:") + with pytest.raises(Exception): + gdal.Open("AVIF:1") + with pytest.raises(Exception): + gdal.Open("AVIF:1:") + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +def test_avif_single_band(): + tst = gdaltest.GDALTest( + "AVIF", + "byte.tif", + 1, + 4672, + ) + tst.testCreateCopy(vsimem=1, check_checksum_not_null=True, check_minmax=False) + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +@pytest.mark.require_driver("PNG") +def test_avif_gray_alpha(): + tst = gdaltest.GDALTest( + "AVIF", + "wms/gray+alpha.png", + 1, + 39910, + ) + tst.testCreateCopy(vsimem=1, check_checksum_not_null=True, check_minmax=False) + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +def test_avif_rgb(): + tst = gdaltest.GDALTest( + "AVIF", + "rgbsmall.tif", + 1, + 21212, + options=["QUALITY=100", "NUM_THREADS=1"], + ) + tst.testCreateCopy(vsimem=1) + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +def test_avif_rgba(): + tst = gdaltest.GDALTest( + "AVIF", + "../../gcore/data/stefan_full_rgba.tif", + 1, + 12603, + options=["QUALITY=100"], + ) + tst.testCreateCopy(vsimem=1) + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +def test_avif_uint16(): + tst = gdaltest.GDALTest( + "AVIF", "../../gcore/data/uint16.tif", 1, 4672, options=["NBITS=10"] + ) + tst.testCreateCopy(vsimem=1, check_checksum_not_null=True, check_minmax=False) + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +@pytest.mark.parametrize("yuv_subsampling", ["444", "422", "420"]) +def test_avif_yuv_subsampling(tmp_vsimem, yuv_subsampling): + + src_ds = gdal.Open("data/rgbsmall.tif") + out_filename = str(tmp_vsimem / "out.avif") + gdal.GetDriverByName("AVIF").CreateCopy( + out_filename, src_ds, options=["YUV_SUBSAMPLING=" + yuv_subsampling] + ) + ds = gdal.Open(out_filename) + assert ds.GetMetadataItem("YUV_SUBSAMPLING", "IMAGE_STRUCTURE") == yuv_subsampling + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +def test_avif_nbits_from_src_ds(tmp_vsimem): + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 1, gdal.GDT_UInt16) + src_ds.GetRasterBand(1).SetMetadataItem("NBITS", "12", "IMAGE_STRUCTURE") + out_filename = str(tmp_vsimem / "out.avif") + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + ds = gdal.Open(out_filename) + assert ds.GetRasterBand(1).GetMetadataItem("NBITS", "IMAGE_STRUCTURE") == "12" + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +def test_avif_exif_xmp(tmp_vsimem): + + src_ds = gdal.Open("data/avif/colors-animated-8bpc-alpha-exif-xmp.avif") + out_filename = str(tmp_vsimem / "out.avif") + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + if gdal.VSIStatL(out_filename + ".aux.xml"): + gdal.Unlink(out_filename + ".aux.xml") + ds = gdal.Open(out_filename) + exif_mdd = ds.GetMetadata("EXIF") + assert exif_mdd + assert exif_mdd["EXIF_LensMake"] == "Google" + xmp = ds.GetMetadata("xml:XMP") + assert xmp + assert xmp[0].startswith("= 1.0") + + f = open("data/sRGB.icc", "rb") + data = f.read() + icc = base64.b64encode(data).decode("ascii") + f.close() + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 3) + src_ds.GetRasterBand(1).SetColorInterpretation(gdal.GCI_RedBand) + src_ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_GreenBand) + src_ds.GetRasterBand(3).SetColorInterpretation(gdal.GCI_BlueBand) + src_ds.SetMetadataItem("SOURCE_ICC_PROFILE", icc, "COLOR_PROFILE") + + out_filename = str(tmp_vsimem / "out.avif") + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + ds = gdal.Open(out_filename) + assert ds.GetMetadataItem("SOURCE_ICC_PROFILE", "COLOR_PROFILE") == icc + + +@pytest.mark.skipif(not has_avif_encoder(), reason="libavif encoder missing") +def test_avif_creation_errors(tmp_vsimem): + + out_filename = str(tmp_vsimem / "out.avif") + + src_ds = gdal.GetDriverByName("MEM").Create("", 65537, 1) + with pytest.raises( + Exception, + match="Too big source dataset. Maximum AVIF image dimension is 65,536 x 65,536 pixels", + ): + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 65537) + with pytest.raises( + Exception, + match="Too big source dataset. Maximum AVIF image dimension is 65,536 x 65,536 pixels", + ): + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 5) + with pytest.raises(Exception, match="Unsupported number of bands"): + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 1, gdal.GDT_Float32) + with pytest.raises( + Exception, + match="Unsupported data type: only Byte or UInt16 bands are supported", + ): + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + with pytest.raises( + Exception, match="Invalid/inconsistent bit depth w.r.t data type" + ): + gdal.GetDriverByName("AVIF").CreateCopy( + out_filename, src_ds, options=["NBITS=10"] + ) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 1, gdal.GDT_UInt16) + with pytest.raises( + Exception, match="Invalid/inconsistent bit depth w.r.t data type" + ): + gdal.GetDriverByName("AVIF").CreateCopy( + out_filename, src_ds, options=["NBITS=8"] + ) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + src_ds.GetRasterBand(1).SetColorTable(gdal.ColorTable()) + with pytest.raises( + Exception, + match="Source dataset with color table unsupported. Use gdal_translate -expand rgb|rgba first", + ): + gdal.GetDriverByName("AVIF").CreateCopy(out_filename, src_ds) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 3) + with pytest.raises( + Exception, match="Only YUV_SUBSAMPLING=444 is supported for lossless encoding" + ): + gdal.GetDriverByName("AVIF").CreateCopy( + out_filename, src_ds, options=["QUALITY=100", "YUV_SUBSAMPLING=422"] + ) + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + with pytest.raises(Exception, match="Cannot create file /i_do/not/exist.avif"): + gdal.GetDriverByName("AVIF").CreateCopy("/i_do/not/exist.avif", src_ds) diff --git a/autotest/gdrivers/avif_heif.py b/autotest/gdrivers/avif_heif.py new file mode 100755 index 000000000000..3d549f5e1752 --- /dev/null +++ b/autotest/gdrivers/avif_heif.py @@ -0,0 +1,63 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# Project: GDAL/OGR Test Suite +# Purpose: Test read functionality for AVIF_HEIF driver. +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2024, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import os +import subprocess +import sys + +import pytest + + +def test_avif_heif(): + + from osgeo import gdal + + drv = gdal.GetDriverByName("HEIF") + if drv is None: + pytest.skip("HEIF driver must be available") + if drv.GetMetadataItem("SUPPORTS_AVIF", "HEIF") is None: + pytest.skip("libheif has no AVIF support") + + subprocess.check_call( + [ + sys.executable, + "avif_heif.py", + "subprocess", + ] + ) + + +if __name__ == "__main__": + + os.environ["GDAL_SKIP"] = "AVIF" + from osgeo import gdal + + gdal.UseExceptions() + ds = gdal.Open("data/avif/byte.avif") + assert ds.GetRasterBand(1).Checksum() == 4672 diff --git a/autotest/gdrivers/data/avif/byte.avif b/autotest/gdrivers/data/avif/byte.avif new file mode 100644 index 000000000000..499891bcf0bd Binary files /dev/null and b/autotest/gdrivers/data/avif/byte.avif differ diff --git a/autotest/gdrivers/data/avif/colors-animated-8bpc-alpha-exif-xmp.avif b/autotest/gdrivers/data/avif/colors-animated-8bpc-alpha-exif-xmp.avif new file mode 100644 index 000000000000..afe6a2a60658 Binary files /dev/null and b/autotest/gdrivers/data/avif/colors-animated-8bpc-alpha-exif-xmp.avif differ diff --git a/autotest/gdrivers/data/jpeg/byte_corrupted3.jpg b/autotest/gdrivers/data/jpeg/byte_corrupted3.jpg new file mode 100644 index 000000000000..d6962b48f04d Binary files /dev/null and b/autotest/gdrivers/data/jpeg/byte_corrupted3.jpg differ diff --git a/autotest/gdrivers/data/s102/MD_test_s102_v3.0_with_QualityOfBathymetryCoverage.xml b/autotest/gdrivers/data/s102/MD_test_s102_v3.0_with_QualityOfBathymetryCoverage.xml new file mode 100644 index 000000000000..7d5c8daa45e2 --- /dev/null +++ b/autotest/gdrivers/data/s102/MD_test_s102_v3.0_with_QualityOfBathymetryCoverage.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/autotest/gdrivers/data/s102/generate_test.py b/autotest/gdrivers/data/s102/generate_test.py index c4d3f81a8ed3..c87dbc755d86 100755 --- a/autotest/gdrivers/data/s102/generate_test.py +++ b/autotest/gdrivers/data/s102/generate_test.py @@ -35,7 +35,13 @@ import numpy as np -def generate(filename, version, with_QualityOfSurvey=False): +def generate( + filename, + version, + with_QualityOfSurvey=False, + with_QualityOfCoverage=False, + use_compound_type_for_Quality=False, +): f = h5py.File(os.path.join(os.path.dirname(__file__), f"{filename}.h5"), "w") BathymetryCoverage = f.create_group("BathymetryCoverage") BathymetryCoverage_01 = BathymetryCoverage.create_group("BathymetryCoverage.01") @@ -71,7 +77,7 @@ def generate(filename, version, with_QualityOfSurvey=False): f.attrs["issueDate"] = "2023-12-31" f.attrs["geographicIdentifier"] = "Somewhere" f.attrs["verticalDatum"] = np.int16(12) - if version == "INT.IHO.S-102.2.2": + if version >= "INT.IHO.S-102.2.2": f.attrs["horizontalCRS"] = np.int32(4326) f.attrs["verticalCS"] = np.int32(6498) # Depth, metres, orientation down f.attrs["verticalCoordinateBase"] = np.int32(2) @@ -90,9 +96,12 @@ def generate(filename, version, with_QualityOfSurvey=False): b"" ) - if with_QualityOfSurvey: - QualityOfSurvey = f.create_group("QualityOfSurvey") - QualityOfSurvey_01 = QualityOfSurvey.create_group("QualityOfSurvey.01") + if with_QualityOfSurvey or with_QualityOfCoverage: + quality_name = ( + "QualityOfSurvey" if with_QualityOfSurvey else "QualityOfBathymetryCoverage" + ) + qualityGroup = f.create_group(quality_name) + quality01Group = qualityGroup.create_group(quality_name + ".01") for attr_name in ( "gridOriginLongitude", @@ -102,16 +111,26 @@ def generate(filename, version, with_QualityOfSurvey=False): "numPointsLongitudinal", "numPointsLatitudinal", ): - QualityOfSurvey_01.attrs[attr_name] = BathymetryCoverage_01.attrs[attr_name] - - Group_001 = QualityOfSurvey_01.create_group("Group_001") - - values = Group_001.create_dataset("values", (2, 3), dtype=np.uint32) - data = np.array( - [0, 1, 2, 1000000, 3, 2], - dtype=np.uint32, - ).reshape(values.shape) - values[...] = data + quality01Group.attrs[attr_name] = BathymetryCoverage_01.attrs[attr_name] + + Group_001 = quality01Group.create_group("Group_001") + + if use_compound_type_for_Quality: + # As found in product 102DE00CA22_UNC_MD.H5 + data_struct_type = np.dtype([("iD", "u4")]) + values = Group_001.create_dataset("values", (2, 3), dtype=data_struct_type) + data = np.array( + [(0), (1), (2), (1000000), (3), (2)], + dtype=data_struct_type, + ).reshape(values.shape) + values[...] = data + else: + values = Group_001.create_dataset("values", (2, 3), dtype=np.uint32) + data = np.array( + [0, 1, 2, 1000000, 3, 2], + dtype=np.uint32, + ).reshape(values.shape) + values[...] = data featureAttributeTable_struct_type = np.dtype( [ @@ -121,7 +140,7 @@ def generate(filename, version, with_QualityOfSurvey=False): ] ) - featureAttributeTable = QualityOfSurvey.create_dataset( + featureAttributeTable = qualityGroup.create_dataset( "featureAttributeTable", (5,), dtype=featureAttributeTable_struct_type ) @@ -137,7 +156,7 @@ def generate(filename, version, with_QualityOfSurvey=False): ) featureAttributeTable[...] = data - GroupFQualityOfSurvey_struct_type = np.dtype( + GroupFQuality_struct_type = np.dtype( [ ("code", "S16"), ("name", "S16"), @@ -149,12 +168,12 @@ def generate(filename, version, with_QualityOfSurvey=False): ("closure", "S16"), ] ) - GroupFQualityOfSurvey = group_f.create_dataset( - "QualityOfSurvey", (1,), dtype=GroupFQualityOfSurvey_struct_type + GroupFQuality = group_f.create_dataset( + quality_name, (1,), dtype=GroupFQuality_struct_type ) - GroupFQualityOfSurvey[...] = np.array( + GroupFQuality[...] = np.array( [("id", "", "", "0", "H5T_INTEGER", "1", "", "geSemiInterval")], - dtype=GroupFQualityOfSurvey_struct_type, + dtype=GroupFQuality_struct_type, ) @@ -166,3 +185,10 @@ def generate(filename, version, with_QualityOfSurvey=False): "INT.IHO.S-102.2.2", with_QualityOfSurvey=True, ) + +generate( + "test_s102_v3.0_with_QualityOfBathymetryCoverage", + "INT.IHO.S-102.3.0.0", + with_QualityOfCoverage=True, + use_compound_type_for_Quality=True, +) diff --git a/autotest/gdrivers/data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5 b/autotest/gdrivers/data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5 new file mode 100644 index 000000000000..8877ede735bb Binary files /dev/null and b/autotest/gdrivers/data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5 differ diff --git a/autotest/gdrivers/envi.py b/autotest/gdrivers/envi.py index 9cc41b9f074e..23af13136c28 100755 --- a/autotest/gdrivers/envi.py +++ b/autotest/gdrivers/envi.py @@ -1056,3 +1056,120 @@ def test_envi_read_metadata_with_leading_space(): assert ds.GetRasterBand(1).GetMetadataItem("wavelength") == "3" ds = None gdal.GetDriverByName("ENVI").Delete("/vsimem/test.bin") + + +############################################################################### +# Test wavelength / fwhm + + +def test_envi_read_wavelength_fwhm_um(): + + gdal.FileFromMemBuffer( + "/vsimem/test.hdr", + """ENVI +samples = 1 +lines = 1 +bands = 3 +header offset = 0 +file type = ENVI Standard +data type = 1 +interleave = bip +sensor type = Unknown +byte order = 0 +wavelength units = um +wavelength = {3, 2, 1} +fwhm = {.3, .2, .1}""", + ) + gdal.FileFromMemBuffer("/vsimem/test.bin", "xyz") + + ds = gdal.Open("/vsimem/test.bin") + assert ( + ds.GetRasterBand(1).GetMetadataItem("CENTRAL_WAVELENGTH_UM", "IMAGERY") + == "3.000" + ) + assert ds.GetRasterBand(1).GetMetadataItem("FWHM_UM", "IMAGERY") == "0.300" + assert ( + ds.GetRasterBand(2).GetMetadataItem("CENTRAL_WAVELENGTH_UM", "IMAGERY") + == "2.000" + ) + assert ds.GetRasterBand(2).GetMetadataItem("FWHM_UM", "IMAGERY") == "0.200" + ds = None + gdal.GetDriverByName("ENVI").Delete("/vsimem/test.bin") + + +############################################################################### +# Test wavelength / fwhm + + +def test_envi_read_wavelength_fwhm_nm(): + + gdal.FileFromMemBuffer( + "/vsimem/test.hdr", + """ENVI +samples = 1 +lines = 1 +bands = 3 +header offset = 0 +file type = ENVI Standard +data type = 1 +interleave = bip +sensor type = Unknown +byte order = 0 +wavelength units = nm +wavelength = {3000, 2000, 1000} +fwhm = {300, 200, 100}""", + ) + gdal.FileFromMemBuffer("/vsimem/test.bin", "xyz") + + ds = gdal.Open("/vsimem/test.bin") + assert ( + ds.GetRasterBand(1).GetMetadataItem("CENTRAL_WAVELENGTH_UM", "IMAGERY") + == "3.000" + ) + assert ds.GetRasterBand(1).GetMetadataItem("FWHM_UM", "IMAGERY") == "0.300" + assert ( + ds.GetRasterBand(2).GetMetadataItem("CENTRAL_WAVELENGTH_UM", "IMAGERY") + == "2.000" + ) + assert ds.GetRasterBand(2).GetMetadataItem("FWHM_UM", "IMAGERY") == "0.200" + ds = None + gdal.GetDriverByName("ENVI").Delete("/vsimem/test.bin") + + +############################################################################### +# Test wavelength / fwhm + + +def test_envi_read_wavelength_fwhm_mm(): + + gdal.FileFromMemBuffer( + "/vsimem/test.hdr", + """ENVI +samples = 1 +lines = 1 +bands = 3 +header offset = 0 +file type = ENVI Standard +data type = 1 +interleave = bip +sensor type = Unknown +byte order = 0 +wavelength units = mm +wavelength = {0.003, 0.002, 0.001} +fwhm = {0.0003, 0.0002, 0.0001}""", + ) + gdal.FileFromMemBuffer("/vsimem/test.bin", "xyz") + + ds = gdal.Open("/vsimem/test.bin") + assert ( + ds.GetRasterBand(1).GetMetadataItem("CENTRAL_WAVELENGTH_UM", "IMAGERY") + == "3.000" + ) + assert ds.GetRasterBand(1).GetMetadataItem("FWHM_UM", "IMAGERY") == "0.300" + assert ( + ds.GetRasterBand(2).GetMetadataItem("CENTRAL_WAVELENGTH_UM", "IMAGERY") + == "2.000" + ) + assert ds.GetRasterBand(2).GetMetadataItem("FWHM_UM", "IMAGERY") == "0.200" + ds = None + gdal.GetDriverByName("ENVI").Delete("/vsimem/test.bin") diff --git a/autotest/gdrivers/gti.py b/autotest/gdrivers/gti.py index 75719ac07d26..2bab88273269 100755 --- a/autotest/gdrivers/gti.py +++ b/autotest/gdrivers/gti.py @@ -2947,3 +2947,38 @@ def test_gti_read_multi_threaded_disabled_because_truncated_source(tmp_vsimem): assert vrt_ds.GetMetadataItem("MULTI_THREADED_RASTERIO_LAST_USED", "__DEBUG__") == ( "1" if gdal.GetNumCPUs() >= 2 else "0" ) + + +############################################################################### + + +@pytest.mark.require_curl() +@pytest.mark.require_driver("Parquet") +def test_gti_stac_geoparquet(): + + url = ( + "https://github.com/stac-utils/stac-geoparquet/raw/main/tests/data/naip.parquet" + ) + + conn = gdaltest.gdalurlopen(url, timeout=4) + if conn is None: + pytest.skip("cannot open URL") + + ds = gdal.Open("GTI:/vsicurl/" + url) + assert ds.GetSpatialRef().GetAuthorityCode(None) == "26914" + assert ds.GetGeoTransform() == pytest.approx( + (408231.0, 1.0, 0.0, 3873862.0, 0.0, -1.0), rel=1e-5 + ) + assert ds.RasterCount == 4 + assert [band.GetColorInterpretation() for band in ds] == [ + gdal.GCI_RedBand, + gdal.GCI_GreenBand, + gdal.GCI_BlueBand, + gdal.GCI_Undefined, + ] + assert [band.GetDescription() for band in ds] == [ + "Red", + "Green", + "Blue", + "NIR (near-infrared)", + ] diff --git a/autotest/gdrivers/jp2openjpeg.py b/autotest/gdrivers/jp2openjpeg.py index 46820ee810e7..cc64df8dabf1 100755 --- a/autotest/gdrivers/jp2openjpeg.py +++ b/autotest/gdrivers/jp2openjpeg.py @@ -2744,10 +2744,10 @@ def test_jp2openjpeg_45(): ) del out_ds - dircontent = gdal.ReadDir("/vsimem/") + dircontent = gdal.ReadDir("/vsimem/.#!HIDDEN!#.") if dircontent: for filename in dircontent: - assert not filename.startswith("gmljp2") + assert "gmljp2" not in filename ds = ogr.Open("/vsimem/jp2openjpeg_45.jp2") assert ds.GetLayerCount() == 1 diff --git a/autotest/gdrivers/jpeg.py b/autotest/gdrivers/jpeg.py index 7ec2b3b5736e..2dae55fe1e1d 100755 --- a/autotest/gdrivers/jpeg.py +++ b/autotest/gdrivers/jpeg.py @@ -593,7 +593,7 @@ def test_jpeg_17(): assert not ( gdal.GetLastErrorType() != gdal.CE_Failure or gdal.GetLastErrorMsg() == "" - ) + ), "Premature end of file should be a failure by default" gdal.ErrorReset() ds = gdal.Open("data/jpeg/byte_corrupted2.jpg") @@ -603,7 +603,7 @@ def test_jpeg_17(): assert not ( gdal.GetLastErrorType() != gdal.CE_Failure or gdal.GetLastErrorMsg() == "" - ) + ), "Premature end of file should be a failure with GDAL_ERROR_ON_LIBJPEG_WARNING = TRUE" gdal.ErrorReset() ds = gdal.Open("data/jpeg/byte_corrupted2.jpg") @@ -613,7 +613,35 @@ def test_jpeg_17(): assert not ( gdal.GetLastErrorType() != gdal.CE_Warning or gdal.GetLastErrorMsg() == "" - ) + ), "Premature end of file should be a warning with GDAL_ERROR_ON_LIBJPEG_WARNING = FALSE" + + gdal.ErrorReset() + with gdaltest.error_handler("CPLQuietErrorHandler"): + ds = gdal.Open("data/jpeg/byte_corrupted3.jpg") + assert ds.GetRasterBand(1).Checksum() != 0 + + assert not ( + gdal.GetLastErrorType() != gdal.CE_Warning or gdal.GetLastErrorMsg() == "" + ), "Extraneous bytes before marker should be a warning by default" + + gdal.ErrorReset() + with gdaltest.error_handler("CPLQuietErrorHandler"): + with gdaltest.config_option("GDAL_ERROR_ON_LIBJPEG_WARNING", "TRUE"): + ds = gdal.Open("data/jpeg/byte_corrupted3.jpg") + + assert not ( + gdal.GetLastErrorType() != gdal.CE_Failure or gdal.GetLastErrorMsg() == "" + ), "Extraneous bytes before marker should be a failure with GDAL_ERROR_ON_LIBJPEG_WARNING = TRUE" + + gdal.ErrorReset() + with gdaltest.error_handler("CPLQuietErrorHandler"): + with gdaltest.config_option("GDAL_ERROR_ON_LIBJPEG_WARNING", "FALSE"): + ds = gdal.Open("data/jpeg/byte_corrupted3.jpg") + assert ds.GetRasterBand(1).Checksum() != 0 + + assert not ( + gdal.GetLastErrorType() != gdal.CE_Warning or gdal.GetLastErrorMsg() == "" + ), "Extraneous bytes before marker should be a warning with GDAL_ERROR_ON_LIBJPEG_WARNING = FALSE" ############################################################################### diff --git a/autotest/gdrivers/memmultidim.py b/autotest/gdrivers/memmultidim.py index 0c59a4367536..96d15806d8d0 100755 --- a/autotest/gdrivers/memmultidim.py +++ b/autotest/gdrivers/memmultidim.py @@ -90,7 +90,7 @@ def test_mem_md_subgroup(): with gdal.quiet_errors(): assert not rg.CreateGroup("") # unnamed group not supported - with pytest.raises(ValueError): + with pytest.raises(Exception): assert not rg.CreateGroup(None) subg = rg.CreateGroup("subgroup") @@ -339,12 +339,12 @@ def test_mem_md_datatypes(): assert dt_byte.GetNumericDataType() == gdal.GDT_Byte assert dt_byte.GetSize() == 1 assert dt_byte.CanConvertTo(dt_byte) - with pytest.raises(ValueError): + with pytest.raises(Exception): assert dt_byte.CanConvertTo(None) assert dt_byte == gdal.ExtendedDataType.Create(gdal.GDT_Byte) assert not dt_byte != gdal.ExtendedDataType.Create(gdal.GDT_Byte) assert dt_byte.Equals(dt_byte) - with pytest.raises(ValueError): + with pytest.raises(Exception): assert dt_byte.Equals(None) assert not dt_byte.GetComponents() @@ -762,9 +762,9 @@ def test_mem_md_array_invalid_args(): rg.CreateMDArray("myarray", [None], edt) with pytest.raises((TypeError, SystemError)): rg.CreateMDArray("myarray", [1], edt) - with pytest.raises(ValueError): + with pytest.raises(Exception): rg.CreateMDArray("myarray", [dim], None) - with pytest.raises(ValueError): + with pytest.raises(Exception): rg.CreateMDArray(None, [dim], edt) @@ -837,7 +837,7 @@ def test_mem_md_group_attribute_single_numeric(): float64dt = gdal.ExtendedDataType.Create(gdal.GDT_Float64) with gdal.quiet_errors(): assert not rg.CreateAttribute("", [1], float64dt) # unnamed attr not supported - with pytest.raises(ValueError): + with pytest.raises(Exception): rg.CreateAttribute(None, [1], float64dt) attr = rg.CreateAttribute("attr", [1], float64dt) @@ -955,7 +955,7 @@ def test_mem_md_array_attribute(): assert not myarray.CreateAttribute( "", [1], float64dt ) # unnamed attr not supported - with pytest.raises(ValueError): + with pytest.raises(Exception): myarray.CreateAttribute(None, [1], float64dt) attr = myarray.CreateAttribute("attr", [1], float64dt) diff --git a/autotest/gdrivers/ogcapi.py b/autotest/gdrivers/ogcapi.py index 7fec70675248..fdebf3a42427 100644 --- a/autotest/gdrivers/ogcapi.py +++ b/autotest/gdrivers/ogcapi.py @@ -382,7 +382,10 @@ def test_ogr_ogcapi_raster(api, collection, tmp_path): ) def test_ogc_api_wrong_collection(api, of_type): - with pytest.raises(Exception, match="Invalid data collection"): + with pytest.raises( + Exception, + match=r"HTTP error code : 400,

    GNOSIS Map Server \(OGCAPI\) - 400 Bad Request

    Invalid data collection

    ", + ): gdal.OpenEx( f"OGCAPI:http://127.0.0.1:{gdaltest.webserver_port}/fakeogcapi/collections/NOT_EXISTS", of_type, diff --git a/autotest/gdrivers/s102.py b/autotest/gdrivers/s102.py index 3e3010202319..90e7374ee2a1 100755 --- a/autotest/gdrivers/s102.py +++ b/autotest/gdrivers/s102.py @@ -211,34 +211,40 @@ def test_s102_multidim(): ############################################################################### -def test_s102_QualityOfSurvey(): +@pytest.mark.parametrize( + "filename,quality_group_name", + [ + ("data/s102/test_s102_v2.2_with_QualityOfSurvey.h5", "QualityOfSurvey"), + ( + "data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5", + "QualityOfBathymetryCoverage", + ), + ], +) +def test_s102_QualityOfSurvey(filename, quality_group_name): - ds = gdal.Open("data/s102/test_s102_v2.2_with_QualityOfSurvey.h5") + ds = gdal.Open(filename) assert ds.GetSubDatasets() == [ ( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":BathymetryCoverage', + f'S102:"{filename}":BathymetryCoverage', "Bathymetric gridded data", ), ( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":QualityOfSurvey', - "Georeferenced metadata QualityOfSurvey", + f'S102:"{filename}":{quality_group_name}', + f"Georeferenced metadata {quality_group_name}", ), ] with pytest.raises(Exception, match="Unsupported subdataset component"): - gdal.Open('S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":invalid') + gdal.Open(f'S102:"{filename}":invalid') - ds = gdal.Open( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":BathymetryCoverage' - ) + ds = gdal.Open(f'S102:"{filename}":BathymetryCoverage') assert len(ds.GetSubDatasets()) == 0 assert ds.RasterCount == 2 assert ds.RasterXSize == 3 assert ds.RasterYSize == 2 - ds = gdal.Open( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":QualityOfSurvey' - ) + ds = gdal.Open(f'S102:"{filename}":{quality_group_name}') assert len(ds.GetSubDatasets()) == 0 assert ds.RasterCount == 1 assert ds.RasterXSize == 3 @@ -278,7 +284,7 @@ def test_s102_QualityOfSurvey(): assert rat.GetValueAsString(4, 2) == "e" ds = gdal.OpenEx( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":QualityOfSurvey', + f'S102:"{filename}":{quality_group_name}', open_options=["NORTH_UP=NO"], ) assert ds.GetGeoTransform() == pytest.approx((1.8, 0.4, 0.0, 47.75, 0.0, 0.5)) diff --git a/autotest/gdrivers/sentinel2.py b/autotest/gdrivers/sentinel2.py index 63cbb9e6b836..4d5901feda47 100755 --- a/autotest/gdrivers/sentinel2.py +++ b/autotest/gdrivers/sentinel2.py @@ -244,6 +244,9 @@ def test_sentinel2_l1c_2(): pprint.pprint(got_md) pytest.fail() + assert band.GetMetadataItem("CENTRAL_WAVELENGTH_UM", "IMAGERY") == "0.665" + assert band.GetMetadataItem("FWHM_UM", "IMAGERY") == "0.030" + assert band.GetColorInterpretation() == gdal.GCI_RedBand assert band.DataType == gdal.GDT_UInt16 @@ -252,7 +255,7 @@ def test_sentinel2_l1c_2(): band = ds.GetRasterBand(4) - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_NIRBand got_md = band.GetMetadata() expected_md = { @@ -843,7 +846,7 @@ def test_sentinel2_l1c_tile_3(): band = ds.GetRasterBand(4) - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_NIRBand got_md = band.GetMetadata() expected_md = { @@ -2618,7 +2621,7 @@ def test_sentinel2_l1c_safe_compact_2(): band = ds.GetRasterBand(4) - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_NIRBand got_md = band.GetMetadata() expected_md = { @@ -2804,7 +2807,7 @@ def test_sentinel2_l1c_processing_baseline_5_09__1(): band = ds.GetRasterBand(4) - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_NIRBand got_md = band.GetMetadata() expected_md = { @@ -2900,13 +2903,13 @@ def test_sentinel2_l1c_processing_baseline_5_09__2(): pprint.pprint(got_md) pytest.fail() - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_RedEdgeBand assert band.DataType == gdal.GDT_UInt16 band = ds.GetRasterBand(4) - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_NIRBand got_md = band.GetMetadata() expected_md = { @@ -3035,7 +3038,7 @@ def test_sentinel2_l2a_processing_baseline_5_09__1(): band = ds.GetRasterBand(4) - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_NIRBand got_md = band.GetMetadata() expected_md = { @@ -3158,13 +3161,13 @@ def test_sentinel2_l2a_processing_baseline_5_09__2(): pprint.pprint(got_md) pytest.fail() - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_RedEdgeBand assert band.DataType == gdal.GDT_UInt16 band = ds.GetRasterBand(4) - assert band.GetColorInterpretation() == gdal.GCI_Undefined + assert band.GetColorInterpretation() == gdal.GCI_NIRBand got_md = band.GetMetadata() expected_md = { diff --git a/autotest/gdrivers/stacit.py b/autotest/gdrivers/stacit.py index 8e25ce10ec8f..5f5dec2b7a90 100755 --- a/autotest/gdrivers/stacit.py +++ b/autotest/gdrivers/stacit.py @@ -162,7 +162,7 @@ def test_stacit_overlapping_sources(): # Check that the source covered by another one is not listed vrt = ds.GetMetadata("xml:VRT")[0] only_one_simple_source = """ - Gray + Coastal data/byte.tif 1 @@ -361,3 +361,20 @@ def test_stacit_force_opening_no_match(): drv = gdal.IdentifyDriverEx("data/byte.tif", allowed_drivers=["STACIT"]) assert drv is None + + +############################################################################### +# Test opening a top-level Feature + + +def test_stacit_single_feature(tmp_vsimem): + + j = json.loads(open("data/stacit/test.json", "rb").read()) + j = j["features"][0] + + filename = str(tmp_vsimem / "feature.json") + with gdaltest.tempfile(filename, json.dumps(j)): + ds = gdal.Open(filename) + assert ds is not None + assert ds.RasterXSize == 20 + assert ds.GetRasterBand(1).Checksum() == 4672 diff --git a/autotest/gdrivers/vrtwarp.py b/autotest/gdrivers/vrtwarp.py index 3f37e1d0a6c1..9b843aa81342 100755 --- a/autotest/gdrivers/vrtwarp.py +++ b/autotest/gdrivers/vrtwarp.py @@ -732,3 +732,31 @@ def test_vrtwarp_irasterio_optim_window_splitting(): with gdaltest.config_option("GDAL_VRT_WARP_USE_DATASET_RASTERIO", "NO"): expected_data = warped_vrt_ds.ReadRaster() assert warped_vrt_ds.ReadRaster() == expected_data + + +############################################################################### +# Test gdal.AutoCreateWarpedVRT() on a Int16 band with nodata = 32767 + + +def test_vrtwarp_autocreatewarpedvrt_int16_nodata_32767(): + + ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 1, gdal.GDT_Int16) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).SetNoDataValue(32767) + vrt_ds = gdal.AutoCreateWarpedVRT(ds) + assert vrt_ds.GetRasterBand(1).DataType == gdal.GDT_Int16 + assert vrt_ds.GetRasterBand(1).GetNoDataValue() == 32767 + + +############################################################################### +# Test gdal.AutoCreateWarpedVRT() on a source nodata value that does not fit +# the source band type + + +def test_vrtwarp_autocreatewarpedvrt_invalid_nodata(): + + ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 1, gdal.GDT_Byte) + ds.SetGeoTransform([0, 1, 0, 0, 0, -1]) + ds.GetRasterBand(1).SetNoDataValue(-9999) + vrt_ds = gdal.AutoCreateWarpedVRT(ds) + assert vrt_ds.GetRasterBand(1).DataType == gdal.GDT_Byte diff --git a/autotest/gdrivers/wmts.py b/autotest/gdrivers/wmts.py index 93e117939475..cfa9a1341577 100755 --- a/autotest/gdrivers/wmts.py +++ b/autotest/gdrivers/wmts.py @@ -880,6 +880,7 @@ def test_wmts_15(): + """, ) diff --git a/autotest/generate_parquet_test_file.py b/autotest/generate_parquet_test_file.py index 71e226d4ed9e..fd233ca21e55 100644 --- a/autotest/generate_parquet_test_file.py +++ b/autotest/generate_parquet_test_file.py @@ -1245,6 +1245,91 @@ def __arrow_ext_deserialize__(cls, storage_type, serialized): ) +def generate_arrow_stringview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + stringview = pa.array(["foo", "bar", "looooooooooong string"], pa.string_view()) + list_stringview = pa.array( + [None, [None], ["foo", "bar", "looooooooooong string"]], + pa.list_(pa.string_view()), + ) + list_of_list_stringview = pa.array( + [None, [None], [["foo", "bar", "looooooooooong string"]]], + pa.list_(pa.list_(pa.string_view())), + ) + map_stringview = pa.array( + [None, [], [("x", "x_val"), ("y", None)]], + type=pa.map_(pa.string_view(), pa.string_view()), + ) + + names = [ + "stringview", + "list_stringview", + "list_of_list_stringview", + "map_stringview", + ] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/stringview.feather") + + +def generate_arrow_binaryview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + binaryview = pa.array([b"foo", b"bar", b"looooooooooong binary"], pa.binary_view()) + + names = ["binaryview"] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/binaryview.feather") + + +def generate_arrow_listview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + listview = pa.array([[1]], pa.list_view(pa.int32())) + + names = ["listview"] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/listview.feather") + + +def generate_arrow_largelistview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + largelistview = pa.array([[1]], pa.large_list_view(pa.int32())) + + names = ["largelistview"] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/largelistview.feather") + + if __name__ == "__main__": generate_test_parquet() generate_all_geoms_parquet() @@ -1252,3 +1337,7 @@ def __arrow_ext_deserialize__(cls, storage_type, serialized): generate_nested_types() generate_extension_custom() generate_extension_json() + generate_arrow_stringview() + generate_arrow_binaryview() + generate_arrow_listview() + generate_arrow_largelistview() diff --git a/autotest/ogr/data/arrow/binaryview.feather b/autotest/ogr/data/arrow/binaryview.feather new file mode 100644 index 000000000000..9f62bd82944c Binary files /dev/null and b/autotest/ogr/data/arrow/binaryview.feather differ diff --git a/autotest/ogr/data/arrow/largelistview.feather b/autotest/ogr/data/arrow/largelistview.feather new file mode 100644 index 000000000000..65e2ffc575e1 Binary files /dev/null and b/autotest/ogr/data/arrow/largelistview.feather differ diff --git a/autotest/ogr/data/arrow/listview.feather b/autotest/ogr/data/arrow/listview.feather new file mode 100644 index 000000000000..c9737dff6168 Binary files /dev/null and b/autotest/ogr/data/arrow/listview.feather differ diff --git a/autotest/ogr/data/arrow/stringview.feather b/autotest/ogr/data/arrow/stringview.feather new file mode 100644 index 000000000000..43ab1534e0f8 Binary files /dev/null and b/autotest/ogr/data/arrow/stringview.feather differ diff --git a/autotest/ogr/data/filegdb/arc_segment_interior_point_but_line.gdb.zip b/autotest/ogr/data/filegdb/arc_segment_interior_point_but_line.gdb.zip new file mode 100644 index 000000000000..0c73d8614373 Binary files /dev/null and b/autotest/ogr/data/filegdb/arc_segment_interior_point_but_line.gdb.zip differ diff --git a/autotest/ogr/data/flatgeobuf/test_ogr_flatgeobuf_singlepart_mls_new.fgb b/autotest/ogr/data/flatgeobuf/test_ogr_flatgeobuf_singlepart_mls_new.fgb new file mode 100644 index 000000000000..4a1482c1f1fc Binary files /dev/null and b/autotest/ogr/data/flatgeobuf/test_ogr_flatgeobuf_singlepart_mls_new.fgb differ diff --git a/autotest/ogr/data/gml/billionlaugh.gml b/autotest/ogr/data/gml/billionlaugh.gml new file mode 100644 index 000000000000..6fc911bf771f --- /dev/null +++ b/autotest/ogr/data/gml/billionlaugh.gml @@ -0,0 +1,21 @@ + + + + + + + + + + + +]> + + &lol9; + diff --git a/autotest/ogr/data/gml/billionlaugh.xsd b/autotest/ogr/data/gml/billionlaugh.xsd new file mode 100644 index 000000000000..1e8c85b3fa03 --- /dev/null +++ b/autotest/ogr/data/gml/billionlaugh.xsd @@ -0,0 +1,69 @@ + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/autotest/ogr/data/nas/billionlaugh.xml b/autotest/ogr/data/nas/billionlaugh.xml new file mode 100644 index 000000000000..24f73f6e6614 --- /dev/null +++ b/autotest/ogr/data/nas/billionlaugh.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + +]> + + + &lol9; + <?xml version="1.0" encoding="utf-8"?><Protocol xmlns="http://www.aed-sicad.de/namespaces/va" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema"><auftragsid>AMGR000000161077</auftragsid><profilkennung>NBABenutzerLA</profilkennung><auftragsnummer>Admin_100413082150_001_0006</auftragsnummer><antragsnummer>Admin_100413082150_0006</antragsnummer><nasoperation>NutzerbezogeneBestandsdatenaktualisierung_NBA</nasoperation><status>beendet</status><startzeitreal>2010-10-13T22:29:43.66646+02:00</startzeitreal><endzeitreal>2010-10-13T22:29:43.66646+02:00</endzeitreal><Message><Category>NOTHING</Category><MessageLevel>Info</MessageLevel><MessageObject><FeatureClass /><Id /></MessageObject><MessageText>Das Ergebnis wurde mit der 3A-Version 6.0.9.3 erstellt.</MessageText><ProcessingTime>2010-10-13T22:29:43.66646+02:00</ProcessingTime></Message></Protocol> + true + Admin_100413082150_0006 + + + + + + + + true + + + + + + + + Admin_100413082150_001_0006 + + + NBABenutzerLA + 2010-10-13T20:10:14Z + 16 + 130 + 484560.000 5753705.000 + + + diff --git a/autotest/ogr/ogr_arrow.py b/autotest/ogr/ogr_arrow.py index cba6928589bc..7c6f9e858df7 100755 --- a/autotest/ogr/ogr_arrow.py +++ b/autotest/ogr/ogr_arrow.py @@ -794,3 +794,58 @@ def test_ogr_arrow_vsi_arrow_file_system(): pytest.skip("requires Arrow >= 16.0.0") ogr.Open("vsi://data/arrow/test.feather") + + +############################################################################### + + +@gdaltest.enable_exceptions() +def test_ogr_arrow_string_view(): + + version = int( + ogr.GetDriverByName("ARROW").GetMetadataItem("ARROW_VERSION").split(".")[0] + ) + if version < 15: + pytest.skip("requires Arrow >= 15") + + with ogr.Open("data/arrow/stringview.feather") as ds: + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f["stringview"] == "foo" + assert f["list_stringview"] is None + assert f["list_of_list_stringview"] is None + assert f["map_stringview"] is None + + f = lyr.GetNextFeature() + assert f["stringview"] == "bar" + assert f["list_stringview"] == [""] + assert f["list_of_list_stringview"] == "[null]" + assert f["map_stringview"] == "{}" + + f = lyr.GetNextFeature() + assert f["stringview"] == "looooooooooong string" + assert f["list_stringview"] == ["foo", "bar", "looooooooooong string"] + assert f["list_of_list_stringview"] == '[["foo","bar","looooooooooong string"]]' + assert f["map_stringview"] == '{"x":"x_val","y":null}' + + +############################################################################### + + +@gdaltest.enable_exceptions() +def test_ogr_arrow_binary_view(): + + version = int( + ogr.GetDriverByName("ARROW").GetMetadataItem("ARROW_VERSION").split(".")[0] + ) + if version < 15: + pytest.skip("requires Arrow >= 15") + + with ogr.Open("data/arrow/binaryview.feather") as ds: + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f.GetFieldAsBinary("binaryview") == b"foo" + f = lyr.GetNextFeature() + assert f.GetFieldAsBinary("binaryview") == b"bar" + f = lyr.GetNextFeature() + assert f.GetFieldAsBinary("binaryview") == b"looooooooooong binary" diff --git a/autotest/ogr/ogr_csv.py b/autotest/ogr/ogr_csv.py index 6b8924722eee..0ab07472aed8 100755 --- a/autotest/ogr/ogr_csv.py +++ b/autotest/ogr/ogr_csv.py @@ -713,6 +713,9 @@ def test_ogr_csv_17(): def test_ogr_csv_write_to_stdout(): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + python_exe = sys.executable if sys.platform == "win32": python_exe = python_exe.replace("\\", "/") diff --git a/autotest/ogr/ogr_csw.py b/autotest/ogr/ogr_csw.py index 653de9c73981..b6fb800d4688 100755 --- a/autotest/ogr/ogr_csw.py +++ b/autotest/ogr/ogr_csw.py @@ -46,9 +46,14 @@ def module_disable_exceptions(): ############################################################################### @pytest.fixture(autouse=True, scope="module") def setup_and_cleanup(): + + vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") + with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"): yield + assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before + ############################################################################### # Test underlying OGR drivers diff --git a/autotest/ogr/ogr_flatgeobuf.py b/autotest/ogr/ogr_flatgeobuf.py index a37597cc044c..78a1c029eb2c 100644 --- a/autotest/ogr/ogr_flatgeobuf.py +++ b/autotest/ogr/ogr_flatgeobuf.py @@ -41,12 +41,6 @@ pytestmark = pytest.mark.require_driver("FlatGeobuf") -############################################################################### -@pytest.fixture(autouse=True, scope="module") -def module_disable_exceptions(): - with gdaltest.disable_exceptions(): - yield - ############################################################################### @pytest.fixture(autouse=True, scope="module") @@ -648,11 +642,10 @@ def test_ogr_flatgeobuf_huge_number_of_columns(): lyr.CreateField(ogr.FieldDefn("col%d" % i, ogr.OFTInteger)) == ogr.OGRERR_NONE ), i - with gdal.quiet_errors(): - assert ( - lyr.CreateField(ogr.FieldDefn("col65536", ogr.OFTInteger)) - == ogr.OGRERR_FAILURE - ) + with pytest.raises( + Exception, match="Cannot create features with more than 65536 columns" + ): + lyr.CreateField(ogr.FieldDefn("col65536", ogr.OFTInteger)) f = ogr.Feature(lyr.GetLayerDefn()) f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (0 0)")) for i in range(65536): @@ -765,7 +758,8 @@ def test_ogr_flatgeobuf_editing(): assert lyr.TestCapability(ogr.OLCDeleteFeature) == 1 assert lyr.DeleteFeature(1) == 0 - assert lyr.DeleteFeature(1) == ogr.OGRERR_NON_EXISTING_FEATURE + with pytest.raises(Exception, match="Non existing feature"): + lyr.DeleteFeature(1) assert lyr.TestCapability(ogr.OLCReorderFields) == 1 # assert lyr.ReorderFields([0, 1]) == 0 assert lyr.DeleteField(1) == 0 @@ -797,8 +791,8 @@ def test_ogr_flatgeobuf_editing(): f = ogr.Feature(lyr.GetLayerDefn()) f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (1 1)")) - with gdal.quiet_errors(): - assert lyr.CreateFeature(f) != ogr.OGRERR_NONE + with pytest.raises(Exception, match="not supported on read-only layer"): + lyr.CreateFeature(f) ogr.GetDriverByName("FlatGeobuf").DeleteDataSource("/vsimem/test.fgb") assert not gdal.VSIStatL("/vsimem/test.fgb") @@ -871,8 +865,27 @@ def test_ogr_flatgeobuf_read_invalid_geometries(filename): with gdal.quiet_errors(): ds = gdal.OpenEx(filename) lyr = ds.GetLayer(0) - for f in lyr: - pass + with pytest.raises(Exception, match="Fatal error parsing feature"): + for f in lyr: + pass + + +############################################################################### +# Check that we can read multilinestrings with a single part, without the +# "ends" array (cf https://github.com/OSGeo/gdal/issues/10774) + + +@pytest.mark.parametrize( + "filename", + [ + "data/flatgeobuf/test_ogr_flatgeobuf_singlepart_mls_new.fgb", + ], +) +def test_ogr_flatgeobuf_read_singlepart_mls_new(filename): + with gdal.OpenEx(filename) as ds: + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + ogrtest.check_feature_geometry(f, "MULTILINESTRING ((0 0,1 1))") ############################################################################### @@ -959,8 +972,8 @@ def test_ogr_flatgeobuf_coordinate_epoch_custom_wkt(): def test_ogr_flatgeobuf_invalid_output_filename(): ds = ogr.GetDriverByName("FlatGeobuf").CreateDataSource("/i_do/not_exist/my.fgb") - with gdal.quiet_errors(): - assert ds.CreateLayer("foo") is None + with pytest.raises(Exception, match="Failed to create"): + ds.CreateLayer("foo") ############################################################################### @@ -1208,12 +1221,11 @@ def test_ogr_flatgeobuf_issue_7401(): f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (0 0)")) lyr.CreateFeature(f) f = ogr.Feature(lyr.GetLayerDefn()) - lyr.CreateFeature(f) + with pytest.raises( + Exception, match="NULL geometry not supported with spatial index" + ): + lyr.CreateFeature(f) ds = None - assert ( - gdal.GetLastErrorMsg() - == "ICreateFeature: NULL geometry not supported with spatial index" - ) ogr.GetDriverByName("FlatGeobuf").DeleteDataSource("/vsimem/test.fgb") assert not gdal.VSIStatL("/vsimem/test.fgb") @@ -1419,3 +1431,152 @@ def test_ogr_flatgeobuf_write_mismatch_geom_type(tmp_vsimem): match="ICreateFeature: Mismatched geometry type. Feature geometry type is Line String, expected layer geometry type is Point", ): lyr.CreateFeature(f) + + +############################################################################### +# Test OGRGenSQLResultLayer::GetArrowStream() implementation. +# There isn't much specific of the FlatGeoBuf driver, except it is the +# only one in a default build that implements OLCFastGetArrowStream and doesn't +# have a specialized ExecuteSQL() implementation. + + +@gdaltest.enable_exceptions() +def test_ogr_flatgeobuf_sql_arrow(tmp_vsimem): + + filename = str(tmp_vsimem / "temp.fgb") + with ogr.GetDriverByName("FlatGeoBuf").CreateDataSource(filename) as ds: + lyr = ds.CreateLayer("test", geom_type=ogr.wkbPoint) + lyr.CreateField(ogr.FieldDefn("foo")) + lyr.CreateField(ogr.FieldDefn("bar")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "bar" + f["bar"] = "baz" + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (1 2)")) + lyr.CreateFeature(f) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "bar2" + f["bar"] = "baz2" + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (3 4)")) + lyr.CreateFeature(f) + + with ogr.Open(filename) as ds: + with ds.ExecuteSQL("SELECT 'a' FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + f = tmp_lyr.GetNextFeature() + assert f["FIELD_1"] == "a" + + with ds.ExecuteSQL("SELECT foo, foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT CONCAT(foo, 'x') FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT foo AS renamed, foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT bar, foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT CAST(foo AS float) FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT MIN(foo) FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT COUNT(*) FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test a JOIN test b ON a.foo = b.foo") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test OFFSET 1") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test ORDER BY foo") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT *, OGR_STYLE HIDDEN FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT DISTINCT foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test") as lyr: + try: + stream = lyr.GetArrowStreamAsNumPy() + except ImportError: + stream = None + if stream: + with pytest.raises( + Exception, + match=r"Calling get_next\(\) on a freed OGRLayer is not supported", + ): + [batch for batch in stream] + + sql = "SELECT foo, bar AS bar_renamed FROM test" + with ds.ExecuteSQL(sql) as lyr: + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetLayerDefn().GetFieldCount() == 2 + assert tmp_lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "foo" + assert tmp_lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "bar_renamed" + assert tmp_lyr.GetFeatureCount() == 2 + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar2" + assert f["bar_renamed"] == "baz2" + assert f.GetGeometryRef().ExportToWkt() == "POINT (3 4)" + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar" + assert f["bar_renamed"] == "baz" + assert f.GetGeometryRef().ExportToWkt() == "POINT (1 2)" + + sql = "SELECT bar FROM test LIMIT 1" + with ds.ExecuteSQL(sql) as lyr: + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetLayerDefn().GetFieldCount() == 1 + assert tmp_lyr.GetFeatureCount() == 1 + f = tmp_lyr.GetNextFeature() + assert f["bar"] == "baz2" + assert f.GetGeometryRef().ExportToWkt() == "POINT (3 4)" + + sql = "SELECT * EXCLUDE (\"_ogr_geometry_\") FROM test WHERE foo = 'bar'" + with ds.ExecuteSQL(sql) as lyr: + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetFeatureCount() == 1 + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar" + assert f["bar"] == "baz" + assert f.GetGeometryRef() is None + + sql = "SELECT * FROM test" + with ds.ExecuteSQL(sql) as lyr: + lyr.SetSpatialFilterRect(1, 2, 1, 2) + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetLayerDefn().GetFieldCount() == 2 + assert tmp_lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "foo" + assert tmp_lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "bar" + assert tmp_lyr.GetFeatureCount() == 1 + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar" + assert f["bar"] == "baz" + assert f.GetGeometryRef().ExportToWkt() == "POINT (1 2)" + f = tmp_lyr.GetNextFeature() diff --git a/autotest/ogr/ogr_geojson.py b/autotest/ogr/ogr_geojson.py index 8f1d9a254ec6..4bc636f9e01a 100755 --- a/autotest/ogr/ogr_geojson.py +++ b/autotest/ogr/ogr_geojson.py @@ -1806,7 +1806,11 @@ def test_ogr_geojson_48(tmp_vsimem): # Test UpdateFeature() support -def test_ogr_geojson_update_feature(tmp_vsimem): +@pytest.mark.parametrize("check_after_update_before_reopen", [True, False]) +@pytest.mark.parametrize("sync_to_disk_after_update", [True, False]) +def test_ogr_geojson_update_feature( + tmp_vsimem, check_after_update_before_reopen, sync_to_disk_after_update +): filename = str(tmp_vsimem / "test.json") @@ -1821,13 +1825,21 @@ def test_ogr_geojson_update_feature(tmp_vsimem): lyr = ds.GetLayer(0) f = ogr.Feature(lyr.GetLayerDefn()) f.SetFID(0) - f["int64list"] = [123456790123, -123456790123] + f["int64list"] = [-123456790123, 123456790123] lyr.UpdateFeature(f, [0], [], False) + if sync_to_disk_after_update: + lyr.SyncToDisk() + + if check_after_update_before_reopen: + lyr.ResetReading() + f = lyr.GetNextFeature() + assert f["int64list"] == [-123456790123, 123456790123] + with ogr.Open(filename) as ds: lyr = ds.GetLayer(0) f = lyr.GetNextFeature() - assert f["int64list"] == [123456790123, -123456790123] + assert f["int64list"] == [-123456790123, 123456790123] ############################################################################### @@ -2659,15 +2671,32 @@ def test_ogr_geojson_57(tmp_vsimem): got = read_file(tmp_vsimem / "out.json") gdal.Unlink(tmp_vsimem / "out.json") - expected = """{ -"type": "FeatureCollection", -"bbox": [ 45.0000000, 64.3861643, 135.0000000, 90.0000000 ], -"features": [ -{ "type": "Feature", "properties": { }, "bbox": [ 45.0, 64.3861643, 135.0, 90.0 ], "geometry": { "type": "Polygon", "coordinates": [ [ [ 135.0, 64.3861643 ], [ 135.0, 90.0 ], [ 45.0, 90.0 ], [ 45.0, 64.3861643 ], [ 135.0, 64.3861643 ] ] ] } } -] -} -""" - assert json.loads(got) == json.loads(expected) + expected = { + "type": "FeatureCollection", + "bbox": [45.0, 64.3861643, 135.0, 90.0], + "features": [ + { + "type": "Feature", + "properties": {}, + "bbox": [45.0, 64.3861643, 135.0, 90.0], + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [135.0, 64.3861643], + [135.0, 90.0], + [45.0, 90.0], + [45.0, 64.3861643], + [135.0, 64.3861643], + ] + ] + ], + }, + } + ], + } + assert json.loads(got) == expected # Polar case: slice of spherical cap crossing the antimeridian src_ds = gdal.GetDriverByName("Memory").Create("", 0, 0, 0) diff --git a/autotest/ogr/ogr_geojsonseq.py b/autotest/ogr/ogr_geojsonseq.py index 5d0862999e21..89acd3a61ca9 100755 --- a/autotest/ogr/ogr_geojsonseq.py +++ b/autotest/ogr/ogr_geojsonseq.py @@ -515,3 +515,25 @@ def test_ogr_geojsonseq_force_opening(tmp_vsimem): drv = gdal.IdentifyDriverEx("http://example.com", allowed_drivers=["GeoJSONSeq"]) assert drv.GetDescription() == "GeoJSONSeq" + + +############################################################################### +# Test WRITE_BBOX option + + +def test_ogr_geojsonseq_WRITE_BBOX(tmp_vsimem): + + filename = str(tmp_vsimem / "test.geojsonl") + ds = gdal.GetDriverByName("GeoJSONSeq").Create(filename, 0, 0, 0, gdal.GDT_Unknown) + lyr = ds.CreateLayer("test", options=["WRITE_BBOX=YES"]) + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("LINESTRING(2 49,3 50)")) + lyr.CreateFeature(f) + ds.Close() + + f = gdal.VSIFOpenL(filename, "rb") + assert f + data = gdal.VSIFReadL(1, 10000, f) + gdal.VSIFCloseL(f) + + assert b'"bbox": [ 2.0, 49.0, 3.0, 50.0 ]' in data diff --git a/autotest/ogr/ogr_geom.py b/autotest/ogr/ogr_geom.py index 935176d7c4f2..67911355b88c 100755 --- a/autotest/ogr/ogr_geom.py +++ b/autotest/ogr/ogr_geom.py @@ -3986,21 +3986,57 @@ def test_ogr_geom_makevalid(): g, "MULTIPOLYGON (((0 0,5 5,10 0,0 0)),((5 5,0 10,10 10,5 5)))" ) - if ( - ogr.GetGEOSVersionMajor() * 10000 - + ogr.GetGEOSVersionMinor() * 100 - + ogr.GetGEOSVersionMicro() - >= 31000 - ): - g = ogr.CreateGeometryFromWkt( - "POLYGON ((0 0,0 10,10 10,10 0,0 0),(5 5,15 10,15 0,5 5))" - ) - # Only since GEOS 3.10 - g = g.MakeValid(["METHOD=STRUCTURE"]) - if g is not None: - ogrtest.check_feature_geometry( - g, "POLYGON ((0 10,10 10,10.0 7.5,5 5,10.0 2.5,10 0,0 0,0 10))" - ) + +############################################################################### + + +@pytest.mark.require_geos(3, 10, 0) +def test_ogr_geom_makevalid_structure(): + + g = ogr.CreateGeometryFromWkt( + "POLYGON ((0 0,0 10,10 10,10 0,0 0),(5 5,15 10,15 0,5 5))" + ) + g = g.MakeValid(["METHOD=STRUCTURE"]) + ogrtest.check_feature_geometry( + g, "POLYGON ((0 10,10 10,10.0 7.5,5 5,10.0 2.5,10 0,0 0,0 10))" + ) + + # Already valid multi-polygon made of a single-part + g = ogr.CreateGeometryFromWkt("MULTIPOLYGON (((0 0,1 0,1 1,0 1,0 0)))") + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,1 0,1 1,0 1,0 0)))" + or g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,0 1,1 1,1 0,0 0)))" + ) + + # Already valid multi-polygon made of a single-part, with duplicated point + g = ogr.CreateGeometryFromWkt("MULTIPOLYGON (((0 0,1 0,1 0,1 1,0 1,0 0)))") + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,1 0,1 1,0 1,0 0)))" + or g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,0 1,1 1,1 0,0 0)))" + ) + + # Already valid multi-polygon made of a single-part + g = ogr.CreateGeometryFromWkt( + "MULTIPOLYGON Z (((0 0 10,1 0 10,1 1 10,0 1 10,0 0 10)))" + ) + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "MULTIPOLYGON Z (((0 0 10,1 0 10,1 1 10,0 1 10,0 0 10)))" + or g.ExportToIsoWkt() + == "MULTIPOLYGON Z (((0 0 10,0 1 10,1 1 10,1 0 10,0 0 10)))" + ) + + # Already valid geometry collection + g = ogr.CreateGeometryFromWkt( + "GEOMETRYCOLLECTION (POLYGON ((0 0,1 0,1 1,0 1,0 0)))" + ) + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "GEOMETRYCOLLECTION (POLYGON ((0 0,1 0,1 1,0 1,0 0)))" + or g.ExportToIsoWkt() == "GEOMETRYCOLLECTION (POLYGON ((0 0,0 1,1 1,1 0,0 0)))" + ) ############################################################################### diff --git a/autotest/ogr/ogr_gml.py b/autotest/ogr/ogr_gml.py index d4624b100a7b..4c60b5dc0dcf 100755 --- a/autotest/ogr/ogr_gml.py +++ b/autotest/ogr/ogr_gml.py @@ -2558,6 +2558,25 @@ def test_ogr_gml_64(parser): assert feat is not None, parser +############################################################################### +# Test we don't spend too much time parsing documents featuring the billion +# laugh attack + + +@gdaltest.enable_exceptions() +@pytest.mark.parametrize("parser", ("XERCES", "EXPAT")) +def test_ogr_gml_billion_laugh(parser): + + with gdal.config_option("GML_PARSER", parser), pytest.raises( + Exception, match="File probably corrupted" + ): + with gdal.OpenEx("data/gml/billionlaugh.gml") as ds: + assert ds.GetDriver().GetDescription() == "GML" + for lyr in ds: + for f in lyr: + pass + + ############################################################################### # Test SRSDIMENSION_LOC=GEOMETRY option (#5606) @@ -4626,3 +4645,18 @@ def test_ogr_gml_get_layers_by_name_from_imported_schema_more_tests(tmp_path): assert isinstance( dat_erst, list ), f"Expected 'dat_erst' to be of type 'list', but got {type(dat_erst)}" + + +############################################################################### +# Test force opening a GML file + + +def test_ogr_gml_force_opening(tmp_vsimem): + + filename = str(tmp_vsimem / "test.gml") + gdal.FileFromMemBuffer(filename, open("data/nas/empty_nas.xml", "rb").read()) + + # Would be opened by NAS driver if not forced + with gdal.config_option("NAS_GFS_TEMPLATE", ""): + ds = gdal.OpenEx(filename, allowed_drivers=["GML"]) + assert ds.GetDriver().GetDescription() == "GML" diff --git a/autotest/ogr/ogr_gmlas.py b/autotest/ogr/ogr_gmlas.py index d3a82e6467c9..5cafaeafde7f 100755 --- a/autotest/ogr/ogr_gmlas.py +++ b/autotest/ogr/ogr_gmlas.py @@ -3485,3 +3485,19 @@ def test_ogr_gmlas_ossfuzz_70511(): Exception, match="Cannot get type definition for attribute y" ): gdal.OpenEx("GMLAS:", open_options=["XSD=data/gmlas/test_ossfuzz_70511.xsd"]) + + +############################################################################### +# Test we don't spend too much time parsing documents featuring the billion +# laugh attack + + +@gdaltest.enable_exceptions() +def test_ogr_gmlas_billion_laugh(): + + with gdal.quiet_errors(), pytest.raises(Exception, match="File probably corrupted"): + with gdal.OpenEx("GMLAS:data/gml/billionlaugh.gml") as ds: + assert ds.GetDriver().GetDescription() == "GMLAS" + for lyr in ds: + for f in lyr: + pass diff --git a/autotest/ogr/ogr_index_test.py b/autotest/ogr/ogr_index_test.py index a96af7e7c72b..0044b44ccbe0 100755 --- a/autotest/ogr/ogr_index_test.py +++ b/autotest/ogr/ogr_index_test.py @@ -48,18 +48,16 @@ def startup_and_cleanup(): @contextlib.contextmanager def create_index_p_test_file(): drv = ogr.GetDriverByName("MapInfo File") - p_ds = drv.CreateDataSource("index_p.mif") - p_lyr = p_ds.CreateLayer("index_p") + with drv.CreateDataSource("index_p.mif") as p_ds: + p_lyr = p_ds.CreateLayer("index_p") - ogrtest.quick_create_layer_def(p_lyr, [("PKEY", ogr.OFTInteger)]) - ogrtest.quick_create_feature(p_lyr, [5], None) - ogrtest.quick_create_feature(p_lyr, [10], None) - ogrtest.quick_create_feature(p_lyr, [9], None) - ogrtest.quick_create_feature(p_lyr, [4], None) - ogrtest.quick_create_feature(p_lyr, [3], None) - ogrtest.quick_create_feature(p_lyr, [1], None) - - p_ds.Release() + ogrtest.quick_create_layer_def(p_lyr, [("PKEY", ogr.OFTInteger)]) + ogrtest.quick_create_feature(p_lyr, [5], None) + ogrtest.quick_create_feature(p_lyr, [10], None) + ogrtest.quick_create_feature(p_lyr, [9], None) + ogrtest.quick_create_feature(p_lyr, [4], None) + ogrtest.quick_create_feature(p_lyr, [3], None) + ogrtest.quick_create_feature(p_lyr, [1], None) yield @@ -69,21 +67,19 @@ def create_index_p_test_file(): @contextlib.contextmanager def create_join_t_test_file(create_index=False): drv = ogr.GetDriverByName("ESRI Shapefile") - s_ds = drv.CreateDataSource("join_t.dbf") - s_lyr = s_ds.CreateLayer("join_t", geom_type=ogr.wkbNone) - - ogrtest.quick_create_layer_def( - s_lyr, [("SKEY", ogr.OFTInteger), ("VALUE", ogr.OFTString, 16)] - ) + with drv.CreateDataSource("join_t.dbf") as s_ds: + s_lyr = s_ds.CreateLayer("join_t", geom_type=ogr.wkbNone) - for i in range(20): - ogrtest.quick_create_feature(s_lyr, [i, "Value " + str(i)], None) + ogrtest.quick_create_layer_def( + s_lyr, [("SKEY", ogr.OFTInteger), ("VALUE", ogr.OFTString, 16)] + ) - if create_index: - s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") - s_ds.ExecuteSQL("CREATE INDEX ON join_t USING skey") + for i in range(20): + ogrtest.quick_create_feature(s_lyr, [i, "Value " + str(i)], None) - s_ds.Release() + if create_index: + s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") + s_ds.ExecuteSQL("CREATE INDEX ON join_t USING skey") yield @@ -188,12 +184,9 @@ def test_ogr_index_indexed_join_works(): def test_ogr_index_drop_index_removes_files(): with create_join_t_test_file(create_index=True): - s_ds = ogr.OpenShared("join_t.dbf", update=1) - - s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") - s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") - - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") + s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") # After dataset closing, check that the index files do not exist after # dropping the index @@ -228,17 +221,13 @@ def test_ogr_index_attribute_filter_works_after_drop_index(): def test_ogr_index_recreating_index_causes_index_files_to_be_created(): with create_join_t_test_file(create_index=True): - s_ds = ogr.OpenShared("join_t.dbf", update=1) - - s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") - s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") - - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") + s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") # Re-create an index - s_ds = ogr.OpenShared("join_t.dbf", update=1) - s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") for filename in ["join_t.idm", "join_t.ind"]: try: @@ -255,17 +244,13 @@ def test_ogr_index_recreating_index_causes_index_files_to_be_created(): def test_ogr_index_recreating_index_causes_index_to_be_populated(): with create_join_t_test_file(create_index=True): - s_ds = ogr.OpenShared("join_t.dbf", update=1) - - s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") - s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") - - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") + s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") # Re-create an index - s_ds = ogr.OpenShared("join_t.dbf", update=1) - s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") with open("join_t.idm", "rt") as f: xml = f.read() @@ -280,25 +265,19 @@ def test_ogr_index_recreating_index_causes_index_to_be_populated(): def test_ogr_index_creating_index_in_separate_steps_works(): with create_join_t_test_file(create_index=True): - s_ds = ogr.OpenShared("join_t.dbf", update=1) - - s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") - s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") - - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + s_ds.ExecuteSQL("DROP INDEX ON join_t USING value") + s_ds.ExecuteSQL("DROP INDEX ON join_t USING skey") # Re-create an index - s_ds = ogr.OpenShared("join_t.dbf", update=1) - s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + s_ds.ExecuteSQL("CREATE INDEX ON join_t USING value") # Close the dataset and re-open - s_ds = ogr.OpenShared("join_t.dbf", update=1) - # At this point the .ind was opened in read-only. Now it - # will be re-opened in read-write mode - s_ds.ExecuteSQL("CREATE INDEX ON join_t USING skey") - - s_ds.Release() + with ogr.OpenShared("join_t.dbf", update=1) as s_ds: + # At this point the .ind was opened in read-only. Now it + # will be re-opened in read-write mode + s_ds.ExecuteSQL("CREATE INDEX ON join_t USING skey") with open("join_t.idm", "rt") as f: xml = f.read() diff --git a/autotest/ogr/ogr_jsonfg.py b/autotest/ogr/ogr_jsonfg.py index a94636f7af96..e95a7e5b92db 100755 --- a/autotest/ogr/ogr_jsonfg.py +++ b/autotest/ogr/ogr_jsonfg.py @@ -950,7 +950,7 @@ def test_jsonfg_write_several_layers(): [["GEOMETRYCOLLECTION (POINT (1 2))"], ogr.wkbGeometryCollection], [["GEOMETRYCOLLECTION Z (POINT Z (1 2 3))"], ogr.wkbGeometryCollection25D], [["POINT (1 2)", "LINESTRING (1 2,3 4)"], ogr.wkbUnknown], - [["POLYHEDRALSURFACE EMPTY"], ogr.wkbPolyhedralSurface], + [["POLYHEDRALSURFACE Z EMPTY"], ogr.wkbPolyhedralSurfaceZ], [ [ "POLYHEDRALSURFACE Z (((0 0 0,0 1 0,1 1 0,0 0 0)),((0 0 0,1 0 0,0 0 1,0 0 0)),((0 0 0,0 1 0,0 0 1,0 0 0)),((0 1 0,1 0 0,0 0 1,0 1 0)))" diff --git a/autotest/ogr/ogr_lvbag.py b/autotest/ogr/ogr_lvbag.py index 24154c580b6d..250aeeafffbb 100644 --- a/autotest/ogr/ogr_lvbag.py +++ b/autotest/ogr/ogr_lvbag.py @@ -603,3 +603,14 @@ def test_ogr_lvbag_test_ogrsf_num(): ) assert "INFO" in ret and "ERROR" not in ret + + +############################################################################### +# Test force opening + + +def test_ogr_lvbag_force_opening(): + + # Would be opened by GML driver if not forced + ds = gdal.OpenEx("data/gml/empty.gml", allowed_drivers=["LVBAG"]) + assert ds.GetDriver().GetDescription() == "LVBAG" diff --git a/autotest/ogr/ogr_nas.py b/autotest/ogr/ogr_nas.py index c548635f9706..2190fe6d47f0 100755 --- a/autotest/ogr/ogr_nas.py +++ b/autotest/ogr/ogr_nas.py @@ -304,3 +304,17 @@ def test_ogr_nas_force_opening(tmp_vsimem): with gdal.quiet_errors(): ds = gdal.OpenEx(filename, allowed_drivers=["NAS"]) assert ds.GetDriver().GetDescription() == "NAS" + + +############################################################################### +# Test we don't spend too much time parsing documents featuring the billion +# laugh attack + + +def test_ogr_nas_billion_laugh(): + + with gdal.config_option("NAS_GFS_TEMPLATE", ""): + with gdal.quiet_errors(), pytest.raises( + Exception, match="File probably corrupted" + ): + ogr.Open("data/nas/billionlaugh.xml") diff --git a/autotest/ogr/ogr_oapif.py b/autotest/ogr/ogr_oapif.py index b9b76ab58e71..53156723229e 100755 --- a/autotest/ogr/ogr_oapif.py +++ b/autotest/ogr/ogr_oapif.py @@ -66,7 +66,15 @@ def test_ogr_oapif_errors(): handler = webserver.SequentialHandler() handler.add("GET", "/oapif/collections", 404) with webserver.install_http_handler(handler): - with pytest.raises(Exception): + with pytest.raises(Exception, match="HTTP error code : 404"): + ogr.Open("OAPIF:http://localhost:%d/oapif" % gdaltest.webserver_port) + + handler = webserver.SequentialHandler() + handler.add("GET", "/oapif/collections", 404, {}, "unavailable resource") + with webserver.install_http_handler(handler): + with pytest.raises( + Exception, match="HTTP error code : 404, unavailable resource" + ): ogr.Open("OAPIF:http://localhost:%d/oapif" % gdaltest.webserver_port) # No Content-Type diff --git a/autotest/ogr/ogr_ogrtest.py b/autotest/ogr/ogr_ogrtest.py index ae0a80ff5e33..3389bf833877 100644 --- a/autotest/ogr/ogr_ogrtest.py +++ b/autotest/ogr/ogr_ogrtest.py @@ -87,7 +87,7 @@ def test_check_geometry_equals_ngeoms_mismatch(): def test_check_geometry_equals_orientation_differs(): poly_ccw = ogr.CreateGeometryFromWkt("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))") - poly_cw = ogr.CreateGeometryFromWkt("POLYGON ((0 0, 0 1, 1 1, 0 1, 0 0))") + poly_cw = ogr.CreateGeometryFromWkt("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))") if ogrtest.have_geos(): ogrtest.check_feature_geometry(poly_ccw, poly_cw) diff --git a/autotest/ogr/ogr_openfilegdb.py b/autotest/ogr/ogr_openfilegdb.py index 7d3f41f63591..2d1724f7d1e8 100755 --- a/autotest/ogr/ogr_openfilegdb.py +++ b/autotest/ogr/ogr_openfilegdb.py @@ -2870,3 +2870,92 @@ def test_ogr_openfilegdb_read_objectid_64bit_sparse_test_ogrsf(ogrsf_path): success = "INFO" in ret and "ERROR" not in ret assert success + + +############################################################################### +# Test reading http:// resource + + +@pytest.mark.require_curl() +@pytest.mark.require_driver("HTTP") +def test_ogr_openfilegdb_read_from_http(): + + import webserver + + (webserver_process, webserver_port) = webserver.launch( + handler=webserver.DispatcherHttpHandler + ) + if webserver_port == 0: + pytest.skip() + + response = open("data/filegdb/testopenfilegdb.gdb.zip", "rb").read() + + try: + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/foo", + 200, + { + "Content-Disposition": 'attachment; filename="foo.gdb.zip"; size=' + + str(len(response)) + }, + response, + ) + with webserver.install_http_handler(handler): + ds = gdal.OpenEx( + "http://localhost:%d/foo" % webserver_port, + allowed_drivers=["OpenFileGDB", "HTTP"], + ) + assert ds is not None + assert ds.GetLayerCount() != 0 + + # If we have the GeoJSON driver, there will be one initial GET done by it + if ogr.GetDriverByName("GeoJSON"): + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/foo", + 200, + { + "Content-Disposition": 'attachment; filename="foo.gdb.zip"; size=' + + str(len(response)) + }, + response, + ) + handler.add( + "GET", + "/foo", + 200, + { + "Content-Disposition": 'attachment; filename="foo.gdb.zip"; size=' + + str(len(response)) + }, + response, + ) + with webserver.install_http_handler(handler): + ds = gdal.OpenEx( + "http://localhost:%d/foo" % webserver_port, + allowed_drivers=["GeoJSON", "OpenFileGDB", "HTTP"], + ) + assert ds is not None + assert ds.GetLayerCount() != 0 + + finally: + webserver.server_stop(webserver_process, webserver_port) + + +############################################################################### +# Test reading a geometry where there is an arc with an interior point, but +# it is actually flagged as a line + + +def test_ogr_openfilegdb_arc_interior_point_bug_line(): + + with ogr.Open("data/filegdb/arc_segment_interior_point_but_line.gdb.zip") as ds: + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + ogrtest.check_feature_geometry( + f, + "MULTILINESTRING ((37252520.1717 7431529.9154,38549084.9654 758964.7573))", + ) diff --git a/autotest/ogr/ogr_parquet.py b/autotest/ogr/ogr_parquet.py index 2563010535ac..0d34e48f9a54 100755 --- a/autotest/ogr/ogr_parquet.py +++ b/autotest/ogr/ogr_parquet.py @@ -3870,6 +3870,8 @@ def check(lyr): lyr = ds.GetLayer(0) lyr.SetIgnoredFields(["foo"]) check(lyr) + lyr.SetSpatialFilter(geom) + assert lyr.GetFeatureCount() == (3 if geom.GetGeometryCount() > 1 else 2) ds = ogr.Open(filename_to_open) lyr = ds.GetLayer(0) @@ -4125,3 +4127,42 @@ def test_ogr_parquet_vsi_arrow_file_system(): ds = ogr.Open("PARQUET:vsi://data/parquet/test.parquet") lyr = ds.GetLayer(0) assert lyr.GetFeatureCount() > 0 + + +############################################################################### + + +@gdaltest.enable_exceptions() +@pytest.mark.require_driver("ARROW") +@pytest.mark.parametrize( + "src_filename,expected_error_msg", + [ + ("data/arrow/stringview.feather", "StringView not supported"), + ("data/arrow/binaryview.feather", "BinaryView not supported"), + ], +) +def test_ogr_parquet_IsArrowSchemaSupported_arrow_15_types( + src_filename, expected_error_msg, tmp_vsimem +): + + version = int( + ogr.GetDriverByName("ARROW").GetMetadataItem("ARROW_VERSION").split(".")[0] + ) + if version < 15: + pytest.skip("requires Arrow >= 15.0.0") + + src_ds = ogr.Open(src_filename) + src_lyr = src_ds.GetLayer(0) + + outfilename = str(tmp_vsimem / "test.parquet") + with ogr.GetDriverByName("Parquet").CreateDataSource(outfilename) as dst_ds: + dst_lyr = dst_ds.CreateLayer( + "test", srs=src_lyr.GetSpatialRef(), geom_type=ogr.wkbPoint, options=[] + ) + + stream = src_lyr.GetArrowStream() + schema = stream.GetSchema() + + success, error_msg = dst_lyr.IsArrowSchemaSupported(schema) + assert not success + assert error_msg == expected_error_msg diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index 9c6b39eec499..65e1ac68ffaa 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -247,17 +247,21 @@ def pg_version(pg_autotest_ds): feat = sql_lyr.GetNextFeature() v = feat.GetFieldAsString("version") - pos = v.find(" ") # "PostgreSQL 12.0beta1" or "PostgreSQL 12.2 ...." + # return of version() is something like "PostgreSQL 12.0[rcX|betaX] ...otherstuff..." + + tokens = v.split(" ") + assert len(tokens) >= 2 + # First token is "PostgreSQL" (or some enterprise DB alternative name) + v = tokens[1] + pos = v.find("beta") if pos > 0: - v = v[pos + 1 :] - pos = v.find("beta") - if pos > 0: - v = v[0:pos] - pos = v.find(" ") + v = v[0:pos] + else: + pos = v.find("rc") if pos > 0: v = v[0:pos] - return tuple([int(x) for x in v.split(".")]) + return tuple([int(x) for x in v.split(".")]) @pytest.fixture(scope="module") diff --git a/autotest/ogr/ogr_refcount.py b/autotest/ogr/ogr_refcount.py index fac4027994cf..d76c0304ee4b 100755 --- a/autotest/ogr/ogr_refcount.py +++ b/autotest/ogr/ogr_refcount.py @@ -83,15 +83,17 @@ def test_ogr_refcount_2(): # Verify that releasing the datasources has the expected behaviour. +@pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_ogr_refcount_3(): ds_1 = ogr.OpenShared("data/idlink.dbf") - ds_3 = ogr.OpenShared("data/idlink.dbf") - assert ds_1 is not None - assert ds_3 is not None - ds_3.Release() + ds_2 = ogr.OpenShared("data/idlink.dbf") + assert ds_2 is not None + assert ds_1.GetRefCount() == 2 + assert ds_2.GetRefCount() == 2 + ds_2.Release() assert ds_1.GetRefCount() == 1 diff --git a/autotest/ogr/ogr_sqlite.py b/autotest/ogr/ogr_sqlite.py index 762f66c868bc..0a6b72b37e3f 100755 --- a/autotest/ogr/ogr_sqlite.py +++ b/autotest/ogr/ogr_sqlite.py @@ -4149,6 +4149,123 @@ def test_ogr_sqlite_stddev(): assert f.GetField(1) == pytest.approx(0.5**0.5, rel=1e-15) +@pytest.mark.parametrize( + "input_values,expected_res", + [ + ([], None), + ([1], 1), + ([2.5, None, 1], 1.75), + ([3, 2.2, 1], 2.2), + ([1, "invalid"], None), + ], +) +def test_ogr_sqlite_median(input_values, expected_res): + """Test MEDIAN""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + for v in input_values: + ds.ExecuteSQL( + "INSERT INTO test VALUES (%s)" + % ( + "NULL" + if v is None + else ("'" + v + "'") + if isinstance(v, str) + else str(v) + ) + ) + if expected_res is None and input_values: + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT MEDIAN(v) FROM test"): + pass + else: + with ds.ExecuteSQL("SELECT MEDIAN(v) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == pytest.approx(expected_res) + with ds.ExecuteSQL("SELECT PERCENTILE(v, 50) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == pytest.approx(expected_res) + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, 0.5) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == pytest.approx(expected_res) + + +def test_ogr_sqlite_percentile(): + """Test PERCENTILE""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + ds.ExecuteSQL("INSERT INTO test VALUES (5),(6),(4),(7),(3),(8),(2),(9),(1),(10)") + + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, 'invalid') FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, -0.1) FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, 100.1) FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, v) FROM test"): + pass + + +def test_ogr_sqlite_percentile_cont(): + """Test PERCENTILE_CONT""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + ds.ExecuteSQL("INSERT INTO test VALUES (5),(6),(4),(7),(3),(8),(2),(9),(1),(10)") + + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, 'invalid') FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, -0.1) FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, 1.1) FROM test"): + pass + + +@pytest.mark.parametrize( + "input_values,expected_res", + [ + ([], None), + ([1, 2, None, 3, 2], 2), + (["foo", "bar", "baz", "bar"], "bar"), + ([1, "foo", 2, "foo", "bar"], "foo"), + ([1, "foo", 2, "foo", 1], "foo"), + ], +) +def test_ogr_sqlite_mode(input_values, expected_res): + """Test MODE""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + for v in input_values: + ds.ExecuteSQL( + "INSERT INTO test VALUES (%s)" + % ( + "NULL" + if v is None + else ("'" + v + "'") + if isinstance(v, str) + else str(v) + ) + ) + if expected_res is None and input_values: + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT MODE(v) FROM test"): + pass + else: + with ds.ExecuteSQL("SELECT MODE(v) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == expected_res + + def test_ogr_sqlite_run_deferred_actions_before_start_transaction(): ds = ogr.Open(":memory:", update=1) diff --git a/autotest/ogr/ogr_vrt.py b/autotest/ogr/ogr_vrt.py index f93640c197a2..e5afc296eb04 100755 --- a/autotest/ogr/ogr_vrt.py +++ b/autotest/ogr/ogr_vrt.py @@ -1331,11 +1331,13 @@ def test_ogr_vrt_29(tmp_path): sr.ImportFromEPSG(4326) lyr = ds.CreateLayer("ogr_vrt_29", srs=sr) lyr.CreateField(ogr.FieldDefn("id", ogr.OFTInteger)) + lyr.CreateField(ogr.FieldDefn("str", ogr.OFTString)) for i in range(5): for j in range(5): feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField(0, i * 5 + j) + feat["str"] = f"{j}-{i}" feat.SetGeometry( ogr.CreateGeometryFromWkt("POINT(%f %f)" % (2 + i / 5.0, 49 + j / 5.0)) ) @@ -1483,11 +1485,15 @@ def test_ogr_vrt_29(tmp_path): ), "did not get expected extent" feat = lyr.GetNextFeature() + assert feat["id"] == 0 + assert feat["str"] == "0-0" ogrtest.check_feature_geometry( feat, "POINT(426857.987717275274917 5427937.523466162383556)" ) feat = lyr.GetNextFeature() + assert feat["id"] == 1 + assert feat["str"] == "1-0" feat.SetGeometry(None) assert lyr.SetFeature(feat) == 0 diff --git a/autotest/ogr/ogr_wfs.py b/autotest/ogr/ogr_wfs.py index 7873234f9d2a..3fe3b91695ed 100755 --- a/autotest/ogr/ogr_wfs.py +++ b/autotest/ogr/ogr_wfs.py @@ -63,9 +63,13 @@ def ogr_wfs_init(): if gml_ds is None: pytest.skip("cannot read GML files") + vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") + with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"): yield + assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before + @pytest.fixture( params=["NO", None], scope="module", ids=["without-streaming", "with-streaming"] diff --git a/autotest/osr/osr_basic.py b/autotest/osr/osr_basic.py index 5b6a2aef0d20..d49aada966fd 100755 --- a/autotest/osr/osr_basic.py +++ b/autotest/osr/osr_basic.py @@ -1839,6 +1839,9 @@ def threaded_function(arg): ############################################################################### +@pytest.mark.skipif( + gdaltest.is_travis_branch("sanitize"), reason="fails on sanitize for unknown reason" +) def test_Set_PROJ_DATA_config_option_sub_proccess_config_option_ok(): backup_search_paths = osr.GetPROJSearchPaths() @@ -1857,6 +1860,9 @@ def test_Set_PROJ_DATA_config_option_sub_proccess_config_option_ok(): ############################################################################### +@pytest.mark.skipif( + gdaltest.is_travis_branch("sanitize"), reason="fails on sanitize for unknown reason" +) def test_Set_PROJ_DATA_config_option_sub_proccess_config_option_ko(): backup_search_paths = osr.GetPROJSearchPaths() diff --git a/autotest/pyscripts/test_gdal_fillnodata.py b/autotest/pyscripts/test_gdal_fillnodata.py index 84af78214693..eac38284187d 100755 --- a/autotest/pyscripts/test_gdal_fillnodata.py +++ b/autotest/pyscripts/test_gdal_fillnodata.py @@ -30,6 +30,7 @@ import struct +import gdaltest import pytest import test_py_scripts @@ -52,6 +53,9 @@ def script_path(): def test_gdal_fillnodata_help(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdal_fillnodata", "--help" ) @@ -63,6 +67,9 @@ def test_gdal_fillnodata_help(script_path): def test_gdal_fillnodata_version(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdal_fillnodata", "--version" ) diff --git a/autotest/pyscripts/test_gdalcompare.py b/autotest/pyscripts/test_gdalcompare.py index 904e24ea8edd..bfa928f952a1 100644 --- a/autotest/pyscripts/test_gdalcompare.py +++ b/autotest/pyscripts/test_gdalcompare.py @@ -31,6 +31,7 @@ import shutil +import gdaltest import pytest import test_py_scripts @@ -74,6 +75,9 @@ def source_filename(tmp_vsimem): def test_gdalcompare_help(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdalcompare", "--help" ) @@ -85,6 +89,9 @@ def test_gdalcompare_help(script_path): def test_gdalcompare_version(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "gdalcompare", "--version" ) diff --git a/autotest/utilities/test_gdal_rasterize.py b/autotest/utilities/test_gdal_rasterize.py index fa4c510f93be..f6049bd5becb 100755 --- a/autotest/utilities/test_gdal_rasterize.py +++ b/autotest/utilities/test_gdal_rasterize.py @@ -425,7 +425,7 @@ def test_gdal_rasterize_8(gdal_rasterize_path, tmp_path): f.write('"LINESTRING (0 0, 5 5, 10 0, 10 10)",1'.encode("ascii")) f.close() - cmds = f"""{input_csv} {output_tif} -init 0 -burn 1 -tr 1 1""" + cmds = f"""{input_csv} {output_tif} -tr 1 1 -init 0 -burn 1""" gdaltest.runexternal(gdal_rasterize_path + " " + cmds) diff --git a/autotest/utilities/test_gdal_rasterize_lib.py b/autotest/utilities/test_gdal_rasterize_lib.py index 67b2ba8d6d42..fa8103c7adf2 100755 --- a/autotest/utilities/test_gdal_rasterize_lib.py +++ b/autotest/utilities/test_gdal_rasterize_lib.py @@ -792,3 +792,37 @@ def test_gdal_rasterize_lib_dict_arguments(): ind = opt.index("-co") assert opt[ind : ind + 4] == ["-co", "COMPRESS=DEFLATE", "-co", "LEVEL=4"] + + +############################################################################### +# Test doesn't crash without options + + +def test_gdal_rasterize_no_options(tmp_vsimem): + """Test doesn't crash without options""" + + gdal.FileFromMemBuffer( + tmp_vsimem / "test.json", + r"""{ + "type": "FeatureCollection", + "name": "test", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::4326" } }, + "features": [ + { "type": "Feature", "properties": { "id": 1 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 0, 0 ], [ 0, 1 ], [ 1, 1 ], [ 1, 0 ], [ 0, 0 ] ] ] } } + ] + }""", + ) + + # Open the dataset + ds = gdal.OpenEx(tmp_vsimem / "test.json", gdal.OF_VECTOR) + assert ds + + # Create a raster to rasterize into. + target_ds = gdal.GetDriverByName("GTiff").Create( + tmp_vsimem / "out.tif", 10, 10, 1, gdal.GDT_Byte + ) + + assert target_ds + + # Call rasterize + ds = gdal.Rasterize(target_ds, ds) diff --git a/autotest/utilities/test_gdal_viewshed.py b/autotest/utilities/test_gdal_viewshed.py index ca572b803e66..725550858839 100755 --- a/autotest/utilities/test_gdal_viewshed.py +++ b/autotest/utilities/test_gdal_viewshed.py @@ -210,6 +210,39 @@ def test_gdal_viewshed_all_options(gdal_viewshed_path, tmp_path, viewshed_input) assert nodata == 0 +############################################################################### + +# NOTE: Various compilers (notably Intel), may give different values when +# doing floating point math (because of -ffast-math, for example). +# The test below checks that the COUNT of visible cells is the same, but it is +# not the case that the actual cells marked visible aren't different. That the +# count is the same is luck. If changes are made or compilers/options change, +# the expected value in the test below may need to be changed/added to in order +# to accommodate all the compilers with a single test. See +# ViewshedExecutor::setOutput for the comparison that is at issue. +# +def test_gdal_viewshed_cumulative(gdal_viewshed_path, tmp_path, viewshed_input): + + np = pytest.importorskip("numpy") + + viewshed_out = str(tmp_path / "test_gdal_viewshed_out.tif") + + _, err = gdaltest.runexternal_out_and_err( + gdal_viewshed_path + + " -om ACCUM -f GTiff -os 5 -a_nodata 0 {} {}".format( + viewshed_input, viewshed_out + ) + ) + assert err is None or err == "" + ds = gdal.Open(viewshed_out) + assert ds + vis_count = np.count_nonzero(ds.GetRasterBand(1).ReadAsArray()) + nodata = ds.GetRasterBand(1).GetNoDataValue() + ds = None + assert vis_count == 13448 + assert nodata == 0 + + ############################################################################### @@ -330,7 +363,7 @@ def test_gdal_viewshed_missing_ox(gdal_viewshed_path): _, err = gdaltest.runexternal_out_and_err( gdal_viewshed_path + " /dev/null /dev/null" ) - assert "-ox: required" in err + assert "Option -ox is required." in err ############################################################################### @@ -341,7 +374,7 @@ def test_gdal_viewshed_missing_oy(gdal_viewshed_path): _, err = gdaltest.runexternal_out_and_err( gdal_viewshed_path + " -ox 0 /dev/null /dev/null" ) - assert "-oy: required" in err + assert "Option -oy is required." in err ############################################################################### diff --git a/autotest/utilities/test_gdalinfo_lib.py b/autotest/utilities/test_gdalinfo_lib.py index 8c7cfd3ead44..995f6fa4b15d 100755 --- a/autotest/utilities/test_gdalinfo_lib.py +++ b/autotest/utilities/test_gdalinfo_lib.py @@ -341,3 +341,14 @@ def test_gdalinfo_lib_nomask(tmp_path): ret = gdal.Info(ds, format="json", showMask=False) assert "mask" not in ret["bands"][0] + + +############################################################################### + + +def test_gdalinfo_lib_json_stac_common_name(): + + ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + ds.GetRasterBand(1).SetColorInterpretation(gdal.GCI_PanBand) + ret = gdal.Info(ds, options="-json") + assert ret["stac"]["eo:bands"][0]["common_name"] == "pan" diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index 3134d0a8d9dd..2d55e07c212a 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -4102,7 +4102,7 @@ def DISABLED_test_gdalwarp_lib_to_projection_without_inverse_method(): def test_gdalwarp_lib_no_crash_on_none_dst(): ds1 = gdal.Open("../gcore/data/byte.tif") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.Warp(None, ds1) @@ -4259,3 +4259,31 @@ def test_gdalwarp_lib_minus_180_plus_180_to_span_over_180(tmp_vsimem, extra_colu ) == src_ds.GetRasterBand(1).ReadRaster( 0, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize ) + + +############################################################################### +# Test bugfix for https://lists.osgeo.org/pipermail/gdal-dev/2024-September/059512.html + + +@pytest.mark.parametrize("with_tap", [True, False]) +def test_gdalwarp_lib_blank_edge_one_by_one(with_tap): + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + src_ds.SetGeoTransform([6.8688, 0.0009, 0, 51.3747, 0, -0.0009]) + srs = osr.SpatialReference() + srs.SetFromUserInput("WGS84") + srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + src_ds.SetSpatialRef(srs) + options = "-f MEM -tr 1000 1000 -t_srs EPSG:32631" + if with_tap: + options += " -tap" + out_ds = gdal.Warp("", src_ds, options=options) + assert out_ds.RasterXSize == 1 + assert out_ds.RasterYSize == 1 + gt = out_ds.GetGeoTransform() + if with_tap: + assert gt == pytest.approx((769000.0, 1000.0, 0.0, 5699000.0, 0.0, -1000.0)) + else: + assert gt == pytest.approx( + (769234.6506516202, 1000.0, 0.0, 5698603.782217737, 0.0, -1000.0) + ) diff --git a/autotest/utilities/test_ogr2ogr_lib.py b/autotest/utilities/test_ogr2ogr_lib.py index dbfec4cba68f..f51f7cd1ace3 100755 --- a/autotest/utilities/test_ogr2ogr_lib.py +++ b/autotest/utilities/test_ogr2ogr_lib.py @@ -2829,3 +2829,105 @@ def test_ogr2ogr_lib_skip_invalid(tmp_vsimem): lyr = ds.GetLayer(0) assert lyr.GetFeatureCount() == 1 ds = None + + +############################################################################### +# Test -t_srs in Arrow code path + + +@gdaltest.enable_exceptions() +@pytest.mark.parametrize("force_reproj_threading", [False, True]) +@pytest.mark.parametrize("source_driver", ["GPKG", "Parquet"]) +def test_ogr2ogr_lib_reproject_arrow(tmp_vsimem, source_driver, force_reproj_threading): + + src_driver = gdal.GetDriverByName(source_driver) + if src_driver is None: + pytest.skip(f"{source_driver} is not available") + src_filename = str(tmp_vsimem / ("in." + source_driver.lower())) + with src_driver.Create(src_filename, 0, 0, 0, gdal.GDT_Unknown) as srcDS: + srs = osr.SpatialReference() + srs.ImportFromEPSG(32631) + srcLayer = srcDS.CreateLayer("test", srs=srs) + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT(500000 4500000)")) + srcLayer.CreateFeature(f) + f = ogr.Feature(srcLayer.GetLayerDefn()) + srcLayer.CreateFeature(f) + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT(500000 4000000)")) + srcLayer.CreateFeature(f) + + config_options = {"CPL_DEBUG": "ON", "OGR2OGR_USE_ARROW_API": "YES"} + if force_reproj_threading: + config_options["OGR2OGR_MIN_FEATURES_FOR_THREADED_REPROJ"] = "0" + + with gdal.OpenEx(src_filename) as src_ds: + for i in range(2): + + got_msg = [] + + def my_handler(errorClass, errno, msg): + got_msg.append(msg) + return + + with gdaltest.error_handler(my_handler), gdaltest.config_options( + config_options + ): + ds = gdal.VectorTranslate( + "", src_ds, format="Memory", dstSRS="EPSG:4326" + ) + + assert "OGR2OGR: Using WriteArrowBatch()" in got_msg + + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 3 + f = lyr.GetNextFeature() + ogrtest.check_feature_geometry(f, "POINT(3 40.65085651557158)") + f = lyr.GetNextFeature() + assert f.GetGeometryRef() is None + f = lyr.GetNextFeature() + ogrtest.check_feature_geometry(f, "POINT(3 36.14471809881776)") + + +############################################################################### +# Test -t_srs in Arrow code path in a situation where it cannot be triggered +# currently (source CRS is crossing anti-meridian) + + +@gdaltest.enable_exceptions() +@pytest.mark.require_geos +@pytest.mark.require_driver("GPKG") +def test_ogr2ogr_lib_reproject_arrow_optim_cannot_trigger(tmp_vsimem): + + src_filename = str(tmp_vsimem / "in.gpkg") + with gdal.GetDriverByName("GPKG").Create( + src_filename, 0, 0, 0, gdal.GDT_Unknown + ) as srcDS: + srs = osr.SpatialReference() + srs.ImportFromEPSG(32660) + srcLayer = srcDS.CreateLayer("test", srs=srs) + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry( + ogr.CreateGeometryFromWkt( + "LINESTRING(657630.64 4984896.17,815261.43 4990738.26)" + ) + ) + srcLayer.CreateFeature(f) + + got_msg = [] + + def my_handler(errorClass, errno, msg): + got_msg.append(msg) + return + + config_options = {"CPL_DEBUG": "ON", "OGR2OGR_USE_ARROW_API": "YES"} + with gdaltest.error_handler(my_handler), gdaltest.config_options(config_options): + ds = gdal.VectorTranslate("", src_filename, format="Memory", dstSRS="EPSG:4326") + + assert "OGR2OGR: Using WriteArrowBatch()" not in got_msg + + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + f = lyr.GetNextFeature() + assert f.GetGeometryRef().GetGeometryType() == ogr.wkbMultiLineString + assert f.GetGeometryRef().GetGeometryCount() == 2 diff --git a/ci/travis/osx/before_install.sh b/ci/travis/osx/before_install.sh index 0dbb594ab30b..e0f953c54f4b 100755 --- a/ci/travis/osx/before_install.sh +++ b/ci/travis/osx/before_install.sh @@ -8,5 +8,6 @@ conda install -y compilers automake pkgconfig cmake conda config --set channel_priority strict conda install --yes --quiet proj python=3.12 swig lxml jsonschema numpy setuptools conda install --yes --quiet libgdal libgdal-arrow-parquet +conda install --yes --quiet libavif # Now remove all libgdal* packages, but not their dependencies conda remove --yes --force $(conda list libgdal | grep libgdal | awk '{print $1}') diff --git a/cmake/helpers/CheckDependentLibraries.cmake b/cmake/helpers/CheckDependentLibraries.cmake index 122cf53a932e..0ca740ea69e0 100644 --- a/cmake/helpers/CheckDependentLibraries.cmake +++ b/cmake/helpers/CheckDependentLibraries.cmake @@ -449,6 +449,8 @@ gdal_check_package(MONGOCXX "Enable MongoDBV3 driver" CAN_DISABLE) define_find_package2(HEIF libheif/heif.h heif PKGCONFIG_NAME libheif) gdal_check_package(HEIF "HEIF >= 1.1" CAN_DISABLE) +include(CheckDependentLibrariesAVIF) + include(CheckDependentLibrariesOpenJPEG) gdal_check_package(HDFS "Enable Hadoop File System through native library" CAN_DISABLE) diff --git a/cmake/helpers/CheckDependentLibrariesAVIF.cmake b/cmake/helpers/CheckDependentLibrariesAVIF.cmake new file mode 100644 index 000000000000..77f31d5bf5d1 --- /dev/null +++ b/cmake/helpers/CheckDependentLibrariesAVIF.cmake @@ -0,0 +1,2 @@ +define_find_package2(AVIF avif/avif.h avif PKGCONFIG_NAME libavif) +gdal_check_package(AVIF "AVIF" CAN_DISABLE) diff --git a/doc/old_trac_pointers.md b/doc/old_trac_pointers.md new file mode 100644 index 000000000000..b40174b7562b --- /dev/null +++ b/doc/old_trac_pointers.md @@ -0,0 +1,8 @@ +Google Summer of Code ideas: https://web.archive.org/web/20240812224510/https://trac.osgeo.org/gdal/wiki/SummerOfCode +Google Summer of Code selected projects: https://web.archive.org/web/20240812232901/https://trac.osgeo.org/gdal/wiki/SoCProjects +Old Sponsorship program: http://web.archive.org/web/20230921154549/https://trac.osgeo.org/gdal/wiki/Sponsorship +Old Build hints: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/BuildHints +Additional notes on utilities: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/UserDocs +CodeSnippets: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/CodeSnippets +LayerAlgebra: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/LayerAlgebra +Old FAQ: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/FAQ diff --git a/doc/source/_extensions/configoptions.py b/doc/source/_extensions/configoptions.py index 3f1e15c85e9e..a1864376226a 100644 --- a/doc/source/_extensions/configoptions.py +++ b/doc/source/_extensions/configoptions.py @@ -104,7 +104,14 @@ def render(self, app): elif len(self.choices) == 1: text += f"={self.choices[0]}: " else: - text += f'=[{"/".join(self.choices)}]: ' + # Insert Unicode zero-width space around separating forward slash, + # so that all browsers produce line breaks. + # Cf https://github.com/OSGeo/gdal/issues/10778 + zero_width_space = "\u200b" + concatenated_choices = f"{zero_width_space}/{zero_width_space}".join( + self.choices + ) + text += f"=[{concatenated_choices}]: " para += nodes.strong(text, text) diff --git a/doc/source/api/csharp/csharp_compile_legacy.rst b/doc/source/api/csharp/csharp_compile_legacy.rst index da817c98207a..e9579c045e3f 100644 --- a/doc/source/api/csharp/csharp_compile_legacy.rst +++ b/doc/source/api/csharp/csharp_compile_legacy.rst @@ -84,7 +84,7 @@ To test the compiled binaries, you can use: nmake /f makefile.vc test` -This command will invoke some of the sample applications. +This command will invoke some of the sample applications. .. note:: For the tests to work the location of the proj and gdal DLLs should be available in the PATH. @@ -93,7 +93,7 @@ Using MONO on Windows If you have the Windows version of the MONO package installed you can compile the C# code using the MONO compiler. In this case uncomment the following entry in csharp.opt: -:program:`MONO = YES` +:program:`MONO = YES` .. note:: mcs.exe must be in the PATH. @@ -165,7 +165,7 @@ To test the compiled binaries, you can use: nmake test -This command will invoke some of the sample applications. +This command will invoke some of the sample applications. .. note:: For the tests to work the location of the proj and gdal libraries should be available in the PATH. @@ -179,8 +179,4 @@ To run one of the prebuilt executables - you can run them with Mono as follows : :program:`mono GDALInfo.exe` Both the managed libraries (i.e. the DLLs) and the unmanaged libraries must be available to Mono. -This is in more detail in `the Mono documentation `__ - -.. note:: This document was amended from the previous version at `https://trac.osgeo.org/gdal/wiki/GdalOgrCsharpCompile `__ - - +This is in more detail in `the Mono documentation `__ diff --git a/doc/source/api/csharp/csharp_raster.rst b/doc/source/api/csharp/csharp_raster.rst index ac0e23ea40b8..fde6af99db4b 100644 --- a/doc/source/api/csharp/csharp_raster.rst +++ b/doc/source/api/csharp/csharp_raster.rst @@ -15,40 +15,40 @@ The :file:`Band` class contains the following :file:`ReadRaster`/:file:`WriteRas .. code-block:: C# - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, IntPtr buffer, i nt buf_xSize, int buf_ySize, DataType buf_type, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, IntPtr buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, IntPtr buffer, int buf_xSize, int buf_ySize, DataType buf_type, int pixelSpace, int lineSpace){} The only difference between these functions is the actual type of the buffer parameter. @@ -76,7 +76,7 @@ When reading the image this way the C# API will copy the image data between the band.ReadRaster(0, 0, width, height, r, width, height, 0, 0); // Copying the pixels into the C# bitmap int i, j; - for (i = 0; i< width; i++) + for (i = 0; i< width; i++) { for (j=0; j`__ diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst index 127aea2f66a1..cb4d070b787d 100644 --- a/doc/source/api/index.rst +++ b/doc/source/api/index.rst @@ -117,7 +117,6 @@ API Go Julia - Lua Original Node.js bindings Node.js fork with full Promise-based async and TypeScript support Perl @@ -126,10 +125,6 @@ API Ruby Rust - .. warning:: - For Perl, since GDAL 3.5 the link `Perl `__ is deprecated, use above link instead. - - There are also more Pythonic ways of using the vector/OGR functions with diff --git a/doc/source/development/building_from_source.rst b/doc/source/development/building_from_source.rst index 6a067888836e..adaa100d5bb8 100644 --- a/doc/source/development/building_from_source.rst +++ b/doc/source/development/building_from_source.rst @@ -2546,5 +2546,5 @@ crashes will occur at runtime (often at process termination with a Autoconf/nmake (GDAL versions < 3.5.0) -------------------------------------------------------------------------------- -See https://trac.osgeo.org/gdal/wiki/BuildHints for hints for GDAL < 3.5 +See http://web.archive.org/https://trac.osgeo.org/gdal/wiki/BuildHints for hints for GDAL < 3.5 autoconf and nmake build systems. diff --git a/doc/source/development/rfc/index.rst b/doc/source/development/rfc/index.rst index f0763fe8d0ca..73f24bcf634e 100644 --- a/doc/source/development/rfc/index.rst +++ b/doc/source/development/rfc/index.rst @@ -106,3 +106,4 @@ RFC list rfc97_feature_and_fielddefn_sealing rfc98_build_requirements_gdal_3_9 rfc99_geometry_coordinate_precision + rfc101_raster_dataset_threadsafety diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst new file mode 100644 index 000000000000..be9ff44135fe --- /dev/null +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -0,0 +1,482 @@ +.. _rfc-101: + +=================================================================== +RFC 101: Raster dataset read-only thread-safety +=================================================================== + +============== ============================================= +Author: Even Rouault +Contact: even.rouault @ spatialys.com +Started: 2024-Aug-29 +Status: Adopted, implemented +Target: GDAL 3.10 +============== ============================================= + +Summary +------- + +This RFC enables users to get instances of :cpp:class:`GDALDataset` +(and their related objects such as :cpp:class:`GDALRasterBand`) that are +thread-safe for read-only raster operations, that is such instances can +be safely used from multiple threads without locking. + +Terminology +----------- + +The exact meaning of the terms ``thread-safe`` or ``re-entrant`` is not fully +standardized. We will use here the `QT definitions `__. +In particular, a C function or C++ method is said to be re-entrant if it can +be called simultaneously from multiple threads, *but* only if each invocation +uses its own data/instance. On the contrary, it is thread-safe is if can be +called on the same data/instance (so thread-safe is stronger than re-entrant) + +Motivation +---------- + +A number of raster algorithms can be designed to read chunks of a raster in +an independent and concurrent way, with resulting speed-ups when using +multi-threading. Currently, given a GDALDataset instance is not thread-safe, +this requires either to deal with I/O in a single thread, or through a mutex +to protect against concurrent use, or one needs to open a separate GDALDataset +for each worker thread. Both approaches can complicate the writing of such +algorithms. The enhancement of this RFC aims at providing a special GDALDataset +instance that can be used safely from multiple threads. Internally, it does use +one GDALDataset per thread, but hides this implementation detail to the user. + +C and C++ API extensions +------------------------ + +A new ``GDAL_OF_THREAD_SAFE`` opening flag is added to be specified to +:cpp:func:`GDALOpenEx` / :cpp:func:`GDALDataset::Open`. This flag is for now +mutually exclusive with ``GDAL_OF_VECTOR``, ``GDAL_OF_MULTIDIM_RASTER`` and +``GDAL_OF_UPDATE``. That is this flag is only meant for read-only raster +operations (``GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE``). + +To know if a given dataset can be used in a thread-safe way, the following +C++ method is added to the GDALDataset class: + +.. code-block:: c++ + + /** Return whether this dataset, and its related objects (typically raster + * bands), can be called for the intended scope. + * + * Note that in the current implementation, nScopeFlags should be set to + * GDAL_OF_RASTER, as thread-safety is limited to read-only operations and + * excludes operations on vector layers (OGRLayer) or multidimensional API + * (GDALGroup, GDALMDArray, etc.) + * + * This is the same as the C function GDALDatasetIsThreadSafe(). + * + * @since 3.10 + */ + bool IsThreadSafe(int nScopeFlags) const; + + +The corresponding C function is added: + +.. code-block:: c + + bool GDALDatasetIsThreadSafe(GDALDatasetH hDS, int nScopeFlags, + CSLConstList papszOptions); + + +A new C++ function, GDALGetThreadSafeDataset, is added with two forms: + +.. code-block:: c++ + + std::unique_ptr GDALGetThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags); + + GDALDataset* GDALGetThreadSafeDataset(GDALDataset* poDS, int nScopeFlags); + +This function accepts a (generally non thread-safe) source dataset and return +a new dataset that is a thread-safe wrapper around it, or the source dataset if +it is already thread-safe. +The nScopeFlags argument must be compulsory set to GDAL_OF_RASTER to express that +the intended scope is read-only raster operations (other values will result in +an error and a NULL returned dataset). +This function is used internally by GDALOpenEx() when the GDAL_OF_THREAD_SAFE +flag is passed to wrap the dataset returned by the driver. +The first form takes ownership of the source dataset. The second form does not, +but references it internally, and assumes that its lifetime will be longer than +the lifetime of the returned thread-safe dataset. Note that the second form does +increase the reference count on the passed dataset while it is being used, so +patterns like the following one are valid: + +.. code-block:: c++ + + auto poDS = GDALDataset::Open(...); + GDALDataset* poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); + poDS->ReleaseRef(); // can be done here, or any time later + if (poThreadSafeDS ) + { + // ... do something with poThreadSafeDS ... + poThreadSafeDS->ReleaseRef(); + } + + +For proper working both when a new dataset is returned or the passed one if it +is already thread-safe, :cpp:func:`GDALDataset::ReleaseRef()` (and not delete or +GDALClose()) must be called on the returned dataset. + + +The corresponding C function for the second form is added: + +.. code-block:: c + + GDALDatasetH GDALGetThreadSafeDataset(GDALDatasetH hDS, int nScopeFlags, CSLConstList papszOptions); + + +Usage examples +-------------- + +Example of a function processing a whole dataset passed as a filename: + +.. code-block:: c++ + + void foo(const char* pszFilename) + { + auto poDS = std::unique_ptr(GDALDataset::Open( + pszFilename, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE | GDAL_OF_VERBOSE_ERROR)); + if( !poDS ) + { + return; + } + + // TODO: spawn threads using poDS + } + + +Example of a function processing a whole dataset passed as an object: + +.. code-block:: c++ + + void foo(GDALDataset* poDS) + { + GDALDataset* poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER); + if( poThreadSafeDS ) + { + // TODO: spawn threads using poThreadSafeDS + + poThreadSafeDS->ReleaseRef(); + } + else + { + // TODO: Serial version of the algorithm. It can happen if + // poDS is a on-the-fly dataset. + } + } + + +Example of a function processing a single band passed as an object: + +.. code-block:: c++ + + void foo(GDALRasterBand* poBand) + { + GDALDataset* poThreadSafeDS = nullptr; + GDALRasterBand* poThreadSafeBand = nullptr; + GDALDataset* poDS = poBand->GetDataset(); + // Check that poBand has a matching owing dataset + if( poDS && poDS->GetRasterBand(poBand->GetBand()) == poBand ) + { + poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER); + if( poThreadSafeDS ) + poThreadSafeBand = poThreadSafeDS->GetBand(poBand->GetBand()); + } + + if( poThreadSafeBand ) + { + // TODO: spawn threads using poThreadSafeBand + + poThreadSafeDS->ReleaseRef(); + } + else + { + // TODO: Serial version of the algorithm. It can happen if + // poBand is a on-the-fly band, or a "detached" band, such as a + // mask band, or an overview band as returned by some drivers. + } + } + + +SWIG bindings +------------- + +The new C macro and functions are bound to SWIG as: + +- ``gdal.OF_THREAD_SAFE`` +- :py:func:`Dataset.IsThreadSafe(nScopeFlags)` +- :py:func:`Dataset.GetThreadSafeDataset(nScopeFlags)`. The Python + implementation of this method takes care of keeping a reference on the source + dataset in the returned thread-safe dataset, so the user does not have to + care about their respective lifetimes. + +Usage and design limitations +---------------------------- + +* As implied by the RFC title, the scope of thread-safety is restricted to + **raster** and **read-only** operations. + +* For GDALDataset instances pointing to a file on the regular filesystem, the + limitation of the maximum number of file descriptor opened by a process + (1024 on most Linux systems) could be hit if working with a sufficiently large + number of worker threads and/or instances of GDALThreadSafeDataset. + +* The generic implementation of GDALGetThreadSafeDataset assumes that the + source dataset can be re-opened by its name (GetDescription()), which is the + case for datasets opened by GDALOpenEx(). A special implementation is also + made for dataset instances of the MEM driver. But, there is currently no + support for creating a thread-safe dataset wrapper on on-the-fly datasets + returned by some algorithms (e.g GDALTranslate() or GDALWarp() with VRT as + the output driver and with an empty filename, or custom GDALDataset + implementation by external code). + +* Inherent to the selected approach, there is a band block cache per thread, and + thus no sharing of cached blocks between threads. + However, this should not be a too severe limitation for algorithms where + threads process independent regions of the raster, hence reuse of cached blocks + would be non-existent or low. Optimal algorithms will make sure to work on + regions of interest aligned on the block size (this advice also applies for + the current approach of manually opening a dataset for each worker thread). + +* Due to implementation limitations, :cpp:func:`GDALRasterBand::GetDefaultRAT` + on a GDALThreadSafeDataset instance only works if the RAT is an instance of + :cpp:class:`GDALDefaultRasterAttributeTable`. An error is emitted if + this is not the case. This could potentially be extended to work with any + subclass of :cpp:class:`GDALRasterAttributeTable` but with significant + additional coding to create a thread-safe wrapper. (GDALDefaultRasterAttributeTable + is intrinsically thread-safe for read-only operations). This is not perceived + as a limitation for the intended use cases of this RFC (reading pixel values + in parallel). + +* Some drivers, like netCDF, and HDF5 in some builds, use a global lock around + each call to their APIs, due to the underlying libraries not being re-entrant. + Obviously scalability of GDALThreadSafeDataset will be limited by such global + lock. + But this is no different than the approach of opening as many dataset as + worker threads. + +Implementation details +---------------------- + +(This section is mostly of interest for developers familiar with GDAL internals +and may be skipped by users of the GDAL API) + +The gist of the implementation lies in a new file ``gcore/gdalthreadsafedataset.cpp`` +which defines several classes (internal details): + +- ``GDALThreadSafeDataset`` extending :cpp:class:`GDALProxyDataset`. + Instances of that class are returned by GDALGetThreadSafeDataset(). + On instantiation, it creates as many GDALThreadSafeRasterBand instances as + the number of bands of the source dataset. + All virtual methods of GDALDataset are redefined by GDALProxyDataset. + GDALThreadSafeDataset overloads its ReferenceUnderlyingDataset method, so that + a thread-local dataset is opened the first-time a thread calls a method on + the GDALThreadSafeDataset instance, cached for later use, and method call is + generally forwarded to it. There are exceptions for methods like + :cpp:func:`GDALDataset::GetSpatialRef`, :cpp:func:`GDALDataset::GetGCPSpatialRef`, + :cpp:func:`GDALDataset::GetGCPs`, :cpp:func:`GDALDataset::GetMetadata`, + :cpp:func:`GDALDataset::GetMetadataItem` that return non-primitive types + where the calls are forwarded to the dataset used to construct GDALThreadSafeDataset, + with a mutex being taken around them. If the call was otherwise forwarded to + a thread-local instance, there would be a risk of use-after-free situations + when the returned value is used by different threads. + +- ``GDALThreadSafeRasterBand`` extending :cpp:class:`GDALProxyRasterBand`. + On instantiation, it creates child GDALThreadSafeRasterBand instances for + band mask and overviews. + Its ReferenceUnderlyingRasterBand method calls ReferenceUnderlyingDataset + on the GDALThreadSafeDataset instance to get a thread-local dataset, fetches + the appropriate thread-local band and generally forwards its the method call. + There are exceptions for methods like + :cpp:func:`GDALRasterBand::GetUnitType`, :cpp:func:`GDALRasterBand::GetMetadata`, + :cpp:func:`GDALRasterBand::GetMetadataItem` that return non-primitive types where + the calls are forwarded to the band used to construct GDALThreadSafeRasterBand, + with a mutex being taken around them, and the returned value being . + +- ``GDALThreadLocalDatasetCache``. Instances of that class use thread-local + storage. The main member of such instances is a LRU cache that maps + GDALThreadSafeDataset* instances to a thread specific GDALDataset smart pointer. + On GDALThreadSafeDataset destruction, there's code to iterate over all + alive GDALThreadLocalDatasetCache instances and evict no-longer needed entries + in them, within a per-GDALThreadLocalDatasetCache instance mutex, to avoid + issues when dealing with several instances of GDALThreadLocalDatasetCache... + Note that the existence of this mutex should not cause performance issues, since + contention on it, should be very low in real-world use cases (it could become + a bottleneck if GDALThreadSafeDataset were created and destroyed at a very + high pace) + +Two protected virtual methods are added to GDALDataset for GDALThreadSafeDataset +implementation, and may be overloaded by drivers if needed (but it is not +anticipated that drivers but the MEM driver need to do that) + +- ``bool CanBeCloned(int nScopeFlags, bool bCanShareState) const``. + This method determines if a source dataset can be "cloned" (or re-opened). + It returns true for instances returned by GDALOpenEx, for instances of the MEM + driver if ``nScopeFlags`` == ``GDAL_OF_RASTER`` (and ``bCanShareState`` is + true for instances of the MEM driver) + +- ``std::unique_ptr Clone(int nScopeFlags, bool bCanShareState) const``. + This method returns a "clone" of the dataset on which it is called, and is used + by GDALThreadSafeDataset::ReferenceUnderlyingDataset() when a thread-local + dataset is needed. + Implementation of that method must be thread-safe. + The base implementation calls GDALOpenEx() reusing the dataset name, open flags + and open option. It is overloaded in the MEM driver to return a new instance + of MEMDataset, but sharing the memory buffers with the source dataset. + +No code in drivers, but the MEM driver, is modified by the candidate +implementation. + +A few existing non-virtual methods of GDALDataset and GDALRasterBand have been +made virtual (and overloaded by GDALProxyDataset and GDALProxyRasterBand), +to avoid modifying state on the GDALThreadSafeRasterBand instance, which +wouldn't be thread-safe. + +- :cpp:func:`GDALDataset::BlockBasedRasterIO`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::GetLockedBlockRef`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::TryGetLockedBlockRef`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::FlushBlock`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::InterpolateAtPoint`: + it uses a per-band cache +- :cpp:func:`GDALRasterBand::EnablePixelTypeSignedByteWarning`: it should + already have been made virtual for GDALProxyRasterBand needs. + +Non-virtual methods :cpp:func:`GDALDataset::GetProjectionRef` and +:cpp:func:`GDALDataset::GetGCPProjection`, which cache the return value, have +been modify to apply a mutex when run on a dataset that IsThreadSafe() to be +effectively thread-safe. + +A SetThreadSafe() method has been added to :cpp:class:`OGRSpatialReference`. +When it is called, all methods of that class run under a per-instance (recursive) +mutex. This is used by GDALThreadSafeDataset for its implementation of the +:cpp:func:`GDALDataset::GetSpatialRef` and :cpp:func:`GDALDataset::GetGCPSpatialRef` +methods, such that the returned OGRSpatialReference instances are thread-safe. + +Performance +----------- + +The existing multireadtest utility that reads a dataset from multiple threads +has been extended with a -thread_safe flag to asks to use GDAL_OF_THREAD_SAFE +when opening the dataset in the main thread and use it in the worker threads, +instead of the default behavior of opening explicitly a dataset in each thread. +The thread-safe mode shows similar scalability as the default mode, sometimes +with a slightly decreased efficiency, but not in a too problematic way. + +For example on a 20x20 raster: + +.. code-block:: shell + + $ time multireadtest -t 4 -i 1000000 20x20.tif + real 0m2.084s + user 0m8.155s + sys 0m0.020s + + vs + + $ time multireadtest -thread_safe -t 4 -i 1000000 20x20.tif + real 0m2.387s + user 0m9.334s + sys 0m0.029s + + +But on a 4096x4096 raster with a number of iterations reduced to 100, the +timings between the default and thread_safe modes are very similar. + +A Python equivalent of multireadtest has been written. Scalability depends +on how much Python code is executed. If relatively few long-enough calls to +GDAL are done, scalability tends to be good due to the Python Global Interpreter +Lock (GIL) being dropped around them. If many short calls are done, the GIL +itself, or its repeated acquisition and release, becomes the bottleneck. This is +no different than using a GDALDataset per thread. + +Documentation +------------- + +Documentation for the new constant and functions will be added. The +:ref:`multithreading` page will be updated to reflect the new capability +introduced by this RFC. + +Backward compatibility +---------------------- + +No issue anticipated: the C and C++ API are extended. +The C++ ABI is modified due to additions of new virtual methods. + +Testing +------- + +Tests will be added for the new functionality, including stress tests to have +sufficiently high confidence in the correctness of the implementation for common +use cases. + +Risks +----- + +Like all code related to multi-threading, the C++ language and tooling offers +hardly any safety belt against thread-safety programming errors. So it cannot +be excluded that the implementation suffers from bugs in some edge scenarios, +or in the usage of some methods of GDALDataset, GDALRasterBand and related objects +(particularly existing non-virtual methods of those classes that could happen +to have a non thread-safe implementation) + +Design discussion +----------------- + +This paragraph discusses a number of thoughts that arose during the writing of +this RFC. + +1. A significantly different alternative could have consisted in adding native + thread-safety in each driver. But this is not realistic for the following reasons: + + * if that was feasible, it would require considerable development effort to + rework each drivers. So realistically, only a few select drivers would be updated. + + * Even updating a reduced number of drivers would be extremely difficult, in + particular the GeoTIFF one, due to the underlying library not being reentrant, + and deferred loading strategies and many state variables being modified even + by read-only APIs. And this applies to most typical drivers. + + * Due to the inevitable locks, there would be a (small) cost bore by callers + even on single-thread uses of thread-safe native drivers. + + * Some core mechanisms, particularly around the per-band block cache structures, + are not currently thread-safe. + +2. A variant of the proposed implementation that did not use thread-local storage + has been initially attempted. It stored instead a + ``std::map>`` on each GDALThreadSafeDataset + instance. This implementation was simpler, but unfortunately suffered from high + lock contention since a mutex had to be taken around each access to this map, + with the contention increasing with the number of concurrent threads. + +3. For the unusual situations where a dataset cannot be reopened and thus + GDALGetThreadSafeDataset() fails, should we provide an additional ``bForce`` + argument to force it to still return a dataset, where calls to the wrapped + dataset are protected by a mutex? This would enable to always write multi-thread + safe code, even if the access to the dataset is serialized. + Similarly we could have a + ``std::unique_ptr GDALGetThreadSafeRasterBand(GDALRasterBand* poBand, int nOpenFlags, bool bForce)`` + function that would try to use GDALGetThreadSafeDataset() internally if it + manages to identify the dataset to which the band belongs to, and otherwise would + fallback to protecting calls to the wrapped band with a mutex. + + Given the absence of evidence that such option is necessary, this has been excluded + from the scope of this RFC. + + +Related issues and PRs +---------------------- + +- Candidate implementation: https://github.com/OSGeo/gdal/pull/10746 + +- https://github.com/OSGeo/gdal/issues/8448: GTiff: Allow concurrent reading of single blocks + +Voting history +-------------- + ++1 from PSC members KurtS, JukkaR, JavierJS and EvenR diff --git a/doc/source/development/rfc/rfc12_filemanagement.rst b/doc/source/development/rfc/rfc12_filemanagement.rst index 6a7c448ae9ce..7f59bb83618b 100644 --- a/doc/source/development/rfc/rfc12_filemanagement.rst +++ b/doc/source/development/rfc/rfc12_filemanagement.rst @@ -24,7 +24,7 @@ GetFileList() ------------- The following new virtual method is added on the GDALDataset class, with -an analygous C function. +an analogous C function. :: diff --git a/doc/source/development/rfc/rfc48_geographical_networks_support.rst b/doc/source/development/rfc/rfc48_geographical_networks_support.rst index 0e6bcd2d40fb..32a453bf77cd 100644 --- a/doc/source/development/rfc/rfc48_geographical_networks_support.rst +++ b/doc/source/development/rfc/rfc48_geographical_networks_support.rst @@ -19,7 +19,7 @@ project “GDAL/OGR Geography Network support” into GDAL library. GNM create, manage and analyse networks built over spatial data in GDAL. GSoC project description: -`http://trac.osgeo.org/gdal/wiki/geography_network_support `__ +`http://web.archive.org/web/20240812232429/https://trac.osgeo.org/gdal/wiki/geography_network_support `__ GDAL fork with all changes in trunk: `https://github.com/MikhanGusev/gdal `__ diff --git a/doc/source/download.rst b/doc/source/download.rst index 7fb82cf4bd68..4a613a0d8efc 100644 --- a/doc/source/download.rst +++ b/doc/source/download.rst @@ -97,6 +97,25 @@ GDAL packages are available on `Homebrew`_. .. _`Homebrew`: https://formulae.brew.sh/formula/gdal +Android +....... + +GDAL can be installed using :ref:`vcpkg`. You may also refer to `vcpkg Android support `__ for general instructions. + +For example to install default configuration for the ``arm64-android`` target: + +.. code-block:: shell + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh # ./bootstrap-vcpkg.bat for Windows + ./vcpkg integrate install + export ANDROID_NDK_HOME=/path/to/android_ndk_home # to adapt + ./vcpkg search gdal --featurepackages # list optional features + ./vcpkg install gdal:arm64-android # install with default configuration + ./vcpkg install gdal[poppler,netcdf]:arm64-android # install with Poppler and netdf support + + Cross-Platform Package Managers ............................... @@ -193,13 +212,15 @@ vcpkg The GDAL port in the `vcpkg `__ dependency manager is kept up to date by Microsoft team members and community contributors. You can download and install GDAL using the vcpkg as follows: -:: +.. code-block:: shell git clone https://github.com/Microsoft/vcpkg.git cd vcpkg ./bootstrap-vcpkg.sh # ./bootstrap-vcpkg.bat for Windows ./vcpkg integrate install - ./vcpkg install gdal + ./vcpkg search gdal --featurepackages # list optional features + ./vcpkg install gdal # install with default configuration + ./vcpkg install gdal[poppler,netcdf] # install with Poppler and netdf support If the version is out of date, please `create an issue or pull request `__ on the vcpkg repository. diff --git a/doc/source/drivers/raster/avif.rst b/doc/source/drivers/raster/avif.rst new file mode 100644 index 000000000000..22d81a6313e5 --- /dev/null +++ b/doc/source/drivers/raster/avif.rst @@ -0,0 +1,122 @@ +.. _raster.avif: + +================================================================================ +AVIF -- AV1 Image File Format +================================================================================ + +.. versionadded:: 3.10 + +.. shortname:: AVIF + +.. build_dependencies:: libavif + +AV1 Image File Format (AVIF) is an open, royalty-free image file format +specification for storing images or image sequences compressed with AV1 in the +HEIF container format. + +It supports 8-bit, 10-bit and 12-bit images, single-bland, single-band and +alpha channel, RGB and RGBA. + +Files containing animations (several images) will be exposed as GDAL subdatasets. + +Compression and decompression is done on entire images, so the driver can not +handle arbitrary sizes images. + +Note: read-only support for AVIF files is also available through the +:ref:`raster.heif` driver if the AVIF driver is not available, and if libheif +has been compiled with an AV1 compatible decoder. + +Driver capabilities +------------------- + +.. supports_virtualio + +.. supports_createcopy + +Color Profile Metadata +---------------------- + +GDAL can deal with the following color profile +metadata in the COLOR_PROFILE domain: + +- SOURCE_ICC_PROFILE (Base64 encoded ICC profile embedded in file.) + +Creation options +---------------- + +|about-creation-options| +The following creation options are supported: + +- .. co:: CODEC + :choices: AUTO, AOM, RAV1E, SVT + :default: AUTO + + Compression library to use. Choices available depend on how libavif has + been built. + +- .. co:: QUALITY + :choices: [0-100] + :default: 60 + + Quality for non-alpha channels. 0 is the lowest quality, 100 is for + lossless encoding. Default is 60. + +- .. co:: QUALITY_ALPHA + :choices: [0-100] + :default: 100 + + Quality for alpha channel. 0 is the lowest quality, 100 is for + lossless encoding. Default is 100/lossless. + +- .. co:: SPEED + :choices: [0-10] + :default: 6 + + Speed of encoding. 0=slowest. 10=fastest. + +- .. co:: NUM_THREADS + :choices: |ALL_CPUS + :default: ALL_CPUS + + Number of worker threads for compression. + +- .. co:: SOURCE_ICC_PROFILE + + ICC profile encoded in Base64. Can also be + set to empty string to avoid the ICC profile from the source dataset to be used. + +- .. co:: WRITE_EXIF_METADATA + :choices: YES, NO + :default: YES + + Whether to write EXIF metadata present in source file. + +- .. co:: WRITE_XMP + :choices: YES, NO + :default: YES + + Whether to write XMP metadata present in source file. + +- .. co:: NBITS + :choices: 8, 10, 12 + + Bit depth. + +- .. co:: YUV_SUBSAMPLING + :choices: 444, 422, 420 + :default: 444 + + Type of `chroma subsampling ` + to apply to YUV channels for RGB or RGBA images (it is ignored for single + band of single band + alpha images) + 4:4:4 corresponds to full horizontal and vertical resolution for chrominance + channels. + 4:2:2 corresponds to half horizontal and full vertical resolution. + 4:2:0 corresponds to half horizontal and half vertical resolution. + Only 4:4:4 can be used for lossless encoding. + + +See Also +-------- + +- `libavif `__ diff --git a/doc/source/drivers/raster/cog.rst b/doc/source/drivers/raster/cog.rst index 381cecdc9e7a..c7a50573c960 100644 --- a/doc/source/drivers/raster/cog.rst +++ b/doc/source/drivers/raster/cog.rst @@ -53,9 +53,7 @@ General creation options * ``JPEG`` should generally only be used with Byte data (8 bit per channel). But if GDAL is built with internal libtiff and libjpeg, it is possible to read and write TIFF files with 12bit JPEG compressed TIFF - files (seen as UInt16 bands with NBITS=12). See the `"8 and 12 bit - JPEG in TIFF" `__ wiki - page for more details. + files (seen as UInt16 bands with NBITS=12). For the COG driver, JPEG compression for 3 or 4-band images automatically selects the PHOTOMETRIC=YCBCR colorspace with a 4:2:2 subsampling of the Y,Cb,Cr components. @@ -612,8 +610,6 @@ See Also -------- - :ref:`raster.gtiff` driver -- `How to generate and read cloud optimized GeoTIFF - files `__ (before GDAL 3.1) - If your source dataset is an internally tiled geotiff with the desired georeferencing and compression, using `cogger `__ (possibly along with gdaladdo to create overviews) will be much faster than the COG driver. diff --git a/doc/source/drivers/raster/ecw.rst b/doc/source/drivers/raster/ecw.rst index f30d728f1ea4..add3aa839c5a 100644 --- a/doc/source/drivers/raster/ecw.rst +++ b/doc/source/drivers/raster/ecw.rst @@ -327,4 +327,3 @@ See Also `Hexagon Geospatial public forum `__ - Community contributed `patches `__ to apply to ECW SDK 3.3 sources -- `GDAL ECW Build Hints `__ diff --git a/doc/source/drivers/raster/gif.rst b/doc/source/drivers/raster/gif.rst index 5cebd59c805b..0d22cbfa6c40 100644 --- a/doc/source/drivers/raster/gif.rst +++ b/doc/source/drivers/raster/gif.rst @@ -16,8 +16,9 @@ A GIF image with transparency will have that entry marked as having an alpha value of 0.0 (transparent). Also, the transparent value will be returned as the NoData value for the band. -If an ESRI world file exists with the .gfw, .gifw or .wld extension, it -will be read and used to establish the geotransform for the image. +If an ESRI :ref:`world file ` exists with the .gfw, .gifw +or .wld extension, it will be read and used to establish the geotransform +for the image. XMP metadata can be extracted from the file, and will be stored as XML raw content in the xml:XMP metadata domain. @@ -45,6 +46,7 @@ The following creation options are supported: :default: OFF Force the generation of an associated ESRI world file (.wld). + See :ref:`World Files ` section for details. - .. co:: INTERLACING :choices: ON, OFF diff --git a/doc/source/drivers/raster/gti.rst b/doc/source/drivers/raster/gti.rst index a380c8bf7382..f719bea62b24 100644 --- a/doc/source/drivers/raster/gti.rst +++ b/doc/source/drivers/raster/gti.rst @@ -73,6 +73,19 @@ The GTI driver accepts different types of connection strings: For example: ``tileindex.gti`` +STAC GeoParquet support +----------------------- + +.. versionadded:: 3.10 + +The driver can support `STAC GeoParquet catalogs `_, +provided GDAL is build with :ref:`vector.parquet` support. +It can make use of fields ``proj:epsg`` and ``proj:transform`` from the +`Projection Extension Specification `_, +to correctly infer the appropriate projection and resolution. + +Example of a valid connection string: ``GTI:/vsicurl/https://github.com/stac-utils/stac-geoparquet/raw/main/tests/data/naip.parquet`` + Tile index requirements ----------------------- @@ -163,7 +176,7 @@ PostGIS, ...), the following layer metadata items may be set: virtual mosaic (unless SORT_FIELD_ASC=NO is set) * ``SORT_FIELD_ASC=YES|NO``: whether the values in SORT_FIELD should be sorted - in ascendent or descent order. Defaults to YES (ascendent) + in ascending or descending order. Defaults to YES (ascending) * ``BLOCKXSIZE=`` and ``BLOCKYSIZE=``: Block size of bands of the virtual mosaic. Defaults to 256x256. @@ -367,13 +380,13 @@ You can refer to the documentation of the :ref:`VRT ` driver for their syntax and semantics. -How to build a GTI comptatible index ? +How to build a GTI compatible index ? ---------------------------------------- The :ref:`gdaltindex` program may be used to generate both a vector tile index, and optionally a wrapping .gti XML file. -A GTI comptatible index may also be created by any programmatic means, provided +A GTI compatible index may also be created by any programmatic means, provided the above format specifications are met. @@ -413,7 +426,7 @@ also defined as layer metadata items or in the .gti XML file :choices: YES, NO :default: YES - Whether the values in SORT_FIELD should be sorted in ascendent or descent order + Whether the values in SORT_FIELD should be sorted in ascending or descending order - .. oo:: FILTER :choices: diff --git a/doc/source/drivers/raster/gtiff.rst b/doc/source/drivers/raster/gtiff.rst index f338e3d77db3..0ba9c0de89bc 100644 --- a/doc/source/drivers/raster/gtiff.rst +++ b/doc/source/drivers/raster/gtiff.rst @@ -384,9 +384,6 @@ The TIFF format only supports R,G,B components for palettes / color tables. Thus on writing the alpha information will be silently discarded. -You may want to read hints to `generate and read cloud optimized GeoTIFF -files `__ - Creation Options ~~~~~~~~~~~~~~~~ @@ -397,7 +394,7 @@ This driver supports the following creation options: :choices: YES, NO Force the generation of an associated ESRI world file - (.tfw).See the :ref:`World Files ` page for details. + (.tfw). See the :ref:`World Files ` page for details. - .. co:: RPB :choices: YES, NO diff --git a/doc/source/drivers/raster/heif.rst b/doc/source/drivers/raster/heif.rst index 3bf288b3716d..803e281ac8cd 100644 --- a/doc/source/drivers/raster/heif.rst +++ b/doc/source/drivers/raster/heif.rst @@ -26,6 +26,16 @@ matches the one of the full resolution image) If a file contains several top-level images, they will be exposed as GDAL subdatasets. +AVIF support +------------ + +Starting with GDAL 3.10, the AVIF_HEIF companion driver to the HEIF driver may +be used to open images encoding with the AVIF (AV1 Image File) codec if the +:ref:`raster.avif` driver is not available and if libheif has been compiled with +support for one of the libraries it support that are able of AV1 decoding +(libaom or libdav1d). + + Driver capabilities ------------------- diff --git a/doc/source/drivers/raster/index.rst b/doc/source/drivers/raster/index.rst index 849f627d828e..7f2955a11135 100644 --- a/doc/source/drivers/raster/index.rst +++ b/doc/source/drivers/raster/index.rst @@ -26,6 +26,7 @@ Raster drivers adrg aig airsar + avif bag basisu blx diff --git a/doc/source/drivers/raster/jp2ecw.rst b/doc/source/drivers/raster/jp2ecw.rst index fe1df5c8d7d1..3a4c59172e88 100644 --- a/doc/source/drivers/raster/jp2ecw.rst +++ b/doc/source/drivers/raster/jp2ecw.rst @@ -379,4 +379,3 @@ See Also - Support for non-GDAL specific issues should be directed to the `Hexagon Geospatial public forum `__ -- `GDAL ECW Build Hints `__ diff --git a/doc/source/drivers/raster/jp2mrsid.rst b/doc/source/drivers/raster/jp2mrsid.rst index c22375aa98a4..cfbe0def34d3 100644 --- a/doc/source/drivers/raster/jp2mrsid.rst +++ b/doc/source/drivers/raster/jp2mrsid.rst @@ -27,10 +27,10 @@ Georeferencing -------------- Georeferencing information can come from different sources : internal -(GeoJP2 or GMLJP2 boxes), worldfile .j2w/.wld sidecar files, or PAM -(Persistent Auxiliary metadata) .aux.xml sidecar files. By default, -information is fetched in following order (first listed is the highest -priority): PAM, GeoJP2, GMLJP2, WORLDFILE. +(GeoJP2 or GMLJP2 boxes), :ref:`worldfile ` .j2w/.wld sidecar +files, or PAM (Persistent Auxiliary metadata) .aux.xml sidecar files. By +default, information is fetched in following order (first listed is the +highest priority): PAM, GeoJP2, GMLJP2, WORLDFILE. Starting with GDAL 2.2, the allowed sources and their priority order can be changed with the :config:`GDAL_GEOREF_SOURCES` configuration option (or @@ -70,6 +70,7 @@ The following creation options are supported. :choices: YES to write an ESRI world file (with the extension .j2w). + See :ref:`World Files ` section for details. - .. co:: COMPRESSION diff --git a/doc/source/drivers/raster/jpeg.rst b/doc/source/drivers/raster/jpeg.rst index de018de65f1b..1de58c9d189f 100644 --- a/doc/source/drivers/raster/jpeg.rst +++ b/doc/source/drivers/raster/jpeg.rst @@ -20,8 +20,8 @@ IMAGE_STRUCTURE domain. EXIF metadata can be read from JPEG files (but this will not result in a georeferenced image even if the EXIF_GPSLatitude and EXIF_GPSLongitude -tags are set). But if an ESRI world file exists with the .jgw, -.jpgw/.jpegw or .wld suffixes, it will be read and used to establish the +tags are set). But if an ESRI :ref:`world file ` exists with the +.jgw, .jpgw/.jpegw or .wld suffixes, it will be read and used to establish the geotransform for the image. If available a MapInfo .tab file will also be used for georeferencing. Overviews can be built for JPEG files as an external .ovr file. @@ -169,7 +169,8 @@ The following creation options are supported: :choices: YES Force the generation of an associated ESRI world - file (with the extension .wld). + file (with the extension .wld). See :ref:`World Files ` + section for details. - .. co:: QUALITY :choices: 1-100 diff --git a/doc/source/drivers/raster/msg.rst b/doc/source/drivers/raster/msg.rst index 2ad9c9d03740..357c14a4c4e1 100644 --- a/doc/source/drivers/raster/msg.rst +++ b/doc/source/drivers/raster/msg.rst @@ -35,37 +35,8 @@ Driver capabilities Build Instructions ------------------ -CMake builds -++++++++++++ - See the ``GDAL_USE_PUBLICDECOMPWT`` option of :ref:`building_from_source`. -Other build systems -+++++++++++++++++++ - -Clone the EUMETSAT library for wavelet decompression into ``frmts/msg``. - -If you are building with Visual Studio 6.0, extract the .vc makefiles -for the PublicDecompWT from the file `PublicDecompWTMakefiles.zip` -stored in that directory. - -If you build using the GNUMakefile, use *--with-msg* option to enable -MSG driver: - -:: - - ./configure --with-msg - -If you find that some adjustments are needed in the makefile and/or the msg -source files, please "commit" them. The EUMETSAT library promises to be -"platform independent", but as we are working with Microsoft Windows and -Visual Studio 6.0, we did not have the facilities to check if the rest -of the msg driver is. Furthermore, apply steps 4 to 7 from the :ref:`raster_driver_tut`, section "Adding -Driver to GDAL Tree". - -MSG Wiki page is available at http://trac.osgeo.org/gdal/wiki/MSG. It's -dedicated to document building and usage hints - Specification of Source Dataset ------------------------------- diff --git a/doc/source/drivers/raster/netcdf.rst b/doc/source/drivers/raster/netcdf.rst index 78e95193eb70..030f82381d60 100644 --- a/doc/source/drivers/raster/netcdf.rst +++ b/doc/source/drivers/raster/netcdf.rst @@ -750,9 +750,6 @@ This driver is compiled with the UNIDATA NetCDF library. You need to download or compile the NetCDF library before configuring GDAL with NetCDF support. -See `NetCDF GDAL wiki `__ for -build instructions and information regarding HDF4, NetCDF-4 and HDF5. - See Also: --------- diff --git a/doc/source/drivers/raster/nitf.rst b/doc/source/drivers/raster/nitf.rst index 5e2ad83d0629..6db8c38ede1e 100644 --- a/doc/source/drivers/raster/nitf.rst +++ b/doc/source/drivers/raster/nitf.rst @@ -146,7 +146,7 @@ The following creation options are available: :choices: -1, 0, >0 :default: -1 - Restart interval (in MCUs) for JPEG compressoin. + Restart interval (in MCUs) for JPEG compression. -1 for auto, 0 for none, > 0 for user specified. - .. co:: NUMI diff --git a/doc/source/drivers/raster/pds4.rst b/doc/source/drivers/raster/pds4.rst index d584853f04a8..0dd5add475e9 100644 --- a/doc/source/drivers/raster/pds4.rst +++ b/doc/source/drivers/raster/pds4.rst @@ -194,10 +194,7 @@ The following dataset creation options are available: and not creating from an existing PDS4 file, the data/pds4_template.xml file will be used. For GDAL utilities to find this default PDS4 template, GDAL's data directory should be - defined in your environment (typically on Windows builds). Consult - the - `wiki `__ - for more information. + defined in your environment (typically on Windows builds). - .. co:: LATITUDE_TYPE :choices: Planetocentric, Planetographic @@ -537,12 +534,6 @@ Converting a shapefile to a PDS4 dataset with a CSV-delimited table $ ogr2ogr my_out_pds4.xml in.shp -Limitations ------------ - -As a new driver and new format, please report any issues to the bug -tracker, as explained on the `wiki `__ - See Also: --------- diff --git a/doc/source/drivers/raster/png.rst b/doc/source/drivers/raster/png.rst index 6541209cb2f4..a8a74ce8a9d6 100644 --- a/doc/source/drivers/raster/png.rst +++ b/doc/source/drivers/raster/png.rst @@ -74,8 +74,8 @@ The following creation options are available: :default: NO Force the generation of an associated ESRI world - file (with the extension .wld). See `World File <#WLD>`__ section for - details. + file (with the extension .wld). See :ref:`World Files ` + section for details. - .. co:: ZLEVEL=n :choices: [1-9] diff --git a/doc/source/drivers/raster/postgisraster.rst b/doc/source/drivers/raster/postgisraster.rst index 3927e23e5738..55266b389f32 100644 --- a/doc/source/drivers/raster/postgisraster.rst +++ b/doc/source/drivers/raster/postgisraster.rst @@ -130,6 +130,6 @@ See Also -------- - `GDAL PostGISRaster driver - Wiki `__ + Wiki `__ - `PostGIS Raster documentation `__ diff --git a/doc/source/drivers/raster/s102.rst b/doc/source/drivers/raster/s102.rst index 85ccbd6e397c..02663798c46d 100644 --- a/doc/source/drivers/raster/s102.rst +++ b/doc/source/drivers/raster/s102.rst @@ -11,7 +11,7 @@ S102 -- S-102 Bathymetric Surface Product .. versionadded:: 3.8 This driver provides read-only support for bathymetry data in the S-102 format, -which is a specific product profile in an HDF5 file +which is a specific product profile in an HDF5 file. S-102 files have two image bands representing depth (band 1), uncertainty (band 2) values for each cell in a raster grid area. @@ -25,6 +25,9 @@ Georeferencing is reported. Nodata, minimum and maximum values for each band are also reported. +Supported versions of the specification are S-102 v2.1, v2.2 and v3.0 +(support for v3.0 spatial metadata added in GDAL 3.10) + Driver capabilities ------------------- @@ -63,12 +66,16 @@ The following open options are supported: Spatial metadata support ------------------------ -Starting with GDAL 3.9, GDAL can handle QualityOfSurvey spatial metadata. +Starting with GDAL 3.9, GDAL can handle QualityOfSurvey +(or QualityOfBathymetryCoverage in S102 v3.0) spatial metadata. When such spatial metadata is present, the subdataset list will include -a name of the form ``S102:"{filename}":QualityOfSurvey`` +a name of the form ``S102:"{filename}":QualityOfSurvey`` ( +or ``S102:"{filename}":QualityOfBathymetryCoverage`` in S102 v3.0) -The ``/QualityOfSurvey/featureAttributeTable`` dataset is exposed as a +The ``/QualityOfSurvey/featureAttributeTable`` +(``/QualityOfBathymetryCoverage/featureAttributeTable`` in S102 v3.0) +dataset is exposed as a GDAL Raster Attribute Table associated to the GDAL raster band. The pixel values of the raster match the ``id`` column of the Raster Attribute Table. diff --git a/doc/source/drivers/vector/csv.rst b/doc/source/drivers/vector/csv.rst index 1b800434c136..f3e20f391f5b 100644 --- a/doc/source/drivers/vector/csv.rst +++ b/doc/source/drivers/vector/csv.rst @@ -84,7 +84,7 @@ per line must be present. Lines may be terminated by a DOS (CR/LF) or Unix (LF) style line terminators. Each record should have the same number of fields. The driver will also accept a semicolon, a tabulation, a pipe, or a space character as field separator. -Starting with GDAL 3.8, the autodection will select the separator with the +Starting with GDAL 3.8, the autodetection will select the separator with the most occurrences if there are several candidates on the first line of the CSV file (and warn about that). The :oo:`SEPARATOR` open option may also be set to define the desired separator. diff --git a/doc/source/drivers/vector/geojsonseq.rst b/doc/source/drivers/vector/geojsonseq.rst index 83256ac24743..47539dd4734e 100644 --- a/doc/source/drivers/vector/geojsonseq.rst +++ b/doc/source/drivers/vector/geojsonseq.rst @@ -120,6 +120,14 @@ The following layer creation options are supported: if they start and end with brackets and braces, even if they do not have their subtype set to JSON. +- .. lco:: WRITE_BBOX + :choices: YES, NO + :default: NO + :since: 3.10 + + Set to YES to write a bbox property with the bounding box of the + geometry at the feature level. + Geometry coordinate precision ----------------------------- diff --git a/doc/source/drivers/vector/pgeo.rst b/doc/source/drivers/vector/pgeo.rst index 2176296c7f42..f363b2feaead 100644 --- a/doc/source/drivers/vector/pgeo.rst +++ b/doc/source/drivers/vector/pgeo.rst @@ -104,8 +104,7 @@ How to use PGeo driver with unixODBC and MDB Tools (on Unix and Linux) This article gives step-by-step explanation of how to use OGR with unixODBC package and how to access Personal Geodatabase with PGeo -driver. See also `GDAL wiki for other -details `__ +driver. Prerequisites ~~~~~~~~~~~~~ diff --git a/doc/source/drivers/vector/shapefile.rst b/doc/source/drivers/vector/shapefile.rst index 67cd3abee67f..c978c611612b 100644 --- a/doc/source/drivers/vector/shapefile.rst +++ b/doc/source/drivers/vector/shapefile.rst @@ -166,9 +166,7 @@ terminated with an error. Note that this can make it very difficult to translate a mixed geometry layer from another format into Shapefile format using ogr2ogr, since ogr2ogr has no support for separating out geometries from a source -layer. See the -`FAQ `__ -for a solution. +layer. Shapefile feature attributes are stored in an associated .dbf file, and so attributes suffer a number of limitations: @@ -484,5 +482,3 @@ See Also -------- - `Shapelib Page `__ -- `User Notes on OGR Shapefile - Driver `__ diff --git a/doc/source/faq.rst b/doc/source/faq.rst index d49c2f00bf77..d3025583cd26 100644 --- a/doc/source/faq.rst +++ b/doc/source/faq.rst @@ -4,7 +4,7 @@ FAQ ================================================================================ -.. TODO maybe migrate the chapters 2 and following of https://trac.osgeo.org/gdal/wiki/FAQ +.. TODO maybe migrate the chapters 2 and following of http://web.archive.org/web/https://trac.osgeo.org/gdal/wiki/FAQ .. only:: not latex diff --git a/doc/source/gdal_rtd/static/css/gdal.css b/doc/source/gdal_rtd/static/css/gdal.css index 70f9d653f0a4..6cedf6994ceb 100644 --- a/doc/source/gdal_rtd/static/css/gdal.css +++ b/doc/source/gdal_rtd/static/css/gdal.css @@ -335,3 +335,13 @@ div.horizontal-logos::after { clear: both; display: table; } + + +/* background of logo area */ +div.wy-side-nav-search { + background: white +} + +.wy-side-nav-search > div.version { + color: #343131 +} diff --git a/doc/source/programs/gdal_edit.rst b/doc/source/programs/gdal_edit.rst index 983f315c478d..0fa57725523d 100644 --- a/doc/source/programs/gdal_edit.rst +++ b/doc/source/programs/gdal_edit.rst @@ -15,14 +15,14 @@ Synopsis .. code-block:: - gdal_edit [--help] [--help-general] [-ro] [-a_srs ] + gdal_edit [--help] [--help-general] [-ro] [-a_srs ] [-a_ullr ] [-a_ulurll ] [-tr ] [-unsetgt] [-unsetrpc] [-a_nodata ] [-unsetnodata] [-a_coord_epoch ] [-unsetepoch] [-unsetstats] [-stats] [-approx_stats] [-setstats ] [-scale ] [-offset ] [-units ] - [-colorinterp_ {red|green|blue|alpha|gray|undefined}]... + [-colorinterp_ {red|green|blue|alpha|gray|undefined|pan|coastal|rededge|nir|swir|mwir|lwir|...}]... [-gcp []]... [-unsetmd] [-oo =]... [-mo =]... @@ -172,7 +172,7 @@ It works only with raster formats that support update access to existing dataset .. versionadded:: 3.1 -.. option:: -colorinterp_ {red|green|blue|alpha|gray|undefined} +.. option:: -colorinterp_ {red|green|blue|alpha|gray|undefined|pan|coastal|rededge|nir|swir|mwir|lwir|...} Change the color interpretation of band X (where X is a valid band number, starting at 1). diff --git a/doc/source/programs/gdal_translate.rst b/doc/source/programs/gdal_translate.rst index 8c13b82d5e56..1b7d9b39041c 100644 --- a/doc/source/programs/gdal_translate.rst +++ b/doc/source/programs/gdal_translate.rst @@ -32,8 +32,8 @@ Synopsis [-a_gt ] [-a_scale ] [-a_offset ] [-nogcp] [-gcp []]... - |-colorinterp{_bn} {red|green|blue|alpha|gray|undefined}] - |-colorinterp {red|green|blue|alpha|gray|undefined},...] + |-colorinterp{_bn} {red|green|blue|alpha|gray|undefined|pan|coastal|rededge|nir|swir|mwir|lwir|...}] + |-colorinterp {red|green|blue|alpha|gray|undefined|pan|coastal|rededge|nir|swir|mwir|lwir|...},...] [-mo =]... [-dmo "DOMAIN:META-TAG=VALUE"]... [-q] [-sds] [-co =]... [-stats] [-norat] [-noxmp] [-oo =]... @@ -289,14 +289,14 @@ resampling, and rescaling pixels in the process. nodata value, this does not cause pixel values that are equal to that nodata value to be changed to the value specified with this option. -.. option:: -colorinterp_X +.. option:: -colorinterp_X Override the color interpretation of band X (where X is a valid band number, starting at 1) .. versionadded:: 2.3 -.. option:: -colorinterp {red|green|blue|alpha|gray|undefined},... +.. option:: -colorinterp {red|green|blue|alpha|gray|undefined|pan|coastal|rededge|nir|swir|mwir|lwir|...},... Override the color interpretation of all specified bands. For example -colorinterp red,green,blue,alpha for a 4 band output dataset. diff --git a/doc/source/programs/gdal_viewshed.rst b/doc/source/programs/gdal_viewshed.rst index 5ff9316dc74e..26034a29288b 100644 --- a/doc/source/programs/gdal_viewshed.rst +++ b/doc/source/programs/gdal_viewshed.rst @@ -23,6 +23,7 @@ Synopsis -ox -oy [-vv ] [-iv ] [-ov ] [-cc ] + [-os ] [-j ] [-co =]... [-q] [-om ] @@ -62,13 +63,13 @@ Byte. With the -mode flag can also return a minimum visible height raster of typ The X position of the observer (in SRS units). If the coordinate is outside of the raster, all space between the observer and the raster is assumed not to occlude - visibility of the raster. + visibility of the raster. (Not supported in cumulative mode.) .. option:: -oy The Y position of the observer (in SRS units). If the coordinate is outside of the raster, all space between the observer and the raster is assumed not to occlude - visibility of the raster. + visibility of the raster. (Not supported in cumulative mode.) .. option:: -oz @@ -80,8 +81,9 @@ Byte. With the -mode flag can also return a minimum visible height raster of typ .. option:: -md - Maximum distance from observer to compute visibiliy. + Maximum distance from observer to compute visibility. It is also used to clamp the extent of the output raster. + (Not supported in cumulative mode) .. option:: -cc @@ -128,22 +130,22 @@ Byte. With the -mode flag can also return a minimum visible height raster of typ .. option:: -iv - Pixel value to set for invisible areas. Default: 0 + Pixel value to set for invisible areas. (Not supported in cumulative mode) Default: 0 .. option:: -ov Pixel value to set for the cells that fall outside of the range specified by - the observer location and the maximum distance. Default: 0 + the observer location and the maximum distance. (Not supported in cumulative mode) Default: 0 .. option:: -vv - Pixel value to set for visible areas. Default: 255 + Pixel value to set for visible areas. (Not supported in cumulative mode) Default: 255 .. option:: -om Sets what information the output contains. - Possible values: NORMAL, DEM, GROUND + Possible values: NORMAL, DEM, GROUND, ACCUM NORMAL returns a raster of type Byte containing visible locations. @@ -151,8 +153,21 @@ Byte. With the -mode flag can also return a minimum visible height raster of typ height for target to be visible from the DEM surface or ground level respectively. Flags -tz, -iv and -vv will be ignored. + Cumulative (ACCUM) mode will create an eight bit raster the same size as the input raster + where each cell represents the relative observability from a grid of observer points. + See the -os option. + Default NORMAL +.. option:: -os + + Cell Spacing between observers (only supported in cumulative mode) Default: 10 + +.. option:: -j + + Relative number of jobs to run at once. (only supported in cumulative mode) Default: 3 + + C API ----- diff --git a/doc/source/programs/gdalwarp.rst b/doc/source/programs/gdalwarp.rst index 27c5d115e3fb..bccc9778d188 100644 --- a/doc/source/programs/gdalwarp.rst +++ b/doc/source/programs/gdalwarp.rst @@ -214,10 +214,10 @@ with control information. .. option:: -et - Error threshold for transformation approximation (in pixel units - - defaults to 0.125, unless, starting with GDAL 2.1, the RPC_DEM transformer - option is specified, in which case, an exact transformer, i.e. - err_threshold=0, will be used). + Error threshold for transformation approximation, expressed as a number of + source pixels. Defaults to 0.125 pixels unless the ``RPC_DEM`` transformer + option is specified, in which case an exact transformer, i.e. + ``err_threshold=0``, will be used. .. option:: -refine_gcps [] @@ -335,7 +335,7 @@ with control information. resolution. The resampling method used to create those overviews is generally not the one you specify through the :option:`-r` option. Some formats, like JPEG2000, can contain - significant outliers due to wavelet compression works. It might thus be useful in + significant outliers due to how wavelet compression works. It might thus be useful in those situations to use the :option:`-ovr` ``NONE`` option to prevent existing overviews to be used. @@ -490,21 +490,43 @@ with control information. The destination file name. -Mosaicing into an existing output file is supported if the output file -already exists. The spatial extent of the existing file will not -be modified to accommodate new data, so you may have to remove it in that case, or -use the -overwrite option. +Overview +-------- -Polygon cutlines may be used as a mask to restrict the area of the -destination file that may be updated, including blending. If the OGR -layer containing the cutline features has no explicit SRS, the cutline -features must be in the SRS of the destination file. When writing to a -not yet existing target dataset, its extent will be the one of the -original raster unless -te or -crop_to_cutline are specified. +:program:`gdalwarp` transforms images between different coordinate reference +systems and spatial resolutions. + +First, :program:`gdalwarp` must determine the extent and resolution of the +output, if these have not been specified using :option:`-te` and :option:`-tr`. +These are determined by transforming a sample of points from the source CRS to +the destination CRS. Details of the procedure can be found in the documentation +for :cpp:func:`GDALSuggestedWarpOutput`. If multiple inputs are provided to +:program:`gdalwarp`, the output extent will be calculated to cover all of them, +at a resolution consistent with the highest-resolution input. + +Once the dimensions of the output image have been determined, +:program:`gdalwarp` divides the output image into chunks that can be processed +independently within the amount of memory specified by :option:`-wm`. +:program:`gdalwarp` then iterates over scanlines in these chunks, and for each +output pixel determines the set of source pixels that contribute to the value +of the output pixel. These source pixels are provided to a function that +performs the resampling algorithm selected with :option:`-r`. + +Writing to an existing file +--------------------------- + +Mosaicing into an existing output file is supported if the output file already +exists. The spatial extent of the existing file will not be modified to +accommodate new data, so you may have to remove it in that case, or use the +:option:`-overwrite` option. + +Polygon cutlines may be used as a mask to restrict the area of the destination +file that may be updated, including blending. If the OGR layer containing the +cutline features has no explicit SRS, the cutline features are assumed to be in +the SRS of the destination file. When writing to a not yet existing target +dataset, its extent will be the one of the original raster unless :option:`-te` +or :option:`-crop_to_cutline` are specified. -Starting with GDAL 3.1, it is possible to use as output format a driver that -only supports the CreateCopy operation. This may internally imply creation of -a temporary file. .. _gdalwarp_nodata: @@ -559,30 +581,34 @@ Approximate transformation -------------------------- By default :program:`gdalwarp` uses a linear approximator for the -transformations with a permitted error of 0.125 pixels. The approximator -basically transforms three points on a scanline: the start, end and middle. -Then it compares the linear approximation of the center based on the end points -to the real thing and checks the error. If the error is less than the error -threshold then the remaining points are approximated (in two chunks utilizing -the center point). If the error exceeds the threshold, the scanline is split -into two sections, and the approximator is recursively applied to each section -until the error is less than the threshold or all points have been exactly -computed. - -The error threshold (in pixels) can be controlled with the gdalwarp +transformations with a permitted error of 0.125 pixels in the source dataset. +The approximator precisely transforms three points per output scanline (the +start, middle, and end) from a row and column in the output dataset to a +row and column in the source dataset. +It then compares a linear approximation of the center point coordinates to the +precisely transformed value. +If the sum of the horizontal and vertical errors is less than the error +threshold then the remaining source points are approximated using linear +interpolation between the start and middle point, and between the middle and +end point. +If the error exceeds the threshold, the scanline is split into two sections and +the approximator is recursively applied to each section until the error is less +than the threshold or all points have been exactly computed. + +The error threshold (in source dataset pixels) can be controlled with the gdalwarp :option:`-et` switch. If you want to compare a true pixel-by-pixel reprojection use :option:`-et 0` which disables this approximator entirely. Vertical transformation ----------------------- -While gdalwarp can essentially perform coordinate transformations in the 2D -space, it can perform as well vertical transformations. This is automatically -enabled when the 2 following conditions are met: +While gdalwarp is most commonly used to perform coordinate transformations in the 2D +space, it can also perform vertical transformations. Vertical transformations are +automatically performed when the following two conditions are met: - at least one of the source or target CRS has an explicit vertical CRS - (as part of a compound CRS) or is a 3D (generally geographic) CRS, -- and the raster has a single band. + (as part of a compound CRS) or is a 3D (generally geographic) CRS, and +- the raster has a single band This mode can also be forced by using the :option:`-vshift` (this is essentially useful when the CRS involved are not explicitly 3D, but a @@ -752,15 +778,3 @@ C API ----- This utility is also callable from C with :cpp:func:`GDALWarp`. - - -See also --------- - -.. only:: not man - - `Wiki page discussing options and behaviours of gdalwarp `_ - -.. only:: man - - Wiki page discussing options and behaviours of gdalwarp: https://trac.osgeo.org/gdal/wiki/UserDocs/GdalWarp diff --git a/doc/source/spelling_wordlist.txt b/doc/source/spelling_wordlist.txt index 2398cc6a3475..bba8115394f7 100644 --- a/doc/source/spelling_wordlist.txt +++ b/doc/source/spelling_wordlist.txt @@ -58,7 +58,6 @@ Amici amongst amz Amz -analygous analyse Analyse analysing @@ -79,7 +78,6 @@ api Api apoNewDims apploy -appoach approximator AppVeyor APSTILE @@ -97,7 +95,6 @@ arra Arrayname arrayStartIdx ASC -ascendent ascii askin aspatial @@ -133,7 +130,7 @@ autocommit autoconf Autoconf autocorrelation -autodection +autodetection Autodesk autodetect autodetected @@ -200,6 +197,7 @@ BBoxOrder bc bCatchDebug bCopy +bDelete bDstToSrc bedrijven behaviours @@ -438,9 +436,6 @@ ComplexSource compoundcurves compressiblity compressions -compressoin -comptatible -Conceptionally conceptualhelp cond conda @@ -619,6 +614,7 @@ describedby deserialization deserialize deserialized +deserializing desirabled dest destructor @@ -660,6 +656,7 @@ dfTargetHeight dfTimeout dfTolerance dfURY +dfValue dfVisibleVal dfXOff dfXSize @@ -988,6 +985,7 @@ gdalbuildvrt gdalcompare gdalconst gdaldata +GDALDataset gdaldem gdaldriver gdalenhance @@ -1017,6 +1015,7 @@ gdalrasterblock gdalrc gdalsrsinfo gdaltest +GDALThreadLocalDatasetCache gdaltindex gdaltransform gdalvirtualmem @@ -1396,6 +1395,7 @@ InfoFormat InfoOptions Informatisé Informix +InfraRed ing ini init @@ -1512,6 +1512,7 @@ KShortest kslikebase KSpread ktx +Ku Kubernetes Kumar Kuratomi @@ -1543,8 +1544,10 @@ LegalConstraints len Leveller lf +libaom libarchive libarrow +libavif libblosc libbsd libcf @@ -1552,6 +1555,7 @@ libcfitsio libcrypto libcurl libCurl +libdav libde libdeflate libdf @@ -1851,6 +1855,7 @@ MultiPolygon multipolygons multiproc multiprocess +multireadtest multirecords multispectral multistring @@ -2048,6 +2053,7 @@ nRecurseDepth nRefCount nReqOrder nSampleStep +nScopeFlags nSecond nSize nSrcBufferAllocSize @@ -3361,6 +3367,7 @@ UpdateFeature updateTime UpperLeftX UpperLeftY +upsampling upsert uri url diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst index aa2bcaa0209c..164bc6f1ceb7 100644 --- a/doc/source/user/configoptions.rst +++ b/doc/source/user/configoptions.rst @@ -967,36 +967,41 @@ PROJ options - .. config:: CHECK_WITH_INVERT_PROJ :since: 1.7.0 + :default: NO Used by :source_file:`ogr/ogrct.cpp` and :source_file:`apps/gdalwarp_lib.cpp`. - This option can be used to control the behavior of gdalwarp when warping global + This option can be used to control the behavior of :program:`gdalwarp` when warping global datasets or when transforming from/to polar projections, which causes coordinate discontinuities. See http://trac.osgeo.org/gdal/ticket/2305. - The background is that PROJ does not guarantee that converting from src_srs to - dst_srs and then from dst_srs to src_srs will yield to the initial coordinates. + The background is that PROJ does not guarantee that converting from ``src_srs`` to + ``dst_srs`` and then from ``dst_srs`` to ``src_srs`` will yield the initial coordinates. This can lead to errors in the computation of the target bounding box of - gdalwarp, or to visual artifacts. + :program:`gdalwarp`, or to visual artifacts. - If CHECK_WITH_INVERT_PROJ option is not set, gdalwarp will check that the the + If :config:`CHECK_WITH_INVERT_PROJ` option is not set, :program:`gdalwarp` will check that the computed coordinates of the edges of the target image are in the validity area of the target projection. If they are not, it will retry computing them by - setting :config:`CHECK_WITH_INVERT_PROJ=TRUE` that forces ogrct.cpp to check the - consistency of each requested projection result with the invert projection. + setting :config:`CHECK_WITH_INVERT_PROJ=TRUE` that forces + :source_file:`ogr/ogrct.cpp` to check the consistency of each requested + projection result with the inverse projection. - If set to NO, gdalwarp will not attempt to use the invert projection. + If set to ``NO``, :program:`gdalwarp` will not attempt to use the inverse projection. - .. config:: THRESHOLD :since: 1.7.0 + :default: 0.1 for geographic SRS, 10000 otherwise Used by :source_file:`ogr/ogrct.cpp`. - Used in combination with :config:`CHECK_WITH_INVERT_PROJ=TRUE`. Define - the acceptable threshold used to check if the roundtrip from src_srs to - dst_srs and from dst_srs to srs_srs yield to the initial coordinates. The - value must be expressed in the units of the source SRS (typically degrees - for a geographic SRS, meters for a projected SRS) + Used in combination with :config:`CHECK_WITH_INVERT_PROJ=TRUE`. Defines + the acceptable threshold used to check if the round-trip from ``src_srs`` to + ``dst_srs`` and from ``dst_srs`` to ``srs_srs`` yields the initial coordinates. + The round-trip transformation will be considered successful if the ``x`` and ``y`` + values are both within :config:`THRESHOLD` of the original values. + The value must be expressed in the units of the source SRS (typically degrees + for a geographic SRS, meters for a projected SRS). - .. config:: OGR_ENABLE_PARTIAL_REPROJECTION :since: 1.8.0 diff --git a/doc/source/user/multithreading.rst b/doc/source/user/multithreading.rst index a4263362a679..c4318bada342 100644 --- a/doc/source/user/multithreading.rst +++ b/doc/source/user/multithreading.rst @@ -4,8 +4,8 @@ Multi-threading =============== -GDAL API: re-entrant, but not thread-safe ------------------------------------------ +GDAL API: re-entrant, but (generally) not thread-safe +----------------------------------------------------- The exact meaning of the terms ``thread-safe`` or ``re-entrant`` is not fully standardized. We will use here the `QT definitions `__. @@ -43,6 +43,26 @@ that are not thread-safe. Those restrictions apply to the C and C++ ABI, and all languages bindings (unless they would take special precautions to serialize calls) +Thread-safe GDAL dataset instances for raster read-only use cases +----------------------------------------------------------------- + +.. versionadded:: 3.10 + +RFC 101 adds a new capability to open, or obtain, a thread-safe dataset from +any dataset, but only for raster read-only use cases. + +At open time, this can be done by passing ``GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE`` +to :cpp:func:`GDALOpenEx` / :cpp:func:`GDALDataset::Open`. + +Given an existing GDALDataset* instance, :cpp:func:`GDALDataset::IsThreadSafe` +can be used to determine if it is thread-safe or not. If not, +:cpp:func:`GDALDataset::GetThreadSafeDataset` can be used. + +Note that the generic implementation of this capability involves opening one +dataset the first time a thread-safe dataset/raster band is accessed by a thread. +While this is an implementation detail that can be ignored to develop code, it is +important to note regarding potential performance impacts + GDAL block cache and multi-threading ------------------------------------ diff --git a/doc/source/user/raster_data_model.rst b/doc/source/user/raster_data_model.rst index 35b80ae79ca6..03af38ace48b 100644 --- a/doc/source/user/raster_data_model.rst +++ b/doc/source/user/raster_data_model.rst @@ -189,6 +189,21 @@ For satellite or aerial imagery the IMAGERY Domain may be present. It depends on - CLOUDCOVER: Cloud coverage. The value between 0 - 100 or 999 if not available - ACQUISITIONDATETIME: The image acquisition date time in UTC +Starting with GDAL 3.10, there also exists a raster band level IMAGERY metadata domain with the following items: + +- CENTRAL_WAVELENGTH_UM: Central Wavelength in micrometers. +- FWHM_UM: Full-width half-maximum (FWHM) in micrometers. + +Clients can get (resp. set) these metadata items with :cpp:func:`GDALRasterBand::GetMetadataItem()` +(resp. :cpp:func:`GDALRasterBand::SetMetadataItem()`).` + +They are specifically set by the :ref:`raster.sentinel2` and +:ref:`raster.envi` drivers (if corresponding metadata items are found in the ENVI header), +but may also be found in other drivers handling arbitrary GDAL metadata, such as +the one using the GDAL Persistent Auxiliary Mechanism (PAM / .aux.xml side car files) +or :ref:`raster.vrt` drivers. The :ref:`raster.gtiff` driver also supports serializing +and deserializing the band IMAGERY metadata domain in the ``GDAL_METADATA`` TIFF tag. + xml: Domains ++++++++++++ @@ -228,20 +243,48 @@ A raster band has the following properties: - An optional raster unit name. For instance, this might indicate linear units for elevation data. - A color interpretation for the band. This is one of: - * GCI_Undefined: the default, nothing is known. - * GCI_GrayIndex: this is an independent gray-scale image - * GCI_PaletteIndex: this raster acts as an index into a color table - * GCI_RedBand: this raster is the red portion of an RGB or RGBA image - * GCI_GreenBand: this raster is the green portion of an RGB or RGBA image - * GCI_BlueBand: this raster is the blue portion of an RGB or RGBA image - * GCI_AlphaBand: this raster is the alpha portion of an RGBA image - * GCI_HueBand: this raster is the hue of an HLS image - * GCI_SaturationBand: this raster is the saturation of an HLS image - * GCI_LightnessBand: this raster is the lightness of an HLS image - * GCI_CyanBand: this band is the cyan portion of a CMY or CMYK image - * GCI_MagentaBand: this band is the magenta portion of a CMY or CMYK image - * GCI_YellowBand: this band is the yellow portion of a CMY or CMYK image - * GCI_BlackBand: this band is the black portion of a CMYK image. + * GCI_Undefined / "Undefined": default, nothing is known. + * GCI_GrayIndex / "Gray": independent gray-scale image + * GCI_PaletteIndex / "Palette": this raster acts as an index into a color table + * GCI_RedBand / "Red": red portion of an RGB or RGBA image, or red spectral band [0.62 - 0.69 um] + * GCI_GreenBand/ "Green": green portion of an RGB or RGBA image, or green spectral band [0.51 - 0.60 um] + * GCI_BlueBand / "Blue": blue portion of an RGB or RGBA image, or blue spectral band [0.45 - 0.53 um] + * GCI_AlphaBand / "Alpha": alpha portion of an RGBA image + * GCI_HueBand / "Hue": hue of a HLS image + * GCI_SaturationBand / "Saturation": saturation of a HLS image + * GCI_LightnessBand / "Lightness": lightness of a HLS image + * GCI_CyanBand / "Cyan": cyan portion of a CMY or CMYK image + * GCI_MagentaBand / "Magenta": magenta portion of a CMY or CMYK image + * GCI_YellowBand / "Yellow": yellow portion of a CMY or CMYK image, or yellow spectral band [0.58 - 0.62 um] + * GCI_BlackBand / "Black": black portion of a CMYK image. + + Below values have been added in GDAL 3.10: + + * GCI_PanBand / "Pan": Panchromatic band [0.40 - 1.00 um] + * GCI_CoastalBand / "Coastal": Coastal band [0.40 - 0.45 um] + * GCI_RedEdgeBand / "RedEdge": Red-edge band [0.69 - 0.79 um] + * GCI_NIRBand / "NIR": Near-InfraRed (NIR) band [0.75 - 1.40 um] + * GCI_SWIRBand / "SWIR": Short-Wavelength InfraRed (SWIR) band [1.40 - 3.00 um] + * GCI_MWIRBand / "MWIR": Mid-Wavelength InfraRed (MWIR) band [3.00 - 8.00 um] + * GCI_LWIRBand / "LWIR": Long-Wavelength InfraRed (LWIR) band [8.00 - 15 um] + * GCI_TIRBand / "TIR": Thermal InfraRed (TIR) band (MWIR or LWIR) [3 - 15 um] + * GCI_OtherIRBand / "OtherIR": Other infrared band [0.75 - 1000 um] + * GCI_SAR_Ka_Band / "SAR_Ka": Synthetic Aperture Radar (SAR) Ka band [0.8 - 1.1 cm / 27 - 40 GHz] + * GCI_SAR_K_Band / "SAR_K": Synthetic Aperture Radar (SAR) K band [1.1 - 1.7 cm / 18 - 27 GHz] + * GCI_SAR_Ku_Band / "SAR_Ku": Synthetic Aperture Radar (SAR) Ku band [1.7 - 2.4 cm / 12 - 18 GHz] + * GCI_SAR_X_Band / "SAR_X": Synthetic Aperture Radar (SAR) X band [2.4 - 3.8 cm / 8 - 12 GHz] + * GCI_SAR_C_Band / "SAR_C": Synthetic Aperture Radar (SAR) C band [3.8 - 7.5 cm / 4 - 8 GHz] + * GCI_SAR_S_Band / "SAR_S": Synthetic Aperture Radar (SAR) S band [7.5 - 15 cm / 2 - 4 GHz] + * GCI_SAR_L_Band / "SAR_L": Synthetic Aperture Radar (SAR) L band [15 - 30 cm / 1 - 2 GHz] + * GCI_SAR_P_Band / "SAR_P": Synthetic Aperture Radar (SAR) P band [30 - 100 cm / 0.3 - 1 GHz] + + For spectral bands, the wavelength ranges are indicative only, and may vary + depending on sensors. The ``CENTRAL_WAVELENGTH_UM`` and ``FWHM_UM`` metadata + items in the band ``IMAGERY`` metadata domain of the raster band, when present, will + give more accurate characteristics. + + Values belonging to the IR domain are in the [GCI_IR_Start, GCI_IR_End] range. + Values belonging to the SAR domain are in the [GCI_SAR_Start, GCI_SAR_End] range. - A color table, described in more detail later. - Knowledge of reduced resolution overviews (pyramids) if available. diff --git a/doc/source/user/sql_sqlite_dialect.rst b/doc/source/user/sql_sqlite_dialect.rst index 0a539e115d26..9433694b78c0 100644 --- a/doc/source/user/sql_sqlite_dialect.rst +++ b/doc/source/user/sql_sqlite_dialect.rst @@ -208,8 +208,18 @@ Statistics functions In addition to standard COUNT(), SUM(), AVG(), MIN(), MAX(), the following aggregate functions are available: -- STDDEV_POP: (GDAL >= 3.10) numerical population standard deviation. -- STDDEV_SAMP: (GDAL >= 3.10) numerical `sample standard deviation `__ +- ``STDDEV_POP(numeric_value)``: (GDAL >= 3.10) numerical population standard deviation. +- ``STDDEV_SAMP(numeric_value)``: (GDAL >= 3.10) numerical `sample standard deviation `__ + +Ordered-set aggregate functions ++++++++++++++++++++++++++++++++ + +The following aggregate functions are available. Note that they require to allocate an amount of memory proportional to the number of selected rows (for ``MEDIAN``, ``PERCENTILE`` and ``PERCENTILE_CONT``) or to the number of values (for ``MODE``). + +- ``MEDIAN(numeric_value)``: (GDAL >= 3.10) (continuous) median (equivalent to ``PERCENTILE(numeric_value, 50)``). NULL values are ignored. +- ``PERCENTILE(numeric_value, percentage)``: (GDAL >= 3.10) (continuous) percentile, with percentage between 0 and 100 (equivalent to ``PERCENTILE_CONT(numeric_value, percentage / 100)``). NULL values are ignored. +- ``PERCENTILE_CONT(numeric_value, fraction)``: (GDAL >= 3.10) (continuous) percentile, with fraction between 0 and 1. NULL values are ignored. +- ``MODE(value)``: (GDAL >= 3.10): mode, i.e. most frequent input value (strings and numeric values are supported), arbitrarily choosing the first one if there are multiple equally-frequent results. NULL values are ignored. Spatialite SQL functions ++++++++++++++++++++++++ diff --git a/docker/alpine-normal/Dockerfile b/docker/alpine-normal/Dockerfile index fe2358f92c9e..76c6e0373217 100644 --- a/docker/alpine-normal/Dockerfile +++ b/docker/alpine-normal/Dockerfile @@ -44,6 +44,7 @@ RUN apk add --no-cache \ lerc-dev \ libaec-dev \ libarchive-dev \ + libavif-dev \ libdeflate-dev \ libgeotiff-dev \ libheif-dev \ @@ -130,7 +131,7 @@ RUN if test "${OPENDRIVE_VERSION}" != ""; then ( \ && cd .. \ && rm -rf libOpenDRIVE-${OPENDRIVE_VERSION} \ ); fi - + RUN apk add --no-cache rsync ccache ARG RSYNC_REMOTE @@ -321,6 +322,7 @@ RUN apk add --no-cache \ libarchive \ libarrow \ libarrow_dataset \ + libavif \ libcrypto3 \ libcurl \ libdeflate \ diff --git a/docker/ubuntu-full/Dockerfile b/docker/ubuntu-full/Dockerfile index 383b44fdf813..685d1686ed7b 100644 --- a/docker/ubuntu-full/Dockerfile +++ b/docker/ubuntu-full/Dockerfile @@ -68,6 +68,7 @@ RUN . /buildscripts/bh-set-envvars.sh \ libbrotli-dev${APT_ARCH_SUFFIX} \ libarchive-dev${APT_ARCH_SUFFIX} \ libaec-dev${APT_ARCH_SUFFIX} \ + libavif-dev${APT_ARCH_SUFFIX} \ && rm -rf /var/lib/apt/lists/* # Build likbkea @@ -341,6 +342,7 @@ RUN apt-get update \ libbrotli1 \ libarchive13 \ libaec0 \ + libavif16 \ libspdlog1.12 libmagic1t64 \ python-is-python3 \ # pil for antialias option of gdal2tiles diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index 073dfc9667a9..875eaade5385 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -88,6 +88,7 @@ gdal_optional_format(r "R Object Data Store") gdal_optional_format(northwood "NWT_GRD/NWT_GRC -- Northwood/Vertical Mapper File Format") gdal_optional_format(saga "SAGA GIS Binary Driver") gdal_optional_format(xyz "ASCII Gridded XYZ") +include(avif/driver_declaration.cmake) gdal_dependent_format(heif "HEIF" "GDAL_USE_HEIF") gdal_optional_format(esric "ESRI compact cache") gdal_optional_format(hf2 "HF2/HFZ heightfield raster") diff --git a/frmts/avif/CMakeLists.txt b/frmts/avif/CMakeLists.txt new file mode 100644 index 000000000000..ef734f575e31 --- /dev/null +++ b/frmts/avif/CMakeLists.txt @@ -0,0 +1,26 @@ +include("${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/helpers/GdalCMakeMinimumRequired.cmake") +cmake_minimum_required(VERSION ${GDAL_CMAKE_VERSION_MIN}...${GDAL_CMAKE_VERSION_MAX}) + +if(NOT DEFINED PROJECT_SOURCE_DIR) + # Standalone plugin building + project(gdal_AVIF) + set(STRICT_VERSION_CHECK ON) + include("${PROJECT_SOURCE_DIR}/../../cmake/helpers/SetupStandalonePlugin.cmake" ) + include(CheckDependentLibrariesAVIF) + standalone_driver_finalize(GDAL_ENABLE_DRIVER_AVIF) +endif() + +add_gdal_driver(TARGET gdal_AVIF + SOURCES avifdataset.cpp + CORE_SOURCES avifdrivercore.cpp + PLUGIN_CAPABLE + NO_SHARED_SYMBOL_WITH_CORE + STRONG_CXX_WFLAGS) + +if(NOT TARGET gdal_AVIF) + return() +endif() + +gdal_standard_includes(gdal_AVIF) +gdal_target_link_libraries(gdal_AVIF PRIVATE AVIF::AVIF) +target_include_directories(gdal_AVIF PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../mem) diff --git a/frmts/avif/avifdataset.cpp b/frmts/avif/avifdataset.cpp new file mode 100644 index 000000000000..bb94754d211c --- /dev/null +++ b/frmts/avif/avifdataset.cpp @@ -0,0 +1,1179 @@ +/****************************************************************************** + * + * Project: AVIF driver + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "gdal_pam.h" +#include "cpl_minixml.h" +#include "cpl_vsi_virtual.h" + +#include "avifdrivercore.h" +#include "gdalexif.h" +#include "memdataset.h" + +#include + +#include +#include +#include + +constexpr const char *DEFAULT_QUALITY_STR = "60"; +constexpr const char *DEFAULT_QUALITY_ALPHA_STR = "100"; +constexpr const char *DEFAULT_SPEED_STR = "6"; + +/************************************************************************/ +/* GDALAVIFDataset */ +/************************************************************************/ + +class GDALAVIFDataset final : public GDALPamDataset +{ + friend class GDALAVIFRasterBand; + + avifDecoder *m_decoder = nullptr; + bool m_bDecodedDone = false; + bool m_bDecodedOK = false; + int m_iPart = 0; + avifRGBImage m_rgb{}; // memset()' to 0 in constructor + + bool Init(GDALOpenInfo *poOpenInfo); + bool Decode(); + + GDALAVIFDataset(const GDALAVIFDataset &) = delete; + GDALAVIFDataset &operator=(const GDALAVIFDataset &) = delete; + + public: + GDALAVIFDataset() + { + memset(&m_rgb, 0, sizeof(m_rgb)); + } + + ~GDALAVIFDataset(); + + static GDALPamDataset *OpenStaticPAM(GDALOpenInfo *poOpenInfo); + + static GDALDataset *Open(GDALOpenInfo *poOpenInfo) + { + return OpenStaticPAM(poOpenInfo); + } + + static GDALDataset *CreateCopy(const char *, GDALDataset *, int, + char **papszOptions, + GDALProgressFunc pfnProgress, + void *pProgressData); +}; + +/************************************************************************/ +/* GDALAVIFRasterBand */ +/************************************************************************/ + +class GDALAVIFRasterBand final : public MEMRasterBand +{ + public: + GDALAVIFRasterBand(GDALAVIFDataset *poDSIn, int nBandIn, + GDALDataType eDataTypeIn, int nBits); + + GDALColorInterp GetColorInterpretation() override + { + if (poDS->GetRasterCount() == 1) + return GCI_GrayIndex; + else if (poDS->GetRasterCount() == 2) + return nBand == 1 ? GCI_GrayIndex : GCI_AlphaBand; + else + return static_cast(GCI_RedBand + nBand - 1); + } + + protected: + friend class GDALAVIFDataset; + + CPLErr IReadBlock(int, int, void *) override; + CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, GSpacing nPixelSpaceBuf, + GSpacing nLineSpaceBuf, + GDALRasterIOExtraArg *psExtraArg) override; + + void SetData(GByte *pabyDataIn, int nPixelOffsetIn, int nLineOffsetIn); +}; + +/************************************************************************/ +/* GDALAVIFIO */ +/************************************************************************/ + +class GDALAVIFIO +{ + public: + explicit GDALAVIFIO(VSIVirtualHandleUniquePtr fpIn); + + private: + avifIO io{}; // memset()' to 0 in constructor + VSIVirtualHandleUniquePtr fp{}; + vsi_l_offset nFileSize = 0; + std::vector buffer{}; + + static void Destroy(struct avifIO *io); + static avifResult Read(struct avifIO *io, uint32_t readFlags, + uint64_t offset, size_t size, avifROData *out); + + GDALAVIFIO(const GDALAVIFIO &) = delete; + GDALAVIFIO &operator=(const GDALAVIFIO &) = delete; +}; + +/************************************************************************/ +/* ~GDALAVIFDataset() */ +/************************************************************************/ + +GDALAVIFDataset::~GDALAVIFDataset() +{ + if (m_decoder) + { + avifDecoderDestroy(m_decoder); + avifRGBImageFreePixels(&m_rgb); + } +} + +/************************************************************************/ +/* GDALAVIFDataset::Decode() */ +/************************************************************************/ + +bool GDALAVIFDataset::Decode() +{ + if (m_bDecodedDone) + return m_bDecodedOK; + m_bDecodedDone = true; + + auto avifErr = m_iPart == 0 ? avifDecoderNextImage(m_decoder) + : avifDecoderNthImage(m_decoder, m_iPart); + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifDecoderNextImage() failed with: %s", + avifResultToString(avifErr)); + return false; + } + + avifRGBImageSetDefaults(&m_rgb, m_decoder->image); + + m_rgb.format = (nBands == 1 || nBands == 3) ? AVIF_RGB_FORMAT_RGB + : AVIF_RGB_FORMAT_RGBA; + const int nChannels = m_rgb.format == AVIF_RGB_FORMAT_RGB ? 3 : 4; + +#if AVIF_VERSION_MAJOR >= 1 + avifErr = avifRGBImageAllocatePixels(&m_rgb); + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifRGBImageAllocatePixels() failed with: %s", + avifResultToString(avifErr)); + return false; + } +#else + avifRGBImageAllocatePixels(&m_rgb); + if (m_rgb.pixels == nullptr) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifRGBImageAllocatePixels() failed"); + return false; + } +#endif + + avifErr = avifImageYUVToRGB(m_decoder->image, &m_rgb); + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifImageYUVToRGB() failed with: %s", + avifResultToString(avifErr)); + return false; + } + + const auto eDT = papoBands[0]->GetRasterDataType(); + const int nDTSize = GDALGetDataTypeSizeBytes(eDT); + for (int i = 0; i < nBands; ++i) + { + const int iAVIFChannel = (nBands == 2 && i == 1) ? 3 : i; + cpl::down_cast(papoBands[i]) + ->SetData(m_rgb.pixels + iAVIFChannel * nDTSize, + nDTSize * nChannels, static_cast(m_rgb.rowBytes)); + } + + m_bDecodedOK = true; + return m_bDecodedOK; +} + +/************************************************************************/ +/* GDALAVIFRasterBand() */ +/************************************************************************/ + +GDALAVIFRasterBand::GDALAVIFRasterBand(GDALAVIFDataset *poDSIn, int nBandIn, + GDALDataType eDataTypeIn, int nBits) + : MEMRasterBand(poDSIn, nBandIn, nullptr, eDataTypeIn, 0, 0, false) +{ + if (nBits != 8 && nBits != 16) + { + GDALRasterBand::SetMetadataItem("NBITS", CPLSPrintf("%d", nBits), + "IMAGE_STRUCTURE"); + } +} + +/************************************************************************/ +/* SetData() */ +/************************************************************************/ + +void GDALAVIFRasterBand::SetData(GByte *pabyDataIn, int nPixelOffsetIn, + int nLineOffsetIn) +{ + pabyData = pabyDataIn; + nPixelOffset = nPixelOffsetIn; + nLineOffset = nLineOffsetIn; +} + +/************************************************************************/ +/* IReadBlock() */ +/************************************************************************/ + +CPLErr GDALAVIFRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, + void *pImage) +{ + GDALAVIFDataset *poGDS = cpl::down_cast(poDS); + if (!poGDS->Decode()) + return CE_Failure; + return MEMRasterBand::IReadBlock(nBlockXOff, nBlockYOff, pImage); +} + +/************************************************************************/ +/* IRasterIO() */ +/************************************************************************/ + +CPLErr GDALAVIFRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, + int nBufXSize, int nBufYSize, + GDALDataType eBufType, + GSpacing nPixelSpaceBuf, + GSpacing nLineSpaceBuf, + GDALRasterIOExtraArg *psExtraArg) +{ + GDALAVIFDataset *poGDS = cpl::down_cast(poDS); + if (!poGDS->Decode()) + return CE_Failure; + return MEMRasterBand::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, + pData, nBufXSize, nBufYSize, eBufType, + nPixelSpaceBuf, nLineSpaceBuf, psExtraArg); +} + +/************************************************************************/ +/* GDALAVIFIO::GDALAVIFIO() */ +/************************************************************************/ + +GDALAVIFIO::GDALAVIFIO(VSIVirtualHandleUniquePtr fpIn) : fp(std::move(fpIn)) +{ + memset(&io, 0, sizeof(io)); + io.destroy = Destroy; + io.read = Read; + + fp->Seek(0, SEEK_END); + nFileSize = fp->Tell(); + fp->Seek(0, SEEK_SET); + + io.sizeHint = std::min(10 * 1024 * 1024, nFileSize); +} + +/************************************************************************/ +/* GDALAVIFIO::Destroy() */ +/************************************************************************/ + +/* static */ void GDALAVIFIO::Destroy(struct avifIO *io) +{ + GDALAVIFIO *gdalIO = reinterpret_cast(io); + delete gdalIO; +} + +/************************************************************************/ +/* GDALAVIFIO::Read() */ +/************************************************************************/ + +/* static */ avifResult GDALAVIFIO::Read(struct avifIO *io, uint32_t readFlags, + uint64_t offset, size_t size, + avifROData *out) +{ + GDALAVIFIO *gdalIO = reinterpret_cast(io); + if (readFlags != 0) + { + // Unsupported readFlags + return AVIF_RESULT_IO_ERROR; + } + if (offset > gdalIO->nFileSize) + { + return AVIF_RESULT_IO_ERROR; + } + + if (offset == gdalIO->nFileSize) + { + out->data = gdalIO->buffer.data(); + out->size = 0; + return AVIF_RESULT_OK; + } + + const uint64_t availableSize = gdalIO->nFileSize - offset; + size = static_cast(std::min(size, availableSize)); + try + { + gdalIO->buffer.resize(size); + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory in GDALAVIFIO::Read()"); + return AVIF_RESULT_IO_ERROR; + } + + if (gdalIO->fp->Seek(offset, SEEK_SET) != 0 || + gdalIO->fp->Read(gdalIO->buffer.data(), size, 1) != 1) + { + return AVIF_RESULT_IO_ERROR; + } + + out->data = gdalIO->buffer.data(); + out->size = size; + return AVIF_RESULT_OK; +} + +/************************************************************************/ +/* Init() */ +/************************************************************************/ + +bool GDALAVIFDataset::Init(GDALOpenInfo *poOpenInfo) +{ + m_decoder = avifDecoderCreate(); + if (!m_decoder) + return false; + + std::string osFilename(poOpenInfo->pszFilename); + VSIVirtualHandleUniquePtr fp(poOpenInfo->fpL); + poOpenInfo->fpL = nullptr; + + if (STARTS_WITH_CI(poOpenInfo->pszFilename, "AVIF:")) + { + const char *pszPartPos = poOpenInfo->pszFilename + strlen("AVIF:"); + const char *pszNextColumn = strchr(pszPartPos, ':'); + if (pszNextColumn == nullptr) + return false; + m_iPart = atoi(pszPartPos); + if (m_iPart <= 0) + return false; + osFilename = pszNextColumn + 1; + fp.reset(VSIFOpenL(osFilename.c_str(), "rb")); + if (fp == nullptr) + return false; + } + + auto gdalIO = std::make_unique(std::move(fp)); + avifDecoderSetIO(m_decoder, reinterpret_cast(gdalIO.release())); + + const auto avifErr = avifDecoderParse(m_decoder); + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifDecoderParse() failed with: %s", + avifResultToString(avifErr)); + return false; + } + + // AVIF limit is 65,536 x 65,536 pixels; + nRasterXSize = static_cast(m_decoder->image->width); + nRasterYSize = static_cast(m_decoder->image->height); + + if (m_decoder->image->depth > 12) + { + CPLError(CE_Failure, CPLE_NotSupported, "Unsupported AVIF depth: %u", + m_decoder->image->depth); + return false; + } + + const auto eDataType = + (m_decoder->image->depth <= 8) ? GDT_Byte : GDT_UInt16; + const int l_nBands = m_decoder->image->yuvFormat == AVIF_PIXEL_FORMAT_YUV400 + ? (m_decoder->alphaPresent ? 2 : 1) + : m_decoder->alphaPresent ? 4 + : 3; + + if (m_decoder->image->yuvFormat == AVIF_PIXEL_FORMAT_YUV444) + GDALDataset::SetMetadataItem("YUV_SUBSAMPLING", "444", + "IMAGE_STRUCTURE"); + else if (m_decoder->image->yuvFormat == AVIF_PIXEL_FORMAT_YUV422) + GDALDataset::SetMetadataItem("YUV_SUBSAMPLING", "422", + "IMAGE_STRUCTURE"); + else if (m_decoder->image->yuvFormat == AVIF_PIXEL_FORMAT_YUV420) + GDALDataset::SetMetadataItem("YUV_SUBSAMPLING", "420", + "IMAGE_STRUCTURE"); + + for (int i = 0; i < l_nBands; ++i) + { + SetBand(i + 1, new GDALAVIFRasterBand( + this, i + 1, eDataType, + static_cast(m_decoder->image->depth))); + } + + if (m_iPart == 0) + { + if (m_decoder->imageCount > 1) + { + CPLStringList aosSubDS; + for (int i = 0; i < m_decoder->imageCount; i++) + { + aosSubDS.SetNameValue( + CPLSPrintf("SUBDATASET_%d_NAME", i + 1), + CPLSPrintf("AVIF:%d:%s", i + 1, poOpenInfo->pszFilename)); + aosSubDS.SetNameValue(CPLSPrintf("SUBDATASET_%d_DESC", i + 1), + CPLSPrintf("Subdataset %d", i + 1)); + } + GDALDataset::SetMetadata(aosSubDS.List(), "SUBDATASETS"); + } + } + else if (m_iPart > m_decoder->imageCount) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid image part number. Maximum allowed is %d", + m_decoder->imageCount); + return false; + } + else + { + m_iPart--; + } + + if (m_decoder->image->exif.size >= 8) + { + VSILFILE *fpEXIF = + VSIFileFromMemBuffer(nullptr, m_decoder->image->exif.data, + m_decoder->image->exif.size, false); + int nExifOffset = 0; + int nInterOffset = 0; + int nGPSOffset = 0; + char **papszEXIFMetadata = nullptr; +#ifdef CPL_LSB + const bool bSwab = m_decoder->image->exif.data[0] == 0x4d; +#else + const bool bSwab = m_decoder->image->exif.data[0] == 0x49; +#endif + constexpr int nTIFFHEADER = 0; + uint32_t nTiffDirStart; + memcpy(&nTiffDirStart, m_decoder->image->exif.data + 4, + sizeof(uint32_t)); + if (bSwab) + { + CPL_LSBPTR32(&nTiffDirStart); + } + EXIFExtractMetadata(papszEXIFMetadata, fpEXIF, nTiffDirStart, bSwab, + nTIFFHEADER, nExifOffset, nInterOffset, nGPSOffset); + + if (nExifOffset > 0) + { + EXIFExtractMetadata(papszEXIFMetadata, fpEXIF, nExifOffset, bSwab, + nTIFFHEADER, nExifOffset, nInterOffset, + nGPSOffset); + } + if (nInterOffset > 0) + { + EXIFExtractMetadata(papszEXIFMetadata, fpEXIF, nInterOffset, bSwab, + nTIFFHEADER, nExifOffset, nInterOffset, + nGPSOffset); + } + if (nGPSOffset > 0) + { + EXIFExtractMetadata(papszEXIFMetadata, fpEXIF, nGPSOffset, bSwab, + nTIFFHEADER, nExifOffset, nInterOffset, + nGPSOffset); + } + VSIFCloseL(fpEXIF); + GDALDataset::SetMetadata(papszEXIFMetadata, "EXIF"); + CSLDestroy(papszEXIFMetadata); + } + + if (m_decoder->image->xmp.size > 0) + { + const std::string osXMP( + reinterpret_cast(m_decoder->image->xmp.data), + m_decoder->image->xmp.size); + const char *const apszMD[] = {osXMP.c_str(), nullptr}; + GDALDataset::SetMetadata(const_cast(apszMD), "xml:XMP"); + } + + if (m_decoder->image->icc.size > 0) + { + // Escape the profile. + char *pszBase64Profile = + CPLBase64Encode(static_cast(m_decoder->image->icc.size), + m_decoder->image->icc.data); + + // Set ICC profile metadata. + SetMetadataItem("SOURCE_ICC_PROFILE", pszBase64Profile, + "COLOR_PROFILE"); + + CPLFree(pszBase64Profile); + } + + // Initialize any PAM information. + if (m_decoder->imageCount > 1) + { + SetSubdatasetName(CPLSPrintf("%d", m_iPart + 1)); + SetPhysicalFilename(osFilename.c_str()); + } + SetDescription(poOpenInfo->pszFilename); + TryLoadXML(poOpenInfo->GetSiblingFiles()); + + return true; +} + +/************************************************************************/ +/* OpenStaticPAM() */ +/************************************************************************/ + +/* static */ +GDALPamDataset *GDALAVIFDataset::OpenStaticPAM(GDALOpenInfo *poOpenInfo) +{ + if (!AVIFDriverIdentify(poOpenInfo)) + return nullptr; + + if (poOpenInfo->eAccess == GA_Update) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Update of existing AVIF file not supported"); + return nullptr; + } + + auto poDS = std::make_unique(); + if (!poDS->Init(poOpenInfo)) + return nullptr; + + return poDS.release(); +} + +/************************************************************************/ +/* CreateCopy() */ +/************************************************************************/ + +/* static */ +GDALDataset *GDALAVIFDataset::CreateCopy(const char *pszFilename, + GDALDataset *poSrcDS, + int /* bStrict */, char **papszOptions, + GDALProgressFunc pfnProgress, + void *pProgressData) +{ + auto poDrv = GetGDALDriverManager()->GetDriverByName(DRIVER_NAME); + if (poDrv && poDrv->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST) == nullptr) + { + CPLError(CE_Failure, CPLE_NotSupported, + "This build of libavif has been done without any AV1 encoder"); + return nullptr; + } + + // Perform various validations on source dataset + const int nXSize = poSrcDS->GetRasterXSize(); + const int nYSize = poSrcDS->GetRasterYSize(); + const int nBands = poSrcDS->GetRasterCount(); + + if (nXSize > 65536 || nYSize > 65536) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Too big source dataset. Maximum AVIF image dimension is " + "65,536 x 65,536 pixels"); + return nullptr; + } + if (nBands != 1 && nBands != 2 && nBands != 3 && nBands != 4) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Unsupported number of bands: only 1 (Gray), 2 (Graph+Alpha) " + "3 (RGB) or 4 (RGBA) bands are supported"); + return nullptr; + } + + const auto poFirstBand = poSrcDS->GetRasterBand(1); + if (poFirstBand->GetColorTable()) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Source dataset with color table unsupported. Use " + "gdal_translate -expand rgb|rgba first"); + return nullptr; + } + + const auto eDT = poFirstBand->GetRasterDataType(); + if (eDT != GDT_Byte && eDT != GDT_UInt16) + { + CPLError( + CE_Failure, CPLE_NotSupported, + "Unsupported data type: only Byte or UInt16 bands are supported"); + return nullptr; + } + + int nBits = eDT == GDT_Byte ? 8 : 12; + const char *pszNBITS = CSLFetchNameValue(papszOptions, "NBITS"); + if (pszNBITS) + { + nBits = atoi(pszNBITS); + } + else if (eDT == GDT_UInt16) + { + pszNBITS = poFirstBand->GetMetadataItem("NBITS", "IMAGE_STRUCTURE"); + if (pszNBITS) + { + nBits = atoi(pszNBITS); + } + } + if ((eDT == GDT_Byte && nBits != 8) || + (eDT == GDT_UInt16 && nBits != 10 && nBits != 12)) + { + CPLError(CE_Failure, CPLE_FileIO, + "Invalid/inconsistent bit depth w.r.t data type"); + return nullptr; + } + + const int nQuality = + std::clamp(atoi(CSLFetchNameValueDef(papszOptions, "QUALITY", + DEFAULT_QUALITY_STR)), + 0, 100); + const int nQualityAlpha = + std::clamp(atoi(CSLFetchNameValueDef(papszOptions, "QUALITY_ALPHA", + DEFAULT_QUALITY_ALPHA_STR)), + 0, 100); + + // Create AVIF image. + avifPixelFormat ePixelFormat = + nBands <= 2 ? AVIF_PIXEL_FORMAT_YUV400 : AVIF_PIXEL_FORMAT_YUV444; + if (nBands >= 3) + { + const char *pszYUV_SUBSAMPLING = + CSLFetchNameValueDef(papszOptions, "YUV_SUBSAMPLING", "444"); + if (EQUAL(pszYUV_SUBSAMPLING, "422")) + ePixelFormat = AVIF_PIXEL_FORMAT_YUV422; + else if (EQUAL(pszYUV_SUBSAMPLING, "420")) + ePixelFormat = AVIF_PIXEL_FORMAT_YUV420; + + if (nQuality == 100 && nQualityAlpha == 100 && + ePixelFormat != AVIF_PIXEL_FORMAT_YUV444) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Only YUV_SUBSAMPLING=444 is supported for lossless " + "encoding"); + return nullptr; + } + } + + // Create empty output file + VSIVirtualHandleUniquePtr fp(VSIFOpenL(pszFilename, "wb")); + if (!fp) + { + CPLError(CE_Failure, CPLE_FileIO, "Cannot create file %s", pszFilename); + return nullptr; + } + + avifImage *image = avifImageCreate(nXSize, nYSize, nBits, ePixelFormat); + if (!image) + { + return nullptr; + } + + avifRGBImage rgb; + memset(&rgb, 0, sizeof(rgb)); + avifRGBImageSetDefaults(&rgb, image); + + rgb.format = + nBands == 1 || nBands == 3 ? AVIF_RGB_FORMAT_RGB : AVIF_RGB_FORMAT_RGBA; + + avifResult avifErr; + +#if AVIF_VERSION_MAJOR >= 1 + avifErr = avifRGBImageAllocatePixels(&rgb); + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifRGBImageAllocatePixels() failed with: %s", + avifResultToString(avifErr)); + avifImageDestroy(image); + return nullptr; + } +#else + avifRGBImageAllocatePixels(&rgb); + if (!rgb.pixels) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifRGBImageAllocatePixels() failed"); + avifImageDestroy(image); + return nullptr; + } +#endif + + const int nDTSize = GDALGetDataTypeSizeBytes(eDT); + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + + CPLErr eErr; + if (nBands == 1) + { + int anBands[] = {1, 1, 1}; + eErr = poSrcDS->RasterIO(GF_Read, 0, 0, nXSize, nYSize, rgb.pixels, + nXSize, nYSize, eDT, 3, anBands, nDTSize * 3, + static_cast(rgb.rowBytes), nDTSize, + &sExtraArg); + } + else if (nBands == 2) + { + int anBands[] = {1, 1, 1, 2}; + eErr = poSrcDS->RasterIO(GF_Read, 0, 0, nXSize, nYSize, rgb.pixels, + nXSize, nYSize, eDT, 4, anBands, nDTSize * 4, + static_cast(rgb.rowBytes), nDTSize, + &sExtraArg); + } + else + { + eErr = poSrcDS->RasterIO( + GF_Read, 0, 0, nXSize, nYSize, rgb.pixels, nXSize, nYSize, eDT, + nBands, nullptr, nDTSize * nBands, static_cast(rgb.rowBytes), + nDTSize, &sExtraArg); + } + if (eErr != CE_None) + { + avifImageDestroy(image); + avifRGBImageFreePixels(&rgb); + return nullptr; + } + + if (nQuality == 100 && nQualityAlpha == 100) + { + // Cf https://github.com/AOMediaCodec/libavif/blob/0d3e5e215dffbbd6afbf917ce00c84de599ba410/apps/avifenc.c#L1952 + image->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_IDENTITY; + } + else + { + // Cf https://github.com/AOMediaCodec/libavif/blob/0d3e5e215dffbbd6afbf917ce00c84de599ba410/apps/avifenc.c#L1434 + image->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT601; + } + + // Cf https://github.com/AOMediaCodec/libavif/blob/0d3e5e215dffbbd6afbf917ce00c84de599ba410/apps/avifenc.c#L2249 + // The final image has no ICC profile, the user didn't specify any CICP, and the source + // image didn't provide any CICP. Explicitly signal SRGB CP/TC here, as 2/2/x will be + // interpreted as SRGB anyway. + image->colorPrimaries = AVIF_COLOR_PRIMARIES_BT709; + image->transferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB; + + image->yuvRange = AVIF_RANGE_FULL; + image->alphaPremultiplied = 0; + + avifErr = avifImageRGBToYUV(image, &rgb); + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifImageRGBToYUV() failed with: %s", + avifResultToString(avifErr)); + avifImageDestroy(image); + avifRGBImageFreePixels(&rgb); + return nullptr; + } + + avifEncoder *encoder = avifEncoderCreate(); + if (!encoder) + { + avifImageDestroy(image); + avifRGBImageFreePixels(&rgb); + return nullptr; + } + + const char *pszCodec = CSLFetchNameValueDef(papszOptions, "CODEC", "AUTO"); + if (!EQUAL(pszCodec, "AUTO")) + { + encoder->codecChoice = + avifCodecChoiceFromName(CPLString(pszCodec).tolower().c_str()); + } + + const char *pszThreads = CSLFetchNameValueDef( + papszOptions, "NUM_THREADS", + CPLGetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS")); + if (pszThreads && !EQUAL(pszThreads, "ALL_CPUS")) + encoder->maxThreads = atoi(pszThreads); + else + encoder->maxThreads = CPLGetNumCPUs(); + +#if AVIF_VERSION_MAJOR >= 1 + encoder->quality = nQuality; + encoder->qualityAlpha = nQualityAlpha; +#else + // Cf https://github.com/AOMediaCodec/libavif/blob/0d3e5e215dffbbd6afbf917ce00c84de599ba410/src/write.c#L1119 + const int nQuantizer = ((100 - nQuality) * 63 + 50) / 100; + encoder->minQuantizer = nQuantizer; + encoder->maxQuantizer = nQuantizer; + const int nQuantizerAlpha = ((100 - nQualityAlpha) * 63 + 50) / 100; + encoder->minQuantizerAlpha = nQuantizerAlpha; + encoder->maxQuantizerAlpha = nQuantizerAlpha; +#endif + + encoder->speed = std::clamp( + atoi(CSLFetchNameValueDef(papszOptions, "SPEED", DEFAULT_SPEED_STR)), 0, + 10); + + if (CPLTestBool( + CSLFetchNameValueDef(papszOptions, "WRITE_EXIF_METADATA", "YES"))) + { + char **papszEXIFMD = poSrcDS->GetMetadata("EXIF"); + if (papszEXIFMD) + { + GUInt32 nDataSize = 0; + GByte *pabyEXIF = + EXIFCreate(papszEXIFMD, nullptr, 0, 0, 0, &nDataSize); + if (pabyEXIF) + { + CPLAssert(nDataSize > 6 && + memcmp(pabyEXIF, "Exif\0\0", 6) == 0); + +#if AVIF_VERSION_MAJOR >= 1 + CPL_IGNORE_RET_VAL(avifImageSetMetadataExif(image, pabyEXIF + 6, + nDataSize - 6)); +#else + avifImageSetMetadataExif(image, pabyEXIF + 6, nDataSize - 6); +#endif + CPLFree(pabyEXIF); + } + } + } + + if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "WRITE_XMP", "YES"))) + { + CSLConstList papszXMP = poSrcDS->GetMetadata("xml:XMP"); + if (papszXMP && papszXMP[0]) + { +#if AVIF_VERSION_MAJOR >= 1 + CPL_IGNORE_RET_VAL(avifImageSetMetadataXMP( + image, reinterpret_cast(papszXMP[0]), + strlen(papszXMP[0]))); +#else + avifImageSetMetadataXMP( + image, reinterpret_cast(papszXMP[0]), + strlen(papszXMP[0])); +#endif + } + } + +#if AVIF_VERSION_MAJOR >= 1 + const char *pszICCProfile = + CSLFetchNameValue(papszOptions, "SOURCE_ICC_PROFILE"); + if (pszICCProfile == nullptr) + { + pszICCProfile = + poSrcDS->GetMetadataItem("SOURCE_ICC_PROFILE", "COLOR_PROFILE"); + } + if (pszICCProfile && pszICCProfile[0] != '\0') + { + char *pEmbedBuffer = CPLStrdup(pszICCProfile); + const GInt32 nEmbedLen = + CPLBase64DecodeInPlace(reinterpret_cast(pEmbedBuffer)); + CPL_IGNORE_RET_VAL(avifImageSetProfileICC( + image, reinterpret_cast(pEmbedBuffer), nEmbedLen)); + CPLFree(pEmbedBuffer); + } +#endif + + avifErr = + avifEncoderAddImage(encoder, image, 1, AVIF_ADD_IMAGE_FLAG_SINGLE); + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifEncoderAddImage() failed with: %s", + avifResultToString(avifErr)); + avifImageDestroy(image); + avifEncoderDestroy(encoder); + avifRGBImageFreePixels(&rgb); + return nullptr; + } + + avifRWData avifOutput = AVIF_DATA_EMPTY; + avifErr = avifEncoderFinish(encoder, &avifOutput); + + avifEncoderDestroy(encoder); + avifImageDestroy(image); + avifRGBImageFreePixels(&rgb); + + if (avifErr != AVIF_RESULT_OK) + { + CPLError(CE_Failure, CPLE_AppDefined, + "avifEncoderFinish() failed with: %s", + avifResultToString(avifErr)); + return nullptr; + } + + const size_t nSize = static_cast(avifOutput.size); + if (fp->Write(avifOutput.data, 1, nSize) != nSize || fp->Close() != 0) + { + CPLError(CE_Failure, CPLE_FileIO, + "Could not write %" PRIu64 " bytes into file %s", + static_cast(nSize), pszFilename); + avifRWDataFree(&avifOutput); + return nullptr; + } + avifRWDataFree(&avifOutput); + + fp.reset(); + + if (pfnProgress) + pfnProgress(1.0, "", pProgressData); + + // Re-open file and clone missing info to PAM + GDALOpenInfo oOpenInfo(pszFilename, GA_ReadOnly); + auto poDS = OpenStaticPAM(&oOpenInfo); + if (poDS) + { + // Do not create a .aux.xml file just for AREA_OR_POINT=Area + const char *pszAreaOfPoint = + poSrcDS->GetMetadataItem(GDALMD_AREA_OR_POINT); + if (pszAreaOfPoint && EQUAL(pszAreaOfPoint, GDALMD_AOP_AREA)) + { + poDS->SetMetadataItem(GDALMD_AREA_OR_POINT, GDALMD_AOP_AREA); + poDS->SetPamFlags(poDS->GetPamFlags() & ~GPF_DIRTY); + } + + int nPamMask = GCIF_PAM_DEFAULT; + poDS->CloneInfo(poSrcDS, nPamMask); + } + + return poDS; +} + +/************************************************************************/ +/* GDALAVIFDriver */ +/************************************************************************/ + +class GDALAVIFDriver final : public GDALDriver +{ + bool m_bMetadataInitialized = false; + void InitMetadata(); + + public: + const char *GetMetadataItem(const char *pszName, + const char *pszDomain = "") override + { + if (EQUAL(pszName, GDAL_DMD_CREATIONOPTIONLIST)) + { + InitMetadata(); + } + return GDALDriver::GetMetadataItem(pszName, pszDomain); + } + + char **GetMetadata(const char *pszDomain) override + { + InitMetadata(); + return GDALDriver::GetMetadata(pszDomain); + } +}; + +void GDALAVIFDriver::InitMetadata() +{ + if (m_bMetadataInitialized) + return; + m_bMetadataInitialized = true; + + std::vector aosCodecNames; + for (auto eMethod : {AVIF_CODEC_CHOICE_AUTO, AVIF_CODEC_CHOICE_AOM, + AVIF_CODEC_CHOICE_RAV1E, AVIF_CODEC_CHOICE_SVT}) + { + const char *pszName = + avifCodecName(eMethod, AVIF_CODEC_FLAG_CAN_ENCODE); + if (pszName) + { + aosCodecNames.push_back(eMethod == AVIF_CODEC_CHOICE_AUTO + ? CPLString("AUTO") + : CPLString(pszName).toupper()); + } + } + + if (aosCodecNames.empty()) + return; + + CPLXMLTreeCloser oTree( + CPLCreateXMLNode(nullptr, CXT_Element, "CreationOptionList")); + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "CODEC"); + CPLAddXMLAttributeAndValue(psOption, "type", "string-select"); + CPLAddXMLAttributeAndValue(psOption, "description", + "Compression CODEC"); + CPLAddXMLAttributeAndValue(psOption, "default", "AUTO"); + for (const std::string &osCodecName : aosCodecNames) + { + auto poValueNode = CPLCreateXMLNode(psOption, CXT_Element, "Value"); + CPLCreateXMLNode(poValueNode, CXT_Text, osCodecName.c_str()); + } + } + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "QUALITY"); + CPLAddXMLAttributeAndValue(psOption, "type", "int"); + CPLAddXMLAttributeAndValue( + psOption, "description", + "Quality for non-alpha channels (0=worst, 100=best/lossless)"); + CPLAddXMLAttributeAndValue(psOption, "default", DEFAULT_QUALITY_STR); + CPLAddXMLAttributeAndValue(psOption, "min", "0"); + CPLAddXMLAttributeAndValue(psOption, "max", "100"); + } + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "QUALITY_ALPHA"); + CPLAddXMLAttributeAndValue(psOption, "type", "int"); + CPLAddXMLAttributeAndValue( + psOption, "description", + "Quality for alpha channel (0=worst, 100=best/lossless)"); + CPLAddXMLAttributeAndValue(psOption, "default", + DEFAULT_QUALITY_ALPHA_STR); + CPLAddXMLAttributeAndValue(psOption, "min", "0"); + CPLAddXMLAttributeAndValue(psOption, "max", "100"); + } + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "SPEED"); + CPLAddXMLAttributeAndValue(psOption, "type", "int"); + CPLAddXMLAttributeAndValue(psOption, "description", + "Encoder speed (0=slowest, 10=fastest)"); + CPLAddXMLAttributeAndValue(psOption, "default", DEFAULT_SPEED_STR); + CPLAddXMLAttributeAndValue(psOption, "min", "0"); + CPLAddXMLAttributeAndValue(psOption, "max", "10"); + } + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "NUM_THREADS"); + CPLAddXMLAttributeAndValue(psOption, "type", "string"); + CPLAddXMLAttributeAndValue( + psOption, "description", + "Number of worker threads for compression. Can be set to ALL_CPUS"); + CPLAddXMLAttributeAndValue(psOption, "default", "ALL_CPUS"); + } + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "WRITE_EXIF_METADATA"); + CPLAddXMLAttributeAndValue(psOption, "type", "boolean"); + CPLAddXMLAttributeAndValue(psOption, "description", + "Whether to write EXIF metadata"); + CPLAddXMLAttributeAndValue(psOption, "default", "YES"); + } + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "WRITE_XMP"); + CPLAddXMLAttributeAndValue(psOption, "type", "boolean"); + CPLAddXMLAttributeAndValue(psOption, "description", + "Whether to write XMP metadata"); + CPLAddXMLAttributeAndValue(psOption, "default", "YES"); + } + +#if AVIF_VERSION_MAJOR >= 1 + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "SOURCE_ICC_PROFILE"); + CPLAddXMLAttributeAndValue(psOption, "type", "string"); + CPLAddXMLAttributeAndValue(psOption, "description", + "ICC profile encoded in Base64"); + } +#endif + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "NBITS"); + CPLAddXMLAttributeAndValue(psOption, "type", "int"); + CPLAddXMLAttributeAndValue(psOption, "description", + "Bit depth. Valid values are 8, 10, 12."); + } + + { + auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option"); + CPLAddXMLAttributeAndValue(psOption, "name", "YUV_SUBSAMPLING"); + CPLAddXMLAttributeAndValue(psOption, "type", "string-select"); + CPLAddXMLAttributeAndValue( + psOption, "description", + "Subsampling factor for YUV colorspace (for RGB or RGBA)"); + CPLAddXMLAttributeAndValue(psOption, "default", "444"); + + for (const char *pszValue : {"444", "422", "420"}) + { + auto poValueNode = CPLCreateXMLNode(psOption, CXT_Element, "Value"); + CPLCreateXMLNode(poValueNode, CXT_Text, pszValue); + } + } + + char *pszXML = CPLSerializeXMLTree(oTree.get()); + GDALDriver::SetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST, pszXML); + CPLFree(pszXML); +} + +/************************************************************************/ +/* GDALRegister_AVIF() */ +/************************************************************************/ + +void GDALRegister_AVIF() + +{ + if (!GDAL_CHECK_VERSION("AVIF driver")) + return; + + if (GDALGetDriverByName(DRIVER_NAME) != nullptr) + return; + + // Check libavif runtime vs compile-time versions + const char *pszVersion = avifVersion(); + const CPLStringList aosVersionTokens( + CSLTokenizeString2(pszVersion, ".", 0)); + if (aosVersionTokens.size() >= 2 && + std::string(aosVersionTokens[0]) + .append(".") + .append(aosVersionTokens[1]) != + CPLSPrintf("%d.%d", AVIF_VERSION_MAJOR, AVIF_VERSION_MINOR)) + { + const std::string osExpectedVersion( + CPLSPrintf("%d.%d.%d", AVIF_VERSION_MAJOR, AVIF_VERSION_MINOR, + AVIF_VERSION_PATCH)); + CPLError(CE_Warning, CPLE_AppDefined, + "GDAL AVIF driver was built against libavif %s but is running " + "against %s. Runtime issues could occur", + osExpectedVersion.c_str(), avifVersion()); + } + + auto poDriver = std::make_unique(); + auto poDM = GetGDALDriverManager(); + bool bMayHaveWriteSupport = true; + if (!poDM->IsKnownDriver("AVIF")) + { + // If we are not built as a defered plugin, check now if libavif has + // write support + bMayHaveWriteSupport = + poDriver->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST) != nullptr; + } + + AVIFDriverSetCommonMetadata(poDriver.get(), bMayHaveWriteSupport); + + poDriver->pfnOpen = GDALAVIFDataset::Open; + if (bMayHaveWriteSupport) + poDriver->pfnCreateCopy = GDALAVIFDataset::CreateCopy; + + poDM->RegisterDriver(poDriver.release()); +} diff --git a/frmts/avif/avifdrivercore.cpp b/frmts/avif/avifdrivercore.cpp new file mode 100644 index 000000000000..f6b6d9d2578b --- /dev/null +++ b/frmts/avif/avifdrivercore.cpp @@ -0,0 +1,92 @@ +/****************************************************************************** + * + * Project: AVIF Driver + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "avifdrivercore.h" + +/************************************************************************/ +/* AVIFDriverIdentify() */ +/************************************************************************/ + +int AVIFDriverIdentify(GDALOpenInfo *poOpenInfo) + +{ + if (STARTS_WITH_CI(poOpenInfo->pszFilename, "AVIF:")) + return true; + + if (poOpenInfo->nHeaderBytes < 12 || poOpenInfo->fpL == nullptr) + return false; + + return memcmp(poOpenInfo->pabyHeader + 4, "ftypavif", 8) == 0 || + memcmp(poOpenInfo->pabyHeader + 4, "ftypavis", 8) == 0; +} + +/************************************************************************/ +/* AVIFDriverSetCommonMetadata() */ +/************************************************************************/ + +void AVIFDriverSetCommonMetadata(GDALDriver *poDriver, + bool bMayHaveWriteSupport) +{ + poDriver->SetDescription(DRIVER_NAME); + poDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES"); + poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, "AV1 Image File Format"); + poDriver->SetMetadataItem(GDAL_DMD_MIMETYPE, "image/avif"); + poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, "drivers/raster/avif.html"); + poDriver->SetMetadataItem(GDAL_DMD_EXTENSION, "avif"); + poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); + poDriver->SetMetadataItem(GDAL_DMD_SUBDATASETS, "YES"); + + poDriver->pfnIdentify = AVIFDriverIdentify; + poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES"); + + if (bMayHaveWriteSupport) + { + poDriver->SetMetadataItem(GDAL_DMD_CREATIONDATATYPES, "Byte UInt16"); + poDriver->SetMetadataItem(GDAL_DCAP_CREATECOPY, "YES"); + } +} + +/************************************************************************/ +/* DeclareDeferredAVIFPlugin() */ +/************************************************************************/ + +#ifdef PLUGIN_FILENAME +void DeclareDeferredAVIFPlugin() +{ + if (GDALGetDriverByName(DRIVER_NAME) != nullptr) + { + return; + } + auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME); +#ifdef PLUGIN_INSTALLATION_MESSAGE + poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE, + PLUGIN_INSTALLATION_MESSAGE); +#endif + AVIFDriverSetCommonMetadata(poDriver, /* bMayHaveWriteSupport = */ true); + GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver); +} +#endif diff --git a/frmts/avif/avifdrivercore.h b/frmts/avif/avifdrivercore.h new file mode 100644 index 000000000000..8499e6c1bc18 --- /dev/null +++ b/frmts/avif/avifdrivercore.h @@ -0,0 +1,44 @@ +/****************************************************************************** + * + * Project: AVIF Driver + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef AVIFDRIVERCORE_H +#define AVIFDRIVERCORE_H + +#include "gdal_priv.h" + +constexpr const char *DRIVER_NAME = "AVIF"; + +#define AVIFDriverIdentify PLUGIN_SYMBOL_NAME(AVIFDriverIdentify) +#define AVIFDriverSetCommonMetadata \ + PLUGIN_SYMBOL_NAME(AVIFDriverSetCommonMetadata) + +int AVIFDriverIdentify(GDALOpenInfo *poOpenInfo); + +void AVIFDriverSetCommonMetadata(GDALDriver *poDriver, + bool bMayHaveWriteSupport); + +#endif diff --git a/frmts/avif/driver_declaration.cmake b/frmts/avif/driver_declaration.cmake new file mode 100644 index 000000000000..c69e87ec1399 --- /dev/null +++ b/frmts/avif/driver_declaration.cmake @@ -0,0 +1 @@ +gdal_dependent_format(avif "AVIF" "GDAL_USE_AVIF") diff --git a/frmts/cals/calsdataset.cpp b/frmts/cals/calsdataset.cpp index f8267075379f..389308524764 100644 --- a/frmts/cals/calsdataset.cpp +++ b/frmts/cals/calsdataset.cpp @@ -331,7 +331,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo) // Create a TIFF header for a single-strip CCITTFAX4 file. poDS->osTIFFHeaderFilename = - CPLSPrintf("/vsimem/cals/header_%p.tiff", poDS); + VSIMemGenerateHiddenFilename("cals_header.tiff"); VSILFILE *fp = VSIFOpenL(poDS->osTIFFHeaderFilename, "wb"); const int nTagCount = 10; const int nHeaderSize = 4 + 4 + 2 + nTagCount * 12 + 4; @@ -359,7 +359,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo) // Create a /vsisparse/ description file assembling the TIFF header // with the FAX4 codestream that starts at offset 2048 of the CALS file. - poDS->osSparseFilename = CPLSPrintf("/vsimem/cals/sparse_%p.xml", poDS); + poDS->osSparseFilename = VSIMemGenerateHiddenFilename("cals_sparse.xml"); fp = VSIFOpenL(poDS->osSparseFilename, "wb"); CPLAssert(fp); VSIFPrintfL(fp, @@ -473,7 +473,8 @@ GDALDataset *CALSDataset::CreateCopy(const char *pszFilename, // Write a in-memory TIFF with just the TIFF header to figure out // how large it will be. - CPLString osTmpFilename(CPLSPrintf("/vsimem/cals/tmp_%p", poSrcDS)); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("tmp_tif_header")); char **papszOptions = nullptr; papszOptions = CSLSetNameValue(papszOptions, "COMPRESS", "CCITTFAX4"); papszOptions = CSLSetNameValue(papszOptions, "NBITS", "1"); diff --git a/frmts/daas/daasdataset.cpp b/frmts/daas/daasdataset.cpp index 0c68c9e31439..8672c941a699 100644 --- a/frmts/daas/daasdataset.cpp +++ b/frmts/daas/daasdataset.cpp @@ -2420,7 +2420,7 @@ CPLErr GDALDAASRasterBand::GetBlocks(int nBlockXOff, int nBlockYOff, } else { - CPLString osTmpMemFile = CPLSPrintf("/vsimem/daas_%p", this); + const CPLString osTmpMemFile = VSIMemGenerateHiddenFilename("daas"); VSIFCloseL(VSIFileFromMemBuffer( osTmpMemFile, psResult->pasMimePart[iDataPart].pabyData, psResult->pasMimePart[iDataPart].nDataLen, false)); diff --git a/frmts/drivers.ini b/frmts/drivers.ini index 4bcea36fd07d..c05e28e5c948 100644 --- a/frmts/drivers.ini +++ b/frmts/drivers.ini @@ -164,6 +164,7 @@ DAAS NULL SIGDEM EXR +AVIF HEIF TGA OGCAPI diff --git a/frmts/ecw/ecwdataset.cpp b/frmts/ecw/ecwdataset.cpp index dd7bde5a9b1a..6c94add614c1 100644 --- a/frmts/ecw/ecwdataset.cpp +++ b/frmts/ecw/ecwdataset.cpp @@ -2850,10 +2850,11 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000) /* There are issues at least in the 5.x series. */ /* -------------------------------------------------------------------- */ #if ECWSDK_VERSION >= 40 + constexpr const char *szDETECT_BUG_FILENAME = + "__detect_ecw_uint32_bug__.j2k"; if (bIsJPEG2000 && poDS->eNCSRequestDataType == NCSCT_UINT32 && CPLTestBool(CPLGetConfigOption("ECW_CHECK_CORRECT_DECODING", "TRUE")) && - !STARTS_WITH_CI(poOpenInfo->pszFilename, - "/vsimem/detect_ecw_uint32_bug")) + strstr(poOpenInfo->pszFilename, szDETECT_BUG_FILENAME) == nullptr) { static bool bUINT32_Ok = false; { @@ -2878,7 +2879,7 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000) 0xDF, 0xFF, 0x7F, 0x5F, 0xFF, 0xD9}; const std::string osTmpFilename = - CPLSPrintf("/vsimem/detect_ecw_uint32_bug_%p.j2k", poDS); + VSIMemGenerateHiddenFilename(szDETECT_BUG_FILENAME); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), const_cast(abyTestUInt32ImageData), diff --git a/frmts/eeda/eedaidataset.cpp b/frmts/eeda/eedaidataset.cpp index 9a6c41be78b8..e6df9d0d2a8a 100644 --- a/frmts/eeda/eedaidataset.cpp +++ b/frmts/eeda/eedaidataset.cpp @@ -433,7 +433,7 @@ bool GDALEEDAIRasterBand::DecodeGDALDataset(const GByte *pabyData, int nDataLen, { GDALEEDAIDataset *poGDS = reinterpret_cast(poDS); - CPLString osTmpFilename(CPLSPrintf("/vsimem/eeai/%p", this)); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("eedai")); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename, const_cast(pabyData), nDataLen, false)); const char *const apszDrivers[] = {"PNG", "JPEG", "GTIFF", nullptr}; diff --git a/frmts/esric/esric_dataset.cpp b/frmts/esric/esric_dataset.cpp index 7ff5fdd585f4..2d5768762b59 100644 --- a/frmts/esric/esric_dataset.cpp +++ b/frmts/esric/esric_dataset.cpp @@ -896,9 +896,7 @@ CPLErr ECBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pData) GUInt64(size), GUInt64(offset)); return CE_Failure; } - CPLString magic; - // Should use some sort of unique - magic.Printf("/vsimem/esric_%p.tmp", this); + const CPLString magic(VSIMemGenerateHiddenFilename("esric.tmp")); auto mfh = VSIFileFromMemBuffer(magic.c_str(), fbuffer.data(), size, false); VSIFCloseL(mfh); // Can't open a raster by handle? diff --git a/frmts/gdalallregister.cpp b/frmts/gdalallregister.cpp index b7c37bd68bad..376a5fc21199 100644 --- a/frmts/gdalallregister.cpp +++ b/frmts/gdalallregister.cpp @@ -185,6 +185,9 @@ void CPL_STDCALL GDALAllRegister() #if defined(DEFERRED_HANA_DRIVER) DeclareDeferredOGRHANAPlugin(); #endif +#if defined(DEFERRED_AVIF_DRIVER) + DeclareDeferredAVIFPlugin(); +#endif #if defined(DEFERRED_HEIF_DRIVER) DeclareDeferredHEIFPlugin(); #endif @@ -765,6 +768,10 @@ void CPL_STDCALL GDALAllRegister() GDALRegister_EXR(); #endif +#ifdef FRMT_avif + GDALRegister_AVIF(); +#endif + #ifdef FRMT_heif GDALRegister_HEIF(); #endif diff --git a/frmts/georaster/georaster_rasterband.cpp b/frmts/georaster/georaster_rasterband.cpp index 907760453be9..cb0eb8a79648 100644 --- a/frmts/georaster/georaster_rasterband.cpp +++ b/frmts/georaster/georaster_rasterband.cpp @@ -851,7 +851,7 @@ GDALRasterAttributeTable *GeoRasterRasterBand::GetDefaultRAT() } if (!osColumnList.empty()) - osColumnList.resize(osColumnList.size() - 1); // remove the last comma + osColumnList.pop_back(); // remove the last comma // ---------------------------------------------------------- // Read VAT and load RAT diff --git a/frmts/georaster/georaster_wrapper.cpp b/frmts/georaster/georaster_wrapper.cpp index cdb9dbd8c9cb..6e271202a956 100644 --- a/frmts/georaster/georaster_wrapper.cpp +++ b/frmts/georaster/georaster_wrapper.cpp @@ -214,10 +214,10 @@ char **GeoRasterWrapper::ParseIdentificator(const char *pszStringID) char *pszStartPos = (char *)strstr(pszStringID, ":") + 1; - char **papszParam = - CSLTokenizeString2(pszStartPos, ",@", - CSLT_HONOURSTRINGS | CSLT_ALLOWEMPTYTOKENS | - CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES); + char **papszParam = CSLTokenizeString2( + pszStartPos, ",@", + CSLT_HONOURSTRINGS | CSLT_ALLOWEMPTYTOKENS | CSLT_STRIPLEADSPACES | + CSLT_STRIPENDSPACES | CSLT_PRESERVEQUOTES); // ------------------------------------------------------------------- // The "/" should not be catch on the previous parser @@ -4236,13 +4236,13 @@ void GeoRasterWrapper::UncompressJpeg(unsigned long nInSize) // Load JPEG in a virtual file // -------------------------------------------------------------------- - const char *pszMemFile = CPLSPrintf("/vsimem/geor_%p.jpg", pabyBlockBuf); + const CPLString osMemFile = VSIMemGenerateHiddenFilename("geor.jpg"); - VSILFILE *fpImage = VSIFOpenL(pszMemFile, "wb"); + VSILFILE *fpImage = VSIFOpenL(osMemFile, "wb"); VSIFWriteL(pabyBlockBuf, nInSize, 1, fpImage); VSIFCloseL(fpImage); - fpImage = VSIFOpenL(pszMemFile, "rb"); + fpImage = VSIFOpenL(osMemFile, "rb"); // -------------------------------------------------------------------- // Initialize decompressor @@ -4299,7 +4299,7 @@ void GeoRasterWrapper::UncompressJpeg(unsigned long nInSize) VSIFCloseL(fpImage); - VSIUnlink(pszMemFile); + VSIUnlink(osMemFile); } // --------------------------------------------------------------------------- @@ -4312,9 +4312,9 @@ unsigned long GeoRasterWrapper::CompressJpeg(void) // Load JPEG in a virtual file // -------------------------------------------------------------------- - const char *pszMemFile = CPLSPrintf("/vsimem/geor_%p.jpg", pabyBlockBuf); + const CPLString osMemFile = VSIMemGenerateHiddenFilename("geor.jpg"); - VSILFILE *fpImage = VSIFOpenL(pszMemFile, "wb"); + VSILFILE *fpImage = VSIFOpenL(osMemFile, "wb"); bool write_all_tables = TRUE; @@ -4389,11 +4389,11 @@ unsigned long GeoRasterWrapper::CompressJpeg(void) VSIFCloseL(fpImage); - fpImage = VSIFOpenL(pszMemFile, "rb"); + fpImage = VSIFOpenL(osMemFile, "rb"); size_t nSize = VSIFReadL(pabyCompressBuf, 1, nBlockBytes, fpImage); VSIFCloseL(fpImage); - VSIUnlink(pszMemFile); + VSIUnlink(osMemFile); return (unsigned long)nSize; } diff --git a/frmts/grib/degrib/README.TXT b/frmts/grib/degrib/README.TXT index f041fa15e0e6..58d547b16a81 100644 --- a/frmts/grib/degrib/README.TXT +++ b/frmts/grib/degrib/README.TXT @@ -1,4 +1,4 @@ -PROVENANCE +Provenance ---------- This directory contains a *modified* version of degrib 2.14: diff --git a/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp b/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp index a22109e823f8..cf0961c637e8 100644 --- a/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp +++ b/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp @@ -52,8 +52,8 @@ int dec_jpeg2000(const void *injpc,g2int bufsize,g2int **outfld,g2int outpixels) { // create "memory file" from buffer - CPLString osFileName; - osFileName.Printf( "/vsimem/work_grib_%p.jpc", injpc ); + const CPLString osFileName( + VSIMemGenerateHiddenFilename("temp_grib.jpc")); VSIFCloseL( VSIFileFromMemBuffer( osFileName, (unsigned char*)injpc, bufsize, diff --git a/frmts/grib/gribcreatecopy.cpp b/frmts/grib/gribcreatecopy.cpp index 8a0140ff030a..6a9637bca749 100644 --- a/frmts/grib/gribcreatecopy.cpp +++ b/frmts/grib/gribcreatecopy.cpp @@ -1664,7 +1664,7 @@ bool GRIB2Section567Writer::WritePNG() GDALDataset *poMEMDS = WrapArrayAsMemDataset(m_nXSize, m_nYSize, eReducedDT, panData); - CPLString osTmpFile(CPLSPrintf("/vsimem/grib_driver_%p.png", m_poSrcDS)); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("grib_driver.png")); GDALDataset *poPNGDS = poPNGDriver->CreateCopy( osTmpFile, poMEMDS, FALSE, aosPNGOptions.List(), nullptr, nullptr); if (poPNGDS == nullptr) @@ -1850,7 +1850,7 @@ bool GRIB2Section567Writer::WriteJPEG2000(char **papszOptions) GDALDataset *poMEMDS = WrapArrayAsMemDataset(m_nXSize, m_nYSize, eReducedDT, panData); - CPLString osTmpFile(CPLSPrintf("/vsimem/grib_driver_%p.j2k", m_poSrcDS)); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("grib_driver.j2k")); GDALDataset *poJ2KDS = poJ2KDriver->CreateCopy( osTmpFile, poMEMDS, FALSE, aosJ2KOptions.List(), nullptr, nullptr); if (poJ2KDS == nullptr) @@ -2250,7 +2250,7 @@ static void WriteAssembledPDS(VSILFILE *fp, const gtemplate *mappds, else if (nEltSize == 4) { GIntBig nBigVal = CPLAtoGIntBig(papszTokens[i]); - anVals[anVals.size() - 1] = static_cast(nBigVal); + anVals.back() = static_cast(nBigVal); if (nBigVal < 0 || nBigVal > static_cast(UINT_MAX)) { CPLError(CE_Warning, CPLE_AppDefined, diff --git a/frmts/grib/gribdataset.cpp b/frmts/grib/gribdataset.cpp index f098463b23c6..c7a42e496819 100644 --- a/frmts/grib/gribdataset.cpp +++ b/frmts/grib/gribdataset.cpp @@ -1489,10 +1489,7 @@ GDALDataset *GRIBDataset::Open(GDALOpenInfo *poOpenInfo) // for other thread safe formats CPLMutexHolderD(&hGRIBMutex); - CPLString tmpFilename; - tmpFilename.Printf("/vsimem/gribdataset-%p", poOpenInfo); - - VSILFILE *memfp = VSIFileFromMemBuffer(tmpFilename, poOpenInfo->pabyHeader, + VSILFILE *memfp = VSIFileFromMemBuffer(nullptr, poOpenInfo->pabyHeader, poOpenInfo->nHeaderBytes, FALSE); if (memfp == nullptr || ReadSECT0(memfp, &buff, &buffLen, -1, sect0, &gribLen, &version) < 0) @@ -1500,7 +1497,6 @@ GDALDataset *GRIBDataset::Open(GDALOpenInfo *poOpenInfo) if (memfp != nullptr) { VSIFCloseL(memfp); - VSIUnlink(tmpFilename); } free(buff); char *errMsg = errSprintf(nullptr); @@ -1510,7 +1506,6 @@ GDALDataset *GRIBDataset::Open(GDALOpenInfo *poOpenInfo) return nullptr; } VSIFCloseL(memfp); - VSIUnlink(tmpFilename); free(buff); // Confirm the requested access is supported. diff --git a/frmts/gtiff/geotiff.cpp b/frmts/gtiff/geotiff.cpp index 55b1d9ebf6f9..77b6c0ee6b88 100644 --- a/frmts/gtiff/geotiff.cpp +++ b/frmts/gtiff/geotiff.cpp @@ -701,8 +701,8 @@ void GTiffWriteJPEGTables(TIFF *hTIFF, const char *pszPhotometric, if (!TIFFGetField(hTIFF, TIFFTAG_BITSPERSAMPLE, &(l_nBitsPerSample))) l_nBitsPerSample = 1; - CPLString osTmpFilenameIn; - osTmpFilenameIn.Printf("%s%p", szJPEGGTiffDatasetTmpPrefix, hTIFF); + const CPLString osTmpFilenameIn( + VSIMemGenerateHiddenFilename("gtiffdataset_jpg_tmp")); VSILFILE *fpTmp = nullptr; CPLString osTmp; char **papszLocalParameters = nullptr; @@ -723,6 +723,8 @@ void GTiffWriteJPEGTables(TIFF *hTIFF, const char *pszPhotometric, CPLSPrintf("%u", l_nBitsPerSample)); papszLocalParameters = CSLSetNameValue(papszLocalParameters, "JPEGTABLESMODE", pszJPEGTablesMode); + papszLocalParameters = + CSLSetNameValue(papszLocalParameters, "WRITE_JPEGTABLE_TAG", "NO"); TIFF *hTIFFTmp = GTiffDataset::CreateLL(osTmpFilenameIn, nInMemImageWidth, diff --git a/frmts/gtiff/gt_jpeg_copy.cpp b/frmts/gtiff/gt_jpeg_copy.cpp index 73262092f210..2a647ff190fb 100644 --- a/frmts/gtiff/gt_jpeg_copy.cpp +++ b/frmts/gtiff/gt_jpeg_copy.cpp @@ -377,13 +377,10 @@ static void GTIFF_ErrorExitJPEG(j_common_ptr cinfo) /************************************************************************/ static void GTIFF_Set_TIFFTAG_JPEGTABLES(TIFF *hTIFF, - jpeg_decompress_struct &sDInfo, jpeg_compress_struct &sCInfo) { - char szTmpFilename[128] = {'\0'}; - snprintf(szTmpFilename, sizeof(szTmpFilename), "/vsimem/tables_%p", - &sDInfo); - VSILFILE *fpTABLES = VSIFOpenL(szTmpFilename, "wb+"); + const std::string osTmpFilename(VSIMemGenerateHiddenFilename("tables")); + VSILFILE *fpTABLES = VSIFOpenL(osTmpFilename.c_str(), "wb+"); uint16_t nPhotometric = 0; TIFFGetField(hTIFF, TIFFTAG_PHOTOMETRIC, &nPhotometric); @@ -409,11 +406,11 @@ static void GTIFF_Set_TIFFTAG_JPEGTABLES(TIFF *hTIFF, vsi_l_offset nSizeTables = 0; GByte *pabyJPEGTablesData = - VSIGetMemFileBuffer(szTmpFilename, &nSizeTables, FALSE); + VSIGetMemFileBuffer(osTmpFilename.c_str(), &nSizeTables, FALSE); TIFFSetField(hTIFF, TIFFTAG_JPEGTABLES, static_cast(nSizeTables), pabyJPEGTablesData); - VSIUnlink(szTmpFilename); + VSIUnlink(osTmpFilename.c_str()); } /************************************************************************/ @@ -476,7 +473,7 @@ CPLErr GTIFF_CopyFromJPEG_WriteAdditionalTags(TIFF *hTIFF, GDALDataset *poSrcDS) jpeg_CreateCompress(&sCInfo, JPEG_LIB_VERSION, sizeof(sCInfo)); bCallDestroyCompress = true; jpeg_copy_critical_parameters(&sDInfo, &sCInfo); - GTIFF_Set_TIFFTAG_JPEGTABLES(hTIFF, sDInfo, sCInfo); + GTIFF_Set_TIFFTAG_JPEGTABLES(hTIFF, sCInfo); bCallDestroyCompress = false; jpeg_abort_compress(&sCInfo); jpeg_destroy_compress(&sCInfo); @@ -578,8 +575,9 @@ typedef struct static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) { - CPLString osTmpFilename(CPLSPrintf("/vsimem/%p", psArgs->psDInfo)); - VSILFILE *fpMEM = VSIFOpenL(osTmpFilename, "wb+"); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("GTIFF_CopyBlockFromJPEG.tif")); + VSILFILE *fpMEM = VSIFOpenL(osTmpFilename.c_str(), "wb+"); /* -------------------------------------------------------------------- */ /* Initialization of the compressor */ @@ -588,7 +586,7 @@ static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) if (setjmp(setjmp_buffer)) { CPL_IGNORE_RET_VAL(VSIFCloseL(fpMEM)); - VSIUnlink(osTmpFilename); + VSIUnlink(osTmpFilename.c_str()); return CE_Failure; } @@ -775,7 +773,8 @@ static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) /* Write the JPEG content with libtiff raw API */ /* -------------------------------------------------------------------- */ vsi_l_offset nSize = 0; - GByte *pabyJPEGData = VSIGetMemFileBuffer(osTmpFilename, &nSize, FALSE); + GByte *pabyJPEGData = + VSIGetMemFileBuffer(osTmpFilename.c_str(), &nSize, FALSE); CPLErr eErr = CE_None; @@ -794,7 +793,7 @@ static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) eErr = CE_Failure; } - VSIUnlink(osTmpFilename); + VSIUnlink(osTmpFilename.c_str()); return eErr; } diff --git a/frmts/gtiff/gt_wkt_srs.cpp b/frmts/gtiff/gt_wkt_srs.cpp index 919829f2d511..1bffb47d8f1d 100644 --- a/frmts/gtiff/gt_wkt_srs.cpp +++ b/frmts/gtiff/gt_wkt_srs.cpp @@ -42,7 +42,6 @@ #include "cpl_conv.h" #include "cpl_error.h" -#include "cpl_multiproc.h" #include "cpl_string.h" #include "cpl_vsi.h" #include "gt_citation.h" @@ -3513,10 +3512,8 @@ CPLErr GTIFWktFromMemBufEx(int nSize, unsigned char *pabyBuffer, char ***ppapszRPCMD) { - char szFilename[100] = {}; - - snprintf(szFilename, sizeof(szFilename), "/vsimem/wkt_from_mem_buf_%ld.tif", - static_cast(CPLGetPID())); + const std::string osFilename( + VSIMemGenerateHiddenFilename("wkt_from_mem_buf.tif")); /* -------------------------------------------------------------------- */ /* Initialization of libtiff and libgeotiff. */ @@ -3527,20 +3524,21 @@ CPLErr GTIFWktFromMemBufEx(int nSize, unsigned char *pabyBuffer, /* -------------------------------------------------------------------- */ /* Create a memory file from the buffer. */ /* -------------------------------------------------------------------- */ - VSILFILE *fp = VSIFileFromMemBuffer(szFilename, pabyBuffer, nSize, FALSE); + VSILFILE *fp = + VSIFileFromMemBuffer(osFilename.c_str(), pabyBuffer, nSize, FALSE); if (fp == nullptr) return CE_Failure; /* -------------------------------------------------------------------- */ /* Initialize access to the memory geotiff structure. */ /* -------------------------------------------------------------------- */ - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "rc", fp); + TIFF *hTIFF = VSI_TIFFOpen(osFilename.c_str(), "rc", fp); if (hTIFF == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, "TIFF/GeoTIFF structure is corrupt."); - VSIUnlink(szFilename); + VSIUnlink(osFilename.c_str()); CPL_IGNORE_RET_VAL(VSIFCloseL(fp)); return CE_Failure; } @@ -3678,7 +3676,7 @@ CPLErr GTIFWktFromMemBufEx(int nSize, unsigned char *pabyBuffer, XTIFFClose(hTIFF); CPL_IGNORE_RET_VAL(VSIFCloseL(fp)); - VSIUnlink(szFilename); + VSIUnlink(osFilename.c_str()); if (phSRS && *phSRS == nullptr) return CE_Failure; @@ -3709,10 +3707,8 @@ CPLErr GTIFMemBufFromSRS(OGRSpatialReferenceH hSRS, char **papszRPCMD) { - char szFilename[100] = {}; - - snprintf(szFilename, sizeof(szFilename), "/vsimem/wkt_from_mem_buf_%ld.tif", - static_cast(CPLGetPID())); + const std::string osFilename( + VSIMemGenerateHiddenFilename("wkt_from_mem_buf.tif")); /* -------------------------------------------------------------------- */ /* Initialization of libtiff and libgeotiff. */ @@ -3723,17 +3719,18 @@ CPLErr GTIFMemBufFromSRS(OGRSpatialReferenceH hSRS, /* -------------------------------------------------------------------- */ /* Initialize access to the memory geotiff structure. */ /* -------------------------------------------------------------------- */ - VSILFILE *fpL = VSIFOpenL(szFilename, "w"); + VSILFILE *fpL = VSIFOpenL(osFilename.c_str(), "w"); if (fpL == nullptr) return CE_Failure; - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "w", fpL); + TIFF *hTIFF = VSI_TIFFOpen(osFilename.c_str(), "w", fpL); if (hTIFF == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, "TIFF/GeoTIFF structure is corrupt."); CPL_IGNORE_RET_VAL(VSIFCloseL(fpL)); + VSIUnlink(osFilename.c_str()); return CE_Failure; } @@ -3885,7 +3882,7 @@ CPLErr GTIFMemBufFromSRS(OGRSpatialReferenceH hSRS, /* -------------------------------------------------------------------- */ GUIntBig nBigLength = 0; - *ppabyBuffer = VSIGetMemFileBuffer(szFilename, &nBigLength, TRUE); + *ppabyBuffer = VSIGetMemFileBuffer(osFilename.c_str(), &nBigLength, TRUE); *pnSize = static_cast(nBigLength); return CE_None; diff --git a/frmts/gtiff/gtiffdataset.h b/frmts/gtiff/gtiffdataset.h index 2c54e910a31e..ab8af5725f8d 100644 --- a/frmts/gtiff/gtiffdataset.h +++ b/frmts/gtiff/gtiffdataset.h @@ -54,9 +54,6 @@ enum class GTiffProfile : GByte // This must be a #define, since it is used in a XSTRINGIFY() macro #define DEFAULT_WEBP_LEVEL 75 -constexpr const char *const szJPEGGTiffDatasetTmpPrefix = - "/vsimem/gtiffdataset_jpg_tmp_"; - class GTiffBitmapBand; class GTiffDataset; class GTiffJPEGOverviewBand; diff --git a/frmts/gtiff/gtiffdataset_read.cpp b/frmts/gtiff/gtiffdataset_read.cpp index 2cdcdbeada92..c8db2594b24d 100644 --- a/frmts/gtiff/gtiffdataset_read.cpp +++ b/frmts/gtiff/gtiffdataset_read.cpp @@ -704,8 +704,8 @@ static void CPL_STDCALL ThreadDecompressionFuncErrorHandler( { // Generate a dummy in-memory TIFF file that has all the needed tags // from the original file - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/decompress_%p.tif", psJob); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("decompress.tif")); VSILFILE *fpTmp = VSIFOpenL(osTmpFilename.c_str(), "wb+"); TIFF *hTIFFTmp = VSI_TIFFOpen(osTmpFilename.c_str(), @@ -3524,9 +3524,8 @@ static bool GTIFFExtendMemoryFile(const CPLString &osTmpFilename, static bool GTIFFMakeBufferedStream(GDALOpenInfo *poOpenInfo) { - CPLString osTmpFilename; - static int nCounter = 0; - osTmpFilename.Printf("/vsimem/stream_%d.tif", ++nCounter); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("GTIFFMakeBufferedStream.tif")); VSILFILE *fpTemp = VSIFOpenL(osTmpFilename, "wb+"); if (fpTemp == nullptr) return false; @@ -5680,8 +5679,19 @@ CPLErr GTiffDataset::OpenOffset(TIFF *hTIFFIn, toff_t nDirOffsetIn, } else if (EQUAL(pszRole, "colorinterp")) { - poBand->m_eBandInterp = - GDALGetColorInterpretationByName(pszUnescapedValue); + if (EQUAL(pszUnescapedValue, "undefined")) + poBand->m_eBandInterp = GCI_Undefined; + else + { + poBand->m_eBandInterp = + GDALGetColorInterpretationByName( + pszUnescapedValue); + if (poBand->m_eBandInterp == GCI_Undefined) + { + poBand->m_oGTiffMDMD.SetMetadataItem( + "COLOR_INTERPRETATION", pszUnescapedValue); + } + } } else { diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 4146fcc84236..768837cc753c 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -947,8 +947,8 @@ void GTiffDataset::InitCompressionThreads(bool bUpdateMode, i < static_cast(m_asCompressionJobs.size()); ++i) { m_asCompressionJobs[i].pszTmpFilename = - CPLStrdup(CPLSPrintf("/vsimem/gtiff/thread/job/%p", - &m_asCompressionJobs[i])); + CPLStrdup(VSIMemGenerateHiddenFilename( + CPLSPrintf("thread_job_%d.tif", i))); m_asCompressionJobs[i].nStripOrTile = -1; } @@ -1383,7 +1383,7 @@ bool GTiffDataset::SubmitCompressionJob(int nStripOrTile, GByte *pabyData, memset(&sJob, 0, sizeof(sJob)); SetupJob(sJob); sJob.pszTmpFilename = - CPLStrdup(CPLSPrintf("/vsimem/gtiff/%p", this)); + CPLStrdup(VSIMemGenerateHiddenFilename("temp.tif")); ThreadCompressionFunc(&sJob); @@ -3833,7 +3833,7 @@ static void WriteMDMetadata(GDALMultiDomainMetadata *poMDMD, TIFF *hTIFF, for (int iDomain = 0; papszDomainList && papszDomainList[iDomain]; ++iDomain) { - char **papszMD = poMDMD->GetMetadata(papszDomainList[iDomain]); + CSLConstList papszMD = poMDMD->GetMetadata(papszDomainList[iDomain]); bool bIsXML = false; if (EQUAL(papszDomainList[iDomain], "IMAGE_STRUCTURE") || @@ -4065,6 +4065,11 @@ bool GTiffDataset::WriteMetadata(GDALDataset *poSrcDS, TIFF *l_hTIFF, CPLXMLNode *psRoot = nullptr; CPLXMLNode *psTail = nullptr; + const char *pszCopySrcMDD = + CSLFetchNameValueDef(papszCreationOptions, "COPY_SRC_MDD", "AUTO"); + char **papszSrcMDD = + CSLFetchNameValueMultiple(papszCreationOptions, "SRC_MDD"); + if (bSrcIsGeoTIFF) { GTiffDataset *poSrcDSGTiff = cpl::down_cast(poSrcDS); @@ -4074,15 +4079,11 @@ bool GTiffDataset::WriteMetadata(GDALDataset *poSrcDS, TIFF *l_hTIFF, } else { - const char *pszCopySrcMDD = - CSLFetchNameValueDef(papszCreationOptions, "COPY_SRC_MDD", "AUTO"); - char **papszSrcMDD = - CSLFetchNameValueMultiple(papszCreationOptions, "SRC_MDD"); if (EQUAL(pszCopySrcMDD, "AUTO") || CPLTestBool(pszCopySrcMDD) || papszSrcMDD) { GDALMultiDomainMetadata l_oMDMD; - char **papszMD = poSrcDS->GetMetadata(); + CSLConstList papszMD = poSrcDS->GetMetadata(); if (CSLCount(papszMD) > 0 && (!papszSrcMDD || CSLFindString(papszSrcMDD, "") >= 0 || CSLFindString(papszSrcMDD, "_DEFAULT_") >= 0)) @@ -4094,7 +4095,7 @@ bool GTiffDataset::WriteMetadata(GDALDataset *poSrcDS, TIFF *l_hTIFF, papszSrcMDD) { char **papszDomainList = poSrcDS->GetMetadataDomainList(); - for (char **papszIter = papszDomainList; + for (CSLConstList papszIter = papszDomainList; papszIter && *papszIter; ++papszIter) { const char *pszDomain = *papszIter; @@ -4111,7 +4112,6 @@ bool GTiffDataset::WriteMetadata(GDALDataset *poSrcDS, TIFF *l_hTIFF, WriteMDMetadata(&l_oMDMD, l_hTIFF, &psRoot, &psTail, 0, eProfile); } - CSLDestroy(papszSrcMDD); } if (!bExcludeRPBandIMGFileWriting) @@ -4156,13 +4156,44 @@ bool GTiffDataset::WriteMetadata(GDALDataset *poSrcDS, TIFF *l_hTIFF, } else { - char **papszMD = poBand->GetMetadata(); + GDALMultiDomainMetadata l_oMDMD; + bool bOMDMDSet = false; - if (CSLCount(papszMD) > 0) + if (EQUAL(pszCopySrcMDD, "AUTO") && !papszSrcMDD) { - GDALMultiDomainMetadata l_oMDMD; - l_oMDMD.SetMetadata(papszMD); + for (const char *pszDomain : {"", "IMAGERY"}) + { + if (CSLConstList papszMD = poBand->GetMetadata(pszDomain)) + { + if (papszMD[0]) + { + bOMDMDSet = true; + l_oMDMD.SetMetadata(papszMD, pszDomain); + } + } + } + } + else if (CPLTestBool(pszCopySrcMDD) || papszSrcMDD) + { + char **papszDomainList = poBand->GetMetadataDomainList(); + for (const char *pszDomain : + cpl::Iterate(CSLConstList(papszDomainList))) + { + if (pszDomain[0] != 0 && + !EQUAL(pszDomain, "IMAGE_STRUCTURE") && + (!papszSrcMDD || + CSLFindString(papszSrcMDD, pszDomain) >= 0)) + { + bOMDMDSet = true; + l_oMDMD.SetMetadata(poBand->GetMetadata(pszDomain), + pszDomain); + } + } + CSLDestroy(papszDomainList); + } + if (bOMDMDSet) + { WriteMDMetadata(&l_oMDMD, l_hTIFF, &psRoot, &psTail, nBand, eProfile); } @@ -4234,6 +4265,8 @@ bool GTiffDataset::WriteMetadata(GDALDataset *poSrcDS, TIFF *l_hTIFF, } } + CSLDestroy(papszSrcMDD); + const char *pszTilingSchemeName = CSLFetchNameValue(papszCreationOptions, "@TILING_SCHEME_NAME"); if (pszTilingSchemeName) @@ -5122,6 +5155,55 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, return nullptr; } + constexpr int JPEG_MAX_DIMENSION = 65500; // Defined in jpeglib.h + constexpr int WEBP_MAX_DIMENSION = 16383; + + const struct + { + int nCodecID; + const char *pszCodecName; + int nMaxDim; + } asLimitations[] = { + {COMPRESSION_JPEG, "JPEG", JPEG_MAX_DIMENSION}, + {COMPRESSION_WEBP, "WEBP", WEBP_MAX_DIMENSION}, + }; + + for (const auto &sLimitation : asLimitations) + { + if (l_nCompression == sLimitation.nCodecID && !bTiled && + nXSize > sLimitation.nMaxDim) + { + ReportError( + pszFilename, CE_Failure, CPLE_IllegalArg, + "COMPRESS=%s is only compatible of un-tiled images whose " + "width is lesser or equal to %d pixels. " + "To overcome this limitation, set the TILED=YES creation " + "option.", + sLimitation.pszCodecName, sLimitation.nMaxDim); + return nullptr; + } + else if (l_nCompression == sLimitation.nCodecID && bTiled && + l_nBlockXSize > sLimitation.nMaxDim) + { + ReportError(pszFilename, CE_Failure, CPLE_IllegalArg, + "COMPRESS=%s is only compatible of tiled images whose " + "BLOCKXSIZE is lesser or equal to %d pixels.", + sLimitation.pszCodecName, sLimitation.nMaxDim); + return nullptr; + } + else if (l_nCompression == sLimitation.nCodecID && + l_nBlockYSize > sLimitation.nMaxDim) + { + ReportError(pszFilename, CE_Failure, CPLE_IllegalArg, + "COMPRESS=%s is only compatible of images whose " + "BLOCKYSIZE is lesser or equal to %d pixels. " + "To overcome this limitation, set the TILED=YES " + "creation option", + sLimitation.pszCodecName, sLimitation.nMaxDim); + return nullptr; + } + } + /* -------------------------------------------------------------------- */ /* How many bits per sample? We have a special case if NBITS */ /* specified for GDT_Byte, GDT_UInt16, GDT_UInt32. */ @@ -5344,8 +5426,7 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, } if (bStreaming) { - static int nCounter = 0; - l_osTmpFilename = CPLSPrintf("/vsimem/vsistdout_%d.tif", ++nCounter); + l_osTmpFilename = VSIMemGenerateHiddenFilename("vsistdout.tif"); pszFilename = l_osTmpFilename.c_str(); } @@ -5875,7 +5956,6 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, // strip/tile writing, which is too late, since we have already crystalized // the directory. This way we avoid a directory rewriting. if (l_nCompression == COMPRESSION_JPEG && - !STARTS_WITH(pszFilename, szJPEGGTiffDatasetTmpPrefix) && CPLTestBool( CSLFetchNameValueDef(papszParamList, "WRITE_JPEGTABLE_TAG", "YES"))) { @@ -6068,9 +6148,8 @@ int GTiffDataset::GuessJPEGQuality(bool &bOutHasQuantizationTable, papszLocalParameters = CSLSetNameValue(papszLocalParameters, "NBITS", "12"); - CPLString osTmpFilenameIn; - osTmpFilenameIn.Printf("/vsimem/gtiffdataset_guess_jpeg_quality_tmp_%p", - this); + const CPLString osTmpFilenameIn( + VSIMemGenerateHiddenFilename("gtiffdataset_guess_jpeg_quality_tmp")); int nRet = -1; for (int nQuality = 0; nQuality <= 100 && nRet < 0; ++nQuality) diff --git a/frmts/gtiff/gtiffjpegoverviewds.cpp b/frmts/gtiff/gtiffjpegoverviewds.cpp index cc3ca61e3db6..86e4205107e0 100644 --- a/frmts/gtiff/gtiffjpegoverviewds.cpp +++ b/frmts/gtiff/gtiffjpegoverviewds.cpp @@ -71,7 +71,7 @@ GTiffJPEGOverviewDS::GTiffJPEGOverviewDS(GTiffDataset *poParentDSIn, { ShareLockWithParentDataset(poParentDSIn); - m_osTmpFilenameJPEGTable.Printf("/vsimem/jpegtable_%p", this); + m_osTmpFilenameJPEGTable = VSIMemGenerateHiddenFilename("jpegtable"); const GByte abyAdobeAPP14RGB[] = {0xFF, 0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, 0x65, 0x00, 0x64, 0x00, @@ -221,7 +221,7 @@ CPLErr GTiffJPEGOverviewBand::IReadBlock(int nBlockXOff, int nBlockYOff, nByteCount -= 2; CPLString osFileToOpen; - m_poGDS->m_osTmpFilename.Printf("/vsimem/sparse_%p", m_poGDS); + m_poGDS->m_osTmpFilename = VSIMemGenerateHiddenFilename("sparse"); VSILFILE *fp = VSIFOpenL(m_poGDS->m_osTmpFilename, "wb+"); // If the size of the JPEG strip/tile is small enough, we will diff --git a/frmts/gtiff/libtiff/tif_dir.h b/frmts/gtiff/libtiff/tif_dir.h index f4182a2cb7a5..ffad085e02ed 100644 --- a/frmts/gtiff/libtiff/tif_dir.h +++ b/frmts/gtiff/libtiff/tif_dir.h @@ -335,11 +335,10 @@ extern "C" TIFFDataType field_type; /* type of associated data */ uint32_t field_anonymous; /* if true, this is a unknown / anonymous tag */ - TIFFSetGetFieldType - set_field_type; /* type to be passed to TIFFSetField */ - TIFFSetGetFieldType - get_field_type; /* type to be passed to TIFFGetField */ - unsigned short field_bit; /* bit in fieldsset bit vector */ + TIFFSetGetFieldType set_field_type; /* type to be passed to TIFFSetField + and TIFFGetField*/ + TIFFSetGetFieldType get_field_type; /* not used */ + unsigned short field_bit; /* bit in fieldsset bit vector */ unsigned char field_oktochange; /* if true, can change while writing */ unsigned char field_passcount; /* if true, pass dir count on set */ char *field_name; /* ASCII name */ diff --git a/frmts/gtiff/libtiff/tif_dirwrite.c b/frmts/gtiff/libtiff/tif_dirwrite.c index 0701f40d97ef..facdeaf7b161 100644 --- a/frmts/gtiff/libtiff/tif_dirwrite.c +++ b/frmts/gtiff/libtiff/tif_dirwrite.c @@ -869,7 +869,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, if ((o->field_bit >= FIELD_CODEC) && (TIFFFieldSet(tif, o->field_bit))) { - switch (o->get_field_type) + switch (o->set_field_type) { case TIFF_SETGET_ASCII: { diff --git a/frmts/gtiff/libtiff/tif_jpeg.c b/frmts/gtiff/libtiff/tif_jpeg.c index 10aed5463589..c14ca522fe94 100644 --- a/frmts/gtiff/libtiff/tif_jpeg.c +++ b/frmts/gtiff/libtiff/tif_jpeg.c @@ -2191,9 +2191,12 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); } - if (segment_width > 65535 || segment_height > 65535) + if (segment_width > (uint32_t)JPEG_MAX_DIMENSION || + segment_height > (uint32_t)JPEG_MAX_DIMENSION) { - TIFFErrorExtR(tif, module, "Strip/tile too large for JPEG"); + TIFFErrorExtR(tif, module, + "Strip/tile too large for JPEG. Maximum dimension is %d", + (int)JPEG_MAX_DIMENSION); return (0); } sp->cinfo.c.image_width = segment_width; diff --git a/frmts/hdf5/hdf5multidim.cpp b/frmts/hdf5/hdf5multidim.cpp index f5a0704c458f..5d8b2a1d64cc 100644 --- a/frmts/hdf5/hdf5multidim.cpp +++ b/frmts/hdf5/hdf5multidim.cpp @@ -1070,17 +1070,28 @@ HDF5Array::HDF5Array(const std::string &osParentName, const std::string &osName, } // Special case for S102 QualityOfSurvey nodata value that is typically at 0 - if (GetFullName() == - "/QualityOfSurvey/QualityOfSurvey.01/Group_001/values" && - m_dt.GetClass() == GEDTC_NUMERIC && - m_dt.GetNumericDataType() == GDT_UInt32) + const bool bIsQualityOfSurvey = + (GetFullName() == + "/QualityOfSurvey/QualityOfSurvey.01/Group_001/values"); + const bool bIsQualityOfBathymetryCoverage = + (GetFullName() == "/QualityOfBathymetryCoverage/" + "QualityOfBathymetryCoverage.01/Group_001/values"); + if ((bIsQualityOfSurvey || bIsQualityOfBathymetryCoverage) && + ((m_dt.GetClass() == GEDTC_NUMERIC && + m_dt.GetNumericDataType() == GDT_UInt32) || + (m_dt.GetClass() == GEDTC_COMPOUND && + m_dt.GetComponents().size() == 1 && + m_dt.GetComponents()[0]->GetType().GetClass() == GEDTC_NUMERIC && + m_dt.GetComponents()[0]->GetType().GetNumericDataType() == + GDT_UInt32))) { if (auto poRootGroup = HDF5Array::GetRootGroup()) { if (const auto poGroupF = poRootGroup->OpenGroup("Group_F")) { - const auto poGroupFArray = - poGroupF->OpenMDArray("QualityOfSurvey"); + const auto poGroupFArray = poGroupF->OpenMDArray( + bIsQualityOfSurvey ? "QualityOfSurvey" + : "QualityOfBathymetryCoverage"); if (poGroupFArray && poGroupFArray->GetDataType().GetClass() == GEDTC_COMPOUND && poGroupFArray->GetDataType().GetComponents().size() == 8 && diff --git a/frmts/hdf5/s102dataset.cpp b/frmts/hdf5/s102dataset.cpp index fb84cc49cbb1..428ec45cc894 100644 --- a/frmts/hdf5/s102dataset.cpp +++ b/frmts/hdf5/s102dataset.cpp @@ -46,8 +46,8 @@ class S102Dataset final : public S100BaseDataset { - bool OpenQualityOfSurvey(GDALOpenInfo *poOpenInfo, - const std::shared_ptr &poRootGroup); + bool OpenQuality(GDALOpenInfo *poOpenInfo, + const std::shared_ptr &poRootGroup); public: explicit S102Dataset(const std::string &osFilename) @@ -169,7 +169,7 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) std::string osFilename(poOpenInfo->pszFilename); bool bIsSubdataset = false; - bool bIsQualityOfSurvey = false; + bool bIsQuality = false; if (STARTS_WITH(poOpenInfo->pszFilename, "S102:")) { const CPLStringList aosTokens( @@ -188,9 +188,10 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) { // Default dataset } - else if (EQUAL(aosTokens[2], "QualityOfSurvey")) + else if (EQUAL(aosTokens[2], "QualityOfSurvey") || // < v3 + EQUAL(aosTokens[2], "QualityOfBathymetryCoverage")) // v3 { - bIsQualityOfSurvey = true; + bIsQuality = true; } else { @@ -220,9 +221,9 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) const bool bNorthUp = CPLTestBool( CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, "NORTH_UP", "YES")); - if (bIsQualityOfSurvey) + if (bIsQuality) { - if (!poDS->OpenQualityOfSurvey(poOpenInfo, poRootGroup)) + if (!poDS->OpenQuality(poOpenInfo, poRootGroup)) return nullptr; // Setup/check for pam .aux.xml. @@ -376,12 +377,21 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) poDS->GDALDataset::SetMetadataItem(GDALMD_AREA_OR_POINT, GDALMD_AOP_POINT); - auto poGroupQualityOfSurvey = poRootGroup->OpenGroup("QualityOfSurvey"); - if (!bIsSubdataset && poGroupQualityOfSurvey) + auto poGroupQuality = poRootGroup->OpenGroup("QualityOfSurvey"); + const bool bIsNamedQualityOfSurvey = poGroupQuality != nullptr; + if (!bIsNamedQualityOfSurvey) { - auto poGroupQualityOfSurvey01 = - poGroupQualityOfSurvey->OpenGroup("QualityOfSurvey.01"); - if (poGroupQualityOfSurvey01) + // S102 v3 now uses QualityOfBathymetryCoverage instead of QualityOfSurvey + poGroupQuality = poRootGroup->OpenGroup("QualityOfBathymetryCoverage"); + } + if (!bIsSubdataset && poGroupQuality) + { + const char *pszNameOfQualityGroup = bIsNamedQualityOfSurvey + ? "QualityOfSurvey" + : "QualityOfBathymetryCoverage"; + auto poGroupQuality01 = poGroupQuality->OpenGroup( + CPLSPrintf("%s.01", pszNameOfQualityGroup)); + if (poGroupQuality01) { poDS->GDALDataset::SetMetadataItem( "SUBDATASET_1_NAME", @@ -393,10 +403,12 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) poDS->GDALDataset::SetMetadataItem( "SUBDATASET_2_NAME", - CPLSPrintf("S102:\"%s\":QualityOfSurvey", osFilename.c_str()), + CPLSPrintf("S102:\"%s\":%s", osFilename.c_str(), + pszNameOfQualityGroup), "SUBDATASETS"); poDS->GDALDataset::SetMetadataItem( - "SUBDATASET_2_DESC", "Georeferenced metadata QualityOfSurvey", + "SUBDATASET_2_DESC", + CPLSPrintf("Georeferenced metadata %s", pszNameOfQualityGroup), "SUBDATASETS"); } } @@ -412,34 +424,45 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) } /************************************************************************/ -/* OpenQualityOfSurvey() */ +/* OpenQuality() */ /************************************************************************/ -bool S102Dataset::OpenQualityOfSurvey( - GDALOpenInfo *poOpenInfo, const std::shared_ptr &poRootGroup) +bool S102Dataset::OpenQuality(GDALOpenInfo *poOpenInfo, + const std::shared_ptr &poRootGroup) { const bool bNorthUp = CPLTestBool( CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, "NORTH_UP", "YES")); - auto poGroupQualityOfSurvey = poRootGroup->OpenGroup("QualityOfSurvey"); - if (!poGroupQualityOfSurvey) + const char *pszNameOfQualityGroup = "QualityOfSurvey"; + auto poGroupQuality = poRootGroup->OpenGroup(pszNameOfQualityGroup); + if (!poGroupQuality) { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find group /QualityOfSurvey"); - return false; + pszNameOfQualityGroup = "QualityOfBathymetryCoverage"; + poGroupQuality = poRootGroup->OpenGroup(pszNameOfQualityGroup); + if (!poGroupQuality) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot find group /QualityOfSurvey or " + "/QualityOfBathymetryCoverage"); + return false; + } } - auto poGroupQualityOfSurvey01 = - poGroupQualityOfSurvey->OpenGroup("QualityOfSurvey.01"); - if (!poGroupQualityOfSurvey01) + const std::string osQuality01Name = + std::string(pszNameOfQualityGroup).append(".01"); + const std::string osQuality01FullName = std::string("/") + .append(pszNameOfQualityGroup) + .append("/") + .append(osQuality01Name); + auto poGroupQuality01 = poGroupQuality->OpenGroup(osQuality01Name); + if (!poGroupQuality01) { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find group /QualityOfSurvey/QualityOfSurvey.01"); + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find group %s", + osQuality01FullName.c_str()); return false; } - if (auto poStartSequence = - poGroupQualityOfSurvey01->GetAttribute("startSequence")) + if (auto poStartSequence = poGroupQuality01->GetAttribute("startSequence")) { const char *pszStartSequence = poStartSequence->ReadAsString(); if (pszStartSequence && !EQUAL(pszStartSequence, "0,0")) @@ -452,15 +475,14 @@ bool S102Dataset::OpenQualityOfSurvey( } // Compute geotransform - m_bHasGT = S100GetGeoTransform(poGroupQualityOfSurvey01.get(), - m_adfGeoTransform, bNorthUp); + m_bHasGT = S100GetGeoTransform(poGroupQuality01.get(), m_adfGeoTransform, + bNorthUp); - auto poGroup001 = poGroupQualityOfSurvey01->OpenGroup("Group_001"); + auto poGroup001 = poGroupQuality01->OpenGroup("Group_001"); if (!poGroup001) { - CPLError( - CE_Failure, CPLE_AppDefined, - "Cannot find group /QualityOfSurvey/QualityOfSurvey.01/Group_001"); + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find group %s/Group_001", + osQuality01FullName.c_str()); return false; } @@ -469,14 +491,41 @@ bool S102Dataset::OpenQualityOfSurvey( { CPLError(CE_Failure, CPLE_AppDefined, "Cannot find array " - "/QualityOfSurvey/QualityOfSurvey.01/Group_001/values"); + "%s/Group_001/values", + osQuality01FullName.c_str()); return false; } { const auto &oType = poValuesArray->GetDataType(); - if (oType.GetClass() != GEDTC_NUMERIC && - oType.GetNumericDataType() != GDT_UInt32) + if (oType.GetClass() == GEDTC_NUMERIC && + oType.GetNumericDataType() == GDT_UInt32) + { + // ok + } + else if (oType.GetClass() == GEDTC_COMPOUND && + oType.GetComponents().size() == 1 && + oType.GetComponents()[0]->GetType().GetClass() == + GEDTC_NUMERIC && + oType.GetComponents()[0]->GetType().GetNumericDataType() == + GDT_UInt32) + { + // seen in a S102 v3 product (102DE00CA22_UNC_MD.H5), although + // I believe this is non-conformant. + + // Escape potentials single-quote and double-quote with back-slash + CPLString osEscapedCompName(oType.GetComponents()[0]->GetName()); + osEscapedCompName.replaceAll("\\", "\\\\") + .replaceAll("'", "\\'") + .replaceAll("\"", "\\\""); + + // Gets a view with that single component extracted. + poValuesArray = poValuesArray->GetView( + std::string("['").append(osEscapedCompName).append("']")); + if (!poValuesArray) + return false; + } + else { CPLError(CE_Failure, CPLE_NotSupported, "Unsupported data type for %s", @@ -494,11 +543,12 @@ bool S102Dataset::OpenQualityOfSurvey( } auto poFeatureAttributeTable = - poGroupQualityOfSurvey->OpenMDArray("featureAttributeTable"); + poGroupQuality->OpenMDArray("featureAttributeTable"); if (!poFeatureAttributeTable) { CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find array /QualityOfSurvey/featureAttributeTable"); + "Cannot find array /%s/featureAttributeTable", + pszNameOfQualityGroup); return false; } @@ -527,6 +577,8 @@ bool S102Dataset::OpenQualityOfSurvey( auto poDS = std::unique_ptr(poValuesArray->AsClassicDataset(1, 0)); + if (!poDS) + return false; nRasterXSize = poDS->GetRasterXSize(); nRasterYSize = poDS->GetRasterYSize(); diff --git a/frmts/heif/heifdataset.cpp b/frmts/heif/heifdataset.cpp index 71280d197c6f..86e00c177b9b 100644 --- a/frmts/heif/heifdataset.cpp +++ b/frmts/heif/heifdataset.cpp @@ -76,7 +76,10 @@ class GDALHEIFDataset final : public GDALPamDataset GDALHEIFDataset(); ~GDALHEIFDataset(); - static GDALDataset *Open(GDALOpenInfo *poOpenInfo); + static GDALDataset *OpenHEIF(GDALOpenInfo *poOpenInfo); +#if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 12, 0) + static GDALDataset *OpenAVIF(GDALOpenInfo *poOpenInfo); +#endif }; /************************************************************************/ @@ -164,7 +167,7 @@ int64_t GDALHEIFDataset::GetPositionCbk(void *userdata) int GDALHEIFDataset::ReadCbk(void *data, size_t size, void *userdata) { GDALHEIFDataset *poThis = static_cast(userdata); - return VSIFReadL(data, size, 1, poThis->m_fpL) == 1 ? 0 : -1; + return VSIFReadL(data, 1, size, poThis->m_fpL) == size ? 0 : -1; } /************************************************************************/ @@ -314,6 +317,12 @@ bool GDALHEIFDataset::Init(GDALOpenInfo *poOpenInfo) OpenThumbnails(); + if (poOpenInfo->nHeaderBytes > 12 && + memcmp(poOpenInfo->pabyHeader + 4, "ftypavif", 8) == 0) + { + poDriver = GetGDALDriverManager()->GetDriverByName("AVIF_HEIF"); + } + // Initialize any PAM information. if (nSubdatasets > 1) { @@ -383,8 +392,8 @@ void GDALHEIFDataset::ReadMetadata() } } - CPLString osTempFile; - osTempFile.Printf("/vsimem/heif_exif_%p.tif", this); + const CPLString osTempFile( + VSIMemGenerateHiddenFilename("heif_exif.tif")); VSILFILE *fpTemp = VSIFileFromMemBuffer(osTempFile, &data[nTIFFFileOffset], nCount - nTIFFFileOffset, FALSE); @@ -517,6 +526,7 @@ static int HEIFDriverIdentify(GDALOpenInfo *poOpenInfo) if (poOpenInfo->nHeaderBytes < 12 || poOpenInfo->fpL == nullptr) return false; + #if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 4, 0) const auto res = heif_check_filetype(poOpenInfo->pabyHeader, poOpenInfo->nHeaderBytes); @@ -561,10 +571,10 @@ static int HEIFDriverIdentify(GDALOpenInfo *poOpenInfo) } /************************************************************************/ -/* Open() */ +/* OpenHEIF() */ /************************************************************************/ -GDALDataset *GDALHEIFDataset::Open(GDALOpenInfo *poOpenInfo) +GDALDataset *GDALHEIFDataset::OpenHEIF(GDALOpenInfo *poOpenInfo) { if (!HEIFDriverIdentify(poOpenInfo)) return nullptr; @@ -582,6 +592,44 @@ GDALDataset *GDALHEIFDataset::Open(GDALOpenInfo *poOpenInfo) return poDS.release(); } +#if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 12, 0) + +/************************************************************************/ +/* HEIFIdentifyOnlyAVIF() */ +/************************************************************************/ + +static int HEIFIdentifyOnlyAVIF(GDALOpenInfo *poOpenInfo) +{ + if (poOpenInfo->nHeaderBytes < 12 || poOpenInfo->fpL == nullptr) + return false; + if (memcmp(poOpenInfo->pabyHeader + 4, "ftypavif", 8) == 0) + return true; + return false; +} + +/************************************************************************/ +/* OpenAVIF() */ +/************************************************************************/ + +GDALDataset *GDALHEIFDataset::OpenAVIF(GDALOpenInfo *poOpenInfo) +{ + if (!HEIFIdentifyOnlyAVIF(poOpenInfo)) + return nullptr; + if (poOpenInfo->eAccess == GA_Update) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Update of existing AVIF file not supported"); + return nullptr; + } + + auto poDS = std::make_unique(); + if (!poDS->Init(poOpenInfo)) + return nullptr; + + return poDS.release(); +} +#endif + /************************************************************************/ /* GDALHEIFRasterBand() */ /************************************************************************/ @@ -700,10 +748,45 @@ void GDALRegister_HEIF() if (GDALGetDriverByName(DRIVER_NAME) != nullptr) return; - GDALDriver *poDriver = new GDALDriver(); - HEIFDriverSetCommonMetadata(poDriver); + auto poDM = GetGDALDriverManager(); + { + GDALDriver *poDriver = new GDALDriver(); + HEIFDriverSetCommonMetadata(poDriver); + +#if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 12, 0) + // If the AVIF dedicated driver is not available, register an AVIF driver, + // called AVIF_HEIF, based on libheif, if it has AV1 decoding capabilities. + if (heif_have_decoder_for_format(heif_compression_AV1)) + { + poDriver->SetMetadataItem("SUPPORTS_AVIF", "YES", "HEIF"); + } +#endif - poDriver->pfnOpen = GDALHEIFDataset::Open; + poDriver->pfnOpen = GDALHEIFDataset::OpenHEIF; + poDM->RegisterDriver(poDriver); + } - GetGDALDriverManager()->RegisterDriver(poDriver); +#if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 12, 0) + // If the AVIF dedicated driver is not available, register an AVIF driver, + // called AVIF_HEIF, based on libheif, if it has AV1 decoding capabilities. + if (heif_have_decoder_for_format(heif_compression_AV1) && + !poDM->IsKnownDriver("AVIF") && !poDM->IsKnownDriver("AVIF_HEIF")) + { + GDALDriver *poAVIF_HEIFDriver = new GDALDriver(); + poAVIF_HEIFDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES"); + poAVIF_HEIFDriver->SetDescription("AVIF_HEIF"); + poAVIF_HEIFDriver->SetMetadataItem( + GDAL_DMD_LONGNAME, "AV1 Image File Format (using libheif)"); + poAVIF_HEIFDriver->SetMetadataItem(GDAL_DMD_MIMETYPE, "image/avif"); + poAVIF_HEIFDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, + "drivers/raster/heif.html"); + poAVIF_HEIFDriver->SetMetadataItem(GDAL_DMD_EXTENSION, "avif"); + poAVIF_HEIFDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); + + poAVIF_HEIFDriver->pfnOpen = GDALHEIFDataset::OpenAVIF; + poAVIF_HEIFDriver->pfnIdentify = HEIFIdentifyOnlyAVIF; + + poDM->RegisterDriver(poAVIF_HEIFDriver); + } +#endif } diff --git a/frmts/http/httpdriver.cpp b/frmts/http/httpdriver.cpp index 075030090d63..153a4863b6c5 100644 --- a/frmts/http/httpdriver.cpp +++ b/frmts/http/httpdriver.cpp @@ -29,15 +29,26 @@ #include "cpl_string.h" #include "cpl_http.h" -#include "cpl_atomic_ops.h" #include "gdal_frmts.h" #include "gdal_pam.h" +static std::string SanitizeDispositionFilename(const std::string &osVal) +{ + std::string osRet(osVal); + if (!osRet.empty() && osRet[0] == '"') + { + const auto nEnd = osRet.find('"', 1); + if (nEnd != std::string::npos) + return osRet.substr(1, nEnd - 1); + } + return osRet; +} + /************************************************************************/ /* HTTPFetchContentDispositionFilename() */ /************************************************************************/ -static const char *HTTPFetchContentDispositionFilename(char **papszHeaders) +static std::string HTTPFetchContentDispositionFilename(char **papszHeaders) { char **papszIter = papszHeaders; while (papszIter && *papszIter) @@ -47,25 +58,25 @@ static const char *HTTPFetchContentDispositionFilename(char **papszHeaders) if (STARTS_WITH(*papszIter, "Content-Disposition: attachment; filename=")) { - return *papszIter + 42; + return SanitizeDispositionFilename(*papszIter + 42); } /* For single part, the headers are in KEY=VAL format, but with e-o-l * ... */ else if (STARTS_WITH(*papszIter, "Content-Disposition=attachment; filename=")) { - char *pszVal = (char *)(*papszIter + 41); + char *pszVal = (*papszIter + 41); char *pszEOL = strchr(pszVal, '\r'); if (pszEOL) *pszEOL = 0; pszEOL = strchr(pszVal, '\n'); if (pszEOL) *pszEOL = 0; - return pszVal; + return SanitizeDispositionFilename(pszVal); } papszIter++; } - return nullptr; + return std::string(); } /************************************************************************/ @@ -75,8 +86,6 @@ static const char *HTTPFetchContentDispositionFilename(char **papszHeaders) static GDALDataset *HTTPOpen(GDALOpenInfo *poOpenInfo) { - static volatile int nCounter = 0; - if (poOpenInfo->nHeaderBytes != 0) return nullptr; @@ -104,21 +113,19 @@ static GDALDataset *HTTPOpen(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ /* Create a memory file from the result. */ /* -------------------------------------------------------------------- */ - CPLString osResultFilename; - - int nNewCounter = CPLAtomicInc(&nCounter); - - const char *pszFilename = + std::string osFilename = HTTPFetchContentDispositionFilename(psResult->papszHeaders); - if (pszFilename == nullptr) + if (osFilename.empty()) { - pszFilename = CPLGetFilename(poOpenInfo->pszFilename); + osFilename = CPLGetFilename(poOpenInfo->pszFilename); /* If we have special characters, let's default to a fixed name */ - if (strchr(pszFilename, '?') || strchr(pszFilename, '&')) - pszFilename = "file.dat"; + if (strchr(osFilename.c_str(), '?') || strchr(osFilename.c_str(), '&')) + osFilename = "file.dat"; } - osResultFilename.Printf("/vsimem/http_%d/%s", nNewCounter, pszFilename); + // If changing the _gdal_http_ marker, change jpgdataset.cpp that tests for it + const CPLString osResultFilename = VSIMemGenerateHiddenFilename( + std::string("_gdal_http_").append(osFilename).c_str()); VSILFILE *fp = VSIFileFromMemBuffer(osResultFilename, psResult->pabyData, psResult->nDataLen, TRUE); diff --git a/frmts/jpeg/jpgdataset.cpp b/frmts/jpeg/jpgdataset.cpp index 3016819e28c4..a9a8029c2dc9 100644 --- a/frmts/jpeg/jpgdataset.cpp +++ b/frmts/jpeg/jpgdataset.cpp @@ -2859,7 +2859,8 @@ GDALDataset *JPGDatasetCommon::OpenFLIRRawThermalImage() GByte *pabyData = static_cast(CPLMalloc(m_abyRawThermalImage.size())); - const std::string osTmpFilename(CPLSPrintf("/vsimem/jpeg/%p", pabyData)); + const std::string osTmpFilename( + VSIMemGenerateHiddenFilename("jpeg_flir_raw")); memcpy(pabyData, m_abyRawThermalImage.data(), m_abyRawThermalImage.size()); VSILFILE *fpRaw = VSIFileFromMemBuffer(osTmpFilename.c_str(), pabyData, m_abyRawThermalImage.size(), true); @@ -3235,7 +3236,8 @@ JPGDatasetCommon *JPGDataset::OpenStage2(JPGDatasetOpenArgs *psArgs, // will unlink the temporary /vsimem file just after GDALOpen(), so // later VSIFOpenL() when reading internal overviews would fail. // Initialize them now. - if (STARTS_WITH(real_filename, "/vsimem/http_")) + if (STARTS_WITH(real_filename, "/vsimem/") && + strstr(real_filename, "_gdal_http_")) { poDS->InitInternalOverviews(); } @@ -3744,7 +3746,7 @@ void JPGDataset::EmitMessage(j_common_ptr cinfo, int msg_level) buffer); } } - else if (pszVal == nullptr || CPLTestBool(pszVal)) + else if (pszVal == nullptr || !CPLTestBool(pszVal)) { if (pszVal == nullptr) { @@ -4097,7 +4099,7 @@ void JPGAddEXIF(GDALDataType eWorkDT, GDALDataset *poSrcDS, char **papszOptions, return; } - CPLString osTmpFile(CPLSPrintf("/vsimem/ovrjpg%p", poMemDS)); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("ovrjpg")); GDALDataset *poOutDS = pCreateCopy(osTmpFile, poMemDS, 0, nullptr, GDALDummyProgress, nullptr); const bool bExifOverviewSuccess = poOutDS != nullptr; diff --git a/frmts/jpegxl/jpegxl.cpp b/frmts/jpegxl/jpegxl.cpp index 17b6ba51dc6a..826315d228ba 100644 --- a/frmts/jpegxl/jpegxl.cpp +++ b/frmts/jpegxl/jpegxl.cpp @@ -358,11 +358,9 @@ bool JPEGXLDataset::Open(GDALOpenInfo *poOpenInfo) { CPL_LSBPTR32(&nTiffDirStart); } - const std::string osTmpFilename = - CPLSPrintf("/vsimem/jxl/%p", this); - VSILFILE *fpEXIF = VSIFileFromMemBuffer( - osTmpFilename.c_str(), abyBoxBuffer.data() + 4, - abyBoxBuffer.size() - 4, false); + VSILFILE *fpEXIF = + VSIFileFromMemBuffer(nullptr, abyBoxBuffer.data() + 4, + abyBoxBuffer.size() - 4, false); int nExifOffset = 0; int nInterOffset = 0; int nGPSOffset = 0; diff --git a/frmts/jpipkak/jpipkakdataset.cpp b/frmts/jpipkak/jpipkakdataset.cpp index 9057a3f7e198..f8195d9d19c0 100644 --- a/frmts/jpipkak/jpipkakdataset.cpp +++ b/frmts/jpipkak/jpipkakdataset.cpp @@ -749,8 +749,7 @@ int JPIPKAKDataset::Initialize(const char *pszDatasetName, int bReinitializing) } // create in memory file using vsimem - CPLString osFileBoxName; - osFileBoxName.Printf("/vsimem/jpip/%s.dat", pszCid); + const CPLString osFileBoxName(VSIMemGenerateHiddenFilename("jpip")); VSILFILE *fpLL = VSIFOpenL(osFileBoxName.c_str(), "w+"); poCache->set_read_scope(KDU_META_DATABIN, nCodestream, 0); kdu_byte *pabyBuffer = (kdu_byte *)CPLMalloc(nLen); diff --git a/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp b/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp index eeab23d79b87..4b06cb443f31 100644 --- a/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp +++ b/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp @@ -31,7 +31,6 @@ #include "kmlsuperoverlaydataset.h" #include -#include #include #include #include @@ -1799,9 +1798,7 @@ KmlSuperOverlayLoadIcon(const char *pszBaseFilename, const char *pszIcon) return nullptr; } - static std::atomic nInc = 0; - osSubFilename = - CPLSPrintf("/vsimem/kmlsuperoverlay_%d_%p", nInc++, pszBaseFilename); + osSubFilename = VSIMemGenerateHiddenFilename("kmlsuperoverlay"); VSIFCloseL(VSIFileFromMemBuffer(osSubFilename, pabyBuffer, nRead, TRUE)); auto poDSIcon = std::unique_ptr(GDALDataset::Open( @@ -2793,6 +2790,9 @@ KmlSuperOverlayReadDataset::Open(const char *pszFilename, auto poOvrDS = std::make_unique(); poOvrDS->bIsOvr = true; + // The life-time of objects is such that poOvrDS is destroyed when + // poDS is destroyed. + // coverity[escape] poOvrDS->poParent = poDS.get(); poOvrDS->nFactor = nFactor; poOvrDS->nRasterXSize = nFactor * poDSIcon->GetRasterXSize(); diff --git a/frmts/l1b/l1bdataset.cpp b/frmts/l1b/l1bdataset.cpp index 1c256eaad9cc..dcb5e8364aa9 100644 --- a/frmts/l1b/l1bdataset.cpp +++ b/frmts/l1b/l1bdataset.cpp @@ -3310,7 +3310,7 @@ GDALDataset *L1BDataset::Open(GDALOpenInfo *poOpenInfo) pszFilename++; osFilename = pszFilename; if (!osFilename.empty() && osFilename.back() == '"') - osFilename.resize(osFilename.size() - 1); + osFilename.pop_back(); fp = VSIFOpenL(osFilename, "rb"); if (!fp) { diff --git a/frmts/mbtiles/mbtilesdataset.cpp b/frmts/mbtiles/mbtilesdataset.cpp index 1d3549ca0d53..6ef43a125b20 100644 --- a/frmts/mbtiles/mbtilesdataset.cpp +++ b/frmts/mbtiles/mbtilesdataset.cpp @@ -1710,8 +1710,8 @@ GIntBig MBTilesVectorLayer::GetFeatureCount(int bForce) { VSIUnlink(m_osTmpFilename); } - m_osTmpFilename = - CPLSPrintf("/vsimem/mvt_%p_%d_%d.pbf", this, m_nX, m_nY); + m_osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("mvt_%d_%d.pbf", m_nX, m_nY)); VSIFCloseL(VSIFileFromMemBuffer(m_osTmpFilename, pabyDataDup, nDataSize, true)); @@ -1794,8 +1794,8 @@ OGRFeature *MBTilesVectorLayer::GetNextSrcFeature() { VSIUnlink(m_osTmpFilename); } - m_osTmpFilename = - CPLSPrintf("/vsimem/mvt_%p_%d_%d.pbf", this, m_nX, m_nY); + m_osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("mvt_%d_%d.pbf", m_nX, m_nY)); VSIFCloseL(VSIFileFromMemBuffer(m_osTmpFilename, pabyDataDup, nDataSize, true)); @@ -1903,8 +1903,8 @@ OGRFeature *MBTilesVectorLayer::GetFeature(GIntBig nFID) OGR_F_Destroy(hFeat); OGR_DS_ReleaseResultSet(m_poDS->hDS, hSQLLyr); - CPLString osTmpFilename = - CPLSPrintf("/vsimem/mvt_getfeature_%p_%d_%d.pbf", this, nX, nY); + const CPLString osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("mvt_get_feature_%d_%d.pbf", m_nX, m_nY)); VSIFCloseL( VSIFileFromMemBuffer(osTmpFilename, pabyDataDup, nDataSize, true)); @@ -1979,7 +1979,8 @@ void MBTilesDataset::InitVector(double dfMinX, double dfMinY, double dfMaxX, OGR_DS_ReleaseResultSet(hDS, hSQLLyr); } - m_osMetadataMemFilename = CPLSPrintf("/vsimem/%p_metadata.json", this); + m_osMetadataMemFilename = + VSIMemGenerateHiddenFilename("mbtiles_metadata.json"); oDoc.Save(m_osMetadataMemFilename); CPLJSONArray oVectorLayers; @@ -2556,8 +2557,7 @@ static int MBTilesGetBandCountAndTileSize(bool bIsVSICURL, OGRDataSourceH &hDS, break; } - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/%p", hSQLLyr); + const CPLString osMemFileName(VSIMemGenerateHiddenFilename("mvt_temp.db")); int nDataSize = 0; GByte *pabyData = OGR_F_GetFieldAsBinary(hFeat, 0, &nDataSize); diff --git a/frmts/mem/memdataset.cpp b/frmts/mem/memdataset.cpp index 2d624dafeab4..bf1502a42ea5 100644 --- a/frmts/mem/memdataset.cpp +++ b/frmts/mem/memdataset.cpp @@ -89,10 +89,10 @@ GDALRasterBandH MEMCreateRasterBandEx(GDALDataset *poDS, int nBand, /************************************************************************/ MEMRasterBand::MEMRasterBand(GByte *pabyDataIn, GDALDataType eTypeIn, - int nXSizeIn, int nYSizeIn) + int nXSizeIn, int nYSizeIn, bool bOwnDataIn) : GDALPamRasterBand(FALSE), pabyData(pabyDataIn), nPixelOffset(GDALGetDataTypeSizeBytes(eTypeIn)), nLineOffset(0), - bOwnData(true) + bOwnData(bOwnDataIn) { eAccess = GA_Update; eDataType = eTypeIn; @@ -463,7 +463,7 @@ int MEMRasterBand::GetOverviewCount() MEMDataset *poMemDS = dynamic_cast(poDS); if (poMemDS == nullptr) return 0; - return poMemDS->m_nOverviewDSCount; + return static_cast(poMemDS->m_apoOverviewDS.size()); } /************************************************************************/ @@ -476,9 +476,9 @@ GDALRasterBand *MEMRasterBand::GetOverview(int i) MEMDataset *poMemDS = dynamic_cast(poDS); if (poMemDS == nullptr) return nullptr; - if (i < 0 || i >= poMemDS->m_nOverviewDSCount) + if (i < 0 || i >= static_cast(poMemDS->m_apoOverviewDS.size())) return nullptr; - return poMemDS->m_papoOverviewDS[i]->GetRasterBand(nBand); + return poMemDS->m_apoOverviewDS[i]->GetRasterBand(nBand); } /************************************************************************/ @@ -504,8 +504,8 @@ CPLErr MEMRasterBand::CreateMaskBand(int nFlagsIn) return CE_Failure; nMaskFlags = nFlagsIn; - auto poMemMaskBand = - new MEMRasterBand(pabyMaskData, GDT_Byte, nRasterXSize, nRasterYSize); + auto poMemMaskBand = new MEMRasterBand(pabyMaskData, GDT_Byte, nRasterXSize, + nRasterYSize, /* bOwnData= */ true); poMemMaskBand->m_bIsMask = true; poMask.reset(poMemMaskBand, true); if ((nFlagsIn & GMF_PER_DATASET) != 0 && nBand == 1 && poMemDS != nullptr) @@ -542,8 +542,7 @@ bool MEMRasterBand::IsMaskBand() const /************************************************************************/ MEMDataset::MEMDataset() - : GDALDataset(FALSE), bGeoTransformSet(FALSE), m_nOverviewDSCount(0), - m_papoOverviewDS(nullptr), m_poPrivate(new Private()) + : GDALDataset(FALSE), bGeoTransformSet(FALSE), m_poPrivate(new Private()) { adfGeoTransform[0] = 0.0; adfGeoTransform[1] = 1.0; @@ -565,10 +564,6 @@ MEMDataset::~MEMDataset() bSuppressOnClose = true; FlushCache(true); bSuppressOnClose = bSuppressOnCloseBackup; - - for (int i = 0; i < m_nOverviewDSCount; ++i) - delete m_papoOverviewDS[i]; - CPLFree(m_papoOverviewDS); } #if 0 @@ -824,11 +819,7 @@ CPLErr MEMDataset::IBuildOverviews(const char *pszResampling, int nOverviews, if (nOverviews == 0) { // Cleanup existing overviews - for (int i = 0; i < m_nOverviewDSCount; ++i) - delete m_papoOverviewDS[i]; - CPLFree(m_papoOverviewDS); - m_nOverviewDSCount = 0; - m_papoOverviewDS = nullptr; + m_apoOverviewDS.clear(); return CE_None; } @@ -901,7 +892,7 @@ CPLErr MEMDataset::IBuildOverviews(const char *pszResampling, int nOverviews, // Create new overview dataset if needed. if (!bExisting) { - MEMDataset *poOvrDS = new MEMDataset(); + auto poOvrDS = std::make_unique(); poOvrDS->eAccess = GA_Update; poOvrDS->nRasterXSize = (nRasterXSize + panOverviewList[i] - 1) / panOverviewList[i]; @@ -913,14 +904,10 @@ CPLErr MEMDataset::IBuildOverviews(const char *pszResampling, int nOverviews, GetRasterBand(iBand + 1)->GetRasterDataType(); if (poOvrDS->AddBand(eDT, nullptr) != CE_None) { - delete poOvrDS; return CE_Failure; } } - m_nOverviewDSCount++; - m_papoOverviewDS = (GDALDataset **)CPLRealloc( - m_papoOverviewDS, sizeof(GDALDataset *) * m_nOverviewDSCount); - m_papoOverviewDS[m_nOverviewDSCount - 1] = poOvrDS; + m_apoOverviewDS.emplace_back(std::move(poOvrDS)); } } @@ -1053,6 +1040,106 @@ CPLErr MEMDataset::CreateMaskBand(int nFlagsIn) return poFirstBand->CreateMaskBand(nFlagsIn | GMF_PER_DATASET); } +/************************************************************************/ +/* CanBeCloned() */ +/************************************************************************/ + +/** Implements GDALDataset::CanBeCloned() + * + * This method is called by GDALThreadSafeDataset::Create() to determine if + * it is possible to create a thread-safe wrapper for a dataset, which involves + * the ability to Clone() it. + * + * The implementation of this method must be thread-safe. + */ +bool MEMDataset::CanBeCloned(int nScopeFlags, bool bCanShareState) const +{ + return nScopeFlags == GDAL_OF_RASTER && bCanShareState && + typeid(this) == typeid(const MEMDataset *); +} + +/************************************************************************/ +/* Clone() */ +/************************************************************************/ + +/** Implements GDALDataset::Clone() + * + * This method returns a new instance, identical to "this", but which shares the + * same memory buffer as "this". + * + * The implementation of this method must be thread-safe. + */ +std::unique_ptr MEMDataset::Clone(int nScopeFlags, + bool bCanShareState) const +{ + if (MEMDataset::CanBeCloned(nScopeFlags, bCanShareState)) + { + auto poNewDS = std::make_unique(); + poNewDS->poDriver = poDriver; + poNewDS->nRasterXSize = nRasterXSize; + poNewDS->nRasterYSize = nRasterYSize; + poNewDS->bGeoTransformSet = bGeoTransformSet; + memcpy(poNewDS->adfGeoTransform, adfGeoTransform, + sizeof(adfGeoTransform)); + poNewDS->m_oSRS = m_oSRS; + poNewDS->m_aoGCPs = m_aoGCPs; + poNewDS->m_oGCPSRS = m_oGCPSRS; + for (const auto &poOvrDS : m_apoOverviewDS) + { + poNewDS->m_apoOverviewDS.emplace_back( + poOvrDS->Clone(nScopeFlags, bCanShareState)); + } + + poNewDS->SetDescription(GetDescription()); + poNewDS->oMDMD = oMDMD; + + // Clone bands + for (int i = 1; i <= nBands; ++i) + { + auto poSrcMEMBand = + dynamic_cast(papoBands[i - 1]); + CPLAssert(poSrcMEMBand); + auto poNewBand = std::make_unique( + poNewDS.get(), i, poSrcMEMBand->pabyData, + poSrcMEMBand->GetRasterDataType(), poSrcMEMBand->nPixelOffset, + poSrcMEMBand->nLineOffset, + /* bAssumeOwnership = */ false); + + poNewBand->SetDescription(poSrcMEMBand->GetDescription()); + poNewBand->oMDMD = poSrcMEMBand->oMDMD; + + if (poSrcMEMBand->psPam) + { + poNewBand->PamInitialize(); + CPLAssert(poNewBand->psPam); + poNewBand->psPam->CopyFrom(*(poSrcMEMBand->psPam)); + } + + // Instanciates a mask band when needed. + if ((poSrcMEMBand->nMaskFlags & + (GMF_ALL_VALID | GMF_ALPHA | GMF_NODATA)) == 0) + { + auto poSrcMaskBand = dynamic_cast( + poSrcMEMBand->poMask.get()); + if (poSrcMaskBand) + { + auto poMaskBand = new MEMRasterBand( + poSrcMaskBand->pabyData, GDT_Byte, nRasterXSize, + nRasterYSize, /* bOwnData = */ false); + poMaskBand->m_bIsMask = true; + poNewBand->poMask.reset(poMaskBand, true); + poNewBand->nMaskFlags = poSrcMaskBand->nMaskFlags; + } + } + + poNewDS->SetBand(i, std::move(poNewBand)); + } + + return poNewDS; + } + return GDALDataset::Clone(nScopeFlags, bCanShareState); +} + /************************************************************************/ /* Open() */ /************************************************************************/ diff --git a/frmts/mem/memdataset.h b/frmts/mem/memdataset.h index f2843cbf78dd..8a52c0a69f9d 100644 --- a/frmts/mem/memdataset.h +++ b/frmts/mem/memdataset.h @@ -66,8 +66,7 @@ class CPL_DLL MEMDataset CPL_NON_FINAL : public GDALDataset std::vector m_aoGCPs{}; OGRSpatialReference m_oGCPSRS{}; - int m_nOverviewDSCount; - GDALDataset **m_papoOverviewDS; + std::vector> m_apoOverviewDS{}; struct Private; std::unique_ptr m_poPrivate; @@ -85,6 +84,12 @@ class CPL_DLL MEMDataset CPL_NON_FINAL : public GDALDataset int nYSize, int nBands, GDALDataType eType, char **papszParamList); + protected: + bool CanBeCloned(int nScopeFlags, bool bCanShareState) const override; + + std::unique_ptr Clone(int nScopeFlags, + bool bCanShareState) const override; + public: MEMDataset(); virtual ~MEMDataset(); @@ -141,9 +146,6 @@ class CPL_DLL MEMDataset CPL_NON_FINAL : public GDALDataset class CPL_DLL MEMRasterBand CPL_NON_FINAL : public GDALPamRasterBand { private: - MEMRasterBand(GByte *pabyDataIn, GDALDataType eTypeIn, int nXSizeIn, - int nYSizeIn); - CPL_DISALLOW_COPY_ASSIGN(MEMRasterBand) protected: @@ -156,6 +158,9 @@ class CPL_DLL MEMRasterBand CPL_NON_FINAL : public GDALPamRasterBand bool m_bIsMask = false; + MEMRasterBand(GByte *pabyDataIn, GDALDataType eTypeIn, int nXSizeIn, + int nYSizeIn, bool bOwnDataIn); + public: MEMRasterBand(GDALDataset *poDS, int nBand, GByte *pabyData, GDALDataType eType, GSpacing nPixelOffset, diff --git a/frmts/mrf/PNG_band.cpp b/frmts/mrf/PNG_band.cpp index b868e6324232..b0582f346a95 100644 --- a/frmts/mrf/PNG_band.cpp +++ b/frmts/mrf/PNG_band.cpp @@ -153,7 +153,7 @@ CPLErr PNG_Codec::DecompressPNG(buf_mgr &dst, buf_mgr &src) { // Use the PNG driver for decompression of 8-bit images, as it // has optimizations for whole image decompression. - CPLString osTmpFilename(CPLSPrintf("/vsimem/mrf/%p.png", &dst)); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("mrf.png")); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), reinterpret_cast(src_ori.buffer), src_ori.size, false)); diff --git a/frmts/mrf/QB3_band.cpp b/frmts/mrf/QB3_band.cpp index 23fe11bcf639..c751234a5da4 100644 --- a/frmts/mrf/QB3_band.cpp +++ b/frmts/mrf/QB3_band.cpp @@ -84,6 +84,12 @@ CPLErr QB3_Band::Compress(buf_mgr &dst, buf_mgr &src) // Quality of 90 and above trigger the better encoding qb3_set_encoder_mode(pQB3, (img.quality > 90) ? QB3M_BEST : QB3M_BASE); +#if defined(QB3_HAS_FTL) + // Quality below 5 triggers the faster encoding, when available + if (img.quality < 5) + qb3_set_encoder_mode(pQB3, QB3M_FTL); +#endif + dst.size = qb3_encode(pQB3, src.buffer, dst.buffer); if (0 == dst.size) { diff --git a/frmts/mrf/Tif_band.cpp b/frmts/mrf/Tif_band.cpp index 1d69ecabad9b..04a22bc15920 100644 --- a/frmts/mrf/Tif_band.cpp +++ b/frmts/mrf/Tif_band.cpp @@ -51,9 +51,7 @@ #include "marfa.h" NAMESPACE_MRF_START -// Returns a string in /vsimem/ + prefix + count that doesn't exist when this -// function gets called It is not thread safe, open the result as soon as -// possible +// Returns a unique filename static CPLString uniq_memfname(const char *prefix) { // Define MRF_LOCAL_TMP to use local files instead of RAM @@ -61,14 +59,7 @@ static CPLString uniq_memfname(const char *prefix) #if defined(MRF_LOCAL_TMP) return CPLGenerateTempFilename(prefix); #else - CPLString fname; - VSIStatBufL statb; - static unsigned int cnt = 0; - do - { - fname.Printf("/vsimem/%s_%08x", prefix, cnt++); - } while (!VSIStatL(fname, &statb)); - return fname; + return VSIMemGenerateHiddenFilename(prefix); #endif } diff --git a/frmts/mrf/marfa_dataset.cpp b/frmts/mrf/marfa_dataset.cpp index 3f96a07f5d31..6af454ae1702 100644 --- a/frmts/mrf/marfa_dataset.cpp +++ b/frmts/mrf/marfa_dataset.cpp @@ -1389,7 +1389,7 @@ CPLXMLNode *MRFDataset::BuildConfig() options += optlist[i]; options += ' '; } - options.resize(options.size() - 1); + options.pop_back(); CPLCreateXMLElementAndValue(config, "Options", options); } diff --git a/frmts/mrf/mrf_util.cpp b/frmts/mrf/mrf_util.cpp index 8601505680ac..40fc2aebe3d2 100644 --- a/frmts/mrf/mrf_util.cpp +++ b/frmts/mrf/mrf_util.cpp @@ -537,7 +537,7 @@ void XMLSetAttributeVal(CPLXMLNode *parent, const char *pszName, single_val = false; value.append(PrintDouble(values[i]) + " "); } - value.resize(value.size() - 1); // Cut the last space + value.pop_back(); // Cut the last space if (single_val) value = PrintDouble(values[0]); CPLCreateXMLNode(parent, CXT_Attribute, pszName); diff --git a/frmts/msg/PublicDecompWTMakefiles.zip b/frmts/msg/PublicDecompWTMakefiles.zip deleted file mode 100644 index 63d40cd4316f..000000000000 Binary files a/frmts/msg/PublicDecompWTMakefiles.zip and /dev/null differ diff --git a/frmts/netcdf/CMakeLists.txt b/frmts/netcdf/CMakeLists.txt index 506d905fd860..699e24547bd1 100644 --- a/frmts/netcdf/CMakeLists.txt +++ b/frmts/netcdf/CMakeLists.txt @@ -45,12 +45,6 @@ endif () if (HAVE_USERFAULTFD_H) declare_def(-DENABLE_UFFD) endif () -if (HAVE_HDF4) - declare_def(-DHAVE_HDF4) -endif () -if (HAVE_HDF5) - declare_def(-DHAVE_HDF5) -endif () if(NOT TARGET gdal_netCDF) return() diff --git a/frmts/netcdf/netcdfdataset.cpp b/frmts/netcdf/netcdfdataset.cpp index e2dcce35dc6e..e470a6ca724a 100644 --- a/frmts/netcdf/netcdfdataset.cpp +++ b/frmts/netcdf/netcdfdataset.cpp @@ -6451,7 +6451,7 @@ void netCDFDataset::CreateSubDatasetList(int nGroupId) nc_type nVarType; nc_inq_vartype(nGroupId, nVar, &nVarType); // Get rid of the last "x" character. - osDim.resize(osDim.size() - 1); + osDim.pop_back(); const char *pszType = ""; switch (nVarType) { @@ -7649,11 +7649,11 @@ bool netCDFDatasetCreateTempFile(NetCDFFormatEnum eFormat, { if (osVal.back() == ';' || osVal.back() == ' ') { - osVal.resize(osVal.size() - 1); + osVal.pop_back(); } else if (osVal.back() == '"') { - osVal.resize(osVal.size() - 1); + osVal.pop_back(); break; } else @@ -7672,7 +7672,7 @@ bool netCDFDatasetCreateTempFile(NetCDFFormatEnum eFormat, { if (osVal.back() == ';' || osVal.back() == ' ') { - osVal.resize(osVal.size() - 1); + osVal.pop_back(); } else { @@ -7683,12 +7683,12 @@ bool netCDFDatasetCreateTempFile(NetCDFFormatEnum eFormat, if (!osVal.empty() && osVal.back() == 'b') { nc_datatype = NC_BYTE; - osVal.resize(osVal.size() - 1); + osVal.pop_back(); } else if (!osVal.empty() && osVal.back() == 's') { nc_datatype = NC_SHORT; - osVal.resize(osVal.size() - 1); + osVal.pop_back(); } if (CPLGetValueType(osVal) == CPL_VALUE_INTEGER) { @@ -7782,7 +7782,7 @@ bool netCDFDatasetCreateTempFile(NetCDFFormatEnum eFormat, } if (pszLine == nullptr) break; - osAccVal.resize(osAccVal.size() - 1); + osAccVal.pop_back(); const std::vector aoDimIds = oMapVarIdToVectorOfDimId[nVarId]; diff --git a/frmts/netcdf/netcdfdrivercore.cpp b/frmts/netcdf/netcdfdrivercore.cpp index 507ae89b78d0..527c81867900 100644 --- a/frmts/netcdf/netcdfdrivercore.cpp +++ b/frmts/netcdf/netcdfdrivercore.cpp @@ -104,21 +104,20 @@ NetCDFFormatEnum netCDFIdentifyFormat(GDALOpenInfo *poOpenInfo, bool bCheckExt) (poOpenInfo->nHeaderBytes > HDF5_SIG_OFFSET + HDF5_SIG_LEN && memcmp(pszHeader + HDF5_SIG_OFFSET, HDF5_SIG, HDF5_SIG_LEN) == 0)) { + // Requires netCDF-4/HDF5 support in libnetcdf (not just libnetcdf-v4). + // If only the netCDF driver is allowed, immediately recognize the file if (poOpenInfo->IsSingleAllowedDriver("netCDF")) { return NCDF_FORMAT_NC4; } - // Requires netCDF-4/HDF5 support in libnetcdf (not just libnetcdf-v4). - // If HDF5 is not supported in GDAL, this driver will try to open the - // file. Else, make sure this driver does not try to open HDF5 files. If - // user really wants to open with this driver, use NETCDF:file.h5 - // format. This check should be relaxed, but there is no clear way to - // make a difference. - -// Check for HDF5 support in GDAL. -#ifdef HAVE_HDF5 + // If there is a HDF5 GDAL driver available, delegate to it. + // Otherwise open it with this driver. + // If the user really wants to open with this driver, use NETCDF:file.h5 + // format. + + // Check for HDF5 driver availability. if (bCheckExt) { // Check by default. @@ -134,38 +133,35 @@ NetCDFFormatEnum netCDFIdentifyFormat(GDALOpenInfo *poOpenInfo, bool bCheckExt) } } } -#endif return NCDF_FORMAT_NC4; } else if (STARTS_WITH_CI(pszHeader, "\016\003\023\001")) { - // Check for HDF4 support in libnetcdf. #ifdef NETCDF_HAS_HDF4 - // If only the netCDF driver is allowed, immediately recognize the file + // There is HDF4 support in libnetcdf and only the netCDF driver is + // allowed ==> immediately recognize the file. if (poOpenInfo->IsSingleAllowedDriver("netCDF")) { return NCDF_FORMAT_HDF4; } #endif - // Requires HDF4 support in libnetcdf, but if HDF4 is supported by GDAL - // don't try to open. - // If user really wants to open with this driver, use NETCDF:file.hdf - // syntax. + // If there is a HDF4 GDAL driver available, delegate to it. + // Otherwise open it with this driver. + // If the user really wants to open with this driver, use NETCDF:file.hdf + // format. -// Check for HDF4 support in GDAL. -#ifdef HAVE_HDF4 + // Check for HDF4 driver availability. if (bCheckExt && GDALGetDriverByName("HDF4") != nullptr) { // Check by default. // Always treat as HDF4 file. return NCDF_FORMAT_HDF4; } -#endif -// Check for HDF4 support in libnetcdf. #ifdef NETCDF_HAS_HDF4 + // There is HDF4 support in libnetcdf. return NCDF_FORMAT_HDF4; #else return NCDF_FORMAT_NONE; @@ -493,12 +489,7 @@ void netCDFDriverSetCommonMetadata(GDALDriver *poDriver) #ifdef NETCDF_HAS_HDF4 poDriver->SetMetadataItem("NETCDF_HAS_HDF4", "YES"); #endif -#ifdef HAVE_HDF4 - poDriver->SetMetadataItem("GDAL_HAS_HDF4", "YES"); -#endif -#ifdef HAVE_HDF5 - poDriver->SetMetadataItem("GDAL_HAS_HDF5", "YES"); -#endif + poDriver->SetMetadataItem("NETCDF_HAS_NETCDF_MEM", "YES"); #ifdef ENABLE_NCDUMP diff --git a/frmts/nitf/CMakeLists.txt b/frmts/nitf/CMakeLists.txt index 9e4cc0727706..196e4521ae04 100644 --- a/frmts/nitf/CMakeLists.txt +++ b/frmts/nitf/CMakeLists.txt @@ -70,7 +70,7 @@ if (NOT GDAL_USE_TIFF_INTERNAL) gdal_target_link_libraries(gdal_NITF PRIVATE TIFF::TIFF) endif () -add_executable(nitfdump EXCLUDE_FROM_ALL nitfdump.c nitffile.c nitfimage.c rpftocfile.cpp nitfbilevel.cpp nitfaridpcm.cpp mgrs.c) +add_executable(nitfdump EXCLUDE_FROM_ALL nitfdump.c nitffile.c nitfimage.c rpftocfile.cpp nitfbilevel.cpp nitfaridpcm.cpp mgrs.c nitfdes.c) if (GDAL_USE_TIFF_INTERNAL) target_sources(nitfdump PRIVATE $) gdal_add_vendored_lib(nitfdump libtiff) diff --git a/frmts/nitf/nitfbilevel.cpp b/frmts/nitf/nitfbilevel.cpp index 879dbad39582..3370c5eee608 100644 --- a/frmts/nitf/nitfbilevel.cpp +++ b/frmts/nitf/nitfbilevel.cpp @@ -33,7 +33,6 @@ #include #include "cpl_conv.h" -#include "cpl_multiproc.h" #include "cpl_string.h" #include "cpl_vsi.h" #include "gdal.h" @@ -57,10 +56,8 @@ int NITFUncompressBILEVEL(NITFImage *psImage, GByte *pabyInputData, const int nOutputBytes = (psImage->nBlockWidth * psImage->nBlockHeight + 7) / 8; - CPLString osFilename; - - osFilename.Printf("/vsimem/nitf-wrk-%ld.tif", (long)CPLGetPID()); - + const CPLString osFilename( + VSIMemGenerateHiddenFilename("nitf_bilevel.tif")); VSILFILE *fpL = VSIFOpenL(osFilename, "w+"); if (fpL == nullptr) return FALSE; diff --git a/frmts/nitf/nitfdataset.cpp b/frmts/nitf/nitfdataset.cpp index c5835ef47244..c030a1ba82c2 100644 --- a/frmts/nitf/nitfdataset.cpp +++ b/frmts/nitf/nitfdataset.cpp @@ -3366,7 +3366,7 @@ int NITFDataset::CheckForRSets(const char *pszNITFFilename, if (isR0File) { osTarget = pszNITFFilename; - osTarget[osTarget.size() - 1] = static_cast('0' + i); + osTarget.back() = static_cast('0' + i); } else osTarget.Printf("%s.r%d", pszNITFFilename, i); diff --git a/frmts/nitf/nitfdump.c b/frmts/nitf/nitfdump.c index 6d8934490893..8c25e4e08f59 100644 --- a/frmts/nitf/nitfdump.c +++ b/frmts/nitf/nitfdump.c @@ -628,11 +628,12 @@ int main(int nArgc, char **papszArgv) EQUAL(CSLFetchNameValueDef(psDES->papszMetadata, "DESID", ""), "CSSHPA DES")) { - char szFilename[40]; - char szRadix[32]; + char szFilename[512]; + char szRadix[256]; if (bExtractSHPInMem) - snprintf(szRadix, sizeof(szRadix), - "/vsimem/nitf_segment_%d", iSegment + 1); + snprintf(szRadix, sizeof(szRadix), "%s", + VSIMemGenerateHiddenFilename( + CPLSPrintf("nitf_segment_%d", iSegment + 1))); else snprintf(szRadix, sizeof(szRadix), "nitf_segment_%d", iSegment + 1); diff --git a/frmts/nitf/nitfimage.c b/frmts/nitf/nitfimage.c index 185127fc48fd..090edc96b166 100644 --- a/frmts/nitf/nitfimage.c +++ b/frmts/nitf/nitfimage.c @@ -3847,7 +3847,7 @@ static void NITFLoadLocationTable(NITFImage *psImage) GUInt32 nHeaderOffset = 0; int i; int nTRESize; - char szTempFileName[32]; + char szTempFileName[256]; VSILFILE *fpTemp; pszTRE = @@ -3855,7 +3855,8 @@ static void NITFLoadLocationTable(NITFImage *psImage) if (pszTRE == NULL) return; - snprintf(szTempFileName, sizeof(szTempFileName), "/vsimem/%p", pszTRE); + snprintf(szTempFileName, sizeof(szTempFileName), "%s", + VSIMemGenerateHiddenFilename("nitf_tre")); fpTemp = VSIFileFromMemBuffer(szTempFileName, (GByte *)pszTRE, nTRESize, FALSE); psImage->pasLocations = diff --git a/frmts/ogcapi/gdalogcapidataset.cpp b/frmts/ogcapi/gdalogcapidataset.cpp index da3f74d6ac5e..69750ab7526c 100644 --- a/frmts/ogcapi/gdalogcapidataset.cpp +++ b/frmts/ogcapi/gdalogcapidataset.cpp @@ -511,10 +511,15 @@ bool OGCAPIDataset::Download(const CPLString &osURL, const char *pszPostContent, if (psResult->pszErrBuf != nullptr) { - CPLError(CE_Failure, CPLE_AppDefined, "%s", - psResult->pabyData - ? reinterpret_cast(psResult->pabyData) - : psResult->pszErrBuf); + std::string osErrorMsg(psResult->pszErrBuf); + const char *pszData = + reinterpret_cast(psResult->pabyData); + if (pszData) + { + osErrorMsg += ", "; + osErrorMsg.append(pszData, CPLStrnlen(pszData, 1000)); + } + CPLError(CE_Failure, CPLE_AppDefined, "%s", osErrorMsg.c_str()); CPLHTTPDestroyResult(psResult); return false; } @@ -633,8 +638,7 @@ OGCAPIDataset::OpenTile(const CPLString &osURLPattern, int nMatrix, int nColumn, if (bEmptyContent) return nullptr; - CPLString osTempFile; - osTempFile.Printf("/vsimem/ogcapi/%p", this); + const CPLString osTempFile(VSIMemGenerateHiddenFilename("ogcapi")); VSIFCloseL(VSIFileFromMemBuffer(osTempFile.c_str(), reinterpret_cast(&m_osTileData[0]), m_osTileData.size(), false)); diff --git a/frmts/pcraster/pcrasterutil.cpp b/frmts/pcraster/pcrasterutil.cpp index a424278308b2..0228e81ed24f 100644 --- a/frmts/pcraster/pcrasterutil.cpp +++ b/frmts/pcraster/pcrasterutil.cpp @@ -393,8 +393,6 @@ CSF_CR GDALType2CellRepresentation(GDALDataType type, bool exact) /*! \param cellRepresentation Cell representation of the data. \return Missing value. - \exception . - \sa . */ double missingValue(CSF_CR cellRepresentation) { @@ -468,9 +466,6 @@ double missingValue(CSF_CR cellRepresentation) /*! \param filename Filename of raster to open. \return Pointer to CSF MAP structure. - \exception . - \warning . - \sa . */ MAP *mapOpen(std::string const &filename, MOPEN_PERM mode) { diff --git a/frmts/pdf/pdfcreatecopy.cpp b/frmts/pdf/pdfcreatecopy.cpp index 462c2b158bc6..8c72aebf4501 100644 --- a/frmts/pdf/pdfcreatecopy.cpp +++ b/frmts/pdf/pdfcreatecopy.cpp @@ -4311,7 +4311,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( eCompressMethod == COMPRESS_JPEG2000) { GDALDriver *poJPEGDriver = nullptr; - char szTmp[64]; + std::string osTmpfilename; char **papszOptions = nullptr; bool bEcwEncodeKeyRequiredButNotFound = false; @@ -4321,7 +4321,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( if (poJPEGDriver != nullptr && nJPEGQuality > 0) papszOptions = CSLAddString( papszOptions, CPLSPrintf("QUALITY=%d", nJPEGQuality)); - snprintf(szTmp, sizeof(szTmp), "/vsimem/pdftemp/%p.jpg", this); + osTmpfilename = VSIMemGenerateHiddenFilename("pdf_temp.jpg"); } else { @@ -4374,7 +4374,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( papszOptions = CSLAddString(papszOptions, "GMLJP2=OFF"); } } - snprintf(szTmp, sizeof(szTmp), "/vsimem/pdftemp/%p.jp2", this); + osTmpfilename = VSIMemGenerateHiddenFilename("pdf_temp.jp2"); } if (poJPEGDriver == nullptr) @@ -4396,8 +4396,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( } GDALDataset *poJPEGDS = - poJPEGDriver->CreateCopy(szTmp, poBlockSrcDS, FALSE, papszOptions, - pfnProgress, pProgressData); + poJPEGDriver->CreateCopy(osTmpfilename.c_str(), poBlockSrcDS, FALSE, + papszOptions, pfnProgress, pProgressData); CSLDestroy(papszOptions); if (poJPEGDS == nullptr) @@ -4409,7 +4409,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( GDALClose(poJPEGDS); vsi_l_offset nJPEGDataSize = 0; - GByte *pabyJPEGData = VSIGetMemFileBuffer(szTmp, &nJPEGDataSize, TRUE); + GByte *pabyJPEGData = + VSIGetMemFileBuffer(osTmpfilename.c_str(), &nJPEGDataSize, TRUE); VSIFWriteL(pabyJPEGData, static_cast(nJPEGDataSize), 1, m_fp); CPLFree(pabyJPEGData); } diff --git a/frmts/pdf/pdfdataset.cpp b/frmts/pdf/pdfdataset.cpp index cab477ab4da9..845057fd68a9 100644 --- a/frmts/pdf/pdfdataset.cpp +++ b/frmts/pdf/pdfdataset.cpp @@ -2120,7 +2120,7 @@ CPLErr PDFDataset::ReadPixels(int nReqXOff, int nReqYOff, int nReqXSize, } papszArgs = CSLAddString(papszArgs, m_osFilename.c_str()); - osTmpFilename = CPLSPrintf("/vsimem/pdf/temp_%p.ppm", this); + osTmpFilename = VSIMemGenerateHiddenFilename("pdf_temp.ppm"); VSILFILE *fpOut = VSIFOpenL(osTmpFilename, "wb"); if (fpOut != nullptr) { diff --git a/frmts/pds/isis3dataset.cpp b/frmts/pds/isis3dataset.cpp index bacaab84372f..41aa9c2c6808 100644 --- a/frmts/pds/isis3dataset.cpp +++ b/frmts/pds/isis3dataset.cpp @@ -3618,8 +3618,7 @@ void ISIS3Dataset::WriteLabel() CPLString ISIS3Dataset::SerializeAsPDL(const CPLJSONObject &oObj) { - CPLString osTmpFile( - CPLSPrintf("/vsimem/isis3_%p", oObj.GetInternalHandle())); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("isis3_pdl")); VSILFILE *fpTmp = VSIFOpenL(osTmpFile, "wb+"); SerializeAsPDL(fpTmp, oObj); VSIFCloseL(fpTmp); diff --git a/frmts/pds/vicardataset.cpp b/frmts/pds/vicardataset.cpp index 26a51bdf7790..8f022e60f24b 100644 --- a/frmts/pds/vicardataset.cpp +++ b/frmts/pds/vicardataset.cpp @@ -1913,8 +1913,8 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) // Create a in-memory GeoTIFF file - char szFilename[100] = {}; - snprintf(szFilename, sizeof(szFilename), "/vsimem/vicar_tmp_%p.tif", this); + const std::string osTmpFilename( + VSIMemGenerateHiddenFilename("vicar_tmp.tif")); GDALDriver *poGTiffDriver = GDALDriver::FromHandle(GDALGetDriverByName("GTiff")); if (poGTiffDriver == nullptr) @@ -1923,8 +1923,8 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) return; } const char *const apszOptions[] = {"GEOTIFF_VERSION=1.0", nullptr}; - auto poDS = std::unique_ptr( - poGTiffDriver->Create(szFilename, 1, 1, 1, GDT_Byte, apszOptions)); + auto poDS = std::unique_ptr(poGTiffDriver->Create( + osTmpFilename.c_str(), 1, 1, 1, GDT_Byte, apszOptions)); if (!poDS) return; poDS->SetSpatialRef(&m_oSRS); @@ -1935,14 +1935,14 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) poDS.reset(); // Open it with libtiff/libgeotiff - VSILFILE *fpL = VSIFOpenL(szFilename, "r"); + VSILFILE *fpL = VSIFOpenL(osTmpFilename.c_str(), "r"); if (fpL == nullptr) { - VSIUnlink(szFilename); + VSIUnlink(osTmpFilename.c_str()); return; } - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "r", fpL); + TIFF *hTIFF = VSI_TIFFOpen(osTmpFilename.c_str(), "r", fpL); CPLAssert(hTIFF); GTIF *hGTIF = GTIFNew(hTIFF); @@ -2008,7 +2008,7 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) XTIFFClose(hTIFF); CPL_IGNORE_RET_VAL(VSIFCloseL(fpL)); - VSIUnlink(szFilename); + VSIUnlink(osTmpFilename.c_str()); } /************************************************************************/ @@ -2320,8 +2320,8 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() // We will build a in-memory temporary GeoTIFF file from the VICAR GEOTIFF // metadata items. - char szFilename[100] = {}; - snprintf(szFilename, sizeof(szFilename), "/vsimem/vicar_tmp_%p.tif", this); + const std::string osTmpFilename( + VSIMemGenerateHiddenFilename("vicar_tmp.tif")); /* -------------------------------------------------------------------- */ /* Initialization of libtiff and libgeotiff. */ @@ -2332,11 +2332,11 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() /* -------------------------------------------------------------------- */ /* Initialize access to the memory geotiff structure. */ /* -------------------------------------------------------------------- */ - VSILFILE *fpL = VSIFOpenL(szFilename, "w"); + VSILFILE *fpL = VSIFOpenL(osTmpFilename.c_str(), "w"); if (fpL == nullptr) return; - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "w", fpL); + TIFF *hTIFF = VSI_TIFFOpen(osTmpFilename.c_str(), "w", fpL); if (hTIFF == nullptr) { @@ -2451,7 +2451,7 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() /* Get georeferencing from file. */ /* -------------------------------------------------------------------- */ auto poGTiffDS = - std::unique_ptr(GDALDataset::Open(szFilename)); + std::unique_ptr(GDALDataset::Open(osTmpFilename.c_str())); if (poGTiffDS) { auto poSRS = poGTiffDS->GetSpatialRef(); @@ -2469,7 +2469,7 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() GDALDataset::SetMetadataItem(GDALMD_AREA_OR_POINT, pszAreaOrPoint); } - VSIUnlink(szFilename); + VSIUnlink(osTmpFilename.c_str()); } /************************************************************************/ diff --git a/frmts/plmosaic/plmosaicdataset.cpp b/frmts/plmosaic/plmosaicdataset.cpp index fe38f0fc793a..fd759dd1a60c 100644 --- a/frmts/plmosaic/plmosaicdataset.cpp +++ b/frmts/plmosaic/plmosaicdataset.cpp @@ -450,7 +450,7 @@ CPLHTTPResult *PLMosaicDataset::Download(const char *pszURL, int bQuiet404Error) vsi_l_offset nDataLength = 0; CPLString osURL(pszURL); if (osURL.back() == '/') - osURL.resize(osURL.size() - 1); + osURL.pop_back(); GByte *pabyBuf = VSIGetMemFileBuffer(osURL, &nDataLength, FALSE); if (pabyBuf) { @@ -1329,6 +1329,7 @@ GDALDataset *PLMosaicDataset::GetMetaTile(int tile_x, int tile_y) CreateMosaicCachePathIfNecessary(); + bool bUnlink = false; VSILFILE *fp = osCachePathRoot.size() ? VSIFOpenL(osTmpFilename, "wb") : nullptr; if (fp) @@ -1349,9 +1350,10 @@ GDALDataset *PLMosaicDataset::GetMetaTile(int tile_x, int tile_y) FlushDatasetsCache(); nCacheMaxSize = 1; } - osTmpFilename = - CPLSPrintf("/vsimem/single_tile_plmosaic_cache/%s/%d_%d.tif", - osMosaic.c_str(), tile_x, tile_y); + bUnlink = true; + osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("single_tile_plmosaic_cache_%s_%d_%d.tif", + osMosaic.c_str(), tile_x, tile_y)); fp = VSIFOpenL(osTmpFilename, "wb"); if (fp) { @@ -1362,7 +1364,7 @@ GDALDataset *PLMosaicDataset::GetMetaTile(int tile_x, int tile_y) CPLHTTPDestroyResult(psResult); GDALDataset *poDS = OpenAndInsertNewDataset(osTmpFilename, osTilename); - if (STARTS_WITH(osTmpFilename, "/vsimem/single_tile_plmosaic_cache/")) + if (bUnlink) VSIUnlink(osTilename); return poDS; diff --git a/frmts/rasterlite/rasterlitecreatecopy.cpp b/frmts/rasterlite/rasterlitecreatecopy.cpp index fba77104823c..9de3a39f5e0d 100644 --- a/frmts/rasterlite/rasterlitecreatecopy.cpp +++ b/frmts/rasterlite/rasterlitecreatecopy.cpp @@ -581,8 +581,8 @@ GDALDataset *RasterliteCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, return nullptr; } - CPLString osTempFileName; - osTempFileName.Printf("/vsimem/%p", hDS); + const CPLString osTempFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); int nTileId = 0; int nBlocks = 0; diff --git a/frmts/rasterlite/rasterlitedataset.cpp b/frmts/rasterlite/rasterlitedataset.cpp index a85cf6fafa4a..8f12d77d64fd 100644 --- a/frmts/rasterlite/rasterlitedataset.cpp +++ b/frmts/rasterlite/rasterlitedataset.cpp @@ -156,8 +156,8 @@ CPLErr RasterliteBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) return CE_None; } - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); #ifdef RASTERLITE_DEBUG if (nBand == 1) @@ -912,8 +912,8 @@ int RasterliteDataset::GetBlockParams(OGRLayerH hRasterLyr, int nLevelIn, return FALSE; } - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); VSILFILE *fp = VSIFileFromMemBuffer(osMemFileName.c_str(), pabyData, nDataSize, FALSE); VSIFCloseL(fp); diff --git a/frmts/rasterlite/rasterliteoverviews.cpp b/frmts/rasterlite/rasterliteoverviews.cpp index bc6599d9fb0c..2a0c9cc3c8f1 100644 --- a/frmts/rasterlite/rasterliteoverviews.cpp +++ b/frmts/rasterlite/rasterliteoverviews.cpp @@ -350,8 +350,8 @@ CPLErr RasterliteDataset::CreateOverviewLevel(const char *pszResampling, return CE_Failure; } - CPLString osTempFileName; - osTempFileName.Printf("/vsimem/%p", hDS); + const CPLString osTempFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); int nTileId = 0; int nBlocks = 0; diff --git a/frmts/raw/envidataset.cpp b/frmts/raw/envidataset.cpp index 4dba3e5bc89f..13f732150404 100644 --- a/frmts/raw/envidataset.cpp +++ b/frmts/raw/envidataset.cpp @@ -2401,9 +2401,12 @@ ENVIDataset *ENVIDataset::Open(GDALOpenInfo *poOpenInfo, bool bFileSizeCheck) { char **papszBandNames = poDS->SplitList(pszBandNames); char **papszWL = poDS->SplitList(pszWaveLength); + const char *pszFWHM = poDS->m_aosHeader["fwhm"]; + char **papszFWHM = pszFWHM ? poDS->SplitList(pszFWHM) : nullptr; const char *pszWLUnits = nullptr; const int nWLCount = CSLCount(papszWL); + const int nFWHMCount = CSLCount(papszFWHM); if (papszWL) { // If WL information is present, process wavelength units. @@ -2460,6 +2463,29 @@ ENVIDataset *ENVIDataset::Open(GDALOpenInfo *poOpenInfo, bool bFileSizeCheck) CPLString osBandId = CPLSPrintf("Band_%i", i + 1); poDS->SetMetadataItem(osBandId, osBandName.c_str()); + const auto ConvertWaveLength = + [pszWLUnits](double dfVal) -> const char * + { + if (EQUAL(pszWLUnits, "Micrometers") || EQUAL(pszWLUnits, "um")) + { + return CPLSPrintf("%.3f", dfVal); + } + else if (EQUAL(pszWLUnits, "Nanometers") || + EQUAL(pszWLUnits, "nm")) + { + return CPLSPrintf("%.3f", dfVal / 1000); + } + else if (EQUAL(pszWLUnits, "Millimeters") || + EQUAL(pszWLUnits, "mm")) + { + return CPLSPrintf("%.3f", dfVal * 1000); + } + else + { + return nullptr; + } + }; + // Set wavelength metadata to band. if (papszWL && nWLCount > i) { @@ -2470,11 +2496,29 @@ ENVIDataset *ENVIDataset::Open(GDALOpenInfo *poOpenInfo, bool bFileSizeCheck) { poDS->GetRasterBand(i + 1)->SetMetadataItem( "wavelength_units", pszWLUnits); + + if (const char *pszVal = + ConvertWaveLength(CPLAtof(papszWL[i]))) + { + poDS->GetRasterBand(i + 1)->SetMetadataItem( + "CENTRAL_WAVELENGTH_UM", pszVal, "IMAGERY"); + } + } + } + + if (papszFWHM && nFWHMCount > i && pszWLUnits) + { + if (const char *pszVal = + ConvertWaveLength(CPLAtof(papszFWHM[i]))) + { + poDS->GetRasterBand(i + 1)->SetMetadataItem( + "FWHM_UM", pszVal, "IMAGERY"); } } } CSLDestroy(papszWL); CSLDestroy(papszBandNames); + CSLDestroy(papszFWHM); } // Apply "default bands" if we have it to set RGB color interpretation. diff --git a/frmts/rmf/rmfjpeg.cpp b/frmts/rmf/rmfjpeg.cpp index 8c4b7fd78f73..b7dc56abbbcc 100644 --- a/frmts/rmf/rmfjpeg.cpp +++ b/frmts/rmf/rmfjpeg.cpp @@ -31,6 +31,7 @@ #include #include "cpl_conv.h" +#include "cpl_vsi.h" #include "rmfdataset.h" #include "../mem/memdataset.h" @@ -46,13 +47,10 @@ size_t RMFDataset::JPEGDecompress(const GByte *pabyIn, GUInt32 nSizeIn, nSizeIn < 2) return 0; - CPLString osTmpFilename; - VSILFILE *fp; + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("rmfjpeg.jpg")); - osTmpFilename.Printf("/vsimem/rmfjpeg/%p.jpg", pabyIn); - - fp = VSIFileFromMemBuffer(osTmpFilename, const_cast(pabyIn), - nSizeIn, FALSE); + VSILFILE *fp = VSIFileFromMemBuffer( + osTmpFilename, const_cast(pabyIn), nSizeIn, FALSE); if (fp == nullptr) { @@ -165,8 +163,7 @@ size_t RMFDataset::JPEGCompress(const GByte *pabyIn, GUInt32 nSizeIn, poMemDS->AddMEMBand(hBand); } - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/rmfjpeg/%p.jpg", pabyIn); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("rmfjpeg.jpg")); char szQuality[32] = {}; if (poDS != nullptr && poDS->sHeader.iJpegQuality > 0) diff --git a/frmts/sentinel2/sentinel2dataset.cpp b/frmts/sentinel2/sentinel2dataset.cpp index 2de45ee872f2..6be2744bbb4a 100644 --- a/frmts/sentinel2/sentinel2dataset.cpp +++ b/frmts/sentinel2/sentinel2dataset.cpp @@ -78,15 +78,23 @@ typedef struct GDALColorInterp eColorInterp; } SENTINEL2BandDescription; +/* clang-format off */ static const SENTINEL2BandDescription asBandDesc[] = { - {"B1", 60, 443, 20, GCI_Undefined}, {"B2", 10, 490, 65, GCI_BlueBand}, - {"B3", 10, 560, 35, GCI_GreenBand}, {"B4", 10, 665, 30, GCI_RedBand}, - {"B5", 20, 705, 15, GCI_Undefined}, {"B6", 20, 740, 15, GCI_Undefined}, - {"B7", 20, 783, 20, GCI_Undefined}, {"B8", 10, 842, 115, GCI_Undefined}, - {"B8A", 20, 865, 20, GCI_Undefined}, {"B9", 60, 945, 20, GCI_Undefined}, - {"B10", 60, 1375, 30, GCI_Undefined}, {"B11", 20, 1610, 90, GCI_Undefined}, - {"B12", 20, 2190, 180, GCI_Undefined}, + {"B1", 60, 443, 20, GCI_CoastalBand}, + {"B2", 10, 490, 65, GCI_BlueBand}, + {"B3", 10, 560, 35, GCI_GreenBand}, + {"B4", 10, 665, 30, GCI_RedBand}, + {"B5", 20, 705, 15, GCI_RedEdgeBand}, // rededge071 + {"B6", 20, 740, 15, GCI_RedEdgeBand}, // rededge075 + {"B7", 20, 783, 20, GCI_RedEdgeBand}, // rededge078 + {"B8", 10, 842, 115, GCI_NIRBand}, // nir + {"B8A", 20, 865, 20, GCI_NIRBand}, // nir08 + {"B9", 60, 945, 20, GCI_NIRBand}, // nir09 + {"B10", 60, 1375, 30, GCI_OtherIRBand}, // cirrus + {"B11", 20, 1610, 90, GCI_SWIRBand}, // swir16 + {"B12", 20, 2190, 180, GCI_SWIRBand}, // swir11 }; +/* clang-format on */ #define NB_BANDS (sizeof(asBandDesc) / sizeof(asBandDesc[0])) @@ -1971,6 +1979,15 @@ static void SENTINEL2SetBandMetadata(GDALRasterBand *poBand, poBand->SetMetadataItem("WAVELENGTH", CPLSPrintf("%d", psBandDesc->nWaveLength)); poBand->SetMetadataItem("WAVELENGTH_UNIT", "nm"); + + poBand->SetMetadataItem( + "CENTRAL_WAVELENGTH_UM", + CPLSPrintf("%.3f", double(psBandDesc->nWaveLength) / 1000), + "IMAGERY"); + poBand->SetMetadataItem( + "FWHM_UM", + CPLSPrintf("%.3f", double(psBandDesc->nBandWidth) / 1000), + "IMAGERY"); } else { diff --git a/frmts/stacit/stacitdataset.cpp b/frmts/stacit/stacitdataset.cpp index 3522b70108cf..32a092e40934 100644 --- a/frmts/stacit/stacitdataset.cpp +++ b/frmts/stacit/stacitdataset.cpp @@ -178,7 +178,7 @@ static std::string SanitizeCRSValue(const std::string &v) } } if (!ret.empty() && ret.back() == '_') - ret.resize(ret.size() - 1); + ret.pop_back(); return ret; } @@ -602,17 +602,13 @@ bool STACITDataset::SetupDataset( if (!osBandName.empty()) poVRTBand->SetDescription(osBandName.c_str()); - if (eInterp != GCI_Undefined) + const auto osCommonName = eoBand["common_name"].ToString(); + if (!osCommonName.empty()) { - const auto osCommonName = eoBand["common_name"].ToString(); - if (osCommonName == "red") - poVRTBand->SetColorInterpretation(GCI_RedBand); - else if (osCommonName == "green") - poVRTBand->SetColorInterpretation(GCI_GreenBand); - else if (osCommonName == "blue") - poVRTBand->SetColorInterpretation(GCI_BlueBand); - else if (osCommonName == "alpha") - poVRTBand->SetColorInterpretation(GCI_AlphaBand); + const auto eInterpFromCommonName = + GDALGetColorInterpFromSTACCommonName(osCommonName.c_str()); + if (eInterpFromCommonName != GCI_Undefined) + poVRTBand->SetColorInterpretation(eInterpFromCommonName); } for (const auto &eoBandChild : eoBand.GetChildren()) @@ -852,11 +848,19 @@ bool STACITDataset::Open(GDALOpenInfo *poOpenInfo) return false; } const auto oRoot = oDoc.GetRoot(); - const auto oFeatures = oRoot.GetArray("features"); + auto oFeatures = oRoot.GetArray("features"); if (!oFeatures.IsValid()) { - CPLError(CE_Failure, CPLE_AppDefined, "Missing features"); - return false; + if (oRoot.GetString("type") == "Feature") + { + oFeatures = CPLJSONArray(); + oFeatures.Add(oRoot); + } + else + { + CPLError(CE_Failure, CPLE_AppDefined, "Missing features"); + return false; + } } for (const auto &oFeature : oFeatures) { diff --git a/frmts/stacta/stactadataset.cpp b/frmts/stacta/stactadataset.cpp index 55c4cdfc47e9..0603efb561a3 100644 --- a/frmts/stacta/stactadataset.cpp +++ b/frmts/stacta/stactadataset.cpp @@ -489,8 +489,13 @@ CPLErr STACTARawDataset::IRasterIO( return CE_Failure; } VSIFCloseL(fp); - const CPLString osMEMFilename("/vsimem/stacta/" + - osURL); + const CPLString osMEMFilename( + VSIMemGenerateHiddenFilename( + std::string("stacta_") + .append(CPLString(osURL) + .replaceAll("/", "_") + .replaceAll("\\", "_")) + .c_str())); VSIFCloseL(VSIFileFromMemBuffer(osMEMFilename, pabyBuf, nSize, TRUE)); poTileDS = std::unique_ptr( diff --git a/frmts/tga/tgadataset.cpp b/frmts/tga/tgadataset.cpp index cda777088682..d8b4c1b2e9f8 100644 --- a/frmts/tga/tgadataset.cpp +++ b/frmts/tga/tgadataset.cpp @@ -622,7 +622,7 @@ GDALDataset *GDALTGADataset::Open(GDALOpenInfo *poOpenInfo) while (!osAuthorName.empty() && osAuthorName.back() == ' ') { - osAuthorName.resize(osAuthorName.size() - 1); + osAuthorName.pop_back(); } poDS->GDALDataset::SetMetadataItem( "AUTHOR_NAME", osAuthorName.c_str()); @@ -645,7 +645,7 @@ GDALDataset *GDALTGADataset::Open(GDALOpenInfo *poOpenInfo) osLine.resize(strlen(osLine.c_str())); while (!osLine.empty() && osLine.back() == ' ') { - osLine.resize(osLine.size() - 1); + osLine.pop_back(); } if (i > 0) osComments += '\n'; diff --git a/frmts/tiledb/tiledbcommon.cpp b/frmts/tiledb/tiledbcommon.cpp index 120276dcc08d..2c297f139942 100644 --- a/frmts/tiledb/tiledbcommon.cpp +++ b/frmts/tiledb/tiledbcommon.cpp @@ -118,54 +118,24 @@ CPLErr TileDBDataset::AddFilter(tiledb::Context &ctx, int TileDBDataset::Identify(GDALOpenInfo *poOpenInfo) { - if (STARTS_WITH_CI(poOpenInfo->pszFilename, "TILEDB:")) + int nRet = TileDBDriverIdentifySimplified(poOpenInfo); + if (nRet == GDAL_IDENTIFY_UNKNOWN) { - return TRUE; - } - - if (poOpenInfo->IsSingleAllowedDriver("TileDB")) - { - return TRUE; - } - - try - { - const char *pszConfig = - CSLFetchNameValue(poOpenInfo->papszOpenOptions, "TILEDB_CONFIG"); - - if (pszConfig != nullptr) - { - return TRUE; - } - - const bool bIsS3OrGS = - STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || - STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/"); - // If this is a /vsi virtual file systems, bail out, except if it is S3 or GS. - if (!bIsS3OrGS && STARTS_WITH(poOpenInfo->pszFilename, "/vsi")) - { - return false; - } - - if (poOpenInfo->bIsDirectory || - (bIsS3OrGS && - !EQUAL(CPLGetExtension(poOpenInfo->pszFilename), "tif"))) + try { tiledb::Context ctx; CPLString osArrayPath = TileDBDataset::VSI_to_tiledb_uri(poOpenInfo->pszFilename); const auto eType = tiledb::Object::object(ctx, osArrayPath).type(); - if (eType == tiledb::Object::Type::Array || - eType == tiledb::Object::Type::Group) - return true; + nRet = (eType == tiledb::Object::Type::Array || + eType == tiledb::Object::Type::Group); + } + catch (...) + { + nRet = FALSE; } - - return FALSE; - } - catch (...) - { - return FALSE; } + return nRet; } /************************************************************************/ diff --git a/frmts/tiledb/tiledbdrivercore.cpp b/frmts/tiledb/tiledbdrivercore.cpp index 4737945ff783..f3f5917a866f 100644 --- a/frmts/tiledb/tiledbdrivercore.cpp +++ b/frmts/tiledb/tiledbdrivercore.cpp @@ -35,7 +35,7 @@ /* TileDBDriverIdentifySimplified() */ /************************************************************************/ -static int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) +int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) { if (STARTS_WITH_CI(poOpenInfo->pszFilename, "TILEDB:")) @@ -56,6 +56,11 @@ static int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) return TRUE; } + if (!poOpenInfo->bIsDirectory) + { + return false; + } + const bool bIsS3OrGS = STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/"); // If this is a /vsi virtual file systems, bail out, except if it is S3 or GS. @@ -64,17 +69,7 @@ static int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) return false; } - if (poOpenInfo->bIsDirectory) - { - return GDAL_IDENTIFY_UNKNOWN; - } - - if (bIsS3OrGS && !EQUAL(CPLGetExtension(poOpenInfo->pszFilename), "tif")) - { - return GDAL_IDENTIFY_UNKNOWN; - } - - return FALSE; + return GDAL_IDENTIFY_UNKNOWN; } /************************************************************************/ diff --git a/frmts/tiledb/tiledbdrivercore.h b/frmts/tiledb/tiledbdrivercore.h index 26e0e16aea83..632d945bee80 100644 --- a/frmts/tiledb/tiledbdrivercore.h +++ b/frmts/tiledb/tiledbdrivercore.h @@ -36,6 +36,8 @@ constexpr const char *DRIVER_NAME = "TileDB"; #define TileDBDriverSetCommonMetadata \ PLUGIN_SYMBOL_NAME(TileDBDriverSetCommonMetadata) +int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo); + void TileDBDriverSetCommonMetadata(GDALDriver *poDriver); #endif diff --git a/frmts/vrt/CMakeLists.txt b/frmts/vrt/CMakeLists.txt index 447d48eaedbe..777c49af525a 100644 --- a/frmts/vrt/CMakeLists.txt +++ b/frmts/vrt/CMakeLists.txt @@ -20,7 +20,8 @@ add_gdal_driver( gdaltileindexdataset.cpp STRONG_CXX_WFLAGS) gdal_standard_includes(gdal_vrt) -target_include_directories(gdal_vrt PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/raw) +target_include_directories(gdal_vrt PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/raw + $) set(GDAL_DATA_FILES ${CMAKE_CURRENT_SOURCE_DIR}/data/gdalvrt.xsd diff --git a/frmts/vrt/data/gdalvrt.xsd b/frmts/vrt/data/gdalvrt.xsd index 9f9a91d7be63..c2654337d99d 100644 --- a/frmts/vrt/data/gdalvrt.xsd +++ b/frmts/vrt/data/gdalvrt.xsd @@ -442,6 +442,23 @@ + + + + + + + + + + + + + + + + + diff --git a/frmts/vrt/gdaltileindexdataset.cpp b/frmts/vrt/gdaltileindexdataset.cpp index ce2b195aceae..388f45136164 100644 --- a/frmts/vrt/gdaltileindexdataset.cpp +++ b/frmts/vrt/gdaltileindexdataset.cpp @@ -41,12 +41,14 @@ #include "cpl_port.h" #include "cpl_error_internal.h" +#include "cpl_json.h" #include "cpl_mem_cache.h" #include "cpl_minixml.h" #include "cpl_quad_tree.h" #include "vrtdataset.h" #include "vrt_priv.h" #include "ogrsf_frmts.h" +#include "ogrwarpedlayer.h" #include "gdal_proxy.h" #include "gdal_thread_pool.h" #include "gdal_utils.h" @@ -216,6 +218,9 @@ class GDALTileIndexDataset final : public GDALPamDataset //! Vector layer with the sources OGRLayer *m_poLayer = nullptr; + //! When the SRS of m_poLayer is not the one we expose + std::unique_ptr m_poWarpedLayerKeeper{}; + //! Geotransform matrix of the tile index std::array m_adfGeoTransform{0, 0, 0, 0, 0, 0}; @@ -584,6 +589,9 @@ GDALTileIndexDataset::GDALTileIndexDataset() static std::string GetAbsoluteFileName(const char *pszTileName, const char *pszVRTName) { + if (STARTS_WITH(pszTileName, "https://")) + return std::string("/vsicurl/").append(pszTileName); + if (CPLIsFilenameRelative(pszTileName) && !STARTS_WITH(pszTileName, "GetLayerDefn(); + + // Is this a https://stac-utils.github.io/stac-geoparquet/latest/spec/stac-geoparquet-spec ? + const bool bIsStacGeoParquet = + poLayerDefn->GetFieldIndex("assets.image.href") >= 0; + const char *pszLocationFieldName = GetOption(MD_LOCATION_FIELD); if (!pszLocationFieldName) { - constexpr const char *DEFAULT_LOCATION_FIELD_NAME = "location"; - pszLocationFieldName = DEFAULT_LOCATION_FIELD_NAME; + if (bIsStacGeoParquet) + { + pszLocationFieldName = "assets.image.href"; + } + else + { + constexpr const char *DEFAULT_LOCATION_FIELD_NAME = "location"; + pszLocationFieldName = DEFAULT_LOCATION_FIELD_NAME; + } } - auto poLayerDefn = m_poLayer->GetLayerDefn(); + m_nLocationFieldIndex = poLayerDefn->GetFieldIndex(pszLocationFieldName); if (m_nLocationFieldIndex < 0) { @@ -980,8 +1001,8 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) const char *pszMinY = GetOption(MD_MINY); const char *pszMaxX = GetOption(MD_MAXX); const char *pszMaxY = GetOption(MD_MAXY); - const int nCountMinMaxXY = (pszMinX ? 1 : 0) + (pszMinY ? 1 : 0) + - (pszMaxX ? 1 : 0) + (pszMaxY ? 1 : 0); + int nCountMinMaxXY = (pszMinX ? 1 : 0) + (pszMinY ? 1 : 0) + + (pszMaxX ? 1 : 0) + (pszMaxY ? 1 : 0); if (nCountMinMaxXY != 0 && nCountMinMaxXY != 4) { CPLError(CE_Failure, CPLE_AppDefined, @@ -1023,11 +1044,10 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) return false; } } - else + else if (const auto poSRS = m_poLayer->GetSpatialRef()) { - const auto poSRS = m_poLayer->GetSpatialRef(); // Ignore GPKG "Undefined geographic SRS" and "Undefined Cartesian SRS" - if (poSRS && !STARTS_WITH(poSRS->GetName(), "Undefined ")) + if (!STARTS_WITH(poSRS->GetName(), "Undefined ")) m_oSRS = *poSRS; } @@ -1087,6 +1107,108 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) } } + // Take into STAC GeoParquet proj:epsg and proj:transform fields + std::unique_ptr poFeature; + std::string osResX, osResY, osMinX, osMinY, osMaxX, osMaxY; + int iProjEPSG = -1; + int iProjTransform = -1; + if (bIsStacGeoParquet && !pszSRS && !pszResX && !pszResY && !pszMinX && + !pszMinY && !pszMaxX && !pszMaxY && + (iProjEPSG = poLayerDefn->GetFieldIndex("proj:epsg")) >= 0 && + (iProjTransform = poLayerDefn->GetFieldIndex("proj:transform")) >= 0) + { + poFeature.reset(m_poLayer->GetNextFeature()); + if (poFeature && poFeature->IsFieldSet(iProjEPSG) && + poFeature->IsFieldSet(iProjTransform)) + { + const int nEPSGCode = poFeature->GetFieldAsInteger(iProjEPSG); + OGRSpatialReference oSTACSRS; + oSTACSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (nEPSGCode > 0 && + oSTACSRS.importFromEPSG(nEPSGCode) == OGRERR_NONE) + { + int nTransformCount = 0; + const auto padfFeatureTransform = + poFeature->GetFieldAsDoubleList(iProjTransform, + &nTransformCount); + OGREnvelope sEnvelope; + if (nTransformCount >= 6 && m_poLayer->GetSpatialRef() && + m_poLayer->GetExtent(&sEnvelope, /* bForce = */ true) == + OGRERR_NONE) + { + const double dfResX = padfFeatureTransform[0]; + osResX = CPLSPrintf("%.17g", dfResX); + const double dfResY = std::fabs(padfFeatureTransform[4]); + osResY = CPLSPrintf("%.17g", dfResY); + + auto poCT = std::unique_ptr( + OGRCreateCoordinateTransformation( + m_poLayer->GetSpatialRef(), &oSTACSRS)); + auto poInvCT = std::unique_ptr( + poCT ? poCT->GetInverse() : nullptr); + double dfOutMinX = 0; + double dfOutMinY = 0; + double dfOutMaxX = 0; + double dfOutMaxY = 0; + if (dfResX > 0 && dfResY > 0 && poCT && poInvCT && + poCT->TransformBounds(sEnvelope.MinX, sEnvelope.MinY, + sEnvelope.MaxX, sEnvelope.MaxY, + &dfOutMinX, &dfOutMinY, + &dfOutMaxX, &dfOutMaxY, 21)) + { + constexpr double EPSILON = 1e-3; + const bool bTileAlignedOnRes = + (fmod(std::fabs(padfFeatureTransform[3]), dfResX) <= + EPSILON * dfResX && + fmod(std::fabs(padfFeatureTransform[5]), dfResY) <= + EPSILON * dfResY); + + osMinX = CPLSPrintf( + "%.17g", + !bTileAlignedOnRes + ? dfOutMinX + : std::floor(dfOutMinX / dfResX) * dfResX); + osMinY = CPLSPrintf( + "%.17g", + !bTileAlignedOnRes + ? dfOutMinY + : std::floor(dfOutMinY / dfResY) * dfResY); + osMaxX = CPLSPrintf( + "%.17g", + !bTileAlignedOnRes + ? dfOutMaxX + : std::ceil(dfOutMaxX / dfResX) * dfResX); + osMaxY = CPLSPrintf( + "%.17g", + !bTileAlignedOnRes + ? dfOutMaxY + : std::ceil(dfOutMaxY / dfResY) * dfResY); + + m_oSRS = std::move(oSTACSRS); + pszResX = osResX.c_str(); + pszResY = osResY.c_str(); + pszMinX = osMinX.c_str(); + pszMinY = osMinY.c_str(); + pszMaxX = osMaxX.c_str(); + pszMaxY = osMaxY.c_str(); + nCountMinMaxXY = 4; + + poFeature.reset(); + m_poLayer->ResetReading(); + + m_poWarpedLayerKeeper = + std::make_unique( + m_poLayer, /* iGeomField = */ 0, + /* bTakeOwnership = */ false, poCT.release(), + poInvCT.release()); + m_poLayer = m_poWarpedLayerKeeper.get(); + poLayerDefn = m_poLayer->GetLayerDefn(); + } + } + } + } + } + bool bHasMaskBand = false; if ((!pszBandCount && apoXMLNodeBands.empty()) || (!(pszResX && pszResY) && nCountXSizeYSizeGT == 0)) @@ -1094,8 +1216,8 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) CPLDebug("VRT", "Inspecting one feature due to missing metadata items"); m_bScannedOneFeatureAtOpening = true; - auto poFeature = - std::unique_ptr(m_poLayer->GetNextFeature()); + if (!poFeature) + poFeature.reset(m_poLayer->GetNextFeature()); if (!poFeature || !poFeature->IsFieldSetAndNotNull(m_nLocationFieldIndex)) { @@ -1619,6 +1741,66 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) nRasterYSize = static_cast(std::ceil(nRasterYSize / dfOvrFactor)); } + std::vector aosDescriptions; + if (bIsStacGeoParquet && poFeature) + { + const int nEOBandsIdx = + poLayerDefn->GetFieldIndex("assets.image.eo:bands"); + if (nEOBandsIdx >= 0 && + poLayerDefn->GetFieldDefn(nEOBandsIdx)->GetSubType() == OFSTJSON && + poFeature->IsFieldSet(nEOBandsIdx)) + { + const char *pszStr = poFeature->GetFieldAsString(nEOBandsIdx); + CPLJSONDocument oDoc; + if (oDoc.LoadMemory(pszStr) && + oDoc.GetRoot().GetType() == CPLJSONObject::Type::Array) + { + const auto oArray = oDoc.GetRoot().ToArray(); + if (oArray.Size() == nBandCount) + { + int i = 0; + aosDescriptions.resize(nBandCount); + static const std::map + oMapCommonName = { + {"red", GCI_RedBand}, + {"green", GCI_GreenBand}, + {"blue", GCI_BlueBand}, + {"alpha", GCI_AlphaBand}, + }; + for (const auto &oObj : oArray) + { + if (oObj.GetType() == CPLJSONObject::Type::Object) + { + const auto osCommonName = + oObj.GetString("common_name"); + const auto oIter = + oMapCommonName.find(osCommonName); + if (oIter != oMapCommonName.end()) + aeColorInterp[i] = oIter->second; + + const auto osName = oObj.GetString("name"); + aosDescriptions[i] = osName; + + const auto osDescription = + oObj.GetString("description"); + if (!osDescription.empty()) + { + if (aosDescriptions[i].empty()) + aosDescriptions[i] = osDescription; + else + aosDescriptions[i] + .append(" (") + .append(osDescription) + .append(")"); + } + } + ++i; + } + } + } + } + } + GDALTileIndexBand *poFirstBand = nullptr; for (int i = 0; i < nBandCount; ++i) { @@ -1645,6 +1827,10 @@ bool GDALTileIndexDataset::Open(GDALOpenInfo *poOpenInfo) m_bSameDataType = false; } + if (!aosDescriptions.empty() && !aosDescriptions[i].empty()) + { + poBand->GDALRasterBand::SetDescription(aosDescriptions[i].c_str()); + } if (!apoXMLNodeBands.empty()) { const char *pszVal = CPLGetXMLValue( diff --git a/frmts/vrt/vrtdataset.cpp b/frmts/vrt/vrtdataset.cpp index afa44b9b6eb9..5f9c37c333b1 100644 --- a/frmts/vrt/vrtdataset.cpp +++ b/frmts/vrt/vrtdataset.cpp @@ -303,7 +303,7 @@ CPLXMLNode *VRTDataset::SerializeToXML(const char *pszVRTPathIn) if (osCoordinateEpoch.find('.') != std::string::npos) { while (osCoordinateEpoch.back() == '0') - osCoordinateEpoch.resize(osCoordinateEpoch.size() - 1); + osCoordinateEpoch.pop_back(); } CPLAddXMLAttributeAndValue(psSRSNode, "coordinateEpoch", osCoordinateEpoch.c_str()); @@ -2795,7 +2795,7 @@ bool VRTDataset::AddVirtualOverview(int nOvFactor, const char *pszResampling) GDALDatasetH hOverviewDS = GDALTranslate("", GDALDataset::ToHandle(this), psOptions, nullptr); m_bCanTakeRef = true; - m_apoOverviews.resize(m_apoOverviews.size() - 1); + m_apoOverviews.pop_back(); GDALTranslateOptionsFree(psOptions); if (hOverviewDS == nullptr) diff --git a/frmts/vrt/vrtderivedrasterband.cpp b/frmts/vrt/vrtderivedrasterband.cpp index 4db83d4bc31e..9850cb27d669 100644 --- a/frmts/vrt/vrtderivedrasterband.cpp +++ b/frmts/vrt/vrtderivedrasterband.cpp @@ -720,7 +720,7 @@ bool VRTDerivedRasterBand::InitializePython() CPLString osException = GetPyExceptionString(); if (!osException.empty() && osException.back() == '\n') { - osException.resize(osException.size() - 1); + osException.pop_back(); } if (osException.find("ModuleNotFoundError") == 0) { diff --git a/frmts/vrt/vrtmultidim.cpp b/frmts/vrt/vrtmultidim.cpp index 214485ba67b2..c100cb64a0a5 100644 --- a/frmts/vrt/vrtmultidim.cpp +++ b/frmts/vrt/vrtmultidim.cpp @@ -407,7 +407,7 @@ VRTGroup::GetDimensionFromFullName(const std::string &name, return nullptr; } } - auto poDim(curGroup->GetDimension(aosTokens[aosTokens.size() - 1])); + auto poDim(curGroup->GetDimension(aosTokens.back())); if (!poDim) { if (bEmitError) diff --git a/frmts/vrt/vrtwarped.cpp b/frmts/vrt/vrtwarped.cpp index b96cd5071183..ec39b7f1263c 100644 --- a/frmts/vrt/vrtwarped.cpp +++ b/frmts/vrt/vrtwarped.cpp @@ -180,6 +180,9 @@ GDALDatasetH CPL_STDCALL GDALAutoCreateWarpedVRTEx( if (psWO->padfSrcNoDataReal == nullptr && psWO->padfDstNoDataReal == nullptr && psWO->nSrcAlphaBand == 0) { + // If none of the provided input nodata values can be represented in the + // data type of the corresponding source band, ignore them. + int nCountInvalidSrcNoDataReal = 0; for (int i = 0; i < psWO->nBandCount; i++) { GDALRasterBandH rasterBand = @@ -189,22 +192,42 @@ GDALDatasetH CPL_STDCALL GDALAutoCreateWarpedVRTEx( double noDataValue = GDALGetRasterNoDataValue(rasterBand, &hasNoDataValue); - if (hasNoDataValue) + if (hasNoDataValue && + !GDALIsValueExactAs(noDataValue, + GDALGetRasterDataType(rasterBand))) { - // Check if the nodata value is out of range - int bClamped = FALSE; - int bRounded = FALSE; - CPL_IGNORE_RET_VAL(GDALAdjustValueToDataType( - GDALGetRasterDataType(rasterBand), noDataValue, &bClamped, - &bRounded)); - if (!bClamped) + nCountInvalidSrcNoDataReal++; + } + } + + if (nCountInvalidSrcNoDataReal != psWO->nBandCount) + { + for (int i = 0; i < psWO->nBandCount; i++) + { + GDALRasterBandH rasterBand = + GDALGetRasterBand(psWO->hSrcDS, psWO->panSrcBands[i]); + + int hasNoDataValue; + double noDataValue = + GDALGetRasterNoDataValue(rasterBand, &hasNoDataValue); + + if (hasNoDataValue) { - GDALWarpInitNoDataReal(psWO, -1e10); - if (psWO->padfSrcNoDataReal != nullptr && - psWO->padfDstNoDataReal != nullptr) + // Check if the nodata value is out of range + int bClamped = FALSE; + int bRounded = FALSE; + CPL_IGNORE_RET_VAL(GDALAdjustValueToDataType( + GDALGetRasterDataType(rasterBand), noDataValue, + &bClamped, &bRounded)); + if (!bClamped) { - psWO->padfSrcNoDataReal[i] = noDataValue; - psWO->padfDstNoDataReal[i] = noDataValue; + GDALWarpInitNoDataReal(psWO, -1e10); + if (psWO->padfSrcNoDataReal != nullptr && + psWO->padfDstNoDataReal != nullptr) + { + psWO->padfSrcNoDataReal[i] = noDataValue; + psWO->padfDstNoDataReal[i] = noDataValue; + } } } } diff --git a/frmts/wcs/README.md b/frmts/wcs/README.md new file mode 100644 index 000000000000..2dd8345c6bc1 --- /dev/null +++ b/frmts/wcs/README.md @@ -0,0 +1 @@ +WCS Basics and GDAL: http://web.archive.org/web/20240413133419/https://trac.osgeo.org/gdal/wiki/WCS%2Binteroperability diff --git a/frmts/wcs/wcsdataset.cpp b/frmts/wcs/wcsdataset.cpp index 009b035fcfd5..c37e62374a51 100644 --- a/frmts/wcs/wcsdataset.cpp +++ b/frmts/wcs/wcsdataset.cpp @@ -720,7 +720,7 @@ GDALDataset *WCSDataset::GDALOpenResult(CPLHTTPResult *psResult) #endif // Eventually we should be looking at mime info and stuff to figure // out an optimal filename, but for now we just use a fixed one. - osResultFilename = CPLString().Printf("/vsimem/wcs/%p/wcsresult.dat", this); + osResultFilename = VSIMemGenerateHiddenFilename("wcsresult.dat"); VSILFILE *fp = VSIFileFromMemBuffer(osResultFilename.c_str(), pabyData, nDataLen, FALSE); diff --git a/frmts/wms/minidriver_mrf.cpp b/frmts/wms/minidriver_mrf.cpp index 6200e9478f01..f66e4e4926ea 100644 --- a/frmts/wms/minidriver_mrf.cpp +++ b/frmts/wms/minidriver_mrf.cpp @@ -160,7 +160,7 @@ void *SectorCache::data(size_t address) // If this is the last sector, it could be a new sector with invalid data, // so we remove it Otherwise, the previous content is still good if (target == &store.back()) - store.resize(store.size() - 1); + store.pop_back(); // Signal invalid request return nullptr; } diff --git a/frmts/wms/wmsdriver.cpp b/frmts/wms/wmsdriver.cpp index a372989d9b6c..10ab8a53cef1 100644 --- a/frmts/wms/wmsdriver.cpp +++ b/frmts/wms/wmsdriver.cpp @@ -118,7 +118,7 @@ static CPLXMLNode *GDALWMSDatasetGetConfigFromURL(GDALOpenInfo *poOpenInfo) osBaseURL = CPLURLAddKVP(osBaseURL, "BBOXORDER", nullptr); if (!osBaseURL.empty() && osBaseURL.back() == '&') - osBaseURL.resize(osBaseURL.size() - 1); + osBaseURL.pop_back(); if (osVersion.empty()) osVersion = "1.1.1"; diff --git a/frmts/wms/wmsutils.cpp b/frmts/wms/wmsutils.cpp index 75ddddc6598a..b9f044c929d0 100644 --- a/frmts/wms/wmsutils.cpp +++ b/frmts/wms/wmsutils.cpp @@ -70,9 +70,7 @@ void URLPrepare(CPLString &url) CPLString BufferToVSIFile(GByte *buffer, size_t size) { - CPLString file_name; - - file_name.Printf("/vsimem/wms/%p/wmsresult.dat", buffer); + const CPLString file_name(VSIMemGenerateHiddenFilename("wmsresult.dat")); VSILFILE *f = VSIFileFromMemBuffer(file_name.c_str(), buffer, size, false); if (f == nullptr) return CPLString(); diff --git a/frmts/wmts/wmtsdataset.cpp b/frmts/wmts/wmtsdataset.cpp index d2245c9ca301..cc0389d438a9 100644 --- a/frmts/wmts/wmtsdataset.cpp +++ b/frmts/wmts/wmtsdataset.cpp @@ -604,7 +604,7 @@ CPLString WMTSDataset::FixCRSName(const char *pszCRS) while (osRet.size() && (osRet.back() == ' ' || osRet.back() == '\r' || osRet.back() == '\n')) { - osRet.resize(osRet.size() - 1); + osRet.pop_back(); } return osRet; } @@ -1226,20 +1226,21 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) if (STARTS_WITH(osGetCapabilitiesURL, "/vsimem/")) { - const char *pszHref = CPLGetXMLValue( - psXML, "=Capabilities.ServiceMetadataURL.href", nullptr); - if (pszHref) - osGetCapabilitiesURL = pszHref; + osGetCapabilitiesURL = GetOperationKVPURL(psXML, "GetCapabilities"); + if (osGetCapabilitiesURL.empty()) + { + // (ERO) I'm not even sure this is correct at all... + const char *pszHref = CPLGetXMLValue( + psXML, "=Capabilities.ServiceMetadataURL.href", nullptr); + if (pszHref) + osGetCapabilitiesURL = pszHref; + } else { - osGetCapabilitiesURL = GetOperationKVPURL(psXML, "GetCapabilities"); - if (!osGetCapabilitiesURL.empty()) - { - osGetCapabilitiesURL = - CPLURLAddKVP(osGetCapabilitiesURL, "service", "WMTS"); - osGetCapabilitiesURL = CPLURLAddKVP( - osGetCapabilitiesURL, "request", "GetCapabilities"); - } + osGetCapabilitiesURL = + CPLURLAddKVP(osGetCapabilitiesURL, "service", "WMTS"); + osGetCapabilitiesURL = CPLURLAddKVP(osGetCapabilitiesURL, "request", + "GetCapabilities"); } } CPLString osCapabilitiesFilename(osGetCapabilitiesURL); diff --git a/frmts/zarr/zarr_sharedresource.cpp b/frmts/zarr/zarr_sharedresource.cpp index b0daddf2b608..69f739d3b69e 100644 --- a/frmts/zarr/zarr_sharedresource.cpp +++ b/frmts/zarr/zarr_sharedresource.cpp @@ -44,7 +44,7 @@ ZarrSharedResource::ZarrSharedResource(const std::string &osRootDirectoryName, m_osRootDirectoryName = osRootDirectoryName; if (!m_osRootDirectoryName.empty() && m_osRootDirectoryName.back() == '/') { - m_osRootDirectoryName.resize(m_osRootDirectoryName.size() - 1); + m_osRootDirectoryName.pop_back(); } m_poPAM = std::make_shared( CPLFormFilename(m_osRootDirectoryName.c_str(), "pam", nullptr)); diff --git a/frmts/zarr/zarrdriver.cpp b/frmts/zarr/zarrdriver.cpp index 6d399f312e6d..0d2befa2acdf 100644 --- a/frmts/zarr/zarrdriver.cpp +++ b/frmts/zarr/zarrdriver.cpp @@ -58,7 +58,7 @@ GDALDataset *ZarrDataset::OpenMultidim(const char *pszFilename, { CPLString osFilename(pszFilename); if (osFilename.back() == '/') - osFilename.resize(osFilename.size() - 1); + osFilename.pop_back(); auto poSharedResource = ZarrSharedResource::Create(osFilename, bUpdateMode); poSharedResource->SetOpenOptions(papszOpenOptionsIn); diff --git a/fuzzers/build.sh b/fuzzers/build.sh index c6fa8c7f1332..529f3c24b4e7 100755 --- a/fuzzers/build.sh +++ b/fuzzers/build.sh @@ -145,7 +145,12 @@ curl -L https://download.savannah.gnu.org/releases/freetype/freetype-2.13.2.tar. rm freetype-2.13.2.tar.xz rm -rf poppler -git clone --depth 1 https://anongit.freedesktop.org/git/poppler/poppler.git poppler +# Poppler git server is too unreliable. Use a snapshot of a given version +#git clone --depth 1 https://anongit.freedesktop.org/git/poppler/poppler.git poppler +curl -L https://poppler.freedesktop.org/poppler-24.09.0.tar.xz > poppler-24.09.0.tar.xz && \ + tar xJf poppler-24.09.0.tar.xz && \ + mv poppler-24.09.0 poppler && \ + rm poppler-24.09.0.tar.xz # Build xerces-c from source to avoid upstream bugs rm -rf xerces-c diff --git a/gcore/CMakeLists.txt b/gcore/CMakeLists.txt index 7253fd9cea9e..a629c7f163b2 100644 --- a/gcore/CMakeLists.txt +++ b/gcore/CMakeLists.txt @@ -40,6 +40,7 @@ add_library( gdalrelationship.cpp gdalsubdatasetinfo.cpp gdalorienteddataset.cpp + gdalthreadsafedataset.cpp overview.cpp rasterio.cpp rawdataset.cpp diff --git a/gcore/gdal.h b/gcore/gdal.h index 21d1a4da18d7..33b27eeef877 100644 --- a/gcore/gdal.h +++ b/gcore/gdal.h @@ -100,6 +100,7 @@ GDALDataType CPL_DLL CPL_STDCALL GDALFindDataTypeForValue(double dValue, int bComplex); double CPL_DLL GDALAdjustValueToDataType(GDALDataType eDT, double dfValue, int *pbClamped, int *pbRounded); +bool CPL_DLL GDALIsValueExactAs(double dfValue, GDALDataType eDT); GDALDataType CPL_DLL CPL_STDCALL GDALGetNonComplexDataType(GDALDataType); int CPL_DLL CPL_STDCALL GDALDataTypeIsConversionLossy(GDALDataType eTypeFrom, GDALDataType eTypeTo); @@ -222,27 +223,127 @@ typedef struct (s).bFloatingPointWindowValidity = FALSE; \ } while (0) -/*! Types of color interpretation for raster bands. */ +/** Value indicating the start of the range for color interpretations belonging + * to the InfraRed (IR) domain. All constants of the GDALColorInterp enumeration + * in the IR domain are in the [GCI_IR_Start, GCI_IR_End] range. + * + * @since 3.10 + */ +#define GCI_IR_Start 20 + +/** Value indicating the end of the range for color interpretations belonging + * to the InfraRed (IR) domain. All constants of the GDALColorInterp enumeration + * in the IR domain are in the [GCI_IR_Start, GCI_IR_End] range. + * + * @since 3.10 + */ +#define GCI_IR_End 29 + +/** Value indicating the start of the range for color interpretations belonging + * to the Synthetic Aperture Radar (SAR) domain. + * All constants of the GDALColorInterp enumeration + * in the SAR domain are in the [GCI_SAR_Start, GCI_SAR_End] range. + * + * @since 3.10 + */ +#define GCI_SAR_Start 30 + +/** Value indicating the end of the range for color interpretations belonging + * to the Synthetic Aperture Radar (SAR) domain. + * All constants of the GDALColorInterp enumeration + * in the SAR domain are in the [GCI_SAR_Start, GCI_SAR_End] range. + * + * @since 3.10 + */ +#define GCI_SAR_End 39 + +/** Types of color interpretation for raster bands. + * + * For spectral bands, the wavelength ranges are indicative only, and may vary + * depending on sensors. The CENTRAL_WAVELENGTH_UM and FWHM_UM metadata + * items in the IMAGERY metadata domain of the raster band, when present, will + * give more accurate characteristics. + * + * Values belonging to the IR domain are in the [GCI_IR_Start, GCI_IR_End] range. + * Values belonging to the SAR domain are in the [GCI_SAR_Start, GCI_SAR_End] range. + * + * Values between GCI_PanBand to GCI_SAR_Reserved_2 have been added in GDAL 3.10. + */ typedef enum { /*! Undefined */ GCI_Undefined = 0, /*! Greyscale */ GCI_GrayIndex = 1, /*! Paletted (see associated color table) */ GCI_PaletteIndex = 2, - /*! Red band of RGBA image */ GCI_RedBand = 3, - /*! Green band of RGBA image */ GCI_GreenBand = 4, - /*! Blue band of RGBA image */ GCI_BlueBand = 5, + /*! Red band of RGBA image, or red spectral band [0.62 - 0.69 um]*/ + GCI_RedBand = 3, + /*! Green band of RGBA image, or green spectral band [0.51 - 0.60 um]*/ + GCI_GreenBand = 4, + /*! Blue band of RGBA image, or blue spectral band [0.45 - 0.53 um] */ + GCI_BlueBand = 5, /*! Alpha (0=transparent, 255=opaque) */ GCI_AlphaBand = 6, /*! Hue band of HLS image */ GCI_HueBand = 7, /*! Saturation band of HLS image */ GCI_SaturationBand = 8, /*! Lightness band of HLS image */ GCI_LightnessBand = 9, /*! Cyan band of CMYK image */ GCI_CyanBand = 10, /*! Magenta band of CMYK image */ GCI_MagentaBand = 11, - /*! Yellow band of CMYK image */ GCI_YellowBand = 12, + /*! Yellow band of CMYK image, or yellow spectral band [0.58 - 0.62 um] */ + GCI_YellowBand = 12, /*! Black band of CMYK image */ GCI_BlackBand = 13, /*! Y Luminance */ GCI_YCbCr_YBand = 14, /*! Cb Chroma */ GCI_YCbCr_CbBand = 15, /*! Cr Chroma */ GCI_YCbCr_CrBand = 16, - /*! Max current value (equals to GCI_YCbCr_CrBand currently) */ GCI_Max = 16 + + /* GDAL 3.10 addition: begin */ + /*! Panchromatic band [0.40 - 1.00 um] */ GCI_PanBand = 17, + /*! Coastal band [0.40 - 0.45 um] */ GCI_CoastalBand = 18, + /*! Red-edge band [0.69 - 0.79 um] */ GCI_RedEdgeBand = 19, + + /*! Near-InfraRed (NIR) band [0.75 - 1.40 um] */ GCI_NIRBand = + GCI_IR_Start + 0, + /*! Short-Wavelength InfraRed (SWIR) band [1.40 - 3.00 um] */ GCI_SWIRBand = + GCI_IR_Start + 1, + /*! Mid-Wavelength InfraRed (MWIR) band [3.00 - 8.00 um] */ GCI_MWIRBand = + GCI_IR_Start + 2, + /*! Long-Wavelength InfraRed (LWIR) band [8.00 - 15 um] */ GCI_LWIRBand = + GCI_IR_Start + 3, + /*! Thermal InfraRed (TIR) band (MWIR or LWIR) [3 - 15 um] */ GCI_TIRBand = + GCI_IR_Start + 4, + /*! Other infrared band [0.75 - 1000 um] */ GCI_OtherIRBand = + GCI_IR_Start + 5, + /*! Reserved value. Do not set it ! */ + GCI_IR_Reserved_1 = GCI_IR_Start + 6, + /*! Reserved value. Do not set it ! */ + GCI_IR_Reserved_2 = GCI_IR_Start + 7, + /*! Reserved value. Do not set it ! */ + GCI_IR_Reserved_3 = GCI_IR_Start + 8, + /*! Reserved value. Do not set it ! */ + GCI_IR_Reserved_4 = GCI_IR_Start + 9, + + /*! Synthetic Aperture Radar (SAR) Ka band [0.8 - 1.1 cm / 27 - 40 GHz] */ + GCI_SAR_Ka_Band = GCI_SAR_Start + 0, + /*! Synthetic Aperture Radar (SAR) K band [1.1 - 1.7 cm / 18 - 27 GHz] */ + GCI_SAR_K_Band = GCI_SAR_Start + 1, + /*! Synthetic Aperture Radar (SAR) Ku band [1.7 - 2.4 cm / 12 - 18 GHz] */ + GCI_SAR_Ku_Band = GCI_SAR_Start + 2, + /*! Synthetic Aperture Radar (SAR) X band [2.4 - 3.8 cm / 8 - 12 GHz] */ + GCI_SAR_X_Band = GCI_SAR_Start + 3, + /*! Synthetic Aperture Radar (SAR) C band [3.8 - 7.5 cm / 4 - 8 GHz] */ + GCI_SAR_C_Band = GCI_SAR_Start + 4, + /*! Synthetic Aperture Radar (SAR) S band [7.5 - 15 cm / 2 - 4 GHz] */ + GCI_SAR_S_Band = GCI_SAR_Start + 5, + /*! Synthetic Aperture Radar (SAR) L band [15 - 30 cm / 1 - 2 GHz] */ + GCI_SAR_L_Band = GCI_SAR_Start + 6, + /*! Synthetic Aperture Radar (SAR) P band [30 - 100 cm / 0.3 - 1 GHz] */ + GCI_SAR_P_Band = GCI_SAR_Start + 7, + /*! Reserved value. Do not set it ! */ + GCI_SAR_Reserved_1 = GCI_SAR_Start + 8, + /*! Reserved value. Do not set it ! */ + GCI_SAR_Reserved_2 = GCI_SAR_Start + 9, + + /* GDAL 3.10 addition: end */ + + /*! Max current value (equals to GCI_SAR_Reserved_2 currently) */ GCI_Max = + GCI_SAR_Reserved_2 } GDALColorInterp; const char CPL_DLL *GDALGetColorInterpretationName(GDALColorInterp); @@ -1030,6 +1131,14 @@ GDALDatasetH CPL_DLL CPL_STDCALL GDALOpenShared(const char *, GDALAccess) #define GDAL_OF_FROM_GDALOPEN 0x400 #endif +/** Open in thread-safe mode. Not compatible with + * GDAL_OF_VECTOR, GDAL_OF_MULTIDIM_RASTER or GDAL_OF_UPDATE + * + * Used by GDALOpenEx(). + * @since GDAL 3.10 + */ +#define GDAL_OF_THREAD_SAFE 0x800 + GDALDatasetH CPL_DLL CPL_STDCALL GDALOpenEx( const char *pszFilename, unsigned int nOpenFlags, const char *const *papszAllowedDrivers, const char *const *papszOpenOptions, @@ -1140,6 +1249,11 @@ int CPL_DLL CPL_STDCALL GDALGetRasterYSize(GDALDatasetH); int CPL_DLL CPL_STDCALL GDALGetRasterCount(GDALDatasetH); GDALRasterBandH CPL_DLL CPL_STDCALL GDALGetRasterBand(GDALDatasetH, int); +bool CPL_DLL GDALDatasetIsThreadSafe(GDALDatasetH, int nScopeFlags, + CSLConstList papszOptions); +GDALDatasetH CPL_DLL GDALGetThreadSafeDataset(GDALDatasetH, int nScopeFlags, + CSLConstList papszOptions); + CPLErr CPL_DLL CPL_STDCALL GDALAddBand(GDALDatasetH hDS, GDALDataType eType, CSLConstList papszOptions); diff --git a/gcore/gdal_frmts.h b/gcore/gdal_frmts.h index 129845bbe006..2bf67d7527f9 100644 --- a/gcore/gdal_frmts.h +++ b/gcore/gdal_frmts.h @@ -228,6 +228,8 @@ void CPL_DLL GDALRegister_COG(void); void CPL_DLL GDALRegister_RDB(void); void CPL_DLL GDALRegister_EXR(void); void DeclareDeferredEXRPlugin(void); +void CPL_DLL GDALRegister_AVIF(void); +void DeclareDeferredAVIFPlugin(void); void CPL_DLL GDALRegister_HEIF(void); void DeclareDeferredHEIFPlugin(void); void CPL_DLL GDALRegister_TGA(void); diff --git a/gcore/gdal_misc.cpp b/gcore/gdal_misc.cpp index af3d01d6e96b..50e1c65793db 100644 --- a/gcore/gdal_misc.cpp +++ b/gcore/gdal_misc.cpp @@ -140,6 +140,11 @@ GDALDataType CPL_STDCALL GDALDataTypeUnion(GDALDataType eType1, GDALDataType eType2) { + if (eType1 == GDT_Unknown) + return eType2; + if (eType2 == GDT_Unknown) + return eType1; + const int panBits[] = {GetDataTypeElementSizeBits(eType1), GetDataTypeElementSizeBits(eType2)}; @@ -169,19 +174,89 @@ GDALDataType CPL_STDCALL GDALDataTypeUnion(GDALDataType eType1, * \brief Union a data type with the one found for a value * * @param eDT the first data type - * @param dValue the value for which to find a data type and union with eDT + * @param dfValue the value for which to find a data type and union with eDT * @param bComplex if the value is complex * - * @return a data type able to express eDT and dValue. + * @return a data type able to express eDT and dfValue. * @since GDAL 2.3 */ GDALDataType CPL_STDCALL GDALDataTypeUnionWithValue(GDALDataType eDT, - double dValue, int bComplex) + double dfValue, + int bComplex) { - if (eDT == GDT_Float32 && !bComplex && static_cast(dValue) == dValue) - return eDT; + if (!bComplex && !GDALDataTypeIsComplex(eDT)) + { + switch (eDT) + { + case GDT_Byte: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_Int8: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_UInt16: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_Int16: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_UInt32: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_Int32: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_UInt64: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_Int64: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_Float32: + { + if (GDALIsValueExactAs(dfValue)) + return eDT; + break; + } + case GDT_Float64: + { + return eDT; + } + case GDT_Unknown: + case GDT_CInt16: + case GDT_CInt32: + case GDT_CFloat32: + case GDT_CFloat64: + case GDT_TypeCount: + break; + } + } - const GDALDataType eDT2 = GDALFindDataTypeForValue(dValue, bComplex); + const GDALDataType eDT2 = GDALFindDataTypeForValue(dfValue, bComplex); return GDALDataTypeUnion(eDT, eDT2); } @@ -313,7 +388,12 @@ GDALDataType CPL_STDCALL GDALFindDataType(int nBits, int bSigned, int bFloating, */ GDALDataType CPL_STDCALL GDALFindDataTypeForValue(double dValue, int bComplex) { - const bool bFloating = round(dValue) != dValue; + const bool bFloating = + round(dValue) != dValue || + dValue > + static_cast(std::numeric_limits::max()) || + dValue < + static_cast(std::numeric_limits::lowest()); const bool bSigned = bFloating || dValue < 0; const int nBits = GetMinBitsForValue(dValue); @@ -868,6 +948,58 @@ double GDALAdjustValueToDataType(GDALDataType eDT, double dfValue, return dfValue; } +/************************************************************************/ +/* GDALIsValueExactAs() */ +/************************************************************************/ + +/** + * \brief Check whether the provided value can be exactly represented in a + * data type. + * + * Only implemented for non-complex data types + * + * @param dfValue value to check. + * @param eDT target data type. + * + * @return true if the provided value can be exactly represented in the + * data type. + * @since GDAL 3.10 + */ +bool GDALIsValueExactAs(double dfValue, GDALDataType eDT) +{ + switch (eDT) + { + case GDT_Byte: + return GDALIsValueExactAs(dfValue); + case GDT_Int8: + return GDALIsValueExactAs(dfValue); + case GDT_UInt16: + return GDALIsValueExactAs(dfValue); + case GDT_Int16: + return GDALIsValueExactAs(dfValue); + case GDT_UInt32: + return GDALIsValueExactAs(dfValue); + case GDT_Int32: + return GDALIsValueExactAs(dfValue); + case GDT_UInt64: + return GDALIsValueExactAs(dfValue); + case GDT_Int64: + return GDALIsValueExactAs(dfValue); + case GDT_Float32: + return GDALIsValueExactAs(dfValue); + case GDT_Float64: + return true; + case GDT_Unknown: + case GDT_CInt16: + case GDT_CInt32: + case GDT_CFloat32: + case GDT_CFloat64: + case GDT_TypeCount: + break; + } + return true; +} + /************************************************************************/ /* GDALGetNonComplexDataType() */ /************************************************************************/ @@ -1046,10 +1178,15 @@ const char *GDALGetPaletteInterpretationName(GDALPaletteInterp eInterp) const char *GDALGetColorInterpretationName(GDALColorInterp eInterp) { + static_assert(GCI_IR_Start == GCI_RedEdgeBand + 1); + static_assert(GCI_NIRBand == GCI_IR_Start); + static_assert(GCI_SAR_Start == GCI_IR_End + 1); + static_assert(GCI_Max == GCI_SAR_End); + switch (eInterp) { case GCI_Undefined: - return "Undefined"; + break; case GCI_GrayIndex: return "Gray"; @@ -1099,9 +1236,76 @@ const char *GDALGetColorInterpretationName(GDALColorInterp eInterp) case GCI_YCbCr_CrBand: return "YCbCr_Cr"; - default: - return "Unknown"; + case GCI_PanBand: + return "Pan"; + + case GCI_CoastalBand: + return "Coastal"; + + case GCI_RedEdgeBand: + return "RedEdge"; + + case GCI_NIRBand: + return "NIR"; + + case GCI_SWIRBand: + return "SWIR"; + + case GCI_MWIRBand: + return "MWIR"; + + case GCI_LWIRBand: + return "LWIR"; + + case GCI_TIRBand: + return "TIR"; + + case GCI_OtherIRBand: + return "OtherIR"; + + case GCI_IR_Reserved_1: + return "IR_Reserved_1"; + + case GCI_IR_Reserved_2: + return "IR_Reserved_2"; + + case GCI_IR_Reserved_3: + return "IR_Reserved_3"; + + case GCI_IR_Reserved_4: + return "IR_Reserved_4"; + + case GCI_SAR_Ka_Band: + return "SAR_Ka"; + + case GCI_SAR_K_Band: + return "SAR_K"; + + case GCI_SAR_Ku_Band: + return "SAR_Ku"; + + case GCI_SAR_X_Band: + return "SAR_X"; + + case GCI_SAR_C_Band: + return "SAR_C"; + + case GCI_SAR_S_Band: + return "SAR_S"; + + case GCI_SAR_L_Band: + return "SAR_L"; + + case GCI_SAR_P_Band: + return "SAR_P"; + + case GCI_SAR_Reserved_1: + return "SAR_Reserved_1"; + + case GCI_SAR_Reserved_2: + return "SAR_Reserved_2"; } + return "Undefined"; } /************************************************************************/ @@ -1138,9 +1342,85 @@ GDALColorInterp GDALGetColorInterpretationByName(const char *pszName) } } + // Accept British English spelling + if (EQUAL(pszName, "grey")) + return GCI_GrayIndex; + + return GCI_Undefined; +} + +/************************************************************************/ +/* GDALGetColorInterpFromSTACCommonName() */ +/************************************************************************/ + +static const struct +{ + const char *pszName; + GDALColorInterp eInterp; +} asSTACCommonNames[] = { + {"pan", GCI_PanBand}, + {"coastal", GCI_CoastalBand}, + {"blue", GCI_BlueBand}, + {"green", GCI_GreenBand}, + {"green05", GCI_GreenBand}, // no exact match + {"yellow", GCI_YellowBand}, + {"red", GCI_RedBand}, + {"rededge", GCI_RedEdgeBand}, + {"rededge071", GCI_RedEdgeBand}, // no exact match + {"rededge075", GCI_RedEdgeBand}, // no exact match + {"rededge078", GCI_RedEdgeBand}, // no exact match + {"nir", GCI_NIRBand}, + {"nir08", GCI_NIRBand}, // no exact match + {"nir09", GCI_NIRBand}, // no exact match + {"cirrus", GCI_NIRBand}, // no exact match + {nullptr, + GCI_SWIRBand}, // so that GDALGetSTACCommonNameFromColorInterp returns null on GCI_SWIRBand + {"swir16", GCI_SWIRBand}, // no exact match + {"swir22", GCI_SWIRBand}, // no exact match + {"lwir", GCI_LWIRBand}, + {"lwir11", GCI_LWIRBand}, // no exact match + {"lwir12", GCI_LWIRBand}, // no exact match +}; + +/** Get color interpreetation from STAC eo:common_name + * + * Cf https://github.com/stac-extensions/eo?tab=readme-ov-file#common-band-names + * + * @since GDAL 3.10 + */ +GDALColorInterp GDALGetColorInterpFromSTACCommonName(const char *pszName) +{ + + for (const auto &sAssoc : asSTACCommonNames) + { + if (sAssoc.pszName && EQUAL(pszName, sAssoc.pszName)) + return sAssoc.eInterp; + } return GCI_Undefined; } +/************************************************************************/ +/* GDALGetSTACCommonNameFromColorInterp() */ +/************************************************************************/ + +/** Get STAC eo:common_name from GDAL color interpretation + * + * Cf https://github.com/stac-extensions/eo?tab=readme-ov-file#common-band-names + * + * @return nullptr if there is no match + * + * @since GDAL 3.10 + */ +const char *GDALGetSTACCommonNameFromColorInterp(GDALColorInterp eInterp) +{ + for (const auto &sAssoc : asSTACCommonNames) + { + if (eInterp == sAssoc.eInterp) + return sAssoc.pszName; + } + return nullptr; +} + /************************************************************************/ /* GDALGetRandomRasterSample() */ /************************************************************************/ diff --git a/gcore/gdal_pam.h b/gcore/gdal_pam.h index 6d5ebec81671..e0c0660dd5dc 100644 --- a/gcore/gdal_pam.h +++ b/gcore/gdal_pam.h @@ -34,6 +34,7 @@ #include "cpl_minixml.h" #include "gdal_priv.h" +#include #include #include #include @@ -95,7 +96,7 @@ class GDALDatasetPamInfo OGRSpatialReference *poSRS = nullptr; int bHaveGeoTransform = false; - double adfGeoTransform[6]{0, 0, 0, 0, 0, 0}; + std::array adfGeoTransform{}; std::vector asGCPs{}; OGRSpatialReference *poGCP_SRS = nullptr; @@ -268,6 +269,8 @@ struct GDALRasterBandPamInfo bool bOffsetSet = false; bool bScaleSet = false; + + void CopyFrom(const GDALRasterBandPamInfo &sOther); }; //! @endcond diff --git a/gcore/gdal_priv.h b/gcore/gdal_priv.h index fa2b4ef0447b..9ad11d1bfb89 100644 --- a/gcore/gdal_priv.h +++ b/gcore/gdal_priv.h @@ -141,9 +141,6 @@ class CPL_DLL GDALMultiDomainMetadata { Clear(); } - - private: - CPL_DISALLOW_COPY_ASSIGN(GDALMultiDomainMetadata) }; //! @endcond @@ -579,7 +576,8 @@ class CPL_DLL GDALDataset : public GDALMajorObject GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg) CPL_WARN_UNUSED_RESULT; - CPLErr + /* This method should only be be overloaded by GDALProxyDataset */ + virtual CPLErr BlockBasedRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize, void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, int nBandCount, @@ -619,6 +617,15 @@ class CPL_DLL GDALDataset : public GDALMajorObject void ShareLockWithParentDataset(GDALDataset *poParentDataset); + bool m_bCanBeReopened = false; + + virtual bool CanBeCloned(int nScopeFlags, bool bCanShareState) const; + + friend class GDALThreadSafeDataset; + friend class MEMDataset; + virtual std::unique_ptr Clone(int nScopeFlags, + bool bCanShareState) const; + //! @endcond void CleanupPostFileClosing(); @@ -830,7 +837,7 @@ class CPL_DLL GDALDataset : public GDALMajorObject /** Return MarkSuppressOnClose flag. * @return MarkSuppressOnClose flag. */ - bool IsMarkedSuppressOnClose() + bool IsMarkedSuppressOnClose() const { return bSuppressOnClose; } @@ -843,6 +850,8 @@ class CPL_DLL GDALDataset : public GDALMajorObject return papszOpenOptions; } + bool IsThreadSafe(int nScopeFlags) const; + #ifndef DOXYGEN_SKIP /** Return open options. * @return open options. @@ -1582,6 +1591,11 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject m_poBandRef = bOwned ? nullptr : poBand; } + const GDALRasterBand *get() const + { + return static_cast(*this); + } + GDALRasterBand *get() { return static_cast(*this); @@ -1592,6 +1606,11 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject return m_poBandOwned != nullptr; } + operator const GDALRasterBand *() const + { + return m_poBandOwned ? m_poBandOwned.get() : m_poBandRef; + } + operator GDALRasterBand *() { return m_poBandOwned ? m_poBandOwned.get() : m_poBandRef; @@ -1625,7 +1644,7 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject void InitRWLock(); void SetValidPercent(GUIntBig nSampleCount, GUIntBig nValidCount); - mutable std::unique_ptr m_oPointsCache{}; + mutable GDALDoublePointsCache *m_poPointsCache = nullptr; //! @endcond @@ -1679,15 +1698,15 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject ~GDALRasterBand() override; - int GetXSize(); - int GetYSize(); - int GetBand(); - GDALDataset *GetDataset(); + int GetXSize() const; + int GetYSize() const; + int GetBand() const; + GDALDataset *GetDataset() const; - GDALDataType GetRasterDataType(void); - void GetBlockSize(int *pnXSize, int *pnYSize); + GDALDataType GetRasterDataType(void) const; + void GetBlockSize(int *pnXSize, int *pnYSize) const; CPLErr GetActualBlockSize(int nXBlockOff, int nYBlockOff, int *pnXValid, - int *pnYValid); + int *pnYValid) const; virtual GDALSuggestedBlockAccessPattern GetSuggestedBlockAccessPattern() const; @@ -1750,13 +1769,19 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject CPLErr WriteBlock(int nXBlockOff, int nYBlockOff, void *pImage) CPL_WARN_UNUSED_RESULT; - GDALRasterBlock * + // This method should only be overloaded by GDALProxyRasterBand + virtual GDALRasterBlock * GetLockedBlockRef(int nXBlockOff, int nYBlockOff, int bJustInitialize = FALSE) CPL_WARN_UNUSED_RESULT; - GDALRasterBlock *TryGetLockedBlockRef(int nXBlockOff, int nYBlockYOff) - CPL_WARN_UNUSED_RESULT; - CPLErr FlushBlock(int nXBlockOff, int nYBlockOff, - int bWriteDirtyBlock = TRUE); + + // This method should only be overloaded by GDALProxyRasterBand + virtual GDALRasterBlock * + TryGetLockedBlockRef(int nXBlockOff, + int nYBlockYOff) CPL_WARN_UNUSED_RESULT; + + // This method should only be overloaded by GDALProxyRasterBand + virtual CPLErr FlushBlock(int nXBlockOff, int nYBlockOff, + int bWriteDirtyBlock = TRUE); unsigned char * GetIndexColorTranslationTo(/* const */ GDALRasterBand *poReferenceBand, @@ -1857,14 +1882,14 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject std::shared_ptr AsMDArray() const; - CPLErr InterpolateAtPoint(double dfPixel, double dfLine, - GDALRIOResampleAlg eInterpolation, - double *pdfRealValue, - double *pdfImagValue = nullptr) const; + virtual CPLErr InterpolateAtPoint(double dfPixel, double dfLine, + GDALRIOResampleAlg eInterpolation, + double *pdfRealValue, + double *pdfImagValue = nullptr) const; #ifndef DOXYGEN_XML - void ReportError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt, ...) - CPL_PRINT_FUNC_FORMAT(4, 5); + void ReportError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt, + ...) const CPL_PRINT_FUNC_FORMAT(4, 5); #endif /** Convert a GDALRasterBand* to a GDALRasterBandH. @@ -1886,7 +1911,7 @@ class CPL_DLL GDALRasterBand : public GDALMajorObject //! @cond Doxygen_Suppress // Remove me in GDAL 4.0. See GetMetadataItem() implementation // Internal use in GDAL only ! - void EnablePixelTypeSignedByteWarning(bool b) + virtual void EnablePixelTypeSignedByteWarning(bool b) #ifndef GDAL_COMPILATION CPL_WARN_DEPRECATED("Do not use that method outside of GDAL!") #endif @@ -2525,6 +2550,7 @@ class CPL_DLL GDALDriverManager : public GDALMajorObject //! @cond Doxygen_Suppress int GetDriverCount(bool bIncludeHidden) const; GDALDriver *GetDriver(int iDriver, bool bIncludeHidden); + bool IsKnownDriver(const char *pszDriverName) const; //! @endcond }; @@ -4463,6 +4489,12 @@ void CPL_DLL GDALCopyRasterIOExtraArg(GDALRasterIOExtraArg *psDestArg, CPL_C_END +std::unique_ptr CPL_DLL +GDALGetThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags); + +GDALDataset CPL_DLL *GDALGetThreadSafeDataset(GDALDataset *poDS, + int nScopeFlags); + void GDALNullifyOpenDatasetsList(); CPLMutex **GDALGetphDMMutex(); CPLMutex **GDALGetphDLMutex(); @@ -4563,6 +4595,11 @@ GDALRasterAttributeTable CPL_DLL *GDALCreateRasterAttributeTableFromMDArrays( const std::vector> &apoArrays, const std::vector &aeUsages); +GDALColorInterp CPL_DLL +GDALGetColorInterpFromSTACCommonName(const char *pszName); +const char CPL_DLL * +GDALGetSTACCommonNameFromColorInterp(GDALColorInterp eInterp); + // Macro used so that Identify and driver metadata methods in drivers built // as plugin can be duplicated in libgdal core and in the driver under different // names diff --git a/gcore/gdal_proxy.h b/gcore/gdal_proxy.h index 518deb4b2d32..76c85026c257 100644 --- a/gcore/gdal_proxy.h +++ b/gcore/gdal_proxy.h @@ -59,6 +59,13 @@ class CPL_DLL GDALProxyDataset : public GDALDataset CPLErr IRasterIO(GDALRWFlag, int, int, int, int, void *, int, int, GDALDataType, int, BANDMAP_TYPE, GSpacing, GSpacing, GSpacing, GDALRasterIOExtraArg *psExtraArg) override; + CPLErr BlockBasedRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, + int nXSize, int nYSize, void *pData, + int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + const int *panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg) override; public: char **GetMetadataDomainList() override; @@ -140,6 +147,16 @@ class CPL_DLL GDALProxyRasterBand : public GDALRasterBand const char *pszDomain) override; CPLErr SetMetadataItem(const char *pszName, const char *pszValue, const char *pszDomain) override; + + GDALRasterBlock *GetLockedBlockRef(int nXBlockOff, int nYBlockOff, + int bJustInitialize) override; + + GDALRasterBlock *TryGetLockedBlockRef(int nXBlockOff, + int nYBlockYOff) override; + + CPLErr FlushBlock(int nXBlockOff, int nYBlockOff, + int bWriteDirtyBlock) override; + CPLErr FlushCache(bool bAtClosing) override; char **GetCategoryNames() override; double GetNoDataValue(int *pbSuccess = nullptr) override; @@ -206,6 +223,13 @@ class CPL_DLL GDALProxyRasterBand : public GDALRasterBand GIntBig *pnLineSpace, char **papszOptions) override; + CPLErr InterpolateAtPoint(double dfPixel, double dfLine, + GDALRIOResampleAlg eInterpolation, + double *pdfRealValue, + double *pdfImagValue) const override; + + void EnablePixelTypeSignedByteWarning(bool b) override; + private: CPL_DISALLOW_COPY_ASSIGN(GDALProxyRasterBand) }; diff --git a/gcore/gdaldataset.cpp b/gcore/gdaldataset.cpp index 712157b057ca..cc5289d644f5 100644 --- a/gcore/gdaldataset.cpp +++ b/gcore/gdaldataset.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,8 @@ class GDALDataset::Private GIntBig nTotalFeatures = TOTAL_FEATURES_NOT_INIT; OGRLayer *poCurrentLayer = nullptr; + std::mutex m_oMutexWKT{}; + char *m_pszWKTCached = nullptr; OGRSpatialReference *m_poSRSCached = nullptr; char *m_pszWKTGCPCached = nullptr; @@ -373,11 +376,13 @@ GDALDataset::~GDALDataset() if (m_poPrivate->hMutex != nullptr) CPLDestroyMutex(m_poPrivate->hMutex); + // coverity[missing_lock] CPLFree(m_poPrivate->m_pszWKTCached); if (m_poPrivate->m_poSRSCached) { m_poPrivate->m_poSRSCached->Release(); } + // coverity[missing_lock] CPLFree(m_poPrivate->m_pszWKTGCPCached); if (m_poPrivate->m_poSRSGCPCached) { @@ -544,7 +549,7 @@ void GDALDataset::AddToDatasetOpenList() * * The default implementation of this method just calls the FlushCache() method * on each of the raster bands and the SyncToDisk() method - * on each of the layers. Conceptionally, calling FlushCache() on a dataset + * on each of the layers. Conceptually, calling FlushCache() on a dataset * should include any work that might be accomplished by calling SyncToDisk() * on layers in that dataset. * @@ -748,9 +753,7 @@ CPLErr GDALDataset::BlockBasedFlushCache(bool bAtClosing) { for (int iBand = 0; iBand < nBands; ++iBand) { - GDALRasterBand *poBand = GetRasterBand(iBand + 1); - - const CPLErr eErr = poBand->FlushBlock(iX, iY); + const CPLErr eErr = papoBands[iBand]->FlushBlock(iX, iY); if (eErr != CE_None) return CE_Failure; @@ -1168,6 +1171,11 @@ const char *GDALDataset::GetProjectionRef() const { return ""; } + + // If called on a thread-safe dataset, we might be called by several + // threads, so make sure our accesses to m_pszWKTCached are protected + // by a mutex. + std::lock_guard oLock(m_poPrivate->m_oMutexWKT); if (m_poPrivate->m_pszWKTCached && strcmp(pszWKT, m_poPrivate->m_pszWKTCached) == 0) { @@ -1833,6 +1841,11 @@ const char *GDALDataset::GetGCPProjection() { return ""; } + + // If called on a thread-safe dataset, we might be called by several + // threads, so make sure our accesses to m_pszWKTCached are protected + // by a mutex. + std::lock_guard oLock(m_poPrivate->m_oMutexWKT); if (m_poPrivate->m_pszWKTGCPCached && strcmp(pszWKT, m_poPrivate->m_pszWKTGCPCached) == 0) { @@ -2598,6 +2611,12 @@ CPLErr GDALDataset::ValidateRasterIOOrAdviseReadParameters( * buffer size (nBufXSize x nBufYSize) is different than the size of the * region being accessed (nXSize x nYSize). * + * The window of interest expressed by (nXOff, nYOff, nXSize, nYSize) should be + * fully within the raster space, that is nXOff >= 0, nYOff >= 0, + * nXOff + nXSize <= GetRasterXSize() and nYOff + nYSize <= GetRasterYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * The nPixelSpace, nLineSpace and nBandSpace parameters allow reading into or * writing from various organization of buffers. * @@ -3527,18 +3546,29 @@ static GDALDataset *GetSharedDS(const char *pszFilename, *
  • GDAL_OF_VECTOR for vector drivers,
    • *
  • GDAL_OF_GNM for Geographic Network Model drivers.
    • * - * GDAL_OF_RASTER and GDAL_OF_MULTIDIM_RASTER are generally mutually + * GDAL_OF_RASTER and GDAL_OF_MULTIDIM_RASTER are generally mutually * exclusive. If none of the value is specified, GDAL_OF_RASTER | GDAL_OF_VECTOR - * | GDAL_OF_GNM is implied.
  • Access mode: GDAL_OF_READONLY - * (exclusive)or GDAL_OF_UPDATE.
  • Shared mode: GDAL_OF_SHARED. If set, + * | GDAL_OF_GNM is implied. + *
    • + *
  • Access mode: GDAL_OF_READONLY (exclusive)or GDAL_OF_UPDATE. + *
    • + *
  • Shared mode: GDAL_OF_SHARED. If set, * it allows the sharing of GDALDataset handles for a dataset with other callers * that have set GDAL_OF_SHARED. In particular, GDALOpenEx() will first consult * its list of currently open and shared GDALDataset's, and if the * GetDescription() name for one exactly matches the pszFilename passed to * GDALOpenEx() it will be referenced and returned, if GDALOpenEx() is called - * from the same thread.
  • Verbose error: GDAL_OF_VERBOSE_ERROR. If set, + * from the same thread. + *
    • + *
  • Thread safe mode: GDAL_OF_THREAD_SAFE (added in 3.10). + * This must be use in combination with GDAL_OF_RASTER, and is mutually + * exclusive with GDAL_OF_UPDATE, GDAL_OF_VECTOR, GDAL_OF_MULTIDIM_RASTER or + * GDAL_OF_GNM. + *
    • + *
  • Verbose error: GDAL_OF_VERBOSE_ERROR. If set, * a failed attempt to open the file will lead to an error message to be - * reported.
    • + * reported. + * * * * @param papszAllowedDrivers NULL to consider all candidate drivers, or a NULL @@ -3577,6 +3607,33 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, { VALIDATE_POINTER1(pszFilename, "GDALOpen", nullptr); + // Do some sanity checks on incompatible flags with thread-safe mode. + if ((nOpenFlags & GDAL_OF_THREAD_SAFE) != 0) + { + const struct + { + int nFlag; + const char *pszFlagName; + } asFlags[] = { + {GDAL_OF_UPDATE, "GDAL_OF_UPDATE"}, + {GDAL_OF_VECTOR, "GDAL_OF_VECTOR"}, + {GDAL_OF_MULTIDIM_RASTER, "GDAL_OF_MULTIDIM_RASTER"}, + {GDAL_OF_GNM, "GDAL_OF_GNM"}, + }; + + for (const auto &asFlag : asFlags) + { + if ((nOpenFlags & asFlag.nFlag) != 0) + { + CPLError(CE_Failure, CPLE_IllegalArg, + "GDAL_OF_THREAD_SAFE and %s are mutually " + "exclusive", + asFlag.pszFlagName); + return nullptr; + } + } + } + // If no driver kind is specified, assume all are to be probed. if ((nOpenFlags & GDAL_OF_KIND_MASK) == 0) nOpenFlags |= GDAL_OF_KIND_MASK & ~GDAL_OF_MULTIDIM_RASTER; @@ -3868,7 +3925,12 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, } else { - if (!(nOpenFlags & GDAL_OF_INTERNAL)) + // For thread-safe opening, currently poDS is what will be + // the "master" dataset owned by the thread-safe dataset + // returned to the user, hence we do not register it as a + // visible one in the open dataset list, or mark it as shared. + if (!(nOpenFlags & GDAL_OF_INTERNAL) && + !(nOpenFlags & GDAL_OF_THREAD_SAFE)) { poDS->AddToDatasetOpenList(); } @@ -3877,7 +3939,8 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, CSLDestroy(poDS->papszOpenOptions); poDS->papszOpenOptions = CSLDuplicate(papszOpenOptions); poDS->nOpenFlags = nOpenFlags; - poDS->MarkAsShared(); + if (!(nOpenFlags & GDAL_OF_THREAD_SAFE)) + poDS->MarkAsShared(); } } } @@ -3891,8 +3954,11 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, "and description (%s)", pszFilename, poDS->GetDescription()); } - else + else if (!(nOpenFlags & GDAL_OF_THREAD_SAFE)) { + // For thread-safe opening, currently poDS is what will be + // the "master" dataset owned by the thread-safe dataset + // returned to the user, hence we do not or mark it as shared. poDS->MarkAsShared(); } } @@ -3910,6 +3976,29 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, } #endif + if (poDS) + { + poDS->m_bCanBeReopened = true; + + if ((nOpenFlags & GDAL_OF_THREAD_SAFE) != 0) + { + poDS = + GDALGetThreadSafeDataset( + std::unique_ptr(poDS), GDAL_OF_RASTER) + .release(); + if (poDS) + { + poDS->m_bCanBeReopened = true; + poDS->poDriver = poDriver; + poDS->nOpenFlags = nOpenFlags; + if (!(nOpenFlags & GDAL_OF_INTERNAL)) + poDS->AddToDatasetOpenList(); + if (nOpenFlags & GDAL_OF_SHARED) + poDS->MarkAsShared(); + } + } + } + return poDS; } @@ -8176,7 +8265,8 @@ bool GDALDataset::SetQueryLoggerFunc(CPL_UNUSED GDALQueryLoggerFunc callback, int GDALDataset::EnterReadWrite(GDALRWFlag eRWFlag) { - if (m_poPrivate == nullptr) + if (m_poPrivate == nullptr || + IsThreadSafe(GDAL_OF_RASTER | (nOpenFlags & GDAL_OF_UPDATE))) return FALSE; if (m_poPrivate->poParentDataset) @@ -10132,3 +10222,75 @@ CPLErr GDALDatasetReadCompressedData(GDALDatasetH hDS, const char *pszFormat, pszFormat, nXOff, nYOff, nXSize, nYSize, nBandCount, panBandList, ppBuffer, pnBufferSize, ppszDetailedFormat); } + +/************************************************************************/ +/* CanBeCloned() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress + +/** This method is called by GDALThreadSafeDataset::Create() to determine if + * it is possible to create a thread-safe wrapper for a dataset, which involves + * the ability to Clone() it. + * + * Implementations of this method must be thread-safe. + * + * @param nScopeFlags Combination of GDAL_OF_RASTER, GDAL_OF_VECTOR, etc. flags, + * expressing the intended use for thread-safety. + * Currently, the only valid scope is in the base + * implementation is GDAL_OF_RASTER. + * @param bCanShareState Determines if cloned datasets are allowed to share + * state with the dataset they have been cloned from. + * If set to true, the dataset from which they have been + * cloned from must remain opened during the lifetime of + * its clones. + * @return true if the Clone() method is expecte to succeed with the same values + * of nScopeFlags and bCanShareState. + */ +bool GDALDataset::CanBeCloned(int nScopeFlags, + [[maybe_unused]] bool bCanShareState) const +{ + return m_bCanBeReopened && nScopeFlags == GDAL_OF_RASTER; +} + +//! @endcond + +/************************************************************************/ +/* Clone() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress + +/** This method "clones" the current dataset, that is it returns a new instance + * that is opened on the same underlying "file". + * + * The base implementation uses GDALDataset::Open() to re-open the dataset. + * The MEM driver has a specialized implementation that returns a new instance, + * but which shares the same memory buffer as this. + * + * Implementations of this method must be thread-safe. + * + * @param nScopeFlags Combination of GDAL_OF_RASTER, GDAL_OF_VECTOR, etc. flags, + * expressing the intended use for thread-safety. + * Currently, the only valid scope is in the base + * implementation is GDAL_OF_RASTER. + * @param bCanShareState Determines if cloned datasets are allowed to share + * state with the dataset they have been cloned from. + * If set to true, the dataset from which they have been + * cloned from must remain opened during the lifetime of + * its clones. + * @return a new instance, or nullptr in case of error. + */ +std::unique_ptr +GDALDataset::Clone(int nScopeFlags, [[maybe_unused]] bool bCanShareState) const +{ + CPLStringList aosAllowedDrivers; + if (poDriver) + aosAllowedDrivers.AddString(poDriver->GetDescription()); + return std::unique_ptr(GDALDataset::Open( + GetDescription(), + nScopeFlags | GDAL_OF_INTERNAL | GDAL_OF_VERBOSE_ERROR, + aosAllowedDrivers.List(), papszOpenOptions)); +} + +//! @endcond diff --git a/gcore/gdaldriver.cpp b/gcore/gdaldriver.cpp index c1de9b891886..f801283bdc4a 100644 --- a/gcore/gdaldriver.cpp +++ b/gcore/gdaldriver.cpp @@ -123,7 +123,13 @@ GDALDataset *GDALDriver::Open(GDALOpenInfo *poOpenInfo, bool bSetOpenOptions) if (poDS) { - poDS->nOpenFlags = poOpenInfo->nOpenFlags & ~GDAL_OF_FROM_GDALOPEN; + // Only set GDAL_OF_THREAD_SAFE if the driver itself has set it in + // poDS->nOpenFlags + int nOpenFlags = poOpenInfo->nOpenFlags & + ~(GDAL_OF_FROM_GDALOPEN | GDAL_OF_THREAD_SAFE); + if (poDS->nOpenFlags & GDAL_OF_THREAD_SAFE) + nOpenFlags |= GDAL_OF_THREAD_SAFE; + poDS->nOpenFlags = nOpenFlags; if (strlen(poDS->GetDescription()) == 0) poDS->SetDescription(poOpenInfo->pszFilename); diff --git a/gcore/gdaldrivermanager.cpp b/gcore/gdaldrivermanager.cpp index 0adb110bbb9d..b378d7ff0250 100644 --- a/gcore/gdaldrivermanager.cpp +++ b/gcore/gdaldrivermanager.cpp @@ -367,6 +367,26 @@ int GDALDriverManager::GetDriverCount(bool bIncludeHidden) const //! @endcond +/************************************************************************/ +/* IsKnownDriver() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress +bool GDALDriverManager::IsKnownDriver(const char *pszDriverName) const +{ + CPLMutexHolderD(&hDMMutex); + if (cpl::contains(oMapNameToDrivers, CPLString(pszDriverName).toupper())) + return true; + for (const auto &poDriver : m_aoHiddenDrivers) + { + if (EQUAL(poDriver->GetDescription(), pszDriverName)) + return true; + } + return false; +} + +//! @endcond + /************************************************************************/ /* GDALGetDriverCount() */ /************************************************************************/ @@ -525,7 +545,9 @@ int GDALDriverManager::RegisterDriver(GDALDriver *poDriver, bool bHidden) if (poDriver->pfnVectorTranslateFrom != nullptr) poDriver->SetMetadataItem(GDAL_DCAP_VECTOR_TRANSLATE_FROM, "YES"); - if (m_bInDeferredDriverLoading) + if (m_bInDeferredDriverLoading && + cpl::contains(oMapNameToDrivers, + CPLString(poDriver->GetDescription()).toupper())) { if (cpl::contains(m_oMapRealDrivers, poDriver->GetDescription())) { diff --git a/gcore/gdaljp2abstractdataset.cpp b/gcore/gdaljp2abstractdataset.cpp index fc63d9e28578..98b37bec1270 100644 --- a/gcore/gdaljp2abstractdataset.cpp +++ b/gcore/gdaljp2abstractdataset.cpp @@ -368,6 +368,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) return; } + const std::string osTmpDir = VSIMemGenerateHiddenFilename("gmljp2"); + // Find feature collections. int nLayersAtCC = 0; int nLayersAtGC = 0; @@ -451,7 +453,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) if (psFC != nullptr) { - osGMLTmpFile = CPLSPrintf("/vsimem/gmljp2_%p/my.gml", this); + osGMLTmpFile = + CPLFormFilename(osTmpDir.c_str(), "my.gml", nullptr); // Create temporary .gml file. CPLSerializeXMLTreeToFile(psFC, osGMLTmpFile); } @@ -486,8 +489,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) CPLSPrintf("xml:%s", pszBoxName)); if (papszBoxData != nullptr) { - osXSDTmpFile = CPLSPrintf( - "/vsimem/gmljp2_%p/my.xsd", this); + osXSDTmpFile = CPLFormFilename( + osTmpDir.c_str(), "my.xsd", nullptr); CPL_IGNORE_RET_VAL( VSIFCloseL(VSIFileFromMemBuffer( osXSDTmpFile, @@ -551,7 +554,7 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) "No GML driver found to read feature collection"); } - VSIRmdirRecursive(CPLSPrintf("/vsimem/gmljp2_%p", this)); + VSIRmdirRecursive(osTmpDir.c_str()); } } @@ -589,8 +592,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) // Create temporary .kml file. CPLXMLNode *const psKML = psGCorGMLJP2FeaturesChildIter->psChild; - CPLString osKMLTmpFile( - CPLSPrintf("/vsimem/gmljp2_%p_my.kml", this)); + const CPLString osKMLTmpFile( + VSIMemGenerateHiddenFilename("my.kml")); CPLSerializeXMLTreeToFile(psKML, osKMLTmpFile); GDALDatasetUniquePtr poTmpDS(GDALDataset::Open( diff --git a/gcore/gdaljp2metadata.cpp b/gcore/gdaljp2metadata.cpp index 198a80ef9d9f..3852ecd93fd1 100644 --- a/gcore/gdaljp2metadata.cpp +++ b/gcore/gdaljp2metadata.cpp @@ -2593,6 +2593,8 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, "\n", osRootGMLId.c_str(), osGridCoverage.c_str()); + const std::string osTmpDir = VSIMemGenerateHiddenFilename("gmljp2"); + /* -------------------------------------------------------------------- */ /* Process metadata, annotations and features collections. */ /* -------------------------------------------------------------------- */ @@ -2756,7 +2758,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, if (hSrcDS) { CPLString osTmpFile = - CPLSPrintf("/vsimem/gmljp2_%p/%d/%s.gml", this, i, + CPLSPrintf("%s/%d/%s.gml", osTmpDir.c_str(), i, CPLGetBasename(aoGMLFiles[i].osFile)); char **papszOptions = nullptr; papszOptions = @@ -2864,7 +2866,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, !aoGMLFiles[i].osRemoteResource.empty()) { osTmpFile = - CPLSPrintf("/vsimem/gmljp2_%p/%d/%s.gml", this, i, + CPLSPrintf("%s/%d/%s.gml", osTmpDir.c_str(), i, CPLGetBasename(aoGMLFiles[i].osFile)); GMLJP2V2BoxDesc oDesc; @@ -3044,7 +3046,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, if (hSrcDS) { CPLString osTmpFile = - CPLSPrintf("/vsimem/gmljp2_%p/%d/%s.kml", this, i, + CPLSPrintf("%s/%d/%s.kml", osTmpDir.c_str(), i, CPLGetBasename(aoAnnotations[i].osFile)); char **papszOptions = nullptr; if (aoAnnotations.size() > 1) @@ -3260,7 +3262,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, for (auto &poGMLBox : apoGMLBoxes) delete poGMLBox; - VSIRmdirRecursive(CPLSPrintf("/vsimem/gmljp2_%p", this)); + VSIRmdirRecursive(osTmpDir.c_str()); return poGMLData; } diff --git a/gcore/gdaljp2structure.cpp b/gcore/gdaljp2structure.cpp index 6520f901ab02..d115e9cc782d 100644 --- a/gcore/gdaljp2structure.cpp +++ b/gcore/gdaljp2structure.cpp @@ -251,8 +251,7 @@ static void DumpGeoTIFFBox(CPLXMLNode *psBox, GDALJP2Box &oBox, static_cast(GDALGetDriverByName("VRT")); if (pabyBoxData && poVRTDriver) { - CPLString osTmpFilename( - CPLSPrintf("/vsimem/tmp_%p.tif", oBox.GetFILE())); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("tmp.tif")); CPL_IGNORE_RET_VAL(VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename, pabyBoxData, nBoxDataLength, FALSE))); CPLPushErrorHandler(CPLQuietErrorHandler); @@ -267,8 +266,8 @@ static void DumpGeoTIFFBox(CPLXMLNode *psBox, GDALJP2Box &oBox, } if (poDS) { - CPLString osTmpVRTFilename( - CPLSPrintf("/vsimem/tmp_%p.vrt", oBox.GetFILE())); + const CPLString osTmpVRTFilename( + CPLResetExtension(osTmpFilename.c_str(), "vrt")); GDALDataset *poVRTDS = poVRTDriver->CreateCopy( osTmpVRTFilename, poDS, FALSE, nullptr, nullptr, nullptr); GDALClose(poVRTDS); diff --git a/gcore/gdalmultidim.cpp b/gcore/gdalmultidim.cpp index b46bf17780fe..77576028b34b 100644 --- a/gcore/gdalmultidim.cpp +++ b/gcore/gdalmultidim.cpp @@ -8187,10 +8187,10 @@ std::shared_ptr GDALMDArrayResampled::Create( "Setting geolocation array from variables %s and %s", poLongVar->GetName().c_str(), poLatVar->GetName().c_str()); - std::string osFilenameLong = - CPLSPrintf("/vsimem/%p/longitude.tif", poParent.get()); - std::string osFilenameLat = - CPLSPrintf("/vsimem/%p/latitude.tif", poParent.get()); + const std::string osFilenameLong = + VSIMemGenerateHiddenFilename("longitude.tif"); + const std::string osFilenameLat = + VSIMemGenerateHiddenFilename("latitude.tif"); std::unique_ptr poTmpLongDS( longDimCount == 1 ? poLongVar->AsClassicDataset(0, 0) diff --git a/gcore/gdalmultidim_gridded.cpp b/gcore/gdalmultidim_gridded.cpp index 4a0741a903c7..56142a31f4fe 100644 --- a/gcore/gdalmultidim_gridded.cpp +++ b/gcore/gdalmultidim_gridded.cpp @@ -665,9 +665,8 @@ GDALMDArray::GetGridded(const std::string &osGridOptions, } // Create a in-memory vector layer with (X,Y) points - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/GDALMDArray::GetGridded_%p_%p.%s", this, - pOptions, pszExt); + const std::string osTmpFilename(VSIMemGenerateHiddenFilename( + std::string("tmp.").append(pszExt).c_str())); auto poDS = std::unique_ptr( poDrv->Create(osTmpFilename.c_str(), 0, 0, 0, GDT_Unknown, nullptr)); if (!poDS) diff --git a/gcore/gdalnodatamaskband.cpp b/gcore/gdalnodatamaskband.cpp index cd3b121acbbd..390957567d47 100644 --- a/gcore/gdalnodatamaskband.cpp +++ b/gcore/gdalnodatamaskband.cpp @@ -147,7 +147,8 @@ static GDALDataType GetWorkDataType(GDALDataType eDataType) eWrkDT = eDataType; break; - default: + case GDT_Unknown: + case GDT_TypeCount: CPLAssert(false); eWrkDT = GDT_Float64; break; diff --git a/gcore/gdalorienteddataset.cpp b/gcore/gdalorienteddataset.cpp index 11e49b4ebcd7..15e00d6941b0 100644 --- a/gcore/gdalorienteddataset.cpp +++ b/gcore/gdalorienteddataset.cpp @@ -149,8 +149,7 @@ CPLErr GDALOrientedRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, } else { - osTmpName = - CPLSPrintf("/vsimem/_gdalorienteddataset/%p.tif", this); + osTmpName = VSIMemGenerateHiddenFilename(nullptr); } } GDALTranslateOptions *psOptions = diff --git a/gcore/gdalpamdataset.cpp b/gcore/gdalpamdataset.cpp index f6759f42f5f4..bce73219368f 100644 --- a/gcore/gdalpamdataset.cpp +++ b/gcore/gdalpamdataset.cpp @@ -1403,7 +1403,8 @@ CPLErr GDALPamDataset::GetGeoTransform(double *padfTransform) { if (psPam && psPam->bHaveGeoTransform) { - memcpy(padfTransform, psPam->adfGeoTransform, sizeof(double) * 6); + memcpy(padfTransform, psPam->adfGeoTransform.data(), + sizeof(psPam->adfGeoTransform)); return CE_None; } @@ -1423,7 +1424,8 @@ CPLErr GDALPamDataset::SetGeoTransform(double *padfTransform) { MarkPamDirty(); psPam->bHaveGeoTransform = TRUE; - memcpy(psPam->adfGeoTransform, padfTransform, sizeof(double) * 6); + memcpy(psPam->adfGeoTransform.data(), padfTransform, + sizeof(psPam->adfGeoTransform)); return (CE_None); } @@ -1689,7 +1691,7 @@ CPLErr GDALPamDataset::TryLoadAux(CSLConstList papszSiblingFiles) /* -------------------------------------------------------------------- */ /* Geotransform. */ /* -------------------------------------------------------------------- */ - if (poAuxDS->GetGeoTransform(psPam->adfGeoTransform) == CE_None) + if (poAuxDS->GetGeoTransform(psPam->adfGeoTransform.data()) == CE_None) psPam->bHaveGeoTransform = TRUE; /* -------------------------------------------------------------------- */ diff --git a/gcore/gdalpamrasterband.cpp b/gcore/gdalpamrasterband.cpp index 670a4d3b38a4..adf8a0ddc4ec 100644 --- a/gcore/gdalpamrasterband.cpp +++ b/gcore/gdalpamrasterband.cpp @@ -50,6 +50,61 @@ #include "gdal_priv.h" #include "gdal_rat.h" +/************************************************************************/ +/* CopyFrom() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress + +void GDALRasterBandPamInfo::CopyFrom(const GDALRasterBandPamInfo &sOther) +{ + bNoDataValueSet = sOther.bNoDataValueSet; + bNoDataValueSetAsInt64 = sOther.bNoDataValueSetAsInt64; + bNoDataValueSetAsUInt64 = sOther.bNoDataValueSetAsUInt64; + + dfNoDataValue = sOther.dfNoDataValue; + nNoDataValueInt64 = sOther.nNoDataValueInt64; + nNoDataValueUInt64 = sOther.nNoDataValueUInt64; + + delete poColorTable; + poColorTable = sOther.poColorTable + ? new GDALColorTable(*(sOther.poColorTable)) + : nullptr; + + eColorInterp = sOther.eColorInterp; + + CPLFree(pszUnitType); + pszUnitType = sOther.pszUnitType ? CPLStrdup(sOther.pszUnitType) : nullptr; + + CSLDestroy(papszCategoryNames); + papszCategoryNames = CSLDuplicate(sOther.papszCategoryNames); + + dfOffset = sOther.dfOffset; + dfScale = sOther.dfScale; + + bHaveMinMax = sOther.bHaveMinMax; + dfMin = sOther.dfMin; + dfMax = sOther.dfMax; + + bHaveStats = sOther.bHaveStats; + dfMean = sOther.dfMean; + dfStdDev = sOther.dfStdDev; + + if (psSavedHistograms) + CPLDestroyXMLNode(psSavedHistograms); + psSavedHistograms = sOther.psSavedHistograms + ? CPLCloneXMLTree(sOther.psSavedHistograms) + : nullptr; + + delete poDefaultRAT; + poDefaultRAT = sOther.poDefaultRAT ? sOther.poDefaultRAT->Clone() : nullptr; + + bOffsetSet = sOther.bOffsetSet; + bScaleSet = sOther.bScaleSet; +} + +//! @endcond + /************************************************************************/ /* GDALPamRasterBand() */ /************************************************************************/ diff --git a/gcore/gdalproxydataset.cpp b/gcore/gdalproxydataset.cpp index cb6fb251653d..5cd560f0024b 100644 --- a/gcore/gdalproxydataset.cpp +++ b/gcore/gdalproxydataset.cpp @@ -137,6 +137,17 @@ CPLErr GDALProxyDataset::IRasterIO( return ret; } +D_PROXY_METHOD_WITH_RET(CPLErr, CE_Failure, BlockBasedRasterIO, + (GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, + int nYSize, void *pData, int nBufXSize, int nBufYSize, + GDALDataType eBufType, int nBandCount, + const int *panBandMap, GSpacing nPixelSpace, + GSpacing nLineSpace, GSpacing nBandSpace, + GDALRasterIOExtraArg *psExtraArg), + (eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, + nBufXSize, nBufYSize, eBufType, nBandCount, panBandMap, + nPixelSpace, nLineSpace, nBandSpace, psExtraArg)) + D_PROXY_METHOD_WITH_RET(CPLErr, CE_Failure, IBuildOverviews, (const char *pszResampling, int nOverviews, const int *panOverviewList, int nListBands, @@ -159,17 +170,8 @@ D_PROXY_METHOD_WITH_RET(CPLErr, CE_Failure, ReadCompressedData, panBandList, ppBuffer, pnBufferSize, ppszDetailedFormat)) -CPLErr GDALProxyDataset::FlushCache(bool bAtClosing) -{ - CPLErr eErr = CE_None; - GDALDataset *poUnderlyingDataset = RefUnderlyingDataset(); - if (poUnderlyingDataset) - { - eErr = poUnderlyingDataset->FlushCache(bAtClosing); - UnrefUnderlyingDataset(poUnderlyingDataset); - } - return eErr; -} +D_PROXY_METHOD_WITH_RET(CPLErr, CE_None, FlushCache, (bool bAtClosing), + (bAtClosing)) D_PROXY_METHOD_WITH_RET(char **, nullptr, GetMetadataDomainList, (), ()) D_PROXY_METHOD_WITH_RET(char **, nullptr, GetMetadata, (const char *pszDomain), @@ -386,6 +388,18 @@ RB_PROXY_METHOD_WITH_RET(CPLErr, CE_Failure, SetMetadataItem, const char *pszDomain), (pszName, pszValue, pszDomain)) +RB_PROXY_METHOD_WITH_RET(GDALRasterBlock *, nullptr, GetLockedBlockRef, + (int nXBlockOff, int nYBlockOff, int bJustInitialize), + (nXBlockOff, nYBlockOff, bJustInitialize)) + +RB_PROXY_METHOD_WITH_RET(GDALRasterBlock *, nullptr, TryGetLockedBlockRef, + (int nXBlockOff, int nYBlockOff), + (nXBlockOff, nYBlockOff)) + +RB_PROXY_METHOD_WITH_RET(CPLErr, CE_Failure, FlushBlock, + (int nXBlockOff, int nYBlockOff, int bWriteDirtyBlock), + (nXBlockOff, nYBlockOff, bWriteDirtyBlock)) + CPLErr GDALProxyRasterBand::FlushCache(bool bAtClosing) { // We need to make sure that all cached bocks at the proxy level are @@ -512,6 +526,22 @@ RB_PROXY_METHOD_WITH_RET(CPLVirtualMem *, nullptr, GetVirtualMemAuto, GIntBig *pnLineSpace, char **papszOptions), (eRWFlag, pnPixelSpace, pnLineSpace, papszOptions)) +RB_PROXY_METHOD_WITH_RET( + CPLErr, CE_Failure, InterpolateAtPoint, + (double dfPixel, double dfLine, GDALRIOResampleAlg eInterpolation, + double *pdfRealValue, double *pdfImagValue = nullptr) const, + (dfPixel, dfLine, eInterpolation, pdfRealValue, pdfImagValue)) + +void GDALProxyRasterBand::EnablePixelTypeSignedByteWarning(bool b) +{ + GDALRasterBand *poSrcBand = RefUnderlyingRasterBand(); + if (poSrcBand) + { + poSrcBand->EnablePixelTypeSignedByteWarning(b); + UnrefUnderlyingRasterBand(poSrcBand); + } +} + /************************************************************************/ /* UnrefUnderlyingRasterBand() */ /************************************************************************/ diff --git a/gcore/gdalpython.cpp b/gcore/gdalpython.cpp index a70756c55e33..4f944c52e3ae 100644 --- a/gcore/gdalpython.cpp +++ b/gcore/gdalpython.cpp @@ -378,7 +378,7 @@ static bool LoadPythonAPI() "-c", pszPrintVersion, nullptr}; const CPLString osTmpFilename( - "/vsimem/LoadPythonAPI/out.txt"); + VSIMemGenerateHiddenFilename("out.txt")); VSILFILE *fout = VSIFOpenL(osTmpFilename, "wb+"); if (CPLSpawn(apszArgv, nullptr, fout, FALSE) == 0) { @@ -428,10 +428,10 @@ static bool LoadPythonAPI() const char *const apszPythonSO[] = { "libpython3.8." SO_EXT, "libpython3.9." SO_EXT, "libpython3.10." SO_EXT, "libpython3.11." SO_EXT, - "libpython3.12." SO_EXT, "libpython3.7m." SO_EXT, - "libpython3.6m." SO_EXT, "libpython3.5m." SO_EXT, - "libpython3.4m." SO_EXT, "libpython3.3." SO_EXT, - "libpython3.2." SO_EXT}; + "libpython3.12." SO_EXT, "libpython3.13." SO_EXT, + "libpython3.7m." SO_EXT, "libpython3.6m." SO_EXT, + "libpython3.5m." SO_EXT, "libpython3.4m." SO_EXT, + "libpython3.3." SO_EXT, "libpython3.2." SO_EXT}; for (size_t i = 0; libHandle == nullptr && i < CPL_ARRAYSIZE(apszPythonSO); ++i) { @@ -623,9 +623,9 @@ static bool LoadPythonAPI() if (libHandle == nullptr) { const char *const apszPythonSO[] = { - "python38.dll", "python39.dll", "python310.dll", "python311.dll", - "python312.dll", "python37.dll", "python36.dll", "python35.dll", - "python34.dll", "python33.dll", "python32.dll"}; + "python38.dll", "python39.dll", "python310.dll", "python311.dll", + "python312.dll", "python313.dll", "python37.dll", "python36.dll", + "python35.dll", "python34.dll", "python33.dll", "python32.dll"}; UINT uOldErrorMode; uOldErrorMode = SetErrorMode(SEM_NOOPENFILEERRORBOX | SEM_FAILCRITICALERRORS); diff --git a/gcore/gdalrasterband.cpp b/gcore/gdalrasterband.cpp index 93dbe886bed4..34914188e5a8 100644 --- a/gcore/gdalrasterband.cpp +++ b/gcore/gdalrasterband.cpp @@ -107,6 +107,8 @@ GDALRasterBand::~GDALRasterBand() InvalidateMaskBand(); nBand = -nBand; + + delete m_poPointsCache; } /************************************************************************/ @@ -131,6 +133,12 @@ GDALRasterBand::~GDALRasterBand() * buffer size (nBufXSize x nBufYSize) is different than the size of the * region being accessed (nXSize x nYSize). * + * The window of interest expressed by (nXOff, nYOff, nXSize, nYSize) should be + * fully within the raster space, that is nXOff >= 0, nYOff >= 0, + * nXOff + nXSize <= GetXSize() and nYOff + nYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * The nPixelSpace and nLineSpace parameters allow reading into or * writing from unusually organized buffers. This is primarily used * for buffers containing more than one bands raster data in interleaved @@ -232,6 +240,12 @@ GDALRasterBand::~GDALRasterBand() * buffer size (nBufXSize x nBufYSize) is different than the size of the * region being accessed (nXSize x nYSize). * + * The window of interest expressed by (nXOff, nYOff, nXSize, nYSize) should be + * fully within the raster space, that is nXOff >= 0, nYOff >= 0, + * nXOff + nXSize <= GetXSize() and nYOff + nYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * The nPixelSpace and nLineSpace parameters allow reading into or * writing from unusually organized buffers. This is primarily used * for buffers containing more than one bands raster data in interleaved @@ -580,6 +594,12 @@ DEFINE_GetGDTFromCppType(std::complex, GDT_CFloat64); * be called on the same GDALRasterBand instance (or another GDALRasterBand * instance of this dataset) concurrently from several threads. * + * The window of interest expressed by (dfXOff, dfYOff, dfXSize, dfYSize) should be + * fully within the raster space, that is dfXOff >= 0, dfYOff >= 0, + * dfXOff + dfXSize <= GetXSize() and dfYOff + dfYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * @param[out] pData The buffer into which the data should be written. * This buffer must contain at least nBufXSize * * nBufYSize words of type T. It is organized in left to right, @@ -796,6 +816,12 @@ INSTANTIATE_READ_RASTER(std::complex) * be called on the same GDALRasterBand instance (or another GDALRasterBand * instance of this dataset) concurrently from several threads. * + * The window of interest expressed by (dfXOff, dfYOff, dfXSize, dfYSize) should be + * fully within the raster space, that is dfXOff >= 0, dfYOff >= 0, + * dfXOff + dfXSize <= GetXSize() and dfYOff + dfYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * @param[out] vData The vector into which the data should be written. * The vector will be resized, if needed, to contain at least nBufXSize * * nBufYSize values. The values in the vector are organized in left to right, @@ -1306,7 +1332,7 @@ CPLErr CPL_STDCALL GDALWriteBlock(GDALRasterBandH hBand, int nXOff, int nYOff, * @since GDAL 2.2 */ CPLErr GDALRasterBand::GetActualBlockSize(int nXBlockOff, int nYBlockOff, - int *pnXValid, int *pnYValid) + int *pnXValid, int *pnYValid) const { if (nXBlockOff < 0 || nBlockXSize == 0 || nXBlockOff >= DIV_ROUND_UP(nRasterXSize, nBlockXSize) || @@ -1408,7 +1434,7 @@ GDALRasterBand::GetSuggestedBlockAccessPattern() const * @return the data type of pixels for this band. */ -GDALDataType GDALRasterBand::GetRasterDataType() +GDALDataType GDALRasterBand::GetRasterDataType() const { return eDataType; @@ -1459,7 +1485,7 @@ GDALDataType CPL_STDCALL GDALGetRasterDataType(GDALRasterBandH hBand) * @param pnYSize integer to put the Y block size into or NULL. */ -void GDALRasterBand::GetBlockSize(int *pnXSize, int *pnYSize) +void GDALRasterBand::GetBlockSize(int *pnXSize, int *pnYSize) const { if (nBlockXSize <= 0 || nBlockYSize <= 0) @@ -3351,12 +3377,10 @@ GDALGetRasterSampleOverviewEx(GDALRasterBandH hBand, GUIntBig nDesiredSamples) * CE_Failure is returned, and CPLGetLastErrorNo() will return * CPLE_NotSupported. * - * WARNING: It is not possible to build overviews for a single band in - * TIFF format, and thus this method does not work for TIFF format, or any - * formats that use the default overview building in TIFF format. Instead - * it is necessary to build overviews on the dataset as a whole using - * GDALDataset::BuildOverviews(). That makes this method pretty useless - * from a practical point of view. + * WARNING: Most formats don't support per-band overview computation, but + * require that overviews are computed for all bands of a dataset, using + * GDALDataset::BuildOverviews(). The only exception for official GDAL drivers + * is the HFA driver which supports this method. * * @param pszResampling one of "NEAREST", "GAUSS", "CUBIC", "AVERAGE", "MODE", * "AVERAGE_MAGPHASE" "RMS" or "NONE" controlling the downsampling method @@ -3708,7 +3732,7 @@ CPLErr CPL_STDCALL GDALSetRasterUnitType(GDALRasterBandH hBand, * @return the width in pixels of this band. */ -int GDALRasterBand::GetXSize() +int GDALRasterBand::GetXSize() const { return nRasterXSize; @@ -3745,7 +3769,7 @@ int CPL_STDCALL GDALGetRasterBandXSize(GDALRasterBandH hBand) * @return the height in pixels of this band. */ -int GDALRasterBand::GetYSize() +int GDALRasterBand::GetYSize() const { return nRasterYSize; @@ -3787,7 +3811,7 @@ int CPL_STDCALL GDALGetRasterBandYSize(GDALRasterBandH hBand) * @return band number (1+) or 0 if the band number isn't known. */ -int GDALRasterBand::GetBand() +int GDALRasterBand::GetBand() const { return nBand; @@ -3828,7 +3852,7 @@ int CPL_STDCALL GDALGetBandNumber(GDALRasterBandH hBand) * NULL if this cannot be determined. */ -GDALDataset *GDALRasterBand::GetDataset() +GDALDataset *GDALRasterBand::GetDataset() const { return poDS; @@ -8413,7 +8437,7 @@ void GDALRasterBand::IncDirtyBlocks(int nInc) */ void GDALRasterBand::ReportError(CPLErr eErrClass, CPLErrorNum err_no, - const char *fmt, ...) + const char *fmt, ...) const { va_list args; @@ -9400,11 +9424,11 @@ CPLErr GDALRasterBand::InterpolateAtPoint(double dfPixel, double dfLine, } GDALRasterBand *pBand = const_cast(this); - if (!m_oPointsCache) - m_oPointsCache = std::make_unique(); + if (!m_poPointsCache) + m_poPointsCache = new GDALDoublePointsCache(); const bool res = - GDALInterpolateAtPoint(pBand, eInterpolation, m_oPointsCache->cache, + GDALInterpolateAtPoint(pBand, eInterpolation, m_poPointsCache->cache, dfPixel, dfLine, pdfRealValue, pdfImagValue); return res ? CE_None : CE_Failure; diff --git a/gcore/gdalthreadsafedataset.cpp b/gcore/gdalthreadsafedataset.cpp new file mode 100644 index 000000000000..3021768ff48c --- /dev/null +++ b/gcore/gdalthreadsafedataset.cpp @@ -0,0 +1,1286 @@ +/****************************************************************************** + * + * Project: GDAL Core + * Purpose: Base class for thread safe dataset + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef DOXYGEN_SKIP + +#include "cpl_mem_cache.h" +#include "gdal_proxy.h" +#include "gdal_rat.h" +#include "gdal_priv.h" + +#include +#include +#include +#include +#include + +/** Design notes of this file. + * + * This file is at the core of the "RFC 101 - Raster dataset read-only thread-safety". + * Please consult it for high level understanding. + * + * 3 classes are involved: + * - GDALThreadSafeDataset whose instances are returned to the user, and can + * use them in a thread-safe way. + * - GDALThreadSafeRasterBand whose instances are created (and owned) by a + * GDALThreadSafeDataset instance, and returned to the user, which can use + * them in a thread-safe way. + * - GDALThreadLocalDatasetCache which is an internal class, which holds the + * thread-local datasets. + */ + +/************************************************************************/ +/* GDALThreadLocalDatasetCache */ +/************************************************************************/ + +class GDALThreadSafeDataset; + +/** This class is instantiated once per thread that uses a + * GDALThreadSafeDataset instance. It holds mostly a cache that maps a + * GDALThreadSafeDataset* pointer to the corresponding per-thread dataset. + */ +class GDALThreadLocalDatasetCache +{ + private: + /** Least-recently-used based cache that maps a GDALThreadSafeDataset* + * instance to the corresponding per-thread dataset. + * It should be noted as this a LRU cache, entries might get evicted when + * its capacity is reached (64 datasets), which might be undesirable. + * Hence it is doubled with m_oMapReferencedDS for datasets that are in + * active used by a thread. + * + * This cache is created as a unique_ptr, and not a standard object, for + * delicate reasons related to application termination, where we might + * want to leak the memory associated to it, to avoid the dataset it + * references from being closed, after GDAL has been "closed" (typically + * GDALDestroyDriverManager() has been called), which would otherwise lead + * to crashes. + */ + std::unique_ptr>> + m_poCache{}; + + GDALThreadLocalDatasetCache(const GDALThreadLocalDatasetCache &) = delete; + GDALThreadLocalDatasetCache & + operator=(const GDALThreadLocalDatasetCache &) = delete; + + public: + GDALThreadLocalDatasetCache(); + ~GDALThreadLocalDatasetCache(); + + /** Thread-id of the thread that instantiated this object. Used only for + * CPLDebug() purposes + */ + GIntBig m_nThreadID = 0; + + /** Mutex that protects access to m_oCache. There is "competition" around + * access to m_oCache since the destructor of a GDALThreadSafeDataset + * instance needs to evict entries corresponding to itself from all + * GDALThreadLocalDatasetCache instances. + */ + std::mutex m_oMutex{}; + + /** This is a reference to *(m_poCache.get()). + */ + lru11::Cache> + &m_oCache; + + /** Pair of shared_ptr with associated thread-local config + * options that were valid in the calling thread at the time + * GDALThreadLocalDatasetCache::RefUnderlyingDataset() was called, so they + * can be restored at UnrefUnderlyingDataset() time. + */ + struct SharedPtrDatasetThreadLocalConfigOptionsPair + { + std::shared_ptr poDS; + CPLStringList aosTLConfigOptions; + + SharedPtrDatasetThreadLocalConfigOptionsPair( + const std::shared_ptr &poDSIn, + CPLStringList &&aosTLConfigOptionsIn) + : poDS(poDSIn), aosTLConfigOptions(std::move(aosTLConfigOptionsIn)) + { + } + }; + + /** Maps a GDALThreadSafeDataset* + * instance to the corresponding per-thread dataset. Insertion into this + * map is done by GDALThreadLocalDatasetCache::RefUnderlyingDataset() and + * removal by UnrefUnderlyingDataset(). In most all use cases, the size of + * this map should be 0 or 1 (not clear if it could be more, that would + * involve RefUnderlyingDataset() being called in nested ways by the same + * thread, but it doesn't hurt from being robust to that potential situation) + */ + std::map + m_oMapReferencedDS{}; + + /** Maps a GDALRasterBand* returned by GDALThreadSafeRasterBand::RefUnderlyingDataset() + * to the (thread-local) dataset that owns it (that is a dataset returned + * by RefUnderlyingDataset(). The size of his map should be 0 or 1 in + * most cases. + */ + std::map m_oMapReferencedDSFromBand{}; +}; + +/************************************************************************/ +/* GDALThreadSafeDataset */ +/************************************************************************/ + +/** Global variable used to determine if the singleton of GlobalCache + * owned by GDALThreadSafeDataset is valid. + * This is needed to avoid issues at process termination where the order + * of destruction between static global instances and TLS instances can be + * tricky. + */ +static bool bGlobalCacheValid = false; + +/** Thread-safe GDALDataset class. + * + * That class delegates all calls to its members to per-thread GDALDataset + * instances. + */ +class GDALThreadSafeDataset final : public GDALProxyDataset +{ + public: + GDALThreadSafeDataset(std::unique_ptr poPrototypeDSUniquePtr, + GDALDataset *poPrototypeDS); + ~GDALThreadSafeDataset() override; + + static std::unique_ptr + Create(std::unique_ptr poPrototypeDS, int nScopeFlags); + + static GDALDataset *Create(GDALDataset *poPrototypeDS, int nScopeFlags); + + /* All below public methods override GDALDataset methods, and instead of + * forwarding to a thread-local dataset, they act on the prototype dataset, + * because they return a non-trivial type, that could be invalidated + * otherwise if the thread-local dataset is evicted from the LRU cache. + */ + const OGRSpatialReference *GetSpatialRef() const override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + if (m_oSRS.IsEmpty()) + { + auto poSRS = m_poPrototypeDS->GetSpatialRef(); + if (poSRS) + { + m_oSRS.AssignAndSetThreadSafe(*poSRS); + } + } + return m_oSRS.IsEmpty() ? nullptr : &m_oSRS; + } + + const OGRSpatialReference *GetGCPSpatialRef() const override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + if (m_oGCPSRS.IsEmpty()) + { + auto poSRS = m_poPrototypeDS->GetGCPSpatialRef(); + if (poSRS) + { + m_oGCPSRS.AssignAndSetThreadSafe(*poSRS); + } + } + return m_oGCPSRS.IsEmpty() ? nullptr : &m_oGCPSRS; + } + + const GDAL_GCP *GetGCPs() override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + return const_cast(m_poPrototypeDS)->GetGCPs(); + } + + const char *GetMetadataItem(const char *pszName, + const char *pszDomain = "") override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + return const_cast(m_poPrototypeDS) + ->GetMetadataItem(pszName, pszDomain); + } + + char **GetMetadata(const char *pszDomain = "") override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + return const_cast(m_poPrototypeDS) + ->GetMetadata(pszDomain); + } + + /* End of methods that forward on the prototype dataset */ + + GDALAsyncReader *BeginAsyncReader(int, int, int, int, void *, int, int, + GDALDataType, int, int *, int, int, int, + char **) override + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeDataset::BeginAsyncReader() not supported"); + return nullptr; + } + + protected: + GDALDataset *RefUnderlyingDataset() const override; + + void + UnrefUnderlyingDataset(GDALDataset *poUnderlyingDataset) const override; + + int CloseDependentDatasets() override; + + private: + friend class GDALThreadSafeRasterBand; + friend class GDALThreadLocalDatasetCache; + + /** Mutex that protects accesses to m_poPrototypeDS */ + mutable std::mutex m_oPrototypeDSMutex{}; + + /** "Prototype" dataset, that is the dataset that was passed to the + * GDALThreadSafeDataset constructor. All calls on to it should be on + * const methods, and should be protected by m_oPrototypeDSMutex (except + * during GDALThreadSafeDataset instance construction) + */ + const GDALDataset *m_poPrototypeDS = nullptr; + + /** Unique pointer for m_poPrototypeDS in the cases where GDALThreadSafeDataset + * has been passed a unique pointer */ + std::unique_ptr m_poPrototypeDSUniquePtr{}; + + /** Thread-local config options at the time where GDALThreadSafeDataset + * has been constructed. + */ + const CPLStringList m_aosThreadLocalConfigOptions{}; + + /** Cached value returned by GetSpatialRef() */ + mutable OGRSpatialReference m_oSRS{}; + + /** Cached value returned by GetGCPSpatialRef() */ + mutable OGRSpatialReference m_oGCPSRS{}; + + /** Structure that references all GDALThreadLocalDatasetCache* instances. + */ + struct GlobalCache + { + /** Mutex that protect access to oSetOfCache */ + std::mutex oMutex{}; + + /** Set of GDALThreadLocalDatasetCache* instances. That is it has + * one entry per thread that has used at least once a + * GDALThreadLocalDatasetCache/GDALThreadSafeRasterBand + */ + std::set oSetOfCache{}; + + GlobalCache() + { + bGlobalCacheValid = true; + } + + ~GlobalCache() + { + bGlobalCacheValid = false; + } + }; + + /** Returns a singleton for a GlobalCache instance that references all + * GDALThreadLocalDatasetCache* instances. + */ + static GlobalCache &GetSetOfCache() + { + static GlobalCache cache; + return cache; + } + + /** Thread-local dataset cache. */ + static thread_local std::unique_ptr tl_poCache; + + void UnrefUnderlyingDataset(GDALDataset *poUnderlyingDataset, + GDALThreadLocalDatasetCache *poCache) const; + + GDALThreadSafeDataset(const GDALThreadSafeDataset &) = delete; + GDALThreadSafeDataset &operator=(const GDALThreadSafeDataset &) = delete; +}; + +/************************************************************************/ +/* GDALThreadSafeRasterBand */ +/************************************************************************/ + +/** Thread-safe GDALRasterBand class. + * + * That class delegates all calls to its members to per-thread GDALDataset + * instances. + */ +class GDALThreadSafeRasterBand final : public GDALProxyRasterBand +{ + public: + GDALThreadSafeRasterBand(GDALThreadSafeDataset *poTSDS, + GDALDataset *poParentDS, int nBandIn, + GDALRasterBand *poPrototypeBand, + int nBaseBandOfMaskBand, int nOvrIdx); + + GDALRasterBand *GetMaskBand() override; + int GetOverviewCount() override; + GDALRasterBand *GetOverview(int idx) override; + GDALRasterBand *GetRasterSampleOverview(GUIntBig nDesiredSamples) override; + + GDALRasterAttributeTable *GetDefaultRAT() override; + + /* All below public methods override GDALRasterBand methods, and instead of + * forwarding to a thread-local dataset, they act on the prototype band, + * because they return a non-trivial type, that could be invalidated + * otherwise if the thread-local dataset is evicted from the LRU cache. + */ + const char *GetMetadataItem(const char *pszName, + const char *pszDomain = "") override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand) + ->GetMetadataItem(pszName, pszDomain); + } + + char **GetMetadata(const char *pszDomain = "") override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand) + ->GetMetadata(pszDomain); + } + + const char *GetUnitType() override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand)->GetUnitType(); + } + + GDALColorTable *GetColorTable() override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand)->GetColorTable(); + } + + /* End of methods that forward on the prototype band */ + + CPLVirtualMem *GetVirtualMemAuto(GDALRWFlag, int *, GIntBig *, + char **) override + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::GetVirtualMemAuto() not supported"); + return nullptr; + } + + protected: + GDALRasterBand *RefUnderlyingRasterBand(bool bForceOpen) const override; + void UnrefUnderlyingRasterBand( + GDALRasterBand *poUnderlyingRasterBand) const override; + + private: + /** Pointer to the thread-safe dataset from which this band has been + *created */ + GDALThreadSafeDataset *m_poTSDS = nullptr; + + /** Pointer to the "prototype" raster band that corresponds to us. + * All calls to m_poPrototypeBand should be protected by + * GDALThreadSafeDataset:m_oPrototypeDSMutex. + */ + const GDALRasterBand *m_poPrototypeBand = nullptr; + + /** 0 for standard bands, otherwise > 0 value that indicates that this + * band is a mask band and m_nBaseBandOfMaskBand is then the number + * of the band that is the parent of the mask band. + */ + const int m_nBaseBandOfMaskBand; + + /** 0 for standard bands, otherwise >= 0 value that indicates that this + * band is an overview band and m_nOvrIdx is then the index of the overview. + */ + const int m_nOvrIdx; + + /** Mask band associated with this band. */ + std::unique_ptr m_poMaskBand{}; + + /** List of overviews associated with this band. */ + std::vector> m_apoOverviews{}; + + GDALThreadSafeRasterBand(const GDALThreadSafeRasterBand &) = delete; + GDALThreadSafeRasterBand & + operator=(const GDALThreadSafeRasterBand &) = delete; +}; + +/************************************************************************/ +/* Global variables initialization. */ +/************************************************************************/ + +/** Instantiation of the TLS cache of datasets */ +thread_local std::unique_ptr + GDALThreadSafeDataset::tl_poCache; + +/************************************************************************/ +/* GDALThreadLocalDatasetCache() */ +/************************************************************************/ + +/** Constructor of GDALThreadLocalDatasetCache. This is called implicitly + * when GDALThreadSafeDataset::tl_poCache is called the first time by a + * thread. + */ +GDALThreadLocalDatasetCache::GDALThreadLocalDatasetCache() + : m_poCache(std::make_unique>>()), + m_nThreadID(CPLGetPID()), m_oCache(*m_poCache.get()) +{ + CPLDebug("GDAL", + "Registering thread-safe dataset cache for thread " CPL_FRMT_GIB, + m_nThreadID); + + // We reference ourselves to the GDALThreadSafeDataset set-of-cache singleton + auto &oSetOfCache = GDALThreadSafeDataset::GetSetOfCache(); + std::lock_guard oLock(oSetOfCache.oMutex); + oSetOfCache.oSetOfCache.insert(this); +} + +/************************************************************************/ +/* ~GDALThreadLocalDatasetCache() */ +/************************************************************************/ + +/** Destructor of GDALThreadLocalDatasetCache. This is called implicitly when a + * thread is terminated. + */ +GDALThreadLocalDatasetCache::~GDALThreadLocalDatasetCache() +{ + // If GDAL has been de-initialized explicitly (ie GDALDestroyDriverManager() + // has been called), or we are during process termination, do not try to + // free m_poCache at all, which would cause the datasets its owned to be + // destroyed, which will generally lead to crashes in those situations where + // GDAL has been de-initialized. + const bool bDriverManagerDestroyed = *GDALGetphDMMutex() == nullptr; + if (bDriverManagerDestroyed || !bGlobalCacheValid) + { + // Leak datasets when GDAL has been de-initialized + if (!m_poCache->empty()) + { + // coverity[leaked_storage] + CPL_IGNORE_RET_VAL(m_poCache.release()); + } + return; + } + + // Unreference ourselves from the GDALThreadSafeDataset set-of-cache singleton + CPLDebug("GDAL", + "Unregistering thread-safe dataset cache for thread " CPL_FRMT_GIB, + m_nThreadID); + { + auto &oSetOfCache = GDALThreadSafeDataset::GetSetOfCache(); + std::lock_guard oLock(oSetOfCache.oMutex); + oSetOfCache.oSetOfCache.erase(this); + } + + // Below code is just for debugging purposes and show which internal + // thread-local datasets are released at thread termination. + const auto lambda = + [this](const lru11::KeyValuePair> &kv) + { + CPLDebug("GDAL", + "~GDALThreadLocalDatasetCache(): GDALClose(%s, this=%p) " + "for thread " CPL_FRMT_GIB, + kv.value->GetDescription(), kv.value.get(), m_nThreadID); + }; + m_oCache.cwalk(lambda); +} + +/************************************************************************/ +/* GDALThreadSafeDataset() */ +/************************************************************************/ + +/** Constructor of GDALThreadSafeDataset. + * It may be called with poPrototypeDSUniquePtr set to null, in the situations + * where GDALThreadSafeDataset doesn't own the prototype dataset. + * poPrototypeDS should always be not-null, and if poPrototypeDSUniquePtr is + * not null, then poPrototypeDS should be equal to poPrototypeDSUniquePtr.get() + */ +GDALThreadSafeDataset::GDALThreadSafeDataset( + std::unique_ptr poPrototypeDSUniquePtr, + GDALDataset *poPrototypeDS) + : m_poPrototypeDS(poPrototypeDS), + m_aosThreadLocalConfigOptions(CPLGetThreadLocalConfigOptions()) +{ + CPLAssert(poPrototypeDS != nullptr); + if (poPrototypeDSUniquePtr) + { + CPLAssert(poPrototypeDS == poPrototypeDSUniquePtr.get()); + } + + // Replicate the characteristics of the prototype dataset onto ourselves + nRasterXSize = poPrototypeDS->GetRasterXSize(); + nRasterYSize = poPrototypeDS->GetRasterYSize(); + for (int i = 1; i <= poPrototypeDS->GetRasterCount(); ++i) + { + SetBand(i, std::make_unique( + this, this, i, poPrototypeDS->GetRasterBand(i), 0, -1)); + } + nOpenFlags = GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE; + SetDescription(poPrototypeDS->GetDescription()); + papszOpenOptions = CSLDuplicate(poPrototypeDS->GetOpenOptions()); + + m_poPrototypeDSUniquePtr = std::move(poPrototypeDSUniquePtr); + + // In the case where we are constructed without owning the prototype + // dataset, let's increase its reference counter though. + if (!m_poPrototypeDSUniquePtr) + const_cast(m_poPrototypeDS)->Reference(); +} + +/************************************************************************/ +/* Create() */ +/************************************************************************/ + +/** Utility method used by GDALGetThreadSafeDataset() to construct a + * GDALThreadSafeDataset instance in the case where the GDALThreadSafeDataset + * instance owns the prototype dataset. + */ + +/* static */ std::unique_ptr +GDALThreadSafeDataset::Create(std::unique_ptr poPrototypeDS, + int nScopeFlags) +{ + if (nScopeFlags != GDAL_OF_RASTER) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Only nScopeFlags == " + "GDAL_OF_RASTER is supported"); + return nullptr; + } + if (poPrototypeDS->IsThreadSafe(nScopeFlags)) + { + return poPrototypeDS; + } + if (!poPrototypeDS->CanBeCloned(nScopeFlags, /* bCanShareState = */ true)) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Source dataset cannot be " + "cloned"); + return nullptr; + } + auto poPrototypeDSRaw = poPrototypeDS.get(); + return std::make_unique(std::move(poPrototypeDS), + poPrototypeDSRaw); +} + +/************************************************************************/ +/* Create() */ +/************************************************************************/ + +/** Utility method used by GDALGetThreadSafeDataset() to construct a + * GDALThreadSafeDataset instance in the case where the GDALThreadSafeDataset + * instance does not own the prototype dataset. + */ + +/* static */ GDALDataset * +GDALThreadSafeDataset::Create(GDALDataset *poPrototypeDS, int nScopeFlags) +{ + if (nScopeFlags != GDAL_OF_RASTER) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Only nScopeFlags == " + "GDAL_OF_RASTER is supported"); + return nullptr; + } + if (poPrototypeDS->IsThreadSafe(nScopeFlags)) + { + poPrototypeDS->Reference(); + return poPrototypeDS; + } + if (!poPrototypeDS->CanBeCloned(nScopeFlags, /* bCanShareState = */ true)) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Source dataset cannot be " + "cloned"); + return nullptr; + } + return std::make_unique(nullptr, poPrototypeDS) + .release(); +} + +/************************************************************************/ +/* ~GDALThreadSafeDataset() */ +/************************************************************************/ + +GDALThreadSafeDataset::~GDALThreadSafeDataset() +{ + // Collect TLS datasets in a vector, and free them after releasing + // g_nInDestructorCounter to limit contention + std::vector, GIntBig>> aoDSToFree; + { + auto &oSetOfCache = GetSetOfCache(); + std::lock_guard oLock(oSetOfCache.oMutex); + for (auto *poCache : oSetOfCache.oSetOfCache) + { + std::unique_lock oLockCache(poCache->m_oMutex); + std::shared_ptr poDS; + if (poCache->m_oCache.tryGet(this, poDS)) + { + aoDSToFree.emplace_back(std::move(poDS), poCache->m_nThreadID); + poCache->m_oCache.remove(this); + } + } + } + + for (const auto &oEntry : aoDSToFree) + { + CPLDebug("GDAL", + "~GDALThreadSafeDataset(): GDALClose(%s, this=%p) for " + "thread " CPL_FRMT_GIB, + GetDescription(), oEntry.first.get(), oEntry.second); + } + // Actually release TLS datasets + aoDSToFree.clear(); + + GDALThreadSafeDataset::CloseDependentDatasets(); +} + +/************************************************************************/ +/* CloseDependentDatasets() */ +/************************************************************************/ + +/** Implements GDALDataset::CloseDependentDatasets() + * + * Takes care of releasing the prototype dataset. + * + * As implied by the contract of CloseDependentDatasets(), returns true if + * the prototype dataset has actually been released (or false if + * CloseDependentDatasets() has already been closed) + */ +int GDALThreadSafeDataset::CloseDependentDatasets() +{ + int bRet = false; + if (m_poPrototypeDSUniquePtr) + { + bRet = true; + } + else if (m_poPrototypeDS) + { + if (const_cast(m_poPrototypeDS)->ReleaseRef()) + { + bRet = true; + } + } + + m_poPrototypeDSUniquePtr.reset(); + m_poPrototypeDS = nullptr; + + return bRet; +} + +/************************************************************************/ +/* RefUnderlyingDataset() */ +/************************************************************************/ + +/** Implements GDALProxyDataset::RefUnderlyingDataset. + * + * This method is called by all virtual methods of GDALDataset overridden by + * RefUnderlyingDataset() when it delegates the calls to the underlying + * dataset. + * + * Our implementation takes care of opening a thread-local dataset, on the + * same underlying dataset of m_poPrototypeDS, if needed, and to insert it + * into a cache for fast later uses by the same thread. + */ +GDALDataset *GDALThreadSafeDataset::RefUnderlyingDataset() const +{ + // Back-up thread-local config options at the time we are called + CPLStringList aosTLConfigOptionsBackup(CPLGetThreadLocalConfigOptions()); + + // Now merge the thread-local config options at the time where this + // instance has been created with the current ones. + const CPLStringList aosMerged( + CSLMerge(CSLDuplicate(m_aosThreadLocalConfigOptions.List()), + aosTLConfigOptionsBackup.List())); + + // And make that merged list active + CPLSetThreadLocalConfigOptions(aosMerged.List()); + + std::shared_ptr poTLSDS; + + // Get the thread-local dataset cache for this thread. + GDALThreadLocalDatasetCache *poCache = tl_poCache.get(); + if (!poCache) + { + auto poCacheUniquePtr = std::make_unique(); + poCache = poCacheUniquePtr.get(); + tl_poCache = std::move(poCacheUniquePtr); + } + + // Check if there's an entry in this cache for our current GDALThreadSafeDataset + // instance. + std::unique_lock oLock(poCache->m_oMutex); + if (poCache->m_oCache.tryGet(this, poTLSDS)) + { + // If so, return it, but before returning, make sure to creates a + // "hard" reference to the thread-local dataset, in case it would + // get evicted from poCache->m_oCache (by other threads that would + // access lots of datasets in between) + CPLAssert(!cpl::contains(poCache->m_oMapReferencedDS, this)); + auto poDSRet = poTLSDS.get(); + poCache->m_oMapReferencedDS.insert( + {this, GDALThreadLocalDatasetCache:: + SharedPtrDatasetThreadLocalConfigOptionsPair( + poTLSDS, std::move(aosTLConfigOptionsBackup))}); + return poDSRet; + } + + // "Clone" the prototype dataset, which in 99% of the cases, involves + // doing a GDALDataset::Open() call to re-open it. Do that by temporarily + // dropping the lock that protects poCache->m_oCache. + // coverity[uninit_use_in_call] + oLock.unlock(); + poTLSDS = m_poPrototypeDS->Clone(GDAL_OF_RASTER, /* bCanShareState=*/true); + if (poTLSDS) + { + CPLDebug("GDAL", "GDALOpen(%s, this=%p) for thread " CPL_FRMT_GIB, + GetDescription(), poTLSDS.get(), CPLGetPID()); + + // Check that the re-openeded dataset has the same characteristics + // as "this" / m_poPrototypeDS + if (poTLSDS->GetRasterXSize() != nRasterXSize || + poTLSDS->GetRasterYSize() != nRasterYSize || + poTLSDS->GetRasterCount() != nBands) + { + poTLSDS.reset(); + CPLError(CE_Failure, CPLE_AppDefined, + "Re-opened dataset for %s does not share the same " + "characteristics has the master dataset", + GetDescription()); + } + } + + // Re-acquire the lok + oLock.lock(); + + // In case of failed closing, restore the thread-local config options that + // were valid at the beginning of this method, and return in error. + if (!poTLSDS) + { + CPLSetThreadLocalConfigOptions(aosTLConfigOptionsBackup.List()); + return nullptr; + } + + // We have managed to get a thread-local dataset. Insert it into the + // LRU cache and the m_oMapReferencedDS map that holds strong references. + auto poDSRet = poTLSDS.get(); + { + poCache->m_oCache.insert(this, poTLSDS); + CPLAssert(!cpl::contains(poCache->m_oMapReferencedDS, this)); + poCache->m_oMapReferencedDS.insert( + {this, GDALThreadLocalDatasetCache:: + SharedPtrDatasetThreadLocalConfigOptionsPair( + poTLSDS, std::move(aosTLConfigOptionsBackup))}); + } + return poDSRet; +} + +/************************************************************************/ +/* UnrefUnderlyingDataset() */ +/************************************************************************/ + +/** Implements GDALProxyDataset::UnrefUnderlyingDataset. + * + * This is called by GDALProxyDataset overridden methods of GDALDataset, when + * they no longer need to access the underlying dataset. + * + * This method actually delegates most of the work to the other + * UnrefUnderlyingDataset() method that takes an explicit GDALThreadLocalDatasetCache* + * instance. + */ +void GDALThreadSafeDataset::UnrefUnderlyingDataset( + GDALDataset *poUnderlyingDataset) const +{ + GDALThreadLocalDatasetCache *poCache = tl_poCache.get(); + CPLAssert(poCache); + std::unique_lock oLock(poCache->m_oMutex); + UnrefUnderlyingDataset(poUnderlyingDataset, poCache); +} + +/************************************************************************/ +/* UnrefUnderlyingDataset() */ +/************************************************************************/ + +/** Takes care of removing the strong reference to a thread-local dataset + * from the TLS cache of datasets. + */ +void GDALThreadSafeDataset::UnrefUnderlyingDataset( + [[maybe_unused]] GDALDataset *poUnderlyingDataset, + GDALThreadLocalDatasetCache *poCache) const +{ + auto oIter = poCache->m_oMapReferencedDS.find(this); + CPLAssert(oIter != poCache->m_oMapReferencedDS.end()); + CPLAssert(oIter->second.poDS.get() == poUnderlyingDataset); + CPLSetThreadLocalConfigOptions(oIter->second.aosTLConfigOptions.List()); + poCache->m_oMapReferencedDS.erase(oIter); +} + +/************************************************************************/ +/* GDALThreadSafeRasterBand() */ +/************************************************************************/ + +GDALThreadSafeRasterBand::GDALThreadSafeRasterBand( + GDALThreadSafeDataset *poTSDS, GDALDataset *poParentDS, int nBandIn, + GDALRasterBand *poPrototypeBand, int nBaseBandOfMaskBand, int nOvrIdx) + : m_poTSDS(poTSDS), m_poPrototypeBand(poPrototypeBand), + m_nBaseBandOfMaskBand(nBaseBandOfMaskBand), m_nOvrIdx(nOvrIdx) +{ + // Replicates characteristics of the prototype band. + poDS = poParentDS; + nBand = nBandIn; + eDataType = poPrototypeBand->GetRasterDataType(); + nRasterXSize = poPrototypeBand->GetXSize(); + nRasterYSize = poPrototypeBand->GetYSize(); + poPrototypeBand->GetBlockSize(&nBlockXSize, &nBlockYSize); + + if (nBandIn > 0) + { + // For regular bands instantiates a (thread-safe) mask band and + // as many overviews as needed. + + m_poMaskBand = std::make_unique( + poTSDS, nullptr, 0, poPrototypeBand->GetMaskBand(), nBandIn, + nOvrIdx); + if (nOvrIdx < 0) + { + const int nOvrCount = poPrototypeBand->GetOverviewCount(); + for (int iOvrIdx = 0; iOvrIdx < nOvrCount; ++iOvrIdx) + { + m_apoOverviews.emplace_back( + std::make_unique( + poTSDS, nullptr, nBandIn, + poPrototypeBand->GetOverview(iOvrIdx), + nBaseBandOfMaskBand, iOvrIdx)); + } + } + } + else if (nBaseBandOfMaskBand > 0) + { + // If we are a mask band, nstanciates a (thread-safe) mask band of + // ourselves, but with the trick of negating nBaseBandOfMaskBand to + // avoid infinite recursion... + m_poMaskBand = std::make_unique( + poTSDS, nullptr, 0, poPrototypeBand->GetMaskBand(), + -nBaseBandOfMaskBand, nOvrIdx); + } +} + +/************************************************************************/ +/* RefUnderlyingRasterBand() */ +/************************************************************************/ + +/** Implements GDALProxyRasterBand::RefUnderlyingDataset. + * + * This method is called by all virtual methods of GDALRasterBand overridden by + * RefUnderlyingRasterBand() when it delegates the calls to the underlying + * band. + */ +GDALRasterBand * +GDALThreadSafeRasterBand::RefUnderlyingRasterBand(bool /*bForceOpen*/) const +{ + // Get a thread-local dataset + auto poTLDS = m_poTSDS->RefUnderlyingDataset(); + if (!poTLDS) + return nullptr; + + // Get corresponding thread-local band. If m_nBaseBandOfMaskBand is not + // zero, then the base band is indicated into it, otherwise use nBand. + const int nTLSBandIdx = + m_nBaseBandOfMaskBand ? std::abs(m_nBaseBandOfMaskBand) : nBand; + auto poTLRasterBand = poTLDS->GetRasterBand(nTLSBandIdx); + if (!poTLRasterBand) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): " + "GetRasterBand(%d) failed", + nTLSBandIdx); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + + // Get the overview level if needed. + if (m_nOvrIdx >= 0) + { + poTLRasterBand = poTLRasterBand->GetOverview(m_nOvrIdx); + if (!poTLRasterBand) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): " + "GetOverview(%d) failed", + m_nOvrIdx); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + } + + // Get the mask band (or the mask band of the mask band) if needed. + if (m_nBaseBandOfMaskBand) + { + poTLRasterBand = poTLRasterBand->GetMaskBand(); + if (m_nBaseBandOfMaskBand < 0) + poTLRasterBand = poTLRasterBand->GetMaskBand(); + } + + // Check that the thread-local band characteristics are identical to the + // ones of the prototype band. + if (m_poPrototypeBand->GetXSize() != poTLRasterBand->GetXSize() || + m_poPrototypeBand->GetYSize() != poTLRasterBand->GetYSize() || + m_poPrototypeBand->GetRasterDataType() != + poTLRasterBand->GetRasterDataType() + // m_poPrototypeBand->GetMaskFlags() is not thread-safe + // || m_poPrototypeBand->GetMaskFlags() != poTLRasterBand->GetMaskFlags() + ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): TLS " + "band has not expected characteristics"); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + int nThisBlockXSize; + int nThisBlockYSize; + int nTLSBlockXSize; + int nTLSBlockYSize; + m_poPrototypeBand->GetBlockSize(&nThisBlockXSize, &nThisBlockYSize); + poTLRasterBand->GetBlockSize(&nTLSBlockXSize, &nTLSBlockYSize); + if (nThisBlockXSize != nTLSBlockXSize || nThisBlockYSize != nTLSBlockYSize) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): TLS " + "band has not expected characteristics"); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + + // Registers the association between the thread-local band and the + // thread-local dataset + { + GDALThreadLocalDatasetCache *poCache = + GDALThreadSafeDataset::tl_poCache.get(); + CPLAssert(poCache); + std::unique_lock oLock(poCache->m_oMutex); + CPLAssert(!cpl::contains(poCache->m_oMapReferencedDSFromBand, + poTLRasterBand)); + poCache->m_oMapReferencedDSFromBand[poTLRasterBand] = poTLDS; + } + // CPLDebug("GDAL", "%p->RefUnderlyingRasterBand() return %p", this, poTLRasterBand); + return poTLRasterBand; +} + +/************************************************************************/ +/* UnrefUnderlyingRasterBand() */ +/************************************************************************/ + +/** Implements GDALProxyRasterBand::UnrefUnderlyingRasterBand. + * + * This is called by GDALProxyRasterBand overridden methods of GDALRasterBand, + * when they no longer need to access the underlying dataset. + */ +void GDALThreadSafeRasterBand::UnrefUnderlyingRasterBand( + GDALRasterBand *poUnderlyingRasterBand) const +{ + // CPLDebug("GDAL", "%p->UnrefUnderlyingRasterBand(%p)", this, poUnderlyingRasterBand); + + // Unregisters the association between the thread-local band and the + // thread-local dataset + { + GDALThreadLocalDatasetCache *poCache = + GDALThreadSafeDataset::tl_poCache.get(); + CPLAssert(poCache); + std::unique_lock oLock(poCache->m_oMutex); + auto oIter = + poCache->m_oMapReferencedDSFromBand.find(poUnderlyingRasterBand); + CPLAssert(oIter != poCache->m_oMapReferencedDSFromBand.end()); + + m_poTSDS->UnrefUnderlyingDataset(oIter->second, poCache); + poCache->m_oMapReferencedDSFromBand.erase(oIter); + } +} + +/************************************************************************/ +/* GetMaskBand() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetMaskBand + */ +GDALRasterBand *GDALThreadSafeRasterBand::GetMaskBand() +{ + return m_poMaskBand ? m_poMaskBand.get() : this; +} + +/************************************************************************/ +/* GetOverviewCount() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetOverviewCount + */ +int GDALThreadSafeRasterBand::GetOverviewCount() +{ + return static_cast(m_apoOverviews.size()); +} + +/************************************************************************/ +/* GetOverview() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetOverview + */ +GDALRasterBand *GDALThreadSafeRasterBand::GetOverview(int nIdx) +{ + if (nIdx < 0 || nIdx >= static_cast(m_apoOverviews.size())) + return nullptr; + return m_apoOverviews[nIdx].get(); +} + +/************************************************************************/ +/* GetRasterSampleOverview() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetRasterSampleOverview + */ +GDALRasterBand * +GDALThreadSafeRasterBand::GetRasterSampleOverview(GUIntBig nDesiredSamples) + +{ + // Call the base implementation, and do not forward to proxy + return GDALRasterBand::GetRasterSampleOverview(nDesiredSamples); +} + +/************************************************************************/ +/* GetDefaultRAT() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetDefaultRAT + * + * This is a bit tricky to do as GDALRasterAttributeTable has virtual methods + * with potential (non thread-safe) side-effects. The clean solution would be + * to implement a GDALThreadSafeRAT wrapper class, but this is a bit too much + * effort. So for now, we check if the RAT returned by the prototype band is + * an instance of GDALDefaultRasterAttributeTable. If it is, given that this + * class has thread-safe getters, we can directly return it. + * Otherwise return in error. + */ +GDALRasterAttributeTable *GDALThreadSafeRasterBand::GetDefaultRAT() +{ + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + const auto poRAT = + const_cast(m_poPrototypeBand)->GetDefaultRAT(); + if (!poRAT) + return nullptr; + + if (dynamic_cast(poRAT)) + return poRAT; + + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::GetDefaultRAT() not supporting a " + "non-GDALDefaultRasterAttributeTable implementation"); + return nullptr; +} + +#endif // DOXYGEN_SKIP + +/************************************************************************/ +/* GDALDataset::IsThreadSafe() */ +/************************************************************************/ + +/** Return whether this dataset, and its related objects (typically raster + * bands), can be called for the intended scope. + * + * Note that in the current implementation, nScopeFlags should be set to + * GDAL_OF_RASTER, as thread-safety is limited to read-only operations and + * excludes operations on vector layers (OGRLayer) or multidimensional API + * (GDALGroup, GDALMDArray, etc.) + * + * This is the same as the C function GDALDatasetIsThreadSafe(). + * + * @since 3.10 + */ +bool GDALDataset::IsThreadSafe(int nScopeFlags) const +{ + return (nOpenFlags & GDAL_OF_THREAD_SAFE) != 0 && + nScopeFlags == GDAL_OF_RASTER && (nOpenFlags & GDAL_OF_RASTER) != 0; +} + +/************************************************************************/ +/* GDALDatasetIsThreadSafe() */ +/************************************************************************/ + +/** Return whether this dataset, and its related objects (typically raster + * bands), can be called for the intended scope. + * + * Note that in the current implementation, nScopeFlags should be set to + * GDAL_OF_RASTER, as thread-safety is limited to read-only operations and + * excludes operations on vector layers (OGRLayer) or multidimensional API + * (GDALGroup, GDALMDArray, etc.) + * + * This is the same as the C++ method GDALDataset::IsThreadSafe(). + * + * @param hDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * @param papszOptions Options. None currently. + * + * @since 3.10 + */ +bool GDALDatasetIsThreadSafe(GDALDatasetH hDS, int nScopeFlags, + CSLConstList papszOptions) +{ + VALIDATE_POINTER1(hDS, __func__, false); + + CPL_IGNORE_RET_VAL(papszOptions); + + return GDALDataset::FromHandle(hDS)->IsThreadSafe(nScopeFlags); +} + +/************************************************************************/ +/* GDALGetThreadSafeDataset() */ +/************************************************************************/ + +/** Return a thread-safe dataset. + * + * In the general case, this thread-safe dataset will open a + * behind-the-scenes per-thread dataset (re-using the name and open options of poDS), + * the first time a thread calls a method on the thread-safe dataset, and will + * transparently redirect calls from the calling thread to this behind-the-scenes + * per-thread dataset. Hence there is an initial setup cost per thread. + * Datasets of the MEM driver cannot be opened by name, but this function will + * take care of "cloning" them, using the same backing memory, when needed. + * + * Ownership of the passed dataset is transferred to the thread-safe dataset. + * + * The function may also return the passed dataset if it is already thread-safe. + * + * @param poDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * + * @return a new thread-safe dataset, or nullptr in case of error. + * + * @since 3.10 + */ +std::unique_ptr +GDALGetThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags) +{ + return GDALThreadSafeDataset::Create(std::move(poDS), nScopeFlags); +} + +/************************************************************************/ +/* GDALGetThreadSafeDataset() */ +/************************************************************************/ + +/** Return a thread-safe dataset. + * + * In the general case, this thread-safe dataset will open a + * behind-the-scenes per-thread dataset (re-using the name and open options of poDS), + * the first time a thread calls a method on the thread-safe dataset, and will + * transparently redirect calls from the calling thread to this behind-the-scenes + * per-thread dataset. Hence there is an initial setup cost per thread. + * Datasets of the MEM driver cannot be opened by name, but this function will + * take care of "cloning" them, using the same backing memory, when needed. + * + * The life-time of the passed dataset must be longer than the one of + * the returned thread-safe dataset. + * + * Note that this function does increase the reference count on poDS while + * it is being used + * + * The function may also return the passed dataset if it is already thread-safe. + * A non-nullptr returned dataset must be released with ReleaseRef(). + * + * Patterns like the following one are valid: + * \code{.cpp} + * auto poDS = GDALDataset::Open(...); + * auto poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); + * poDS->ReleaseRef(); + * if (poThreadSafeDS ) + * { + * // ... do something with poThreadSafeDS ... + * poThreadSafeDS->ReleaseRef(); + * } + * \endcode + * + * @param poDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * + * @return a new thread-safe dataset or poDS, or nullptr in case of error. + + * @since 3.10 + */ +GDALDataset *GDALGetThreadSafeDataset(GDALDataset *poDS, int nScopeFlags) +{ + return GDALThreadSafeDataset::Create(poDS, nScopeFlags); +} + +/************************************************************************/ +/* GDALGetThreadSafeDataset() */ +/************************************************************************/ + +/** Return a thread-safe dataset. + * + * In the general case, this thread-safe dataset will open a + * behind-the-scenes per-thread dataset (re-using the name and open options of hDS), + * the first time a thread calls a method on the thread-safe dataset, and will + * transparently redirect calls from the calling thread to this behind-the-scenes + * per-thread dataset. Hence there is an initial setup cost per thread. + * Datasets of the MEM driver cannot be opened by name, but this function will + * take care of "cloning" them, using the same backing memory, when needed. + * + * The life-time of the passed dataset must be longer than the one of + * the returned thread-safe dataset. + * + * Note that this function does increase the reference count on poDS while + * it is being used + * + * The function may also return the passed dataset if it is already thread-safe. + * A non-nullptr returned dataset must be released with GDALReleaseDataset(). + * + * \code{.cpp} + * hDS = GDALOpenEx(...); + * hThreadSafeDS = GDALGetThreadSafeDataset(hDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE, NULL); + * GDALReleaseDataset(hDS); + * if( hThreadSafeDS ) + * { + * // ... do something with hThreadSafeDS ... + * GDALReleaseDataset(hThreadSafeDS); + * } + * \endcode + * + * @param hDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * @param papszOptions Options. None currently. + * + * @since 3.10 + */ +GDALDatasetH GDALGetThreadSafeDataset(GDALDatasetH hDS, int nScopeFlags, + CSLConstList papszOptions) +{ + VALIDATE_POINTER1(hDS, __func__, nullptr); + + CPL_IGNORE_RET_VAL(papszOptions); + return GDALDataset::ToHandle( + GDALGetThreadSafeDataset(GDALDataset::FromHandle(hDS), nScopeFlags)); +} diff --git a/gcore/overview.cpp b/gcore/overview.cpp index c962aa73bfc2..7b4276d1b5e9 100644 --- a/gcore/overview.cpp +++ b/gcore/overview.cpp @@ -170,35 +170,58 @@ static CPLErr GDALResampleChunk_Near(const GDALOverviewResampleArgs &args, *peDstBufferDataType = args.eWrkDataType; switch (args.eWrkDataType) { + // For nearest resampling, as no computation is done, only the + // size of the data type matters. case GDT_Byte: + case GDT_Int8: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 1); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } + case GDT_Int16: case GDT_UInt16: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 2); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } + case GDT_CInt16: + case GDT_Int32: + case GDT_UInt32: case GDT_Float32: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 4); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } + case GDT_CInt32: + case GDT_CFloat32: + case GDT_Int64: + case GDT_UInt64: case GDT_Float64: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 8); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } - default: + case GDT_CFloat64: + { + return GDALResampleChunk_NearT( + args, static_cast *>(pChunk), + reinterpret_cast **>(ppDstBuffer)); + } + + case GDT_Unknown: + case GDT_TypeCount: break; } CPLAssert(false); @@ -2076,9 +2099,39 @@ static CPLErr GDALResampleChunk_Gauss(const GDALOverviewResampleArgs &args, /* GDALResampleChunk_Mode() */ /************************************************************************/ +template static inline bool IsSame(T a, T b) +{ + return a == b; +} + +template <> bool IsSame(float a, float b) +{ + return a == b || (std::isnan(a) && std::isnan(b)); +} + +template <> bool IsSame(double a, double b) +{ + return a == b || (std::isnan(a) && std::isnan(b)); +} + +template <> +bool IsSame>(std::complex a, std::complex b) +{ + return a == b || (std::isnan(a.real()) && std::isnan(a.imag()) && + std::isnan(b.real()) && std::isnan(b.imag())); +} + +template <> +bool IsSame>(std::complex a, + std::complex b) +{ + return a == b || (std::isnan(a.real()) && std::isnan(a.imag()) && + std::isnan(b.real()) && std::isnan(b.imag())); +} + template -static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, - const T *pChunk, T *const pDstBuffer) +static CPLErr GDALResampleChunk_ModeT(const GDALOverviewResampleArgs &args, + const T *pChunk, T *const pDstBuffer) { const double dfXRatioDstToSrc = args.dfXRatioDstToSrc; @@ -2095,19 +2148,25 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, const int nDstYOff = args.nDstYOff; const int nDstYOff2 = args.nDstYOff2; const bool bHasNoData = args.bHasNoData; - const double dfNoDataValue = args.dfNoDataValue; const GDALColorTable *poColorTable = args.poColorTable; - const GDALDataType eSrcDataType = args.eSrcDataType; const int nDstXSize = nDstXOff2 - nDstXOff; T tNoDataValue; - if (!bHasNoData || !GDALIsValueInRange(dfNoDataValue)) + if constexpr (std::is_same>::value || + std::is_same>::value) + { + using BaseT = typename T::value_type; + tNoDataValue = + std::complex(std::numeric_limits::quiet_NaN(), + std::numeric_limits::quiet_NaN()); + } + else if (!bHasNoData || !GDALIsValueInRange(args.dfNoDataValue)) tNoDataValue = 0; else - tNoDataValue = static_cast(dfNoDataValue); + tNoDataValue = static_cast(args.dfNoDataValue); size_t nMaxNumPx = 0; - T *padfVals = nullptr; + T *paVals = nullptr; int *panSums = nullptr; const int nChunkRightXOff = nChunkXOff + nChunkXSize; @@ -2190,8 +2249,13 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, if (nSrcXOff2 > nChunkRightXOff) nSrcXOff2 = nChunkRightXOff; - if (eSrcDataType != GDT_Byte || - (poColorTable && poColorTable->GetColorEntryCount() > 256)) + bool bRegularProcessing = false; + if constexpr (!std::is_same::value) + bRegularProcessing = true; + else if (poColorTable && poColorTable->GetColorEntryCount() > 256) + bRegularProcessing = true; + + if (bRegularProcessing) { // Not sure how much sense it makes to run a majority // filter on floating point data, but here it is for the sake @@ -2206,7 +2270,7 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, { CPLError(CE_Failure, CPLE_NotSupported, "Too big downsampling factor"); - CPLFree(padfVals); + CPLFree(paVals); CPLFree(panSums); return CE_Failure; } @@ -2217,19 +2281,19 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, size_t iMaxVal = 0; bool biMaxValdValid = false; - if (padfVals == nullptr || nNumPx > nMaxNumPx) + if (paVals == nullptr || nNumPx > nMaxNumPx) { - T *padfValsNew = static_cast( - VSI_REALLOC_VERBOSE(padfVals, nNumPx * sizeof(T))); + T *paValsNew = static_cast( + VSI_REALLOC_VERBOSE(paVals, nNumPx * sizeof(T))); int *panSumsNew = static_cast( VSI_REALLOC_VERBOSE(panSums, nNumPx * sizeof(int))); - if (padfValsNew != nullptr) - padfVals = padfValsNew; + if (paValsNew != nullptr) + paVals = paValsNew; if (panSumsNew != nullptr) panSums = panSumsNew; - if (padfValsNew == nullptr || panSumsNew == nullptr) + if (paValsNew == nullptr || panSumsNew == nullptr) { - CPLFree(padfVals); + CPLFree(paVals); CPLFree(panSums); return CE_Failure; } @@ -2246,12 +2310,12 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, if (pabySrcScanlineNodataMask == nullptr || pabySrcScanlineNodataMask[iX + iTotYOff]) { - const T dfVal = paSrcScanline[iX + iTotYOff]; + const T val = paSrcScanline[iX + iTotYOff]; size_t i = 0; // Used after for. // Check array for existing entry. for (; i < iMaxInd; ++i) - if (padfVals[i] == dfVal && + if (IsSame(paVals[i], val) && ++panSums[i] > panSums[iMaxVal]) { iMaxVal = i; @@ -2262,7 +2326,7 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, // Add to arr if entry not already there. if (i == iMaxInd) { - padfVals[iMaxInd] = dfVal; + paVals[iMaxInd] = val; panSums[iMaxInd] = 1; if (!biMaxValdValid) @@ -2280,9 +2344,10 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, if (!biMaxValdValid) paDstScanline[iDstPixel - nDstXOff] = tNoDataValue; else - paDstScanline[iDstPixel - nDstXOff] = padfVals[iMaxVal]; + paDstScanline[iDstPixel - nDstXOff] = paVals[iMaxVal]; } - else // if( eSrcDataType == GDT_Byte && nEntryCount < 256 ) + else if constexpr (std::is_same::value) + // ( eSrcDataType == GDT_Byte && nEntryCount < 256 ) { // So we go here for a paletted or non-paletted byte band. // The input values are then between 0 and 255. @@ -2325,7 +2390,7 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, } } - CPLFree(padfVals); + CPLFree(paVals); CPLFree(panSums); return CE_None; @@ -2343,38 +2408,89 @@ static CPLErr GDALResampleChunk_Mode(const GDALOverviewResampleArgs &args, return CE_Failure; } + CPLAssert(args.eSrcDataType == args.eWrkDataType); + *peDstBufferDataType = args.eWrkDataType; switch (args.eWrkDataType) { + // For mode resampling, as no computation is done, only the + // size of the data type matters... except for Byte where we have + // special processing. And for floating point values case GDT_Byte: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); + } + + case GDT_Int8: + { + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); } + case GDT_Int16: case GDT_UInt16: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 2); + return GDALResampleChunk_ModeT( + args, static_cast(pChunk), + static_cast(*ppDstBuffer)); + } + + case GDT_CInt16: + case GDT_Int32: + case GDT_UInt32: + { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 4); + return GDALResampleChunk_ModeT( + args, static_cast(pChunk), + static_cast(*ppDstBuffer)); } case GDT_Float32: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 4); + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); + } + + case GDT_CInt32: + case GDT_Int64: + case GDT_UInt64: + { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 8); + return GDALResampleChunk_ModeT( + args, static_cast(pChunk), + static_cast(*ppDstBuffer)); } case GDT_Float64: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 8); + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); } - default: + case GDT_CFloat32: + { + return GDALResampleChunk_ModeT( + args, static_cast *>(pChunk), + static_cast *>(*ppDstBuffer)); + } + + case GDT_CFloat64: + { + return GDALResampleChunk_ModeT( + args, static_cast *>(pChunk), + static_cast *>(*ppDstBuffer)); + } + + case GDT_Unknown: + case GDT_TypeCount: break; } @@ -3032,6 +3148,7 @@ static CPLErr GDALResampleChunk_ConvolutionT( // cppcheck-suppress unreadVariable const int isIntegerDT = GDALDataTypeIsInteger(dstDataType); const auto nNodataValueInt64 = static_cast(dfNoDataValue); + constexpr int nWrkDataTypeSize = static_cast(sizeof(Twork)); // TODO: we should have some generic function to do this. Twork fDstMin = -std::numeric_limits::max(); @@ -3068,6 +3185,20 @@ static CPLErr GDALResampleChunk_ConvolutionT( // cppcheck-suppress unreadVariable fDstMax = static_cast(std::numeric_limits::max()); } + else if (dstDataType == GDT_UInt64) + { + // cppcheck-suppress unreadVariable + fDstMin = static_cast(std::numeric_limits::min()); + // cppcheck-suppress unreadVariable + fDstMax = static_cast(std::numeric_limits::max()); + } + else if (dstDataType == GDT_Int64) + { + // cppcheck-suppress unreadVariable + fDstMin = static_cast(std::numeric_limits::min()); + // cppcheck-suppress unreadVariable + fDstMax = static_cast(std::numeric_limits::max()); + } auto replaceValIfNodata = [bHasNoData, isIntegerDT, fDstMin, fDstMax, nNodataValueInt64, dfNoDataValue, @@ -3585,7 +3716,7 @@ static CPLErr GDALResampleChunk_ConvolutionT( if (pafWrkScanline) { - GDALCopyWords(pafWrkScanline, eWrkDataType, 4, + GDALCopyWords(pafWrkScanline, eWrkDataType, nWrkDataTypeSize, static_cast(pDstBuffer) + static_cast(iDstLine - nDstYOff) * nDstXSize * nDstDataTypeSize, @@ -4101,33 +4232,38 @@ GDALResampleFunction GDALGetResampleFunction(const char *pszResampling, GDALDataType GDALGetOvrWorkDataType(const char *pszResampling, GDALDataType eSrcDataType) { - if ((STARTS_WITH_CI(pszResampling, "NEAR") || - STARTS_WITH_CI(pszResampling, "AVER") || EQUAL(pszResampling, "RMS") || - EQUAL(pszResampling, "CUBIC") || EQUAL(pszResampling, "CUBICSPLINE") || - EQUAL(pszResampling, "LANCZOS") || EQUAL(pszResampling, "BILINEAR") || - EQUAL(pszResampling, "MODE")) && - eSrcDataType == GDT_Byte) + if (STARTS_WITH_CI(pszResampling, "NEAR") || EQUAL(pszResampling, "MODE")) + { + return eSrcDataType; + } + else if (eSrcDataType == GDT_Byte && + (STARTS_WITH_CI(pszResampling, "AVER") || + EQUAL(pszResampling, "RMS") || EQUAL(pszResampling, "CUBIC") || + EQUAL(pszResampling, "CUBICSPLINE") || + EQUAL(pszResampling, "LANCZOS") || + EQUAL(pszResampling, "BILINEAR") || EQUAL(pszResampling, "MODE"))) { return GDT_Byte; } - else if ((STARTS_WITH_CI(pszResampling, "NEAR") || - STARTS_WITH_CI(pszResampling, "AVER") || + else if (eSrcDataType == GDT_UInt16 && + (STARTS_WITH_CI(pszResampling, "AVER") || EQUAL(pszResampling, "RMS") || EQUAL(pszResampling, "CUBIC") || EQUAL(pszResampling, "CUBICSPLINE") || EQUAL(pszResampling, "LANCZOS") || - EQUAL(pszResampling, "BILINEAR") || - EQUAL(pszResampling, "MODE")) && - eSrcDataType == GDT_UInt16) + EQUAL(pszResampling, "BILINEAR") || EQUAL(pszResampling, "MODE"))) { return GDT_UInt16; } else if (EQUAL(pszResampling, "GAUSS")) return GDT_Float64; - if (eSrcDataType == GDT_Float64) - return GDT_Float64; - - return GDT_Float32; + if (eSrcDataType == GDT_Byte || eSrcDataType == GDT_Int8 || + eSrcDataType == GDT_UInt16 || eSrcDataType == GDT_Int16 || + eSrcDataType == GDT_Float32) + { + return GDT_Float32; + } + return GDT_Float64; } namespace @@ -4360,10 +4496,13 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, poSrcBand->GetBlockSize(&nFRXBlockSize, &nFRYBlockSize); const GDALDataType eSrcDataType = poSrcBand->GetRasterDataType(); + const bool bUseGenericResampleFn = STARTS_WITH_CI(pszResampling, "NEAR") || + EQUAL(pszResampling, "MODE") || + !GDALDataTypeIsComplex(eSrcDataType); const GDALDataType eWrkDataType = - GDALDataTypeIsComplex(eSrcDataType) - ? GDT_CFloat32 - : GDALGetOvrWorkDataType(pszResampling, eSrcDataType); + bUseGenericResampleFn + ? GDALGetOvrWorkDataType(pszResampling, eSrcDataType) + : GDT_CFloat32; const int nWidth = poSrcBand->GetXSize(); const int nHeight = poSrcBand->GetYSize(); @@ -4465,6 +4604,7 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, int nDstWidth = 0; GDALOverviewResampleArgs args{}; const void *pChunk = nullptr; + bool bUseGenericResampleFn = false; // Output values of resampling function CPLErr eErr = CE_Failure; @@ -4494,7 +4634,7 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, { OvrJob *poJob = static_cast(pData); - if (poJob->args.eWrkDataType != GDT_CFloat32) + if (poJob->bUseGenericResampleFn) { poJob->eErr = poJob->pfnResampleFn(poJob->args, poJob->pChunk, &(poJob->pDstBuffer), @@ -4818,6 +4958,7 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, auto poJob = std::make_unique(); poJob->pfnResampleFn = pfnResampleFn; + poJob->bUseGenericResampleFn = bUseGenericResampleFn; poJob->args.eOvrDataType = poDstBand->GetRasterDataType(); poJob->args.nOvrXSize = poDstBand->GetXSize(); poJob->args.nOvrYSize = poDstBand->GetYSize(); @@ -5031,13 +5172,14 @@ CPLErr GDALRegenerateOverviewsMultiBand( for (int iOverview = 0; iOverview < nOverviews; ++iOverview) { - const int nDstWidth = papapoOverviewBands[0][iOverview]->GetXSize(); - const int nDstHeight = papapoOverviewBands[0][iOverview]->GetYSize(); + const auto poOvrFirstBand = papapoOverviewBands[0][iOverview]; + const int nDstWidth = poOvrFirstBand->GetXSize(); + const int nDstHeight = poOvrFirstBand->GetYSize(); for (int iBand = 1; iBand < nBands; ++iBand) { - if (papapoOverviewBands[iBand][iOverview]->GetXSize() != - nDstWidth || - papapoOverviewBands[iBand][iOverview]->GetYSize() != nDstHeight) + const auto poOvrBand = papapoOverviewBands[iBand][iOverview]; + if (poOvrBand->GetXSize() != nDstWidth || + poOvrBand->GetYSize() != nDstHeight) { CPLError( CE_Failure, CPLE_NotSupported, @@ -5045,8 +5187,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( "of the same level must have the same dimensions"); return CE_Failure; } - if (papapoOverviewBands[iBand][iOverview]->GetRasterDataType() != - eDataType) + if (poOvrBand->GetRasterDataType() != eDataType) { CPLError( CE_Failure, CPLE_NotSupported, @@ -5076,6 +5217,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( const GDALDataType eWrkDataType = GDALGetOvrWorkDataType(pszResampling, eDataType); + const int nWrkDataTypeSize = GDALGetDataTypeSizeBytes(eWrkDataType); const bool bIsMask = papoSrcBands[0]->IsMaskBand(); @@ -5116,8 +5258,8 @@ CPLErr GDALRegenerateOverviewsMultiBand( : std::unique_ptr(nullptr); // Only configurable for debug / testing - const int nChunkMaxSize = - atoi(CPLGetConfigOption("GDAL_OVR_CHUNK_MAX_SIZE", "10485760")); + const int nChunkMaxSize = std::max( + 100, atoi(CPLGetConfigOption("GDAL_OVR_CHUNK_MAX_SIZE", "10485760"))); // Second pass to do the real job. double dfCurPixelCount = 0; @@ -5127,11 +5269,6 @@ CPLErr GDALRegenerateOverviewsMultiBand( { int iSrcOverview = -1; // -1 means the source bands. - int nDstChunkXSize = 0; - int nDstChunkYSize = 0; - papapoOverviewBands[0][iOverview]->GetBlockSize(&nDstChunkXSize, - &nDstChunkYSize); - const int nDstTotalWidth = papapoOverviewBands[0][iOverview]->GetXSize(); const int nDstTotalHeight = @@ -5182,6 +5319,23 @@ CPLErr GDALRegenerateOverviewsMultiBand( if (nOvrFactor == 0) nOvrFactor = 1; + int nDstChunkXSize = 0; + int nDstChunkYSize = 0; + papapoOverviewBands[0][iOverview]->GetBlockSize(&nDstChunkXSize, + &nDstChunkYSize); + + const char *pszDST_CHUNK_X_SIZE = + CSLFetchNameValue(papszOptions, "DST_CHUNK_X_SIZE"); + const char *pszDST_CHUNK_Y_SIZE = + CSLFetchNameValue(papszOptions, "DST_CHUNK_Y_SIZE"); + if (pszDST_CHUNK_X_SIZE && pszDST_CHUNK_Y_SIZE) + { + nDstChunkXSize = std::max(1, atoi(pszDST_CHUNK_X_SIZE)); + nDstChunkYSize = std::max(1, atoi(pszDST_CHUNK_Y_SIZE)); + CPLDebug("GDAL", "Using dst chunk size %d x %d", nDstChunkXSize, + nDstChunkYSize); + } + // Try to extend the chunk size so that the memory needed to acquire // source pixels goes up to 10 MB. // This can help for drivers that support multi-threaded reading @@ -5198,8 +5352,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( nFullResXChunk + 2 * nKernelRadius * nOvrFactor; if (static_cast(nFullResXChunkQueried) * - nFullResYChunkQueried * nBands * - GDALGetDataTypeSizeBytes(eWrkDataType) > + nFullResYChunkQueried * nBands * nWrkDataTypeSize > nChunkMaxSize) { break; @@ -5214,6 +5367,199 @@ CPLErr GDALRegenerateOverviewsMultiBand( const int nFullResXChunkQueried = nFullResXChunk + 2 * nKernelRadius * nOvrFactor; + // Make sure that the RAM requirements to acquire the source data does + // not exceed nChunkMaxSize + // If so, reduce the destination chunk size, generate overviews in a + // temporary dataset, and copy that temporary dataset over the target + // overview bands (to avoid issues with lossy compression) + const auto nMemRequirement = + static_cast(nFullResXChunkQueried) * + nFullResYChunkQueried * nBands * nWrkDataTypeSize; + if (nMemRequirement > nChunkMaxSize && + !(pszDST_CHUNK_X_SIZE && pszDST_CHUNK_Y_SIZE)) + { + // Compute a smaller destination chunk size + const auto nOverShootFactor = nMemRequirement / nChunkMaxSize; + const auto nSqrtOverShootFactor = std::max( + 4, static_cast(std::ceil( + std::sqrt(static_cast(nOverShootFactor))))); + const int nReducedDstChunkXSize = std::max( + 1, static_cast(nDstChunkXSize / nSqrtOverShootFactor)); + const int nReducedDstChunkYSize = std::max( + 1, static_cast(nDstChunkYSize / nSqrtOverShootFactor)); + if (nReducedDstChunkXSize < nDstChunkXSize || + nReducedDstChunkYSize < nDstChunkYSize) + { + CPLStringList aosOptions(papszOptions); + aosOptions.SetNameValue( + "DST_CHUNK_X_SIZE", + CPLSPrintf("%d", nReducedDstChunkXSize)); + aosOptions.SetNameValue( + "DST_CHUNK_Y_SIZE", + CPLSPrintf("%d", nReducedDstChunkYSize)); + + const auto nTmpDSMemRequirement = + static_cast(nDstTotalWidth) * nDstTotalHeight * + nBands * GDALGetDataTypeSizeBytes(eDataType); + std::unique_ptr poTmpDS; + // Config option mostly/only for autotest purposes + const char *pszGDAL_OVR_TEMP_DRIVER = + CPLGetConfigOption("GDAL_OVR_TEMP_DRIVER", ""); + if ((nTmpDSMemRequirement <= nChunkMaxSize && + !EQUAL(pszGDAL_OVR_TEMP_DRIVER, "GTIFF")) || + EQUAL(pszGDAL_OVR_TEMP_DRIVER, "MEM")) + { + auto poTmpDrv = + GetGDALDriverManager()->GetDriverByName("MEM"); + if (!poTmpDrv) + { + eErr = CE_Failure; + break; + } + poTmpDS.reset(poTmpDrv->Create("", nDstTotalWidth, + nDstTotalHeight, nBands, + eDataType, nullptr)); + } + else + { + auto poTmpDrv = + GetGDALDriverManager()->GetDriverByName("GTiff"); + if (!poTmpDrv) + { + eErr = CE_Failure; + break; + } + std::string osTmpFilename; + auto poDstDS = papapoOverviewBands[0][0]->GetDataset(); + if (poDstDS) + { + osTmpFilename = poDstDS->GetDescription(); + VSIStatBufL sStatBuf; + if (!osTmpFilename.empty() && + VSIStatL(osTmpFilename.c_str(), &sStatBuf) == 0) + osTmpFilename += "_tmp_ovr.tif"; + } + if (osTmpFilename.empty()) + { + osTmpFilename = CPLGenerateTempFilename(nullptr); + osTmpFilename += ".tif"; + } + CPLDebug("GDAL", + "Creating temporary file %s of %d x %d x %d", + osTmpFilename.c_str(), nDstTotalWidth, + nDstTotalHeight, nBands); + CPLStringList aosCO; + poTmpDS.reset(poTmpDrv->Create( + osTmpFilename.c_str(), nDstTotalWidth, nDstTotalHeight, + nBands, eDataType, aosCO.List())); + if (poTmpDS) + { + poTmpDS->MarkSuppressOnClose(); + VSIUnlink(osTmpFilename.c_str()); + } + } + if (!poTmpDS) + { + eErr = CE_Failure; + break; + } + + std::vector apapoOverviewBands(nBands); + for (int i = 0; i < nBands; ++i) + { + apapoOverviewBands[i] = static_cast( + CPLMalloc(sizeof(GDALRasterBand *))); + apapoOverviewBands[i][0] = poTmpDS->GetRasterBand(i + 1); + } + + const double dfExtraPixels = + static_cast(nSrcXSize) / nToplevelSrcWidth * + papapoOverviewBands[0][iOverview]->GetXSize() * + static_cast(nSrcYSize) / nToplevelSrcHeight * + papapoOverviewBands[0][iOverview]->GetYSize(); + + void *pScaledProgressData = GDALCreateScaledProgress( + dfCurPixelCount / dfTotalPixelCount, + (dfCurPixelCount + dfExtraPixels) / dfTotalPixelCount, + pfnProgress, pProgressData); + + // Generate overviews in temporary dataset + eErr = GDALRegenerateOverviewsMultiBand( + nBands, papoSrcBands, 1, apapoOverviewBands.data(), + pszResampling, GDALScaledProgress, pScaledProgressData, + aosOptions.List()); + + GDALDestroyScaledProgress(pScaledProgressData); + + dfCurPixelCount += dfExtraPixels; + + for (int i = 0; i < nBands; ++i) + { + CPLFree(apapoOverviewBands[i]); + } + + // Copy temporary dataset to destination overview bands + + if (eErr == CE_None) + { + // Check if all papapoOverviewBands[][iOverview] bands point + // to the same dataset. If so, we can use + // GDALDatasetCopyWholeRaster() + GDALDataset *poDstOvrBandDS = + papapoOverviewBands[0][iOverview]->GetDataset(); + if (poDstOvrBandDS) + { + if (poDstOvrBandDS->GetRasterCount() != nBands || + poDstOvrBandDS->GetRasterBand(1) != + papapoOverviewBands[0][iOverview]) + { + poDstOvrBandDS = nullptr; + } + else + { + for (int i = 1; poDstOvrBandDS && i < nBands; ++i) + { + GDALDataset *poThisDstOvrBandDS = + papapoOverviewBands[i][iOverview] + ->GetDataset(); + if (poThisDstOvrBandDS == nullptr || + poThisDstOvrBandDS != poDstOvrBandDS || + poThisDstOvrBandDS->GetRasterBand(i + 1) != + papapoOverviewBands[i][iOverview]) + { + poDstOvrBandDS = nullptr; + } + } + } + } + if (poDstOvrBandDS) + { + eErr = GDALDatasetCopyWholeRaster( + GDALDataset::ToHandle(poTmpDS.get()), + GDALDataset::ToHandle(poDstOvrBandDS), nullptr, + nullptr, nullptr); + } + else + { + for (int i = 0; eErr == CE_None && i < nBands; ++i) + { + eErr = GDALRasterBandCopyWholeRaster( + GDALRasterBand::ToHandle( + poTmpDS->GetRasterBand(i + 1)), + GDALRasterBand::ToHandle( + papapoOverviewBands[i][iOverview]), + nullptr, nullptr, nullptr); + } + } + } + + if (eErr != CE_None) + break; + + continue; + } + } + // Structure describing a resampling job struct OvrJob { @@ -5415,7 +5761,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( { apaChunk[iBand] = VSI_MALLOC3_VERBOSE( nFullResXChunkQueried, nFullResYChunkQueried, - GDALGetDataTypeSizeBytes(eWrkDataType)); + nWrkDataTypeSize); if (apaChunk[iBand] == nullptr) { eErr = CE_Failure; diff --git a/gcore/rasterio.cpp b/gcore/rasterio.cpp index c2eae1398301..8f6a99ee48eb 100644 --- a/gcore/rasterio.cpp +++ b/gcore/rasterio.cpp @@ -454,6 +454,7 @@ CPLErr GDALRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, nXOff <= nLBlockX * nBlockXSize && nYOff <= nLBlockY * nBlockYSize && (nXOff + nXSize >= nXRight || + // cppcheck-suppress knownConditionTrueFalse (nXOff + nXSize == GetXSize() && nXRight > GetXSize())) && (nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize || (nYOff + nYSize == GetYSize() && @@ -1005,9 +1006,10 @@ CPLErr GDALRasterBand::RasterIOResampled( GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg) { // Determine if we use warping resampling or overview resampling - bool bUseWarp = false; - if (GDALDataTypeIsComplex(eDataType)) - bUseWarp = true; + const bool bUseWarp = + (GDALDataTypeIsComplex(eDataType) && + psExtraArg->eResampleAlg != GRIORA_NearestNeighbour && + psExtraArg->eResampleAlg != GRIORA_Mode); double dfXOff = nXOff; double dfYOff = nYOff; @@ -4088,7 +4090,7 @@ CPLErr GDALDataset::BlockBasedRasterIO( { GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]); - eErr = poBand->GDALRasterBand::IRasterIO( + eErr = poBand->IRasterIO( eRWFlag, nChunkXOff, nChunkYOff, nChunkXSize, nChunkYSize, pabyChunkData + diff --git a/ogr/ogr_geocoding.cpp b/ogr/ogr_geocoding.cpp index 412d7d7557c1..c9821807a6b0 100644 --- a/ogr/ogr_geocoding.cpp +++ b/ogr/ogr_geocoding.cpp @@ -471,8 +471,9 @@ static OGRLayer *OGRGeocodeGetCacheLayer(OGRGeocodingSessionH hSession, (EQUAL(osExt, "SQLITE") || EQUAL(osExt, "CSV"))) { CPLFree(hSession->pszCacheFilename); - hSession->pszCacheFilename = CPLStrdup(CPLSPrintf( - "/vsimem/%s.%s", CACHE_LAYER_NAME, osExt.c_str())); + hSession->pszCacheFilename = + CPLStrdup(VSIMemGenerateHiddenFilename(CPLSPrintf( + "%s.%s", CACHE_LAYER_NAME, osExt.c_str()))); CPLDebug("OGR", "Switch geocode cache file to %s", hSession->pszCacheFilename); poDS = reinterpret_cast( diff --git a/ogr/ogr_geometry.h b/ogr/ogr_geometry.h index 73db71889bfc..0c680130791d 100644 --- a/ogr/ogr_geometry.h +++ b/ogr/ogr_geometry.h @@ -3047,6 +3047,7 @@ class CPL_DLL OGRGeometryCollection : public OGRGeometry virtual OGRErr addGeometryDirectly(OGRGeometry *); OGRErr addGeometry(std::unique_ptr geom); virtual OGRErr removeGeometry(int iIndex, int bDelete = TRUE); + std::unique_ptr stealGeometry(int iIndex); bool hasEmptyParts() const override; void removeEmptyParts() override; @@ -4284,6 +4285,12 @@ class CPL_DLL OGRGeometryFactory ~TransformWithOptionsCache(); }; + //! @cond Doxygen_Suppress + static bool isTransformWithOptionsRegularTransform( + const OGRSpatialReference *poSourceCRS, + const OGRSpatialReference *poTargetCRS, CSLConstList papszOptions); + //! @endcond + static OGRGeometry *transformWithOptions( const OGRGeometry *poSrcGeom, OGRCoordinateTransformation *poCT, char **papszOptions, diff --git a/ogr/ogr_spatialref.h b/ogr/ogr_spatialref.h index 91d8be2ba575..f1a3b7c91ba0 100644 --- a/ogr/ogr_spatialref.h +++ b/ogr/ogr_spatialref.h @@ -194,6 +194,8 @@ class CPL_DLL OGRSpatialReference OGRSpatialReference &operator=(const OGRSpatialReference &); OGRSpatialReference &operator=(OGRSpatialReference &&); + OGRSpatialReference &AssignAndSetThreadSafe(const OGRSpatialReference &); + int Reference(); int Dereference(); int GetReferenceCount() const; diff --git a/ogr/ogr_srs_erm.cpp b/ogr/ogr_srs_erm.cpp index 27c09433142a..77b91dcf9637 100644 --- a/ogr/ogr_srs_erm.cpp +++ b/ogr/ogr_srs_erm.cpp @@ -120,7 +120,7 @@ OGRErr OGRSpatialReference::importFromERM(const char *pszProj, } // Remove trailing ] - osProjWKT.resize(osProjWKT.size() - 1); + osProjWKT.pop_back(); // Remove any UNIT auto nPos = osProjWKT.find(",UNIT"); diff --git a/ogr/ogr_wkb.cpp b/ogr/ogr_wkb.cpp index 0f98f0766971..c2b39d03b3ce 100644 --- a/ogr/ogr_wkb.cpp +++ b/ogr/ogr_wkb.cpp @@ -891,7 +891,7 @@ static bool OGRWKBIsClockwiseRing(const GByte *data, const uint32_t nPoints, { v = i; vX = x; - vY = y; + // vY = y; bUseFallback = false; } else if (x == vX) @@ -1072,6 +1072,357 @@ void OGRWKBFixupCounterClockWiseExternalRing(GByte *pabyWkb, size_t nWKBSize) pabyWkb, nWKBSize, iOffsetInOut, /* nRec = */ 0); } +/************************************************************************/ +/* OGRWKBPointUpdater() */ +/************************************************************************/ + +OGRWKBPointUpdater::OGRWKBPointUpdater() = default; + +/************************************************************************/ +/* OGRWKBIntersectsPointSequencePessimistic() */ +/************************************************************************/ + +static bool OGRWKBUpdatePointsSequence(uint8_t *data, const size_t size, + OGRWKBPointUpdater &oUpdater, + const OGRwkbByteOrder eByteOrder, + const int nDim, const bool bHasZ, + const bool bHasM, size_t &iOffsetInOut) +{ + const uint32_t nPoints = + OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); + if (nPoints > (size - iOffsetInOut) / (nDim * sizeof(double))) + { + return false; + } + const bool bNeedSwap = OGR_SWAP(eByteOrder); + for (uint32_t j = 0; j < nPoints; j++) + { + void *pdfX = data + iOffsetInOut; + void *pdfY = data + iOffsetInOut + sizeof(double); + void *pdfZ = bHasZ ? data + iOffsetInOut + 2 * sizeof(double) : nullptr; + void *pdfM = + bHasM ? data + iOffsetInOut + (bHasZ ? 3 : 2) * sizeof(double) + : nullptr; + if (!oUpdater.update(bNeedSwap, pdfX, pdfY, pdfZ, pdfM)) + return false; + + iOffsetInOut += nDim * sizeof(double); + } + + return true; +} + +/************************************************************************/ +/* OGRWKBVisitRingSequence() */ +/************************************************************************/ + +static bool OGRWKBVisitRingSequence(uint8_t *data, const size_t size, + OGRWKBPointUpdater &oUpdater, + const OGRwkbByteOrder eByteOrder, + const int nDim, const bool bHasZ, + const bool bHasM, size_t &iOffsetInOut) +{ + const uint32_t nRings = + OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); + if (nRings > (size - iOffsetInOut) / sizeof(uint32_t)) + { + return false; + } + + for (uint32_t i = 0; i < nRings; ++i) + { + if (iOffsetInOut + sizeof(uint32_t) > size) + { + return false; + } + if (!OGRWKBUpdatePointsSequence(data, size, oUpdater, eByteOrder, nDim, + bHasZ, bHasM, iOffsetInOut)) + { + return false; + } + } + return true; +} + +/************************************************************************/ +/* OGRWKBUpdatePoints() */ +/************************************************************************/ + +static bool OGRWKBUpdatePoints(uint8_t *data, const size_t size, + OGRWKBPointUpdater &oUpdater, + size_t &iOffsetInOut, const int nRec) +{ + if (size - iOffsetInOut < MIN_WKB_SIZE) + { + return false; + } + const int nByteOrder = DB2_V72_FIX_BYTE_ORDER(data[iOffsetInOut]); + if (!(nByteOrder == wkbXDR || nByteOrder == wkbNDR)) + { + return false; + } + const OGRwkbByteOrder eByteOrder = static_cast(nByteOrder); + + OGRwkbGeometryType eGeometryType = wkbUnknown; + OGRReadWKBGeometryType(data + iOffsetInOut, wkbVariantIso, &eGeometryType); + iOffsetInOut += 5; + const auto eFlatType = wkbFlatten(eGeometryType); + + if (eFlatType == wkbGeometryCollection || eFlatType == wkbCompoundCurve || + eFlatType == wkbCurvePolygon || eFlatType == wkbMultiPoint || + eFlatType == wkbMultiLineString || eFlatType == wkbMultiPolygon || + eFlatType == wkbMultiCurve || eFlatType == wkbMultiSurface || + eFlatType == wkbPolyhedralSurface || eFlatType == wkbTIN) + { + if (nRec == 128) + return false; + + const uint32_t nParts = + OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); + if (nParts > (size - iOffsetInOut) / MIN_WKB_SIZE) + { + return false; + } + for (uint32_t k = 0; k < nParts; k++) + { + if (!OGRWKBUpdatePoints(data, size, oUpdater, iOffsetInOut, + nRec + 1)) + return false; + } + return true; + } + + const bool bHasZ = OGR_GT_HasZ(eGeometryType); + const bool bHasM = OGR_GT_HasM(eGeometryType); + const int nDim = 2 + (bHasZ ? 1 : 0) + (bHasM ? 1 : 0); + + if (eFlatType == wkbPoint) + { + if (size - iOffsetInOut < nDim * sizeof(double)) + return false; + void *pdfX = data + iOffsetInOut; + void *pdfY = data + iOffsetInOut + sizeof(double); + void *pdfZ = bHasZ ? data + iOffsetInOut + 2 * sizeof(double) : nullptr; + void *pdfM = + bHasM ? data + iOffsetInOut + (bHasZ ? 3 : 2) * sizeof(double) + : nullptr; + const bool bNeedSwap = OGR_SWAP(eByteOrder); + if (!oUpdater.update(bNeedSwap, pdfX, pdfY, pdfZ, pdfM)) + return false; + iOffsetInOut += nDim * sizeof(double); + return true; + } + + if (eFlatType == wkbLineString || eFlatType == wkbCircularString) + { + return OGRWKBUpdatePointsSequence(data, size, oUpdater, eByteOrder, + nDim, bHasZ, bHasM, iOffsetInOut); + } + + if (eFlatType == wkbPolygon || eFlatType == wkbTriangle) + { + return OGRWKBVisitRingSequence(data, size, oUpdater, eByteOrder, nDim, + bHasZ, bHasM, iOffsetInOut); + } + + CPLDebug("OGR", "Unknown WKB geometry type"); + return false; +} + +/** Visit all points of a WKB geometry to update them. + */ +bool OGRWKBUpdatePoints(GByte *pabyWkb, size_t nWKBSize, + OGRWKBPointUpdater &oUpdater) +{ + size_t iOffsetInOut = 0; + return OGRWKBUpdatePoints(pabyWkb, nWKBSize, oUpdater, iOffsetInOut, + /* nRec = */ 0); +} + +/************************************************************************/ +/* OGRWKBTransformCache::clear() */ +/************************************************************************/ + +#ifdef OGR_WKB_TRANSFORM_ALL_AT_ONCE +void OGRWKBTransformCache::clear() +{ + abNeedSwap.clear(); + abIsEmpty.clear(); + apdfX.clear(); + apdfY.clear(); + apdfZ.clear(); + apdfM.clear(); + adfX.clear(); + adfY.clear(); + adfZ.clear(); + adfM.clear(); + anErrorCodes.clear(); +} +#endif + +/************************************************************************/ +/* OGRWKBTransform() */ +/************************************************************************/ + +/** Visit all points of a WKB geometry to transform them. + */ +bool OGRWKBTransform(GByte *pabyWkb, size_t nWKBSize, + OGRCoordinateTransformation *poCT, + [[maybe_unused]] OGRWKBTransformCache &oCache, + OGREnvelope3D &sEnvelope) +{ +#ifndef OGR_WKB_TRANSFORM_ALL_AT_ONCE + struct OGRWKBPointUpdaterReproj final : public OGRWKBPointUpdater + { + OGRCoordinateTransformation *m_poCT; + OGREnvelope3D &m_sEnvelope; + + explicit OGRWKBPointUpdaterReproj(OGRCoordinateTransformation *poCTIn, + OGREnvelope3D &sEnvelopeIn) + : m_poCT(poCTIn), m_sEnvelope(sEnvelopeIn) + { + } + + bool update(bool bNeedSwap, void *x, void *y, void *z, + void * /* m */) override + { + double dfX, dfY, dfZ; + memcpy(&dfX, x, sizeof(double)); + memcpy(&dfY, y, sizeof(double)); + if (bNeedSwap) + { + CPL_SWAP64PTR(&dfX); + CPL_SWAP64PTR(&dfY); + } + if (!(std::isnan(dfX) && std::isnan(dfY))) + { + if (z) + { + memcpy(&dfZ, z, sizeof(double)); + if (bNeedSwap) + { + CPL_SWAP64PTR(&dfZ); + } + } + else + dfZ = 0; + int nErrorCode = 0; + m_poCT->TransformWithErrorCodes(1, &dfX, &dfY, &dfZ, nullptr, + &nErrorCode); + if (nErrorCode) + return false; + m_sEnvelope.Merge(dfX, dfY, dfZ); + if (bNeedSwap) + { + CPL_SWAP64PTR(&dfX); + CPL_SWAP64PTR(&dfY); + CPL_SWAP64PTR(&dfZ); + } + memcpy(x, &dfX, sizeof(double)); + memcpy(y, &dfY, sizeof(double)); + if (z) + memcpy(z, &dfZ, sizeof(double)); + } + return true; + } + + private: + OGRWKBPointUpdaterReproj(const OGRWKBPointUpdaterReproj &) = delete; + OGRWKBPointUpdaterReproj & + operator=(const OGRWKBPointUpdaterReproj &) = delete; + }; + + sEnvelope = OGREnvelope3D(); + OGRWKBPointUpdaterReproj oUpdater(poCT, sEnvelope); + return OGRWKBUpdatePoints(pabyWkb, nWKBSize, oUpdater); + +#else + struct OGRWKBPointUpdaterReproj final : public OGRWKBPointUpdater + { + OGRWKBTransformCache &m_oCache; + + explicit OGRWKBPointUpdaterReproj(OGRWKBTransformCache &oCacheIn) + : m_oCache(oCacheIn) + { + } + + bool update(bool bNeedSwap, void *x, void *y, void *z, + void * /* m */) override + { + m_oCache.abNeedSwap.push_back(bNeedSwap); + m_oCache.apdfX.push_back(x); + m_oCache.apdfY.push_back(y); + m_oCache.apdfZ.push_back(z); + return true; + } + }; + + oCache.clear(); + OGRWKBPointUpdaterReproj oUpdater(oCache); + if (!OGRWKBUpdatePoints(pabyWkb, nWKBSize, oUpdater)) + return false; + + oCache.adfX.resize(oCache.apdfX.size()); + oCache.adfY.resize(oCache.apdfX.size()); + oCache.adfZ.resize(oCache.apdfX.size()); + + for (size_t i = 0; i < oCache.apdfX.size(); ++i) + { + memcpy(&oCache.adfX[i], oCache.apdfX[i], sizeof(double)); + memcpy(&oCache.adfY[i], oCache.apdfY[i], sizeof(double)); + if (oCache.apdfZ[i]) + memcpy(&oCache.adfZ[i], oCache.apdfZ[i], sizeof(double)); + if (oCache.abNeedSwap[i]) + { + CPL_SWAP64PTR(&oCache.adfX[i]); + CPL_SWAP64PTR(&oCache.adfY[i]); + CPL_SWAP64PTR(&oCache.adfZ[i]); + } + oCache.abIsEmpty.push_back(std::isnan(oCache.adfX[i]) && + std::isnan(oCache.adfY[i])); + } + + oCache.anErrorCodes.resize(oCache.apdfX.size()); + poCT->TransformWithErrorCodes(static_cast(oCache.apdfX.size()), + oCache.adfX.data(), oCache.adfY.data(), + oCache.adfZ.data(), nullptr, + oCache.anErrorCodes.data()); + + for (size_t i = 0; i < oCache.apdfX.size(); ++i) + { + if (!oCache.abIsEmpty[i] && oCache.anErrorCodes[i]) + return false; + } + + sEnvelope = OGREnvelope3D(); + for (size_t i = 0; i < oCache.apdfX.size(); ++i) + { + if (oCache.abIsEmpty[i]) + { + oCache.adfX[i] = std::numeric_limits::quiet_NaN(); + oCache.adfY[i] = std::numeric_limits::quiet_NaN(); + oCache.adfZ[i] = std::numeric_limits::quiet_NaN(); + } + else + { + sEnvelope.Merge(oCache.adfX[i], oCache.adfY[i], oCache.adfZ[i]); + } + if (oCache.abNeedSwap[i]) + { + CPL_SWAP64PTR(&oCache.adfX[i]); + CPL_SWAP64PTR(&oCache.adfY[i]); + CPL_SWAP64PTR(&oCache.adfZ[i]); + } + memcpy(oCache.apdfX[i], &oCache.adfX[i], sizeof(double)); + memcpy(oCache.apdfY[i], &oCache.adfY[i], sizeof(double)); + if (oCache.apdfZ[i]) + memcpy(oCache.apdfZ[i], &oCache.adfZ[i], sizeof(double)); + } + + return true; +#endif +} + /************************************************************************/ /* OGRAppendBuffer() */ /************************************************************************/ diff --git a/ogr/ogr_wkb.h b/ogr/ogr_wkb.h index 2c3a9770dc89..ef4bd409702f 100644 --- a/ogr/ogr_wkb.h +++ b/ogr/ogr_wkb.h @@ -32,6 +32,8 @@ #include "cpl_port.h" #include "ogr_core.h" +#include + bool CPL_DLL OGRWKBGetGeomType(const GByte *pabyWkb, size_t nWKBSize, bool &bNeedSwap, uint32_t &nType); bool OGRWKBPolygonGetArea(const GByte *&pabyWkb, size_t &nWKBSize, @@ -62,6 +64,46 @@ void CPL_DLL OGRWKBFixupCounterClockWiseExternalRing(GByte *pabyWkb, const GByte CPL_DLL *WKBFromEWKB(GByte *pabyEWKB, size_t nEWKBSize, size_t &nWKBSizeOut, int *pnSRIDOut); +/** Object to update point coordinates in a WKB geometry */ +class CPL_DLL OGRWKBPointUpdater +{ + public: + OGRWKBPointUpdater(); + virtual ~OGRWKBPointUpdater() = default; + + /** Update method */ + virtual bool update(bool bNeedSwap, void *x, void *y, void *z, void *m) = 0; +}; + +bool CPL_DLL OGRWKBUpdatePoints(GByte *pabyWkb, size_t nWKBSize, + OGRWKBPointUpdater &oUpdater); + +/** Transformation cache */ +struct CPL_DLL OGRWKBTransformCache +{ +#ifdef OGR_WKB_TRANSFORM_ALL_AT_ONCE + std::vector abNeedSwap{}; + std::vector abIsEmpty{}; + std::vector apdfX{}; + std::vector apdfY{}; + std::vector apdfZ{}; + std::vector apdfM{}; + std::vector adfX{}; + std::vector adfY{}; + std::vector adfZ{}; + std::vector adfM{}; + std::vector anErrorCodes{}; + + void clear(); +#endif +}; + +class OGRCoordinateTransformation; +bool CPL_DLL OGRWKBTransform(GByte *pabyWkb, size_t nWKBSize, + OGRCoordinateTransformation *poCT, + OGRWKBTransformCache &oCache, + OGREnvelope3D &sEnvelope); + /************************************************************************/ /* OGRAppendBuffer */ /************************************************************************/ diff --git a/ogr/ogrct.cpp b/ogr/ogrct.cpp index 49b83af8b096..afc3bbb489ea 100644 --- a/ogr/ogrct.cpp +++ b/ogr/ogrct.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include "cpl_conv.h" #include "cpl_error.h" @@ -765,6 +766,9 @@ class OGRProjCT : public OGRCoordinateTransformation double dfThreshold = 0.0; + PJ_CONTEXT *m_psLastContext = nullptr; + std::thread::id m_nLastContextThreadId{}; + PjPtr m_pj{}; bool m_bReversePj = false; @@ -1277,9 +1281,10 @@ OGRProjCT::OGRProjCT(const OGRProjCT &other) m_osTargetSRS(other.m_osTargetSRS), bWebMercatorToWGS84LongLat(other.bWebMercatorToWGS84LongLat), nErrorCount(other.nErrorCount), dfThreshold(other.dfThreshold), - m_pj(other.m_pj), m_bReversePj(other.m_bReversePj), - m_bEmitErrors(other.m_bEmitErrors), bNoTransform(other.bNoTransform), - m_eStrategy(other.m_eStrategy), + m_psLastContext(nullptr), + m_nLastContextThreadId(std::this_thread::get_id()), m_pj(other.m_pj), + m_bReversePj(other.m_bReversePj), m_bEmitErrors(other.m_bEmitErrors), + bNoTransform(other.bNoTransform), m_eStrategy(other.m_eStrategy), m_oTransformations(other.m_oTransformations), m_iCurTransformation(other.m_iCurTransformation), m_options(other.m_options) @@ -2232,6 +2237,18 @@ int OGRCoordinateTransformation::TransformWithErrorCodes(size_t nCount, int *panErrorCodes) { + if (nCount == 1) + { + int nSuccess = 0; + const bool bOverallSuccess = + CPL_TO_BOOL(Transform(nCount, x, y, z, t, &nSuccess)); + if (panErrorCodes) + { + panErrorCodes[0] = nSuccess ? 0 : -1; + } + return bOverallSuccess; + } + std::vector abSuccess; try { @@ -2532,7 +2549,15 @@ int OGRProjCT::TransformWithErrorCodes(size_t nCount, double *x, double *y, /* Select dynamically the best transformation for the data, if */ /* needed. */ /* -------------------------------------------------------------------- */ - auto ctx = OSRGetProjTLSContext(); + PJ_CONTEXT *ctx = m_psLastContext; + const auto nThisThreadId = std::this_thread::get_id(); + if (!ctx || nThisThreadId != m_nLastContextThreadId) + { + m_nLastContextThreadId = nThisThreadId; + m_psLastContext = OSRGetProjTLSContext(); + ctx = m_psLastContext; + } + PJ *pj = m_pj; if (!bTransformDone && !pj) { diff --git a/ogr/ogrfeature.cpp b/ogr/ogrfeature.cpp index 2304e8cbf516..b1afa3fd3473 100644 --- a/ogr/ogrfeature.cpp +++ b/ogr/ogrfeature.cpp @@ -146,10 +146,12 @@ OGRFeature::~OGRFeature() { if (pauFields != nullptr) { - const int nFieldcount = poDefn->GetFieldCount(); + // We can call GetFieldCountUnsafe() as the constructor has called + // the regular GetFieldCount() + const int nFieldcount = poDefn->GetFieldCountUnsafe(); for (int i = 0; i < nFieldcount; i++) { - OGRFieldDefn *poFDefn = poDefn->GetFieldDefn(i); + const OGRFieldDefn *poFDefn = poDefn->GetFieldDefnUnsafe(i); if (!IsFieldSetAndNotNullUnsafe(i)) continue; @@ -254,7 +256,8 @@ OGRFeature *OGRFeature::CreateFeature(OGRFeatureDefn *poDefn) if (poFeature == nullptr) return nullptr; - if ((poFeature->pauFields == nullptr && poDefn->GetFieldCount() != 0) || + if ((poFeature->pauFields == nullptr && + poDefn->GetFieldCountUnsafe() != 0) || (poFeature->papoGeometries == nullptr && poDefn->GetGeomFieldCount() != 0)) { @@ -311,7 +314,7 @@ void OGRFeature::Reset() if (!IsFieldSetAndNotNullUnsafe(i)) continue; - OGRFieldDefn *poFDefn = poDefn->GetFieldDefn(i); + const OGRFieldDefn *poFDefn = poDefn->GetFieldDefnUnsafe(i); switch (poFDefn->GetType()) { case OFTString: @@ -1139,7 +1142,8 @@ OGRFeatureH OGR_F_Clone(OGRFeatureH hFeat) bool OGRFeature::CopySelfTo(OGRFeature *poNew) const { - for (int i = 0; i < poDefn->GetFieldCount(); i++) + const int nFieldCount = poDefn->GetFieldCountUnsafe(); + for (int i = 0; i < nFieldCount; i++) { if (!poNew->SetFieldInternal(i, pauFields + i)) { @@ -1505,7 +1509,7 @@ int OGR_F_GetGeomFieldIndex(OGRFeatureH hFeat, const char *pszName) int OGRFeature::IsFieldSet(int iField) const { - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { // Special field value accessors. @@ -1658,7 +1662,7 @@ void OGR_F_UnsetField(OGRFeatureH hFeat, int iField) bool OGRFeature::IsFieldNull(int iField) const { - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { // FIXME? @@ -1721,7 +1725,7 @@ int OGR_F_IsFieldNull(OGRFeatureH hFeat, int iField) bool OGRFeature::IsFieldSetAndNotNull(int iField) const { - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { return CPL_TO_BOOL(IsFieldSet(iField)); @@ -1997,7 +2001,7 @@ OGRField *OGR_F_GetRawFieldRef(OGRFeatureH hFeat, int iField) int OGRFeature::GetFieldAsInteger(int iField) const { - int iSpecialField = iField - poDefn->GetFieldCount(); + int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { // Special field value accessors. @@ -2137,7 +2141,7 @@ int OGR_F_GetFieldAsInteger(OGRFeatureH hFeat, int iField) GIntBig OGRFeature::GetFieldAsInteger64(int iField) const { - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { // Special field value accessors. @@ -2255,7 +2259,7 @@ GIntBig OGR_F_GetFieldAsInteger64(OGRFeatureH hFeat, int iField) double OGRFeature::GetFieldAsDouble(int iField) const { - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { // Special field value accessors. @@ -2442,7 +2446,7 @@ const char *OGRFeature::GetFieldAsString(int iField) const CPLFree(m_pszTmpFieldValue); m_pszTmpFieldValue = nullptr; - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { // Special field value accessors. @@ -2810,7 +2814,7 @@ const char *OGRFeature::GetFieldAsISO8601DateTime( CPLFree(m_pszTmpFieldValue); m_pszTmpFieldValue = nullptr; - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { return ""; @@ -2912,7 +2916,7 @@ const char *OGR_F_GetFieldAsISO8601DateTime(OGRFeatureH hFeat, int iField, const int *OGRFeature::GetFieldAsIntegerList(int iField, int *pnCount) const { - OGRFieldDefn *poFDefn = poDefn->GetFieldDefn(iField); + const OGRFieldDefn *poFDefn = poDefn->GetFieldDefn(iField); if (poFDefn != nullptr && IsFieldSetAndNotNullUnsafe(iField) && poFDefn->GetType() == OFTIntegerList) @@ -3516,7 +3520,7 @@ static int OGRFeatureGetIntegerValue(OGRFieldDefn *poFDefn, int nValue) char *OGRFeature::GetFieldAsSerializedJSon(int iField) const { - const int iSpecialField = iField - poDefn->GetFieldCount(); + const int iSpecialField = iField - poDefn->GetFieldCountUnsafe(); if (iSpecialField >= 0) { return nullptr; @@ -5657,11 +5661,12 @@ std::string OGRFeature::DumpReadableAsString(CSLConstList papszOptions) const CSLFetchNameValue(papszOptions, "DISPLAY_FIELDS"); if (pszDisplayFields == nullptr || CPLTestBool(pszDisplayFields)) { - for (int iField = 0; iField < GetFieldCount(); iField++) + const int nFieldCount = GetFieldCount(); + for (int iField = 0; iField < nFieldCount; iField++) { if (!IsFieldSet(iField)) continue; - OGRFieldDefn *poFDefn = poDefn->GetFieldDefn(iField); + const OGRFieldDefn *poFDefn = poDefn->GetFieldDefnUnsafe(iField); const char *pszType = (poFDefn->GetSubType() != OFSTNone) @@ -5926,7 +5931,7 @@ OGRBoolean OGRFeature::Equal(const OGRFeature *poFeature) const if (GetDefnRef() != poFeature->GetDefnRef()) return FALSE; - const int nFields = GetDefnRef()->GetFieldCount(); + const int nFields = GetDefnRef()->GetFieldCountUnsafe(); for (int i = 0; i < nFields; i++) { if (IsFieldSet(i) != poFeature->IsFieldSet(i)) @@ -5936,7 +5941,7 @@ OGRBoolean OGRFeature::Equal(const OGRFeature *poFeature) const if (!IsFieldSetAndNotNullUnsafe(i)) continue; - switch (GetDefnRef()->GetFieldDefn(i)->GetType()) + switch (GetDefnRef()->GetFieldDefnUnsafe(i)->GetType()) { case OFTInteger: if (GetFieldAsInteger(i) != poFeature->GetFieldAsInteger(i)) @@ -6806,10 +6811,12 @@ OGRErr OGRFeature::RemapFields(OGRFeatureDefn *poNewDefn, if (poNewDefn == nullptr) poNewDefn = poDefn; - OGRField *pauNewFields = static_cast( - CPLCalloc(poNewDefn->GetFieldCount(), sizeof(OGRField))); + const int nNewFieldCount = poNewDefn->GetFieldCount(); + OGRField *pauNewFields = + static_cast(CPLCalloc(nNewFieldCount, sizeof(OGRField))); - for (int iDstField = 0; iDstField < poDefn->GetFieldCount(); iDstField++) + const int nFieldCount = poDefn->GetFieldCount(); + for (int iDstField = 0; iDstField < nFieldCount; iDstField++) { if (panRemapSource[iDstField] == -1) { @@ -6846,7 +6853,7 @@ OGRErr OGRFeature::RemapFields(OGRFeatureDefn *poNewDefn, void OGRFeature::AppendField() { - int nFieldCount = poDefn->GetFieldCount(); + const int nFieldCount = poDefn->GetFieldCountUnsafe(); pauFields = static_cast( CPLRealloc(pauFields, nFieldCount * sizeof(OGRField))); OGR_RawField_SetUnset(&pauFields[nFieldCount - 1]); @@ -7002,7 +7009,7 @@ void OGRFeature::FillUnsetWithDefault(int bNotNullableOnly, pszDefault[strlen(pszDefault) - 1] == '\'') { CPLString osDefault(pszDefault + 1); - osDefault.resize(osDefault.size() - 1); + osDefault.pop_back(); char *pszTmp = CPLUnescapeString(osDefault, nullptr, CPLES_SQL); SetField(i, pszTmp); CPLFree(pszTmp); diff --git a/ogr/ogrfeaturequery.cpp b/ogr/ogrfeaturequery.cpp index 4cc77749d7f8..371672d89551 100644 --- a/ogr/ogrfeaturequery.cpp +++ b/ogr/ogrfeaturequery.cpp @@ -33,7 +33,6 @@ #include "ogr_swq.h" #include -#include #include #include "cpl_conv.h" @@ -407,18 +406,6 @@ int OGRFeatureQuery::CanUseIndex(const swq_expr_node *psExpr, OGRLayer *poLayer) /* multi-part queries with ranges. */ /************************************************************************/ -static int CompareGIntBig(const void *pa, const void *pb) -{ - const GIntBig a = *(reinterpret_cast(pa)); - const GIntBig b = *(reinterpret_cast(pb)); - if (a < b) - return -1; - else if (a > b) - return 1; - else - return 0; -} - GIntBig *OGRFeatureQuery::EvaluateAgainstIndices(OGRLayer *poLayer, OGRErr *peErr) @@ -668,8 +655,7 @@ GIntBig *OGRFeatureQuery::EvaluateAgainstIndices(const swq_expr_node *psExpr, if (nFIDCount > 1) { // The returned FIDs are expected to be in sorted order. - qsort(panFIDs, static_cast(nFIDCount), sizeof(GIntBig), - CompareGIntBig); + std::sort(panFIDs, panFIDs + nFIDCount); } return panFIDs; } @@ -712,9 +698,7 @@ GIntBig *OGRFeatureQuery::EvaluateAgainstIndices(const swq_expr_node *psExpr, if (nFIDCount > 1) { // The returned FIDs are expected to be sorted. - // TODO(schwehr): Use std::sort. - qsort(panFIDs, static_cast(nFIDCount), sizeof(GIntBig), - CompareGIntBig); + std::sort(panFIDs, panFIDs + nFIDCount); } return panFIDs; } diff --git a/ogr/ogrgeometry.cpp b/ogr/ogrgeometry.cpp index 287f20370bff..46dd0bd1264c 100644 --- a/ogr/ogrgeometry.cpp +++ b/ogr/ogrgeometry.cpp @@ -3862,7 +3862,12 @@ static OGRBoolean OGRGEOSBooleanPredicate( /** * \brief Attempts to make an invalid geometry valid without losing vertices. * - * Already-valid geometries are cloned without further intervention. + * Already-valid geometries are cloned without further intervention + * for default MODE=LINEWORK. Already-valid geometries with MODE=STRUCTURE + * may be subject to non-significant transformations, such as duplicated point + * removal, change in ring winding order, etc. (before GDAL 3.10, single-part + * geometry collections could be returned a single geometry. GDAL 3.10 + * returns the same type of geometry). * * Running OGRGeometryFactory::removeLowerDimensionSubGeoms() as a * post-processing step is often desired. @@ -3973,6 +3978,20 @@ OGRGeometry *OGRGeometry::MakeValid(CSLConstList papszOptions) const poOGRProduct = OGRGeometryRebuildCurves(this, nullptr, poOGRProduct); GEOSGeom_destroy_r(hGEOSCtxt, hGEOSRet); + +#if GEOS_VERSION_MAJOR > 3 || \ + (GEOS_VERSION_MAJOR == 3 && GEOS_VERSION_MINOR >= 10) + // METHOD=STRUCTURE is not guaranteed to return a multiple geometry + // if the input is a multiple geometry + if (poOGRProduct && bStructureMethod && + OGR_GT_IsSubClassOf(getGeometryType(), wkbGeometryCollection) && + !OGR_GT_IsSubClassOf(poOGRProduct->getGeometryType(), + wkbGeometryCollection)) + { + poOGRProduct = OGRGeometryFactory::forceTo(poOGRProduct, + getGeometryType()); + } +#endif } } freeGEOSContext(hGEOSCtxt); diff --git a/ogr/ogrgeometrycollection.cpp b/ogr/ogrgeometrycollection.cpp index 637619f663df..e4666651666b 100644 --- a/ogr/ogrgeometrycollection.cpp +++ b/ogr/ogrgeometrycollection.cpp @@ -439,7 +439,8 @@ OGRErr OGRGeometryCollection::addGeometry(std::unique_ptr geom) * * @param bDelete if TRUE the geometry will be deallocated, otherwise it will * not. The default is TRUE as the container is considered to own the - * geometries in it. + * geometries in it. Note: using stealGeometry() might be a better alternative + * to using bDelete = false. * * @return OGRERR_NONE if successful, or OGRERR_FAILURE if the index is * out of range. @@ -470,6 +471,35 @@ OGRErr OGRGeometryCollection::removeGeometry(int iGeom, int bDelete) return OGRERR_NONE; } +/************************************************************************/ +/* stealGeometry() */ +/************************************************************************/ + +/** + * \brief Remove a geometry from the container and return it to the caller + * + * Removing a geometry will cause the geometry count to drop by one, and all + * "higher" geometries will shuffle down one in index. + * + * There is no SFCOM analog to this method. + * + * @param iGeom the index of the geometry to delete. + * + * @return the sub-geometry, or nullptr in case of error. + * @since 3.10 + */ + +std::unique_ptr OGRGeometryCollection::stealGeometry(int iGeom) +{ + if (iGeom < 0 || iGeom >= nGeomCount) + return nullptr; + + auto poSubGeom = std::unique_ptr(papoGeoms[iGeom]); + papoGeoms[iGeom] = nullptr; + removeGeometry(iGeom); + return poSubGeom; +} + /************************************************************************/ /* hasEmptyParts() */ /************************************************************************/ diff --git a/ogr/ogrgeometryfactory.cpp b/ogr/ogrgeometryfactory.cpp index 756a608d7ad5..ff9bad541cb8 100644 --- a/ogr/ogrgeometryfactory.cpp +++ b/ogr/ogrgeometryfactory.cpp @@ -3302,15 +3302,15 @@ static void AlterPole(OGRGeometry *poGeom, OGRPoint *poPole, } /************************************************************************/ -/* IsPolarToWGS84() */ +/* IsPolarToGeographic() */ /* */ /* Returns true if poCT transforms from a projection that includes one */ /* of the pole in a continuous way. */ /************************************************************************/ -static bool IsPolarToWGS84(OGRCoordinateTransformation *poCT, - OGRCoordinateTransformation *poRevCT, - bool &bIsNorthPolarOut) +static bool IsPolarToGeographic(OGRCoordinateTransformation *poCT, + OGRCoordinateTransformation *poRevCT, + bool &bIsNorthPolarOut) { bool bIsNorthPolar = false; bool bIsSouthPolar = false; @@ -3366,17 +3366,16 @@ static bool IsPolarToWGS84(OGRCoordinateTransformation *poCT, } /************************************************************************/ -/* TransformBeforePolarToWGS84() */ +/* TransformBeforePolarToGeographic() */ /* */ /* Transform the geometry (by intersection), so as to cut each geometry */ /* that crosses the pole, in 2 parts. Do also tricks for geometries */ /* that just touch the pole. */ /************************************************************************/ -static OGRGeometry * -TransformBeforePolarToWGS84(OGRCoordinateTransformation *poRevCT, - bool bIsNorthPolar, OGRGeometry *poDstGeom, - bool &bNeedPostCorrectionOut) +static std::unique_ptr TransformBeforePolarToGeographic( + OGRCoordinateTransformation *poRevCT, bool bIsNorthPolar, + std::unique_ptr poDstGeom, bool &bNeedPostCorrectionOut) { const int nSign = (bIsNorthPolar) ? 1 : -1; @@ -3429,19 +3428,19 @@ TransformBeforePolarToWGS84(OGRCoordinateTransformation *poRevCT, { if (bContainsPole || bContainsNearPoleAntimeridian) { - OGRGeometry *poNewGeom = poDstGeom->Difference(&oCutter); + auto poNewGeom = + std::unique_ptr(poDstGeom->Difference(&oCutter)); if (poNewGeom) { if (bContainsNearPoleAntimeridian) - RemovePoint(poNewGeom, &oPole); - delete poDstGeom; - poDstGeom = poNewGeom; + RemovePoint(poNewGeom.get(), &oPole); + poDstGeom = std::move(poNewGeom); } } if (bRegularTouchesPole) { - AlterPole(poDstGeom, &oPole); + AlterPole(poDstGeom.get(), &oPole); } bNeedPostCorrectionOut = true; @@ -3450,15 +3449,15 @@ TransformBeforePolarToWGS84(OGRCoordinateTransformation *poRevCT, } /************************************************************************/ -/* IsAntimeridianProjToWGS84() */ +/* IsAntimeridianProjToGeographic() */ /* */ /* Returns true if poCT transforms from a projection that includes the */ /* antimeridian in a continuous way. */ /************************************************************************/ -static bool IsAntimeridianProjToWGS84(OGRCoordinateTransformation *poCT, - OGRCoordinateTransformation *poRevCT, - OGRGeometry *poDstGeometry) +static bool IsAntimeridianProjToGeographic(OGRCoordinateTransformation *poCT, + OGRCoordinateTransformation *poRevCT, + OGRGeometry *poDstGeometry) { const bool bBackupEmitErrors = poCT->GetEmitErrors(); poRevCT->SetEmitErrors(false); @@ -3634,15 +3633,15 @@ struct SortPointsByAscendingY }; /************************************************************************/ -/* TransformBeforeAntimeridianToWGS84() */ +/* TransformBeforeAntimeridianToGeographic() */ /* */ /* Transform the geometry (by intersection), so as to cut each geometry */ /* that crosses the antimeridian, in 2 parts. */ /************************************************************************/ -static OGRGeometry *TransformBeforeAntimeridianToWGS84( +static std::unique_ptr TransformBeforeAntimeridianToGeographic( OGRCoordinateTransformation *poCT, OGRCoordinateTransformation *poRevCT, - OGRGeometry *poDstGeom, bool &bNeedPostCorrectionOut) + std::unique_ptr poDstGeom, bool &bNeedPostCorrectionOut) { OGREnvelope sEnvelope; poDstGeom->getEnvelope(&sEnvelope); @@ -3662,7 +3661,7 @@ static OGRGeometry *TransformBeforeAntimeridianToWGS84( // Collect points that are the intersection of the lines of the geometry // with the antimeridian std::vector aoPoints; - CollectPointsOnAntimeridian(poDstGeom, poCT, poRevCT, aoPoints); + CollectPointsOnAntimeridian(poDstGeom.get(), poCT, poRevCT, aoPoints); if (aoPoints.empty()) return poDstGeom; @@ -3726,11 +3725,11 @@ static OGRGeometry *TransformBeforeAntimeridianToWGS84( #endif // Get the geometry without the antimeridian - OGRGeometry *poInter = poDstGeom->Difference(&oPolyToCut); + auto poInter = + std::unique_ptr(poDstGeom->Difference(&oPolyToCut)); if (poInter != nullptr) { - delete poDstGeom; - poDstGeom = poInter; + poDstGeom = std::move(poInter); bNeedPostCorrectionOut = true; } @@ -3821,9 +3820,22 @@ static void SnapCoordsCloseToLatLongBounds(OGRGeometry *poGeom) struct OGRGeometryFactory::TransformWithOptionsCache::Private { + const OGRSpatialReference *poSourceCRS = nullptr; + const OGRSpatialReference *poTargetCRS = nullptr; + const OGRCoordinateTransformation *poCT = nullptr; std::unique_ptr poRevCT{}; bool bIsPolar = false; bool bIsNorthPolar = false; + + void clear() + { + poSourceCRS = nullptr; + poTargetCRS = nullptr; + poCT = nullptr; + poRevCT.reset(); + bIsPolar = false; + bIsNorthPolar = false; + } }; /************************************************************************/ @@ -3843,6 +3855,50 @@ OGRGeometryFactory::TransformWithOptionsCache::~TransformWithOptionsCache() { } +/************************************************************************/ +/* isTransformWithOptionsRegularTransform() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress +/*static */ +bool OGRGeometryFactory::isTransformWithOptionsRegularTransform( + [[maybe_unused]] const OGRSpatialReference *poSourceCRS, + [[maybe_unused]] const OGRSpatialReference *poTargetCRS, + CSLConstList papszOptions) +{ + if (papszOptions) + return false; + +#ifdef HAVE_GEOS + if (poSourceCRS->IsProjected() && poTargetCRS->IsGeographic() && + poTargetCRS->GetAxisMappingStrategy() == OAMS_TRADITIONAL_GIS_ORDER && + // check that angular units is degree + std::fabs(poTargetCRS->GetAngularUnits(nullptr) - + CPLAtof(SRS_UA_DEGREE_CONV)) <= + 1e-8 * CPLAtof(SRS_UA_DEGREE_CONV)) + { + double dfWestLong = 0.0; + double dfSouthLat = 0.0; + double dfEastLong = 0.0; + double dfNorthLat = 0.0; + if (poSourceCRS->GetAreaOfUse(&dfWestLong, &dfSouthLat, &dfEastLong, + &dfNorthLat, nullptr) && + !(dfSouthLat == -90.0 || dfNorthLat == 90.0 || + dfWestLong == -180.0 || dfEastLong == 180.0 || + dfWestLong > dfEastLong)) + { + // Not a global geographic CRS + return true; + } + return false; + } +#endif + + return true; +} + +//! @endcond + /************************************************************************/ /* transformWithOptions() */ /************************************************************************/ @@ -3858,65 +3914,68 @@ OGRGeometry *OGRGeometryFactory::transformWithOptions( const OGRGeometry *poSrcGeom, OGRCoordinateTransformation *poCT, char **papszOptions, CPL_UNUSED const TransformWithOptionsCache &cache) { - OGRGeometry *poDstGeom = poSrcGeom->clone(); - if (poCT != nullptr) + auto poDstGeom = std::unique_ptr(poSrcGeom->clone()); + if (poCT) { #ifdef HAVE_GEOS bool bNeedPostCorrection = false; - - auto poSourceCRS = poCT->GetSourceCS(); - auto poTargetCRS = poCT->GetTargetCS(); - if (poSourceCRS != nullptr && poTargetCRS != nullptr && - poSourceCRS->IsProjected() && poTargetCRS->IsGeographic()) + const auto poSourceCRS = poCT->GetSourceCS(); + const auto poTargetCRS = poCT->GetTargetCS(); + const auto eSrcGeomType = wkbFlatten(poSrcGeom->getGeometryType()); + // Check if we are transforming from projected coordinates to + // geographic coordinates, with a chance that there might be polar or + // anti-meridian discontinuities. If so, create the inverse transform. + if (eSrcGeomType != wkbPoint && eSrcGeomType != wkbMultiPoint && + (poSourceCRS != cache.d->poSourceCRS || + poTargetCRS != cache.d->poTargetCRS || poCT != cache.d->poCT)) { - OGRSpatialReference oSRSWGS84; - oSRSWGS84.SetWellKnownGeogCS("WGS84"); - oSRSWGS84.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (poTargetCRS->IsSame(&oSRSWGS84)) + cache.d->clear(); + cache.d->poSourceCRS = poSourceCRS; + cache.d->poTargetCRS = poTargetCRS; + cache.d->poCT = poCT; + if (poSourceCRS && poTargetCRS && + !isTransformWithOptionsRegularTransform( + poSourceCRS, poTargetCRS, papszOptions)) { - if (cache.d->poRevCT == nullptr || - !cache.d->poRevCT->GetTargetCS()->IsSame(poSourceCRS)) - { - cache.d->poRevCT.reset(OGRCreateCoordinateTransformation( - &oSRSWGS84, poSourceCRS)); - cache.d->bIsNorthPolar = false; - cache.d->bIsPolar = false; - if (cache.d->poRevCT && - IsPolarToWGS84(poCT, cache.d->poRevCT.get(), - cache.d->bIsNorthPolar)) - { - cache.d->bIsPolar = true; - } - } - auto poRevCT = cache.d->poRevCT.get(); - if (poRevCT != nullptr) + cache.d->poRevCT.reset(OGRCreateCoordinateTransformation( + poTargetCRS, poSourceCRS)); + cache.d->bIsNorthPolar = false; + cache.d->bIsPolar = false; + cache.d->poRevCT.reset(poCT->GetInverse()); + if (cache.d->poRevCT && + IsPolarToGeographic(poCT, cache.d->poRevCT.get(), + cache.d->bIsNorthPolar)) { - if (cache.d->bIsPolar) - { - poDstGeom = TransformBeforePolarToWGS84( - poRevCT, cache.d->bIsNorthPolar, poDstGeom, - bNeedPostCorrection); - } - else if (IsAntimeridianProjToWGS84(poCT, poRevCT, - poDstGeom)) - { - poDstGeom = TransformBeforeAntimeridianToWGS84( - poCT, poRevCT, poDstGeom, bNeedPostCorrection); - } + cache.d->bIsPolar = true; } } } + + if (auto poRevCT = cache.d->poRevCT.get()) + { + if (cache.d->bIsPolar) + { + poDstGeom = TransformBeforePolarToGeographic( + poRevCT, cache.d->bIsNorthPolar, std::move(poDstGeom), + bNeedPostCorrection); + } + else if (IsAntimeridianProjToGeographic(poCT, poRevCT, + poDstGeom.get())) + { + poDstGeom = TransformBeforeAntimeridianToGeographic( + poCT, poRevCT, std::move(poDstGeom), bNeedPostCorrection); + } + } #endif OGRErr eErr = poDstGeom->transform(poCT); if (eErr != OGRERR_NONE) { - delete poDstGeom; return nullptr; } #ifdef HAVE_GEOS if (bNeedPostCorrection) { - SnapCoordsCloseToLatLongBounds(poDstGeom); + SnapCoordsCloseToLatLongBounds(poDstGeom.get()); } #endif } @@ -3935,7 +3994,7 @@ OGRGeometry *OGRGeometryFactory::transformWithOptions( "non-geographic CRS"); bHasWarned = true; } - return poDstGeom; + return poDstGeom.release(); } // TODO and we should probably also test that the axis order + data axis // mapping is long-lat... @@ -3958,9 +4017,9 @@ OGRGeometry *OGRGeometryFactory::transformWithOptions( OGREnvelope sEnvelope; poDstGeom->getEnvelope(&sEnvelope); if (sEnvelope.MinX >= -360.0 && sEnvelope.MaxX <= -180.0) - AddOffsetToLon(poDstGeom, 360.0); + AddOffsetToLon(poDstGeom.get(), 360.0); else if (sEnvelope.MinX >= 180.0 && sEnvelope.MaxX <= 360.0) - AddOffsetToLon(poDstGeom, -360.0); + AddOffsetToLon(poDstGeom.get(), -360.0); else { OGRwkbGeometryType eNewType; @@ -3971,38 +4030,35 @@ OGRGeometry *OGRGeometryFactory::transformWithOptions( else eNewType = wkbGeometryCollection; - OGRGeometry *poMultiGeom = createGeometry(eNewType); - OGRGeometryCollection *poMulti = - poMultiGeom->toGeometryCollection(); + auto poMulti = std::unique_ptr( + createGeometry(eNewType)->toGeometryCollection()); double dfDateLineOffset = CPLAtofM( CSLFetchNameValueDef(papszOptions, "DATELINEOFFSET", "10")); if (dfDateLineOffset <= 0.0 || dfDateLineOffset >= 360.0) dfDateLineOffset = 10.0; - CutGeometryOnDateLineAndAddToMulti(poMulti, poDstGeom, - dfDateLineOffset); + CutGeometryOnDateLineAndAddToMulti( + poMulti.get(), poDstGeom.get(), dfDateLineOffset); if (poMulti->getNumGeometries() == 0) { - delete poMultiGeom; + // do nothing } - else if (poMulti->getNumGeometries() == 1) + else if (poMulti->getNumGeometries() == 1 && + (eType == wkbPolygon || eType == wkbLineString)) { - delete poDstGeom; - poDstGeom = poMulti->getGeometryRef(0)->clone(); - delete poMultiGeom; + poDstGeom = poMulti->stealGeometry(0); } else { - delete poDstGeom; - poDstGeom = poMultiGeom; + poDstGeom = std::move(poMulti); } } } } - return poDstGeom; + return poDstGeom.release(); } /************************************************************************/ diff --git a/ogr/ogrpgeogeometry.cpp b/ogr/ogrpgeogeometry.cpp index fab888ab17ad..71f156f3a856 100644 --- a/ogr/ogrpgeogeometry.cpp +++ b/ogr/ogrpgeogeometry.cpp @@ -2451,7 +2451,8 @@ OGRErr OGRCreateFromShapeBin(GByte *pabyShape, OGRGeometry **ppoGeom, if (nBits & EXT_SHAPE_ARC_IP) CPLDebug("OGR", " DefinedIP"); #endif - if ((nBits & EXT_SHAPE_ARC_IP) != 0) + if ((nBits & EXT_SHAPE_ARC_IP) != 0 && + (nBits & EXT_SHAPE_ARC_LINE) == 0) { pasCurves[iCurve].eType = CURVE_ARC_INTERIOR_POINT; pasCurves[iCurve].u.ArcByIntermediatePoint.dfX = dfVal1; diff --git a/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp b/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp index 8e5dad1c0053..4fe00174df31 100644 --- a/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp +++ b/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp @@ -90,12 +90,12 @@ static bool IsArrowIPCStream(GDALOpenInfo *poOpenInfo) } const std::string osTmpFilename( - CPLSPrintf("/vsimem/_arrow/%p", poOpenInfo)); + VSIMemGenerateHiddenFilename("arrow")); auto fp = VSIVirtualHandleUniquePtr(VSIFileFromMemBuffer( osTmpFilename.c_str(), poOpenInfo->pabyHeader, nSizeToRead, false)); - auto infile = - std::make_shared(std::move(fp)); + auto infile = std::make_shared( + osTmpFilename.c_str(), std::move(fp)); auto options = arrow::ipc::IpcReadOptions::Defaults(); auto result = arrow::ipc::RecordBatchStreamReader::Open(infile, options); @@ -113,8 +113,8 @@ static bool IsArrowIPCStream(GDALOpenInfo *poOpenInfo) return false; // Do not give ownership of poOpenInfo->fpL to infile - auto infile = - std::make_shared(poOpenInfo->fpL, false); + auto infile = std::make_shared( + poOpenInfo->pszFilename, poOpenInfo->fpL, false); auto options = arrow::ipc::IpcReadOptions::Defaults(); auto result = arrow::ipc::RecordBatchStreamReader::Open(infile, options); @@ -164,14 +164,16 @@ static GDALDataset *OGRFeatherDriverOpen(GDALOpenInfo *poOpenInfo) osFilename.c_str()); return nullptr; } - infile = std::make_shared(std::move(fp)); + infile = std::make_shared(osFilename.c_str(), + std::move(fp)); } else if (STARTS_WITH(poOpenInfo->pszFilename, "/vsi") || CPLTestBool(CPLGetConfigOption("OGR_ARROW_USE_VSI", "NO"))) { VSIVirtualHandleUniquePtr fp(poOpenInfo->fpL); poOpenInfo->fpL = nullptr; - infile = std::make_shared(std::move(fp)); + infile = std::make_shared( + poOpenInfo->pszFilename, std::move(fp)); } else { diff --git a/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h b/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h index 6179fd009b07..3f09195c1987 100644 --- a/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h +++ b/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h @@ -352,6 +352,13 @@ class OGRArrowDataset CPL_NON_FINAL : public GDALPamDataset std::vector m_aosDomainNames{}; std::map m_oMapDomainNameToCol{}; + protected: + void close() + { + m_poLayer.reset(); + m_poMemoryPool.reset(); + } + public: explicit OGRArrowDataset( const std::shared_ptr &poMemoryPool); diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp index 084a7273acea..cd51cf2905d9 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp @@ -248,6 +248,9 @@ inline bool OGRArrowLayer::IsHandledListOrMapType( itemTypeId == arrow::Type::DECIMAL256 || itemTypeId == arrow::Type::STRING || itemTypeId == arrow::Type::LARGE_STRING || +#if ARROW_VERSION_MAJOR >= 15 + itemTypeId == arrow::Type::STRING_VIEW || +#endif itemTypeId == arrow::Type::STRUCT || (itemTypeId == arrow::Type::MAP && IsHandledMapType( @@ -276,7 +279,12 @@ inline bool OGRArrowLayer::IsHandledListType( inline bool OGRArrowLayer::IsHandledMapType(const std::shared_ptr &mapType) { - return mapType->key_type()->id() == arrow::Type::STRING && + const auto typeId = mapType->key_type()->id(); + return (typeId == arrow::Type::STRING +#if ARROW_VERSION_MAJOR >= 15 + || typeId == arrow::Type::STRING_VIEW +#endif + ) && IsHandledListOrMapType(mapType->item_type()); } @@ -369,6 +377,9 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( break; case arrow::Type::STRING: case arrow::Type::LARGE_STRING: +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: +#endif bTypeOK = true; eType = OFTString; if (osExtensionName == EXTENSION_NAME_ARROW_JSON) @@ -376,6 +387,9 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( break; case arrow::Type::BINARY: case arrow::Type::LARGE_BINARY: +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::BINARY_VIEW: +#endif bTypeOK = true; eType = OFTBinary; break; @@ -476,6 +490,9 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( break; case arrow::Type::STRING: case arrow::Type::LARGE_STRING: +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: +#endif eType = OFTStringList; break; default: @@ -538,8 +555,6 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( case arrow::Type::RUN_END_ENCODED: #endif #if ARROW_VERSION_MAJOR >= 15 - case arrow::Type::STRING_VIEW: - case arrow::Type::BINARY_VIEW: case arrow::Type::LIST_VIEW: case arrow::Type::LARGE_LIST_VIEW: #endif @@ -1321,6 +1336,15 @@ static void AddToArray(CPLJSONArray &oArray, const arrow::Array *array, nIdx)); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + oArray.Add( + static_cast(array)->GetString( + nIdx)); + break; + } +#endif case arrow::Type::LIST: case arrow::Type::LARGE_LIST: case arrow::Type::FIXED_SIZE_LIST: @@ -1491,6 +1515,14 @@ static void AddToDict(CPLJSONObject &oDict, const std::string &osKey, ->GetString(nIdx)); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + oDict.Add(osKey, static_cast(array) + ->GetString(nIdx)); + break; + } +#endif case arrow::Type::LIST: case arrow::Type::LARGE_LIST: case arrow::Type::FIXED_SIZE_LIST: @@ -1514,12 +1546,12 @@ static void AddToDict(CPLJSONObject &oDict, const std::string &osKey, /* GetMapAsJSON() */ /************************************************************************/ +template static CPLJSONObject GetMapAsJSON(const arrow::Array *array, const size_t nIdxInArray) { const auto mapArray = static_cast(array); - const auto keys = - std::static_pointer_cast(mapArray->keys()); + const auto keys = std::static_pointer_cast(mapArray->keys()); const auto values = mapArray->items(); const auto nIdxStart = mapArray->value_offset(nIdxInArray); const int nCount = mapArray->value_length(nIdxInArray); @@ -1538,6 +1570,24 @@ static CPLJSONObject GetMapAsJSON(const arrow::Array *array, return oRoot; } +static CPLJSONObject GetMapAsJSON(const arrow::Array *array, + const size_t nIdxInArray) +{ + const auto mapArray = static_cast(array); + const auto eKeyType = mapArray->keys()->type()->id(); + if (eKeyType == arrow::Type::STRING) + return GetMapAsJSON(array, nIdxInArray); +#if ARROW_VERSION_MAJOR >= 15 + else if (eKeyType == arrow::Type::STRING_VIEW) + return GetMapAsJSON(array, nIdxInArray); +#endif + else + { + CPLAssert(false); + return CPLJSONObject(); + } +} + /************************************************************************/ /* GetStructureAsJSON() */ /************************************************************************/ @@ -1802,6 +1852,27 @@ static void ReadList(OGRFeature *poFeature, int i, int64_t nIdxInArray, poFeature->SetField(i, aosList.List()); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + const auto values = + std::static_pointer_cast( + array->values()); + const auto nIdxStart = array->value_offset(nIdxInArray); + const int nCount = array->value_length(nIdxInArray); + CPLStringList aosList; + for (int k = 0; k < nCount; k++) + { + if (values->IsNull(nIdxStart + k)) + aosList.AddString( + ""); // we cannot have null strings in a list + else + aosList.AddString(values->GetString(nIdxStart + k).c_str()); + } + poFeature->SetField(i, aosList.List()); + break; + } +#endif case arrow::Type::LIST: case arrow::Type::LARGE_LIST: case arrow::Type::FIXED_SIZE_LIST: @@ -2244,6 +2315,31 @@ inline OGRFeature *OGRArrowLayer::ReadFeature( poFeature->SetField(i, out_length, data); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::BINARY_VIEW: + { + const auto castArray = + static_cast(array); + const auto view = castArray->GetView(nIdxInBatch); + poFeature->SetField(i, static_cast(view.size()), + view.data()); + break; + } +#endif +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + const auto castArray = + static_cast(array); + const auto strView = castArray->GetView(nIdxInBatch); + char *pszString = + static_cast(CPLMalloc(strView.length() + 1)); + memcpy(pszString, strView.data(), strView.length()); + pszString[strView.length()] = 0; + poFeature->SetFieldSameTypeUnsafe(i, pszString); + break; + } +#endif case arrow::Type::FIXED_SIZE_BINARY: { const auto castArray = @@ -2424,8 +2520,6 @@ inline OGRFeature *OGRArrowLayer::ReadFeature( case arrow::Type::RUN_END_ENCODED: #endif #if ARROW_VERSION_MAJOR >= 15 - case arrow::Type::STRING_VIEW: - case arrow::Type::BINARY_VIEW: case arrow::Type::LIST_VIEW: case arrow::Type::LARGE_LIST_VIEW: #endif @@ -3919,62 +4013,65 @@ OGRArrowLayer::SetBatch(const std::shared_ptr &poBatch) { const int idx = m_bIgnoredFields ? oIter->second.iArrayIdx : oIter->second.iArrowCol; - CPLAssert(idx >= 0); - CPLAssert(static_cast(idx) < m_poBatchColumns.size()); - m_poArrayBBOX = m_poBatchColumns[idx].get(); - CPLAssert(m_poArrayBBOX->type_id() == arrow::Type::STRUCT); - const auto castArray = - static_cast(m_poArrayBBOX); - const auto &subArrays = castArray->fields(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldXMin) < - subArrays.size()); - const auto xminArray = - subArrays[oIter->second.iArrowSubfieldXMin].get(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldYMin) < - subArrays.size()); - const auto yminArray = - subArrays[oIter->second.iArrowSubfieldYMin].get(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldXMax) < - subArrays.size()); - const auto xmaxArray = - subArrays[oIter->second.iArrowSubfieldXMax].get(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldYMax) < - subArrays.size()); - const auto ymaxArray = - subArrays[oIter->second.iArrowSubfieldYMax].get(); - if (oIter->second.bIsFloat) - { - CPLAssert(xminArray->type_id() == arrow::Type::FLOAT); - m_poArrayXMinFloat = - static_cast(xminArray); - CPLAssert(yminArray->type_id() == arrow::Type::FLOAT); - m_poArrayYMinFloat = - static_cast(yminArray); - CPLAssert(xmaxArray->type_id() == arrow::Type::FLOAT); - m_poArrayXMaxFloat = - static_cast(xmaxArray); - CPLAssert(ymaxArray->type_id() == arrow::Type::FLOAT); - m_poArrayYMaxFloat = - static_cast(ymaxArray); - } - else + if (idx >= 0) { - CPLAssert(xminArray->type_id() == arrow::Type::DOUBLE); - m_poArrayXMinDouble = - static_cast(xminArray); - CPLAssert(yminArray->type_id() == arrow::Type::DOUBLE); - m_poArrayYMinDouble = - static_cast(yminArray); - CPLAssert(xmaxArray->type_id() == arrow::Type::DOUBLE); - m_poArrayXMaxDouble = - static_cast(xmaxArray); - CPLAssert(ymaxArray->type_id() == arrow::Type::DOUBLE); - m_poArrayYMaxDouble = - static_cast(ymaxArray); + CPLAssert(static_cast(idx) < + m_poBatchColumns.size()); + m_poArrayBBOX = m_poBatchColumns[idx].get(); + CPLAssert(m_poArrayBBOX->type_id() == arrow::Type::STRUCT); + const auto castArray = + static_cast(m_poArrayBBOX); + const auto &subArrays = castArray->fields(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldXMin) < + subArrays.size()); + const auto xminArray = + subArrays[oIter->second.iArrowSubfieldXMin].get(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldYMin) < + subArrays.size()); + const auto yminArray = + subArrays[oIter->second.iArrowSubfieldYMin].get(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldXMax) < + subArrays.size()); + const auto xmaxArray = + subArrays[oIter->second.iArrowSubfieldXMax].get(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldYMax) < + subArrays.size()); + const auto ymaxArray = + subArrays[oIter->second.iArrowSubfieldYMax].get(); + if (oIter->second.bIsFloat) + { + CPLAssert(xminArray->type_id() == arrow::Type::FLOAT); + m_poArrayXMinFloat = + static_cast(xminArray); + CPLAssert(yminArray->type_id() == arrow::Type::FLOAT); + m_poArrayYMinFloat = + static_cast(yminArray); + CPLAssert(xmaxArray->type_id() == arrow::Type::FLOAT); + m_poArrayXMaxFloat = + static_cast(xmaxArray); + CPLAssert(ymaxArray->type_id() == arrow::Type::FLOAT); + m_poArrayYMaxFloat = + static_cast(ymaxArray); + } + else + { + CPLAssert(xminArray->type_id() == arrow::Type::DOUBLE); + m_poArrayXMinDouble = + static_cast(xminArray); + CPLAssert(yminArray->type_id() == arrow::Type::DOUBLE); + m_poArrayYMinDouble = + static_cast(yminArray); + CPLAssert(xmaxArray->type_id() == arrow::Type::DOUBLE); + m_poArrayXMaxDouble = + static_cast(xmaxArray); + CPLAssert(ymaxArray->type_id() == arrow::Type::DOUBLE); + m_poArrayYMaxDouble = + static_cast(ymaxArray); + } } } } diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h b/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h index cd16ddd14868..a1c4a463459a 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h @@ -36,6 +36,9 @@ #include "arrow/io/file.h" #include "arrow/io/interfaces.h" +#include +#include + /************************************************************************/ /* OGRArrowRandomAccessFile */ /************************************************************************/ @@ -43,22 +46,58 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile { int64_t m_nSize = -1; + const std::string m_osFilename; VSILFILE *m_fp; - bool m_bOwnFP; + const bool m_bOwnFP; + std::atomic m_bAskedToClosed = false; + +#ifdef OGR_ARROW_USE_PREAD + const bool m_bDebugReadAt; + const bool m_bUsePRead; +#endif OGRArrowRandomAccessFile(const OGRArrowRandomAccessFile &) = delete; OGRArrowRandomAccessFile & operator=(const OGRArrowRandomAccessFile &) = delete; public: - explicit OGRArrowRandomAccessFile(VSILFILE *fp, bool bOwnFP) - : m_fp(fp), m_bOwnFP(bOwnFP) + OGRArrowRandomAccessFile(const std::string &osFilename, VSILFILE *fp, + bool bOwnFP) + : m_osFilename(osFilename), m_fp(fp), m_bOwnFP(bOwnFP) +#ifdef OGR_ARROW_USE_PREAD + , + m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())), + // Due to the lack of caching for current /vsicurl PRead(), do not + // use the PRead() implementation on those files + m_bUsePRead(m_fp->HasPRead() && + CPLTestBool(CPLGetConfigOption( + "OGR_ARROW_USE_PREAD", + VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO"))) +#endif + { + } + + OGRArrowRandomAccessFile(const std::string &osFilename, + VSIVirtualHandleUniquePtr &&fp) + : m_osFilename(osFilename), m_fp(fp.release()), m_bOwnFP(true) +#ifdef OGR_ARROW_USE_PREAD + , + m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())), + // Due to the lack of caching for current /vsicurl PRead(), do not + // use the PRead() implementation on those files + m_bUsePRead(m_fp->HasPRead() && + CPLTestBool(CPLGetConfigOption( + "OGR_ARROW_USE_PREAD", + VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO"))) +#endif { } - explicit OGRArrowRandomAccessFile(VSIVirtualHandleUniquePtr &&fp) - : m_fp(fp.release()), m_bOwnFP(true) + void AskToClose() { + m_bAskedToClosed = true; + if (m_fp) + m_fp->Interrupt(); } ~OGRArrowRandomAccessFile() override @@ -85,11 +124,14 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile bool closed() const override { - return m_fp == nullptr; + return m_bAskedToClosed || m_fp == nullptr; } arrow::Status Seek(int64_t position) override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + if (VSIFSeekL(m_fp, static_cast(position), SEEK_SET) == 0) return arrow::Status::OK(); return arrow::Status::IOError("Error while seeking"); @@ -97,6 +139,9 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile arrow::Result Read(int64_t nbytes, void *out) override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + CPLAssert(static_cast(static_cast(nbytes)) == nbytes); return static_cast( VSIFReadL(out, 1, static_cast(nbytes), m_fp)); @@ -104,12 +149,10 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile arrow::Result> Read(int64_t nbytes) override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + // CPLDebug("ARROW", "Reading %d bytes", int(nbytes)); - // Ugly hack for https://github.com/OSGeo/gdal/issues/9497 - if (CPLGetConfigOption("OGR_ARROW_STOP_IO", nullptr)) - { - return arrow::Result>(); - } auto buffer = arrow::AllocateResizableBuffer(nbytes); if (!buffer.ok()) { @@ -122,8 +165,54 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile return buffer; } +#ifdef OGR_ARROW_USE_PREAD + using arrow::io::RandomAccessFile::ReadAt; + + arrow::Result> + ReadAt(int64_t position, int64_t nbytes) override + { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + + if (m_bUsePRead) + { + auto buffer = arrow::AllocateResizableBuffer(nbytes); + if (!buffer.ok()) + { + return buffer; + } + if (m_bDebugReadAt) + { + CPLDebug( + "ARROW", + "Start ReadAt() called on %s (this=%p) from " + "thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64, + m_osFilename.c_str(), this, CPLGetPID(), position, nbytes); + } + uint8_t *buffer_data = (*buffer)->mutable_data(); + auto nread = m_fp->PRead(buffer_data, static_cast(nbytes), + static_cast(position)); + CPL_IGNORE_RET_VAL( + (*buffer)->Resize(nread)); // shrink --> cannot fail + if (m_bDebugReadAt) + { + CPLDebug( + "ARROW", + "End ReadAt() called on %s (this=%p) from " + "thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64, + m_osFilename.c_str(), this, CPLGetPID(), position, nbytes); + } + return buffer; + } + return arrow::io::RandomAccessFile::ReadAt(position, nbytes); + } +#endif + arrow::Result GetSize() override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + if (m_nSize < 0) { const auto nPos = VSIFTellL(m_fp); diff --git a/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp b/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp index 6a259e5d880b..0f50d4162ebe 100644 --- a/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp +++ b/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp @@ -33,6 +33,12 @@ #include "ograrrowrandomaccessfile.h" +#include +#include +#include +#include +#include + /************************************************************************/ /* VSIArrowFileSystem */ /************************************************************************/ @@ -42,14 +48,54 @@ class VSIArrowFileSystem final : public arrow::fs::FileSystem const std::string m_osEnvVarPrefix; const std::string m_osQueryParameters; + std::atomic m_bAskedToClosed = false; + std::mutex m_oMutex{}; + std::vector>> + m_oSetFiles{}; + public: - explicit VSIArrowFileSystem(const std::string &osEnvVarPrefix, - const std::string &osQueryParameters) + VSIArrowFileSystem(const std::string &osEnvVarPrefix, + const std::string &osQueryParameters) : m_osEnvVarPrefix(osEnvVarPrefix), m_osQueryParameters(osQueryParameters) { } + // Cf comment in OGRParquetDataset::~OGRParquetDataset() for rationale + // for this method + void AskToClose() + { + m_bAskedToClosed = true; + std::vector< + std::pair>> + oSetFiles; + { + std::lock_guard oLock(m_oMutex); + oSetFiles = m_oSetFiles; + } + for (auto &[osName, poFile] : oSetFiles) + { + bool bWarned = false; + while (!poFile.expired()) + { + if (!bWarned) + { + bWarned = true; + auto poFileLocked = poFile.lock(); + if (poFileLocked) + { + CPLDebug("PARQUET", + "Still on-going reads on %s. Waiting for it " + "to be closed.", + osName.c_str()); + poFileLocked->AskToClose(); + } + } + CPLSleep(0.01); + } + } + } + std::string type_name() const override { return "vsi" + m_osEnvVarPrefix; @@ -203,6 +249,10 @@ class VSIArrowFileSystem final : public arrow::fs::FileSystem arrow::Result> OpenInputFile(const std::string &path) override { + if (m_bAskedToClosed) + return arrow::Status::IOError( + "OpenInputFile(): file system in shutdown"); + std::string osPath(path); osPath += m_osQueryParameters; CPLDebugOnly(m_osEnvVarPrefix.c_str(), "Opening %s", osPath.c_str()); @@ -210,7 +260,13 @@ class VSIArrowFileSystem final : public arrow::fs::FileSystem if (fp == nullptr) return arrow::Status::IOError("OpenInputFile() failed for " + osPath); - return std::make_shared(std::move(fp)); + auto poFile = + std::make_shared(osPath, std::move(fp)); + { + std::lock_guard oLock(m_oMutex); + m_oSetFiles.emplace_back(path, poFile); + } + return poFile; } using arrow::fs::FileSystem::OpenOutputStream; diff --git a/ogr/ogrsf_frmts/avc/avc_e00read.cpp b/ogr/ogrsf_frmts/avc/avc_e00read.cpp index 255ac4ba05c6..6c2bc215eda6 100644 --- a/ogr/ogrsf_frmts/avc/avc_e00read.cpp +++ b/ogr/ogrsf_frmts/avc/avc_e00read.cpp @@ -1003,7 +1003,7 @@ static int _AVCE00ReadBuildSqueleton(AVCE00ReadPtr psInfo, char **papszCoverDir) } CPLString osCoverPathTruncated(psInfo->pszCoverPath); - osCoverPathTruncated.resize(osCoverPathTruncated.size() - 1); + osCoverPathTruncated.pop_back(); pszEXPPath = CPLStrdup( CPLSPrintf("EXP 0 %s%s.E00", szCWD, osCoverPathTruncated.c_str())); pcTmp = pszEXPPath; diff --git a/ogr/ogrsf_frmts/cad/libopencad/dwg/CMakeLists.txt b/ogr/ogrsf_frmts/cad/libopencad/dwg/CMakeLists.txt deleted file mode 100644 index 8b137891791f..000000000000 --- a/ogr/ogrsf_frmts/cad/libopencad/dwg/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp index 3e979ce2cbb4..98e387324589 100644 --- a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp +++ b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp @@ -119,7 +119,7 @@ bool OGRCSVLayer::Matches(const char *pszFieldName, char **papszPossibleNames) { // *pattern* CPLString oPattern(pszPattern + 1); - oPattern.resize(oPattern.size() - 1); + oPattern.pop_back(); if (CPLString(pszFieldName).ifind(oPattern) != std::string::npos) return true; @@ -960,7 +960,7 @@ char **OGRCSVLayer::AutodetectFieldTypes(CSLConstList papszOpenOptions, nRead = VSIFReadL(pszData, 1, nRequested, fpCSV); pszData[nRead] = 0; - osTmpMemFile = CPLSPrintf("/vsimem/tmp%p", this); + osTmpMemFile = VSIMemGenerateHiddenFilename("temp.csv"); fp = VSIFileFromMemBuffer(osTmpMemFile.c_str(), reinterpret_cast(pszData), nRead, FALSE); diff --git a/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp b/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp index 82ae661544f0..b3c5d419254e 100644 --- a/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp +++ b/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp @@ -56,6 +56,8 @@ class OGRCSWLayer final : public OGRLayer CPLString osQuery; CPLString osCSWWhere; + std::string m_osTmpDir{}; + GDALDataset *FetchGetRecords(); GIntBig GetFeatureCountWithHits(); void BuildQuery(); @@ -258,6 +260,8 @@ OGRCSWLayer::OGRCSWLayer(OGRCSWDataSource *poDSIn) } poSRS->Release(); + + m_osTmpDir = VSIMemGenerateHiddenFilename("csw"); } /************************************************************************/ @@ -268,8 +272,7 @@ OGRCSWLayer::~OGRCSWLayer() { poFeatureDefn->Release(); GDALClose(poBaseDS); - CPLString osTmpDirName = CPLSPrintf("/vsimem/tempcsw_%p", this); - OGRWFSRecursiveUnlink(osTmpDirName); + VSIRmdirRecursive(m_osTmpDir.c_str()); } /************************************************************************/ @@ -531,8 +534,7 @@ GDALDataset *OGRCSWLayer::FetchGetRecords() return nullptr; } - CPLString osTmpDirName = CPLSPrintf("/vsimem/tempcsw_%p", this); - VSIMkdir(osTmpDirName, 0); + VSIMkdir(m_osTmpDir.c_str(), 0); GByte *pabyData = psResult->pabyData; int nDataLen = psResult->nDataLen; @@ -549,10 +551,10 @@ GDALDataset *OGRCSWLayer::FetchGetRecords() CPLString osTmpFileName; - osTmpFileName = osTmpDirName + "/file.gfs"; + osTmpFileName = m_osTmpDir + "/file.gfs"; VSIUnlink(osTmpFileName); - osTmpFileName = osTmpDirName + "/file.gml"; + osTmpFileName = m_osTmpDir + "/file.gml"; VSILFILE *fp = VSIFileFromMemBuffer(osTmpFileName, pabyData, nDataLen, TRUE); diff --git a/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp b/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp index c79476e1b2d5..1a8ff3bfa579 100644 --- a/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp +++ b/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp @@ -423,7 +423,7 @@ OGRFeature *OGRDWGLayer::TranslateMTEXT(OdDbEntityPtr poEntity) CPLString osText = TextUnescape(poMTE->contents(), true); if (!osText.empty() && osText.back() == '\n') - osText.resize(osText.size() - 1); + osText.pop_back(); poFeature->SetField("Text", osText); @@ -543,7 +543,7 @@ OGRFeature *OGRDWGLayer::TranslateTEXT(OdDbEntityPtr poEntity) CPLString osText = TextUnescape(poText->textString(), false); if (!osText.empty() && osText.back() == '\n') - osText.resize(osText.size() - 1); + osText.pop_back(); poFeature->SetField("Text", osText); diff --git a/ogr/ogrsf_frmts/dxf/ogrdxf_leader.cpp b/ogr/ogrsf_frmts/dxf/ogrdxf_leader.cpp index 8cdb26f6e7dc..e640f1d7b42a 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxf_leader.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxf_leader.cpp @@ -1412,7 +1412,7 @@ static void InterpolateSpline(OGRLineString *const poLine, adfParameters.push_back(dfParameter); } - const double dfTotalChordLength = adfParameters[adfParameters.size() - 1]; + const double dfTotalChordLength = adfParameters.back(); // Start tangent can be worked out from the first chord DXFTriple oStartTangent(aoDataPoints[1].dfX - aoDataPoints[0].dfX, diff --git a/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp b/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp index ba61e2a5b2ea..947144ee689f 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp @@ -554,7 +554,7 @@ OGRDXFFeature *OGRDXFLayer::TranslateMTEXT() /* Apply text after stripping off any extra terminating newline. */ /* -------------------------------------------------------------------- */ if (!osText.empty() && osText.back() == '\n') - osText.resize(osText.size() - 1); + osText.pop_back(); poFeature->SetField("Text", osText); diff --git a/ogr/ogrsf_frmts/filegdb/FGdbDriver.cpp b/ogr/ogrsf_frmts/filegdb/FGdbDriver.cpp index 3b2eafa548f9..aa8861ac42c4 100644 --- a/ogr/ogrsf_frmts/filegdb/FGdbDriver.cpp +++ b/ogr/ogrsf_frmts/filegdb/FGdbDriver.cpp @@ -294,7 +294,7 @@ OGRErr FGdbTransactionManager::StartTransaction(OGRDataSource *&poDSInOut, CPLString osName(poMutexedDS->GetName()); CPLString osNameOri(osName); if (osName.back() == '/' || osName.back() == '\\') - osName.resize(osName.size() - 1); + osName.pop_back(); #ifndef _WIN32 int bPerLayerCopyingForTransaction = @@ -452,7 +452,7 @@ OGRErr FGdbTransactionManager::CommitTransaction(OGRDataSource *&poDSInOut, CPLString osName(poMutexedDS->GetName()); CPLString osNameOri(osName); if (osName.back() == '/' || osName.back() == '\\') - osName.resize(osName.size() - 1); + osName.pop_back(); #ifndef _WIN32 int bPerLayerCopyingForTransaction = @@ -696,7 +696,7 @@ OGRErr FGdbTransactionManager::RollbackTransaction(OGRDataSource *&poDSInOut, CPLString osName(poMutexedDS->GetName()); CPLString osNameOri(osName); if (osName.back() == '/' || osName.back() == '\\') - osName.resize(osName.size() - 1); + osName.pop_back(); // int bPerLayerCopyingForTransaction = // poDS->HasPerLayerCopyingForTransaction(); diff --git a/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp b/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp index 81d3d6cc3bc9..44aab8480767 100644 --- a/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp +++ b/ogr/ogrsf_frmts/filegdb/FGdbLayer.cpp @@ -1837,7 +1837,7 @@ char *FGdbLayer::CreateFieldDefn(OGRFieldDefn &oField, int bApproxOK, if (osVal[0] == '\'' && osVal.back() == '\'') { osVal = osVal.substr(1); - osVal.resize(osVal.size() - 1); + osVal.pop_back(); char *pszTmp = CPLUnescapeString(osVal, nullptr, CPLES_SQL); osVal = pszTmp; CPLFree(pszTmp); diff --git a/ogr/ogrsf_frmts/flatgeobuf/geometryreader.cpp b/ogr/ogrsf_frmts/flatgeobuf/geometryreader.cpp index 45d18518adab..60c370144390 100644 --- a/ogr/ogrsf_frmts/flatgeobuf/geometryreader.cpp +++ b/ogr/ogrsf_frmts/flatgeobuf/geometryreader.cpp @@ -115,22 +115,31 @@ OGRMultiPoint *GeometryReader::readMultiPoint() OGRMultiLineString *GeometryReader::readMultiLineString() { - const auto pEnds = m_geometry->ends(); - if (pEnds == nullptr) - return CPLErrorInvalidPointer("MultiLineString ends data"); + const auto ends = m_geometry->ends(); auto mls = std::make_unique(); - m_offset = 0; - for (uint32_t i = 0; i < pEnds->size(); i++) + if (ends == nullptr || ends->size() < 2) { - const auto e = pEnds->Get(i); - if (e < m_offset) - return CPLErrorInvalidLength("MultiLineString"); - m_length = e - m_offset; - const auto ls = readSimpleCurve(); - if (ls == nullptr) + m_length = m_length / 2; + const auto part = readSimpleCurve(); + if (part == nullptr) return nullptr; - mls->addGeometryDirectly(ls); - m_offset = e; + mls->addGeometryDirectly(part); + } + else + { + m_offset = 0; + for (uint32_t i = 0; i < ends->size(); i++) + { + const auto e = ends->Get(i); + if (e < m_offset) + return CPLErrorInvalidLength("MultiLineString"); + m_length = e - m_offset; + const auto ls = readSimpleCurve(); + if (ls == nullptr) + return nullptr; + mls->addGeometryDirectly(ls); + m_offset = e; + } } return mls.release(); } diff --git a/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobuflayer.cpp b/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobuflayer.cpp index f964afb0be57..7319cc9f03ff 100644 --- a/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobuflayer.cpp +++ b/ogr/ogrsf_frmts/flatgeobuf/ogrflatgeobuflayer.cpp @@ -455,7 +455,7 @@ void OGRFlatGeobufLayer::writeHeader(VSILFILE *poFp, uint64_t featuresCount, if (osCoordinateEpoch.find('.') != std::string::npos) { while (osCoordinateEpoch.back() == '0') - osCoordinateEpoch.resize(osCoordinateEpoch.size() - 1); + osCoordinateEpoch.pop_back(); } std::string osWKT("COORDINATEMETADATA["); diff --git a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp index ec1072ec1f3f..91307d3efb32 100644 --- a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp +++ b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp @@ -32,9 +32,13 @@ #include "ogr_gensql.h" #include "cpl_string.h" #include "ogr_api.h" +#include "ogr_recordbatch.h" +#include "ogrlayerarrow.h" #include "cpl_time.h" #include #include +#include +#include #include //! @cond Doxygen_Suppress @@ -745,7 +749,7 @@ GIntBig OGRGenSQLResultsLayer::GetFeatureCount(int bForce) int OGRGenSQLResultsLayer::TestCapability(const char *pszCap) { - swq_select *psSelectInfo = m_pSelectInfo.get(); + const swq_select *psSelectInfo = m_pSelectInfo.get(); if (EQUAL(pszCap, OLCFastSetNextByIndex)) { @@ -774,6 +778,58 @@ int OGRGenSQLResultsLayer::TestCapability(const char *pszCap) return m_poSrcLayer->TestCapability(pszCap); } + else if (EQUAL(pszCap, OLCFastGetArrowStream)) + { + // Make sure the SQL is something as simple as + // "SELECT field1 [AS renamed], ... FROM ... WHERE ....", without + // duplicated fields + if (m_bForwardWhereToSourceLayer && + psSelectInfo->query_mode == SWQM_RECORDSET && + psSelectInfo->offset == 0 && psSelectInfo->join_count == 0 && + psSelectInfo->order_specs == 0) + { + std::set oSetFieldIndex; + int nLastIdxRegularField = -1; + for (std::size_t iField = 0; + iField < psSelectInfo->column_defs.size(); iField++) + { + const swq_col_def *psColDef = + &psSelectInfo->column_defs[iField]; + if (psColDef->bHidden || psColDef->table_index < 0 || + psColDef->col_func != SWQCF_NONE || + cpl::contains(oSetFieldIndex, psColDef->field_index)) + { + return false; + } + + oSetFieldIndex.insert(psColDef->field_index); + + const auto poLayerDefn = + m_apoTableLayers[psColDef->table_index]->GetLayerDefn(); + + if (psColDef->field_index >= 0 && poLayerDefn != nullptr && + psColDef->field_index < poLayerDefn->GetFieldCount()) + { + // We do not support re-ordered fields + if (psColDef->field_index <= nLastIdxRegularField) + return false; + nLastIdxRegularField = psColDef->field_index; + } + else if (poLayerDefn != nullptr && + IS_GEOM_FIELD_INDEX(poLayerDefn, + psColDef->field_index)) + { + // ok + } + else + { + return false; + } + } + return m_poSrcLayer->TestCapability(pszCap); + } + } + return FALSE; } @@ -2543,14 +2599,47 @@ int OGRGenSQLResultsLayer::Compare(const OGRField *pasFirstTuple, void OGRGenSQLResultsLayer::AddFieldDefnToSet(int iTable, int iColumn, CPLHashSet *hSet) { - if (iTable != -1 && iColumn != -1) + if (iTable != -1) { OGRLayer *poLayer = m_apoTableLayers[iTable]; - if (iColumn < poLayer->GetLayerDefn()->GetFieldCount()) + const auto poLayerDefn = poLayer->GetLayerDefn(); + const int nFieldCount = poLayerDefn->GetFieldCount(); + if (iColumn == -1) + { + for (int i = 0; i < nFieldCount; ++i) + { + OGRFieldDefn *poFDefn = poLayerDefn->GetFieldDefn(i); + CPLHashSetInsert(hSet, poFDefn); + } + + const int nGeomFieldCount = poLayerDefn->GetGeomFieldCount(); + for (int i = 0; i < nGeomFieldCount; ++i) + { + OGRGeomFieldDefn *poFDefn = poLayerDefn->GetGeomFieldDefn(i); + CPLHashSetInsert(hSet, poFDefn); + } + } + else { - OGRFieldDefn *poFDefn = - poLayer->GetLayerDefn()->GetFieldDefn(iColumn); - CPLHashSetInsert(hSet, poFDefn); + if (iColumn < nFieldCount) + { + OGRFieldDefn *poFDefn = poLayerDefn->GetFieldDefn(iColumn); + CPLHashSetInsert(hSet, poFDefn); + } + else if (iColumn == nFieldCount + SPF_OGR_GEOMETRY || + iColumn == nFieldCount + SPF_OGR_GEOM_WKT || + iColumn == nFieldCount + SPF_OGR_GEOM_AREA) + { + auto poSrcGFDefn = poLayerDefn->GetGeomFieldDefn(0); + CPLHashSetInsert(hSet, poSrcGFDefn); + } + else if (IS_GEOM_FIELD_INDEX(poLayerDefn, iColumn)) + { + const int iSrcGeomField = + ALL_FIELD_INDEX_TO_GEOM_FIELD_INDEX(poLayerDefn, iColumn); + auto poSrcGFDefn = poLayerDefn->GetGeomFieldDefn(iSrcGeomField); + CPLHashSetInsert(hSet, poSrcGFDefn); + } } } } @@ -2619,8 +2708,8 @@ void OGRGenSQLResultsLayer::FindAndSetIgnoredFields() OGRLayer *poLayer = m_apoTableLayers[iTable]; OGRFeatureDefn *poSrcFDefn = poLayer->GetLayerDefn(); char **papszIgnoredFields = nullptr; - for (int iSrcField = 0; iSrcField < poSrcFDefn->GetFieldCount(); - iSrcField++) + const int nSrcFieldCount = poSrcFDefn->GetFieldCount(); + for (int iSrcField = 0; iSrcField < nSrcFieldCount; iSrcField++) { OGRFieldDefn *poFDefn = poSrcFDefn->GetFieldDefn(iSrcField); if (CPLHashSetLookup(hSet, poFDefn) == nullptr) @@ -2632,6 +2721,19 @@ void OGRGenSQLResultsLayer::FindAndSetIgnoredFields() // poFDefn->GetNameRef(), poLayer->GetName()); } } + const int nSrcGeomFieldCount = poSrcFDefn->GetGeomFieldCount(); + for (int iSrcField = 0; iSrcField < nSrcGeomFieldCount; iSrcField++) + { + OGRGeomFieldDefn *poFDefn = poSrcFDefn->GetGeomFieldDefn(iSrcField); + if (CPLHashSetLookup(hSet, poFDefn) == nullptr) + { + papszIgnoredFields = + CSLAddString(papszIgnoredFields, poFDefn->GetNameRef()); + // CPLDebug("OGR", "Adding %s to the list of ignored fields of + // layer %s", + // poFDefn->GetNameRef(), poLayer->GetName()); + } + } poLayer->SetIgnoredFields( const_cast(papszIgnoredFields)); CSLDestroy(papszIgnoredFields); @@ -2692,4 +2794,302 @@ void OGRGenSQLResultsLayer::SetSpatialFilter(int iGeomField, OGRLayer::SetSpatialFilter(iGeomField, poGeom); } +/************************************************************************/ +/* OGRGenSQLResultsLayerArrowStreamPrivateData */ +/************************************************************************/ + +// Structure whose instances are set on the ArrowArrayStream::private_data +// member of the ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() +struct OGRGenSQLResultsLayerArrowStreamPrivateData +{ + // Member shared with OGRLayer::m_poSharedArrowArrayStreamPrivateData + // If the layer pointed by poShared->poLayer is destroyed, before its + // destruction, it nullifies poShared->poLayer, which we can detect. + std::shared_ptr poShared{}; + + // ArrowArrayStream to be used with poShared->poLayer + struct ArrowArrayStream *psSrcLayerStream = nullptr; + + // Original release() callback of the ArrowArrayStream passed to + // OGRGenSQLResultsLayer::GetArrowStream() + void (*release_backup)(struct ArrowArrayStream *) = nullptr; + + // Original private_data member of the ArrowArrayStream passed to + // OGRGenSQLResultsLayer::GetArrowStream() + void *private_data_backup = nullptr; + + // Set as the ArrowArrayStream::release callback member of the + // ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() + static void Release(struct ArrowArrayStream *self) + { + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + self->private_data); + + // Release source layer stream + if (psPrivateData->psSrcLayerStream->release) + psPrivateData->psSrcLayerStream->release( + psPrivateData->psSrcLayerStream); + CPLFree(psPrivateData->psSrcLayerStream); + + // Release ourselves using the base method + self->private_data = psPrivateData->private_data_backup; + self->release = psPrivateData->release_backup; + delete psPrivateData; + if (self->release) + self->release(self); + } + + // Set as the ArrowArrayStream::get_schema callback member of the + // ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() + static int GetSchema(struct ArrowArrayStream *self, struct ArrowSchema *out) + { + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + self->private_data); + auto poLayer = dynamic_cast( + psPrivateData->poShared->m_poLayer); + if (!poLayer) + { + CPLError( + CE_Failure, CPLE_NotSupported, + "Calling get_schema() on a freed OGRLayer is not supported"); + return EINVAL; + } + return poLayer->GetArrowSchemaForwarded(self, out); + } + + // Set as the ArrowArrayStream::get_next callback member of the + // ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() + static int GetNext(struct ArrowArrayStream *self, struct ArrowArray *out) + { + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + self->private_data); + auto poLayer = dynamic_cast( + psPrivateData->poShared->m_poLayer); + if (!poLayer) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Calling get_next() on a freed OGRLayer is not supported"); + return EINVAL; + } + return poLayer->GetNextArrowArrayForwarded(self, out); + } +}; + +/************************************************************************/ +/* GetArrowStream() */ +/************************************************************************/ + +bool OGRGenSQLResultsLayer::GetArrowStream(struct ArrowArrayStream *out_stream, + CSLConstList papszOptions) +{ + if (!TestCapability(OLCFastGetArrowStream) || + CPLTestBool(CPLGetConfigOption("OGR_GENSQL_STREAM_BASE_IMPL", "NO"))) + { + CPLStringList aosOptions(papszOptions); + aosOptions.SetNameValue("OGR_GENSQL_STREAM_BASE_IMPL", "YES"); + return OGRLayer::GetArrowStream(out_stream, aosOptions.List()); + } + + const swq_select *psSelectInfo = m_pSelectInfo.get(); + if (m_nIteratedFeatures != -1) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GetArrowStream() not supported on non-rewinded layer"); + return false; + } + CPLStringList aosOptions(papszOptions); + if (psSelectInfo->limit > 0) + { + aosOptions.SetNameValue( + "MAX_FEATURES_IN_BATCH", + CPLSPrintf(CPL_FRMT_GIB, + std::min(psSelectInfo->limit, + CPLAtoGIntBig(aosOptions.FetchNameValueDef( + "MAX_FEATURES_IN_BATCH", "65536"))))); + } + bool bRet = OGRLayer::GetArrowStream(out_stream, aosOptions.List()); + if (bRet) + { + auto psSrcLayerStream = static_cast( + CPLMalloc(sizeof(ArrowArrayStream))); + if (m_poSrcLayer->GetArrowStream(psSrcLayerStream, aosOptions.List())) + { + auto psPrivateData = + new OGRGenSQLResultsLayerArrowStreamPrivateData; + CPLAssert(m_poSharedArrowArrayStreamPrivateData); + psPrivateData->poShared = m_poSharedArrowArrayStreamPrivateData; + psPrivateData->psSrcLayerStream = psSrcLayerStream; + psPrivateData->release_backup = out_stream->release; + psPrivateData->private_data_backup = out_stream->private_data; + out_stream->get_schema = + OGRGenSQLResultsLayerArrowStreamPrivateData::GetSchema; + out_stream->get_next = + OGRGenSQLResultsLayerArrowStreamPrivateData::GetNext; + out_stream->release = + OGRGenSQLResultsLayerArrowStreamPrivateData::Release; + out_stream->private_data = psPrivateData; + } + else + { + if (psSrcLayerStream->release) + psSrcLayerStream->release(psSrcLayerStream); + CPLFree(psSrcLayerStream); + if (out_stream->release) + out_stream->release(out_stream); + bRet = false; + } + } + return bRet; +} + +/************************************************************************/ +/* GetArrowSchema() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetArrowSchema(struct ArrowArrayStream *stream, + struct ArrowSchema *out_schema) +{ + if (m_aosArrowArrayStreamOptions.FetchNameValue( + "OGR_GENSQL_STREAM_BASE_IMPL") || + !TestCapability(OLCFastGetArrowStream)) + { + return OGRLayer::GetArrowSchema(stream, out_schema); + } + + return GetArrowSchemaForwarded(stream, out_schema); +} + +/************************************************************************/ +/* GetArrowSchemaForwarded() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetArrowSchemaForwarded( + struct ArrowArrayStream *stream, struct ArrowSchema *out_schema) const +{ + const swq_select *psSelectInfo = m_pSelectInfo.get(); + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + stream->private_data); + int ret = m_poSrcLayer->GetArrowSchema(psPrivateData->psSrcLayerStream, + out_schema); + if (ret == 0) + { + struct ArrowSchema newSchema; + ret = OGRCloneArrowSchema(out_schema, &newSchema) ? 0 : EIO; + if (out_schema->release) + out_schema->release(out_schema); + if (ret == 0) + { + std::map oMapSrcNameToRenamed; + for (std::size_t iField = 0; + iField < psSelectInfo->column_defs.size(); iField++) + { + const swq_col_def *psColDef = + &psSelectInfo->column_defs[iField]; + CPLAssert(!psColDef->bHidden); + CPLAssert(psColDef->table_index >= 0); + CPLAssert(psColDef->col_func == SWQCF_NONE); + + const auto poLayerDefn = + m_apoTableLayers[psColDef->table_index]->GetLayerDefn(); + CPLAssert(poLayerDefn); + + if (psColDef->field_index >= 0 && + psColDef->field_index < poLayerDefn->GetFieldCount()) + { + const auto poSrcFDefn = + poLayerDefn->GetFieldDefn(psColDef->field_index); + if (psColDef->field_alias) + oMapSrcNameToRenamed[poSrcFDefn->GetNameRef()] = + psColDef->field_alias; + } + else if (IS_GEOM_FIELD_INDEX(poLayerDefn, + psColDef->field_index)) + { + const int iSrcGeomField = + ALL_FIELD_INDEX_TO_GEOM_FIELD_INDEX( + poLayerDefn, psColDef->field_index); + const auto poSrcGFDefn = + poLayerDefn->GetGeomFieldDefn(iSrcGeomField); + if (psColDef->field_alias) + oMapSrcNameToRenamed[poSrcGFDefn->GetNameRef()] = + psColDef->field_alias; + } + } + + for (int i = 0; i < newSchema.n_children; ++i) + { + const auto oIter = + oMapSrcNameToRenamed.find(newSchema.children[i]->name); + if (oIter != oMapSrcNameToRenamed.end()) + { + CPLFree(const_cast(newSchema.children[i]->name)); + newSchema.children[i]->name = + CPLStrdup(oIter->second.c_str()); + } + } + + memcpy(out_schema, &newSchema, sizeof(newSchema)); + } + } + return ret; +} + +/************************************************************************/ +/* GetNextArrowArray() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetNextArrowArray(struct ArrowArrayStream *stream, + struct ArrowArray *out_array) +{ + if (m_aosArrowArrayStreamOptions.FetchNameValue( + "OGR_GENSQL_STREAM_BASE_IMPL") || + !TestCapability(OLCFastGetArrowStream)) + { + return OGRLayer::GetNextArrowArray(stream, out_array); + } + + return GetNextArrowArrayForwarded(stream, out_array); +} + +/************************************************************************/ +/* GetNextArrowArrayForwarded() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetNextArrowArrayForwarded( + struct ArrowArrayStream *stream, struct ArrowArray *out_array) +{ + const swq_select *psSelectInfo = m_pSelectInfo.get(); + if (psSelectInfo->limit >= 0 && m_nIteratedFeatures >= psSelectInfo->limit) + { + memset(out_array, 0, sizeof(*out_array)); + return 0; + } + + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + stream->private_data); + const int ret = m_poSrcLayer->GetNextArrowArray( + psPrivateData->psSrcLayerStream, out_array); + if (ret == 0 && psSelectInfo->limit >= 0) + { + if (m_nIteratedFeatures < 0) + m_nIteratedFeatures = 0; + m_nIteratedFeatures += out_array->length; + if (m_nIteratedFeatures > psSelectInfo->limit) + { + out_array->length -= m_nIteratedFeatures - psSelectInfo->limit; + for (int i = 0; i < out_array->n_children; ++i) + { + out_array->children[i]->length -= + m_nIteratedFeatures - psSelectInfo->limit; + } + } + } + return ret; +} + //! @endcond diff --git a/ogr/ogrsf_frmts/generic/ogr_gensql.h b/ogr/ogrsf_frmts/generic/ogr_gensql.h index 465d025805cd..30c3eacd9948 100644 --- a/ogr/ogrsf_frmts/generic/ogr_gensql.h +++ b/ogr/ogrsf_frmts/generic/ogr_gensql.h @@ -151,6 +151,24 @@ class OGRGenSQLResultsLayer final : public OGRLayer virtual void SetSpatialFilter(int iGeomField, OGRGeometry *) override; virtual OGRErr SetAttributeFilter(const char *) override; + + bool GetArrowStream(struct ArrowArrayStream *out_stream, + CSLConstList papszOptions = nullptr) override; + + int GetArrowSchema(struct ArrowArrayStream *stream, + struct ArrowSchema *out_schema) override; + + protected: + friend struct OGRGenSQLResultsLayerArrowStreamPrivateData; + + int GetArrowSchemaForwarded(struct ArrowArrayStream *stream, + struct ArrowSchema *out_schema) const; + + int GetNextArrowArray(struct ArrowArrayStream *stream, + struct ArrowArray *out_array) override; + + int GetNextArrowArrayForwarded(struct ArrowArrayStream *stream, + struct ArrowArray *out_array); }; /*! @endcond */ diff --git a/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp b/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp index b6f9a8ba1a61..5963df74a273 100644 --- a/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp +++ b/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp @@ -270,25 +270,29 @@ inline void UnsetBit(uint8_t *pabyData, size_t nIdx) /* DefaultReleaseSchema() */ /************************************************************************/ -static void OGRLayerDefaultReleaseSchema(struct ArrowSchema *schema) +static void OGRLayerReleaseSchema(struct ArrowSchema *schema, + bool bFullFreeFormat) { CPLAssert(schema->release != nullptr); - if (STARTS_WITH(schema->format, "w:") || + if (bFullFreeFormat || STARTS_WITH(schema->format, "w:") || STARTS_WITH(schema->format, "tsm:")) { CPLFree(const_cast(schema->format)); } CPLFree(const_cast(schema->name)); CPLFree(const_cast(schema->metadata)); - for (int i = 0; i < static_cast(schema->n_children); ++i) + if (schema->children) { - if (schema->children[i]->release) + for (int i = 0; i < static_cast(schema->n_children); ++i) { - schema->children[i]->release(schema->children[i]); - CPLFree(schema->children[i]); + if (schema->children[i] && schema->children[i]->release) + { + schema->children[i]->release(schema->children[i]); + CPLFree(schema->children[i]); + } } + CPLFree(schema->children); } - CPLFree(schema->children); if (schema->dictionary) { if (schema->dictionary->release) @@ -300,6 +304,16 @@ static void OGRLayerDefaultReleaseSchema(struct ArrowSchema *schema) schema->release = nullptr; } +static void OGRLayerPartialReleaseSchema(struct ArrowSchema *schema) +{ + OGRLayerReleaseSchema(schema, /* bFullFreeFormat = */ false); +} + +static void OGRLayerFullReleaseSchema(struct ArrowSchema *schema) +{ + OGRLayerReleaseSchema(schema, /* bFullFreeFormat = */ true); +} + /** Release a ArrowSchema. * * To be used by driver implementations that have a custom GetArrowStream() @@ -311,7 +325,7 @@ static void OGRLayerDefaultReleaseSchema(struct ArrowSchema *schema) void OGRLayer::ReleaseSchema(struct ArrowSchema *schema) { - OGRLayerDefaultReleaseSchema(schema); + OGRLayerPartialReleaseSchema(schema); } /************************************************************************/ @@ -355,7 +369,7 @@ static void AddDictToSchema(struct ArrowSchema *psChild, auto psChildDict = static_cast( CPLCalloc(1, sizeof(struct ArrowSchema))); psChild->dictionary = psChildDict; - psChildDict->release = OGRLayerDefaultReleaseSchema; + psChildDict->release = OGRLayerPartialReleaseSchema; psChildDict->name = CPLStrdup(poCodedDomain->GetName().c_str()); psChildDict->format = "u"; if (nCountNull) @@ -5464,6 +5478,103 @@ bool OGRCloneArrowArray(const struct ArrowSchema *schema, return OGRCloneArrowArray(schema, src_array, out_array, 0); } +/************************************************************************/ +/* OGRCloneArrowMetadata() */ +/************************************************************************/ + +static void *OGRCloneArrowMetadata(const void *pMetadata) +{ + if (!pMetadata) + return nullptr; + std::vector abyOut; + const GByte *pabyMetadata = static_cast(pMetadata); + int32_t nKVP; + abyOut.insert(abyOut.end(), pabyMetadata, pabyMetadata + sizeof(int32_t)); + memcpy(&nKVP, pabyMetadata, sizeof(int32_t)); + pabyMetadata += sizeof(int32_t); + for (int i = 0; i < nKVP; ++i) + { + int32_t nSizeKey; + abyOut.insert(abyOut.end(), pabyMetadata, + pabyMetadata + sizeof(int32_t)); + memcpy(&nSizeKey, pabyMetadata, sizeof(int32_t)); + pabyMetadata += sizeof(int32_t); + abyOut.insert(abyOut.end(), pabyMetadata, pabyMetadata + nSizeKey); + pabyMetadata += nSizeKey; + + int32_t nSizeValue; + abyOut.insert(abyOut.end(), pabyMetadata, + pabyMetadata + sizeof(int32_t)); + memcpy(&nSizeValue, pabyMetadata, sizeof(int32_t)); + pabyMetadata += sizeof(int32_t); + abyOut.insert(abyOut.end(), pabyMetadata, pabyMetadata + nSizeValue); + pabyMetadata += nSizeValue; + } + + GByte *pabyOut = static_cast(VSI_MALLOC_VERBOSE(abyOut.size())); + if (pabyOut) + memcpy(pabyOut, abyOut.data(), abyOut.size()); + return pabyOut; +} + +/************************************************************************/ +/* OGRCloneArrowSchema() */ +/************************************************************************/ + +/** Full/deep copy of a schema. + * + * In case of failure, out_schema will be let in a released state. + * + * @param schema Schema to clone. Must *NOT* be NULL. + * @param out_schema Output schema. Must *NOT* be NULL (but its content may be random) + * @return true if success. + */ +bool OGRCloneArrowSchema(const struct ArrowSchema *schema, + struct ArrowSchema *out_schema) +{ + memset(out_schema, 0, sizeof(*out_schema)); + out_schema->release = OGRLayerFullReleaseSchema; + out_schema->format = CPLStrdup(schema->format); + out_schema->name = CPLStrdup(schema->name); + out_schema->metadata = static_cast( + const_cast(OGRCloneArrowMetadata(schema->metadata))); + out_schema->flags = schema->flags; + if (schema->n_children) + { + out_schema->children = + static_cast(VSI_CALLOC_VERBOSE( + static_cast(schema->n_children), sizeof(ArrowSchema *))); + if (!out_schema->children) + { + out_schema->release(out_schema); + return false; + } + out_schema->n_children = schema->n_children; + for (int i = 0; i < static_cast(schema->n_children); ++i) + { + out_schema->children[i] = static_cast( + CPLMalloc(sizeof(ArrowSchema))); + if (!OGRCloneArrowSchema(schema->children[i], + out_schema->children[i])) + { + out_schema->release(out_schema); + return false; + } + } + } + if (schema->dictionary) + { + out_schema->dictionary = + static_cast(CPLMalloc(sizeof(ArrowSchema))); + if (!OGRCloneArrowSchema(schema->dictionary, out_schema->dictionary)) + { + out_schema->release(out_schema); + return false; + } + } + return true; +} + /************************************************************************/ /* OGRLayer::IsArrowSchemaSupported() */ /************************************************************************/ diff --git a/ogr/ogrsf_frmts/generic/ogrlayerarrow.h b/ogr/ogrsf_frmts/generic/ogrlayerarrow.h index dc9ed4e726ef..ae45d9135328 100644 --- a/ogr/ogrsf_frmts/generic/ogrlayerarrow.h +++ b/ogr/ogrsf_frmts/generic/ogrlayerarrow.h @@ -47,4 +47,7 @@ bool CPL_DLL OGRCloneArrowArray(const struct ArrowSchema *schema, const struct ArrowArray *array, struct ArrowArray *out_array); +bool CPL_DLL OGRCloneArrowSchema(const struct ArrowSchema *schema, + struct ArrowSchema *out_schema); + #endif // OGRLAYERARROW_H_DEFINED diff --git a/ogr/ogrsf_frmts/generic/ogrsfdriver.cpp b/ogr/ogrsf_frmts/generic/ogrsfdriver.cpp index 3315b5e58dbd..7a7eda1c94d2 100644 --- a/ogr/ogrsf_frmts/generic/ogrsfdriver.cpp +++ b/ogr/ogrsf_frmts/generic/ogrsfdriver.cpp @@ -173,7 +173,8 @@ int OGR_Dr_TestCapability(OGRSFDriverH hDriver, const char *pszCap) GDALDriver *poDriver = reinterpret_cast(hDriver); if (EQUAL(pszCap, ODrCCreateDataSource)) { - return poDriver->pfnCreate != nullptr || + return poDriver->GetMetadataItem(GDAL_DCAP_CREATE) || + poDriver->pfnCreate != nullptr || poDriver->pfnCreateVectorOnly != nullptr; } else if (EQUAL(pszCap, ODrCDeleteDataSource)) diff --git a/ogr/ogrsf_frmts/generic/ogrunionlayer.cpp b/ogr/ogrsf_frmts/generic/ogrunionlayer.cpp index eed6543d2be4..073d7bef5f8d 100644 --- a/ogr/ogrsf_frmts/generic/ogrunionlayer.cpp +++ b/ogr/ogrsf_frmts/generic/ogrunionlayer.cpp @@ -200,7 +200,7 @@ void OGRUnionLayer::SetFeatureCount(int nFeatureCountIn) /************************************************************************/ static void MergeFieldDefn(OGRFieldDefn *poFieldDefn, - OGRFieldDefn *poSrcFieldDefn) + const OGRFieldDefn *poSrcFieldDefn) { if (poFieldDefn->GetType() != poSrcFieldDefn->GetType()) { @@ -306,14 +306,17 @@ OGRFeatureDefn *OGRUnionLayer::GetLayerDefn() } else if (eFieldStrategy == FIELD_FROM_FIRST_LAYER) { - OGRFeatureDefn *poSrcFeatureDefn = papoSrcLayers[0]->GetLayerDefn(); - for (int i = 0; i < poSrcFeatureDefn->GetFieldCount(); i++) + const OGRFeatureDefn *poSrcFeatureDefn = + papoSrcLayers[0]->GetLayerDefn(); + const int nSrcFieldCount = poSrcFeatureDefn->GetFieldCount(); + for (int i = 0; i < nSrcFieldCount; i++) poFeatureDefn->AddFieldDefn(poSrcFeatureDefn->GetFieldDefn(i)); for (int i = 0; nGeomFields != -1 && i < poSrcFeatureDefn->GetGeomFieldCount(); i++) { - OGRGeomFieldDefn *poFldDefn = poSrcFeatureDefn->GetGeomFieldDefn(i); + const OGRGeomFieldDefn *poFldDefn = + poSrcFeatureDefn->GetGeomFieldDefn(i); poFeatureDefn->AddGeomFieldDefn( std::make_unique(poFldDefn)); } @@ -327,20 +330,31 @@ OGRFeatureDefn *OGRUnionLayer::GetLayerDefn() papoGeomFields[0])); } + int nDstFieldCount = 0; + std::map oMapDstFieldNameToIdx; + for (int iLayer = 0; iLayer < nSrcLayers; iLayer++) { - OGRFeatureDefn *poSrcFeatureDefn = + const OGRFeatureDefn *poSrcFeatureDefn = papoSrcLayers[iLayer]->GetLayerDefn(); /* Add any field that is found in the source layers */ - for (int i = 0; i < poSrcFeatureDefn->GetFieldCount(); i++) + const int nSrcFieldCount = poSrcFeatureDefn->GetFieldCount(); + for (int i = 0; i < nSrcFieldCount; i++) { - OGRFieldDefn *poSrcFieldDefn = + const OGRFieldDefn *poSrcFieldDefn = poSrcFeatureDefn->GetFieldDefn(i); - int nIndex = - poFeatureDefn->GetFieldIndex(poSrcFieldDefn->GetNameRef()); + const auto oIter = + oMapDstFieldNameToIdx.find(poSrcFieldDefn->GetNameRef()); + const int nIndex = + oIter == oMapDstFieldNameToIdx.end() ? -1 : oIter->second; if (nIndex < 0) + { + oMapDstFieldNameToIdx[poSrcFieldDefn->GetNameRef()] = + nDstFieldCount; + nDstFieldCount++; poFeatureDefn->AddFieldDefn(poSrcFieldDefn); + } else { OGRFieldDefn *poFieldDefn = @@ -353,7 +367,7 @@ OGRFeatureDefn *OGRUnionLayer::GetLayerDefn() nGeomFields != -1 && i < poSrcFeatureDefn->GetGeomFieldCount(); i++) { - OGRGeomFieldDefn *poSrcFieldDefn = + const OGRGeomFieldDefn *poSrcFieldDefn = poSrcFeatureDefn->GetGeomFieldDefn(i); int nIndex = poFeatureDefn->GetGeomFieldIndex( poSrcFieldDefn->GetNameRef()); @@ -513,17 +527,29 @@ void OGRUnionLayer::ConfigureActiveLayer() /* Establish map */ GetLayerDefn(); - OGRFeatureDefn *poSrcFeatureDefn = papoSrcLayers[iCurLayer]->GetLayerDefn(); + const OGRFeatureDefn *poSrcFeatureDefn = + papoSrcLayers[iCurLayer]->GetLayerDefn(); + const int nSrcFieldCount = poSrcFeatureDefn->GetFieldCount(); + const int nDstFieldCount = poFeatureDefn->GetFieldCount(); + + std::map oMapDstFieldNameToIdx; + for (int i = 0; i < nDstFieldCount; i++) + { + const OGRFieldDefn *poDstFieldDefn = poFeatureDefn->GetFieldDefn(i); + oMapDstFieldNameToIdx[poDstFieldDefn->GetNameRef()] = i; + } + CPLFree(panMap); - panMap = static_cast( - CPLMalloc(poSrcFeatureDefn->GetFieldCount() * sizeof(int))); - for (int i = 0; i < poSrcFeatureDefn->GetFieldCount(); i++) + panMap = static_cast(CPLMalloc(nSrcFieldCount * sizeof(int))); + for (int i = 0; i < nSrcFieldCount; i++) { - OGRFieldDefn *poSrcFieldDefn = poSrcFeatureDefn->GetFieldDefn(i); + const OGRFieldDefn *poSrcFieldDefn = poSrcFeatureDefn->GetFieldDefn(i); if (m_aosIgnoredFields.FindString(poSrcFieldDefn->GetNameRef()) == -1) { + const auto oIter = + oMapDstFieldNameToIdx.find(poSrcFieldDefn->GetNameRef()); panMap[i] = - poFeatureDefn->GetFieldIndex(poSrcFieldDefn->GetNameRef()); + oIter == oMapDstFieldNameToIdx.end() ? -1 : oIter->second; } else { @@ -545,19 +571,28 @@ void OGRUnionLayer::ConfigureActiveLayer() } } + std::map oMapSrcFieldNameToIdx; + for (int i = 0; i < nSrcFieldCount; i++) + { + const OGRFieldDefn *poSrcFieldDefn = + poSrcFeatureDefn->GetFieldDefn(i); + oMapSrcFieldNameToIdx[poSrcFieldDefn->GetNameRef()] = i; + } + /* Attribute fields */ - std::vector abSrcFieldsUsed(poSrcFeatureDefn->GetFieldCount()); - for (int iField = 0; iField < poFeatureDefn->GetFieldCount(); iField++) + std::vector abSrcFieldsUsed(nSrcFieldCount); + for (int iField = 0; iField < nDstFieldCount; iField++) { const OGRFieldDefn *poFieldDefn = poFeatureDefn->GetFieldDefn(iField); + const auto oIter = + oMapSrcFieldNameToIdx.find(poFieldDefn->GetNameRef()); const int iSrcField = - poSrcFeatureDefn->GetFieldIndex(poFieldDefn->GetNameRef()); + oIter == oMapSrcFieldNameToIdx.end() ? -1 : oIter->second; if (iSrcField >= 0) abSrcFieldsUsed[iSrcField] = true; } - for (int iSrcField = 0; iSrcField < poSrcFeatureDefn->GetFieldCount(); - iSrcField++) + for (int iSrcField = 0; iSrcField < nSrcFieldCount; iSrcField++) { if (!abSrcFieldsUsed[iSrcField]) { diff --git a/ogr/ogrsf_frmts/generic/ogrwarpedlayer.cpp b/ogr/ogrsf_frmts/generic/ogrwarpedlayer.cpp index 1e77c950edea..6c1147c70884 100644 --- a/ogr/ogrsf_frmts/generic/ogrwarpedlayer.cpp +++ b/ogr/ogrsf_frmts/generic/ogrwarpedlayer.cpp @@ -155,17 +155,15 @@ void OGRWarpedLayer::SetSpatialFilterRect(int iGeomField, double dfMinX, /* SrcFeatureToWarpedFeature() */ /************************************************************************/ -OGRFeature *OGRWarpedLayer::SrcFeatureToWarpedFeature(OGRFeature *poSrcFeature) +std::unique_ptr +OGRWarpedLayer::SrcFeatureToWarpedFeature(std::unique_ptr poFeature) { - OGRFeature *poFeature = new OGRFeature(GetLayerDefn()); - poFeature->SetFrom(poSrcFeature); - poFeature->SetFID(poSrcFeature->GetFID()); + // This is safe to do here as they have matching attribute and geometry + // fields + poFeature->SetFDefnUnsafe(GetLayerDefn()); OGRGeometry *poGeom = poFeature->GetGeomFieldRef(m_iGeomField); - if (poGeom == nullptr) - return poFeature; - - if (poGeom->transform(m_poCT) != OGRERR_NONE) + if (poGeom && poGeom->transform(m_poCT) != OGRERR_NONE) { delete poFeature->StealGeometry(m_iGeomField); } @@ -177,30 +175,21 @@ OGRFeature *OGRWarpedLayer::SrcFeatureToWarpedFeature(OGRFeature *poSrcFeature) /* WarpedFeatureToSrcFeature() */ /************************************************************************/ -OGRFeature *OGRWarpedLayer::WarpedFeatureToSrcFeature(OGRFeature *poFeature) +std::unique_ptr +OGRWarpedLayer::WarpedFeatureToSrcFeature(std::unique_ptr poFeature) { - OGRFeature *poSrcFeature = - new OGRFeature(m_poDecoratedLayer->GetLayerDefn()); - poSrcFeature->SetFrom(poFeature); - poSrcFeature->SetFID(poFeature->GetFID()); + // This is safe to do here as they have matching attribute and geometry + // fields + poFeature->SetFDefnUnsafe(m_poDecoratedLayer->GetLayerDefn()); - OGRGeometry *poGeom = poSrcFeature->GetGeomFieldRef(m_iGeomField); - if (poGeom != nullptr) + OGRGeometry *poGeom = poFeature->GetGeomFieldRef(m_iGeomField); + if (poGeom && + (!m_poReversedCT || poGeom->transform(m_poReversedCT) != OGRERR_NONE)) { - if (m_poReversedCT == nullptr) - { - delete poSrcFeature; - return nullptr; - } - - if (poGeom->transform(m_poReversedCT) != OGRERR_NONE) - { - delete poSrcFeature; - return nullptr; - } + return nullptr; } - return poSrcFeature; + return poFeature; } /************************************************************************/ @@ -211,21 +200,19 @@ OGRFeature *OGRWarpedLayer::GetNextFeature() { while (true) { - OGRFeature *poFeature = m_poDecoratedLayer->GetNextFeature(); - if (poFeature == nullptr) + auto poFeature = + std::unique_ptr(m_poDecoratedLayer->GetNextFeature()); + if (!poFeature) return nullptr; - OGRFeature *poFeatureNew = SrcFeatureToWarpedFeature(poFeature); - delete poFeature; - - OGRGeometry *poGeom = poFeatureNew->GetGeomFieldRef(m_iGeomField); + auto poFeatureNew = SrcFeatureToWarpedFeature(std::move(poFeature)); + const OGRGeometry *poGeom = poFeatureNew->GetGeomFieldRef(m_iGeomField); if (m_poFilterGeom != nullptr && !FilterGeometry(poGeom)) { - delete poFeatureNew; continue; } - return poFeatureNew; + return poFeatureNew.release(); } } @@ -235,14 +222,13 @@ OGRFeature *OGRWarpedLayer::GetNextFeature() OGRFeature *OGRWarpedLayer::GetFeature(GIntBig nFID) { - OGRFeature *poFeature = m_poDecoratedLayer->GetFeature(nFID); - if (poFeature != nullptr) + auto poFeature = + std::unique_ptr(m_poDecoratedLayer->GetFeature(nFID)); + if (poFeature) { - OGRFeature *poFeatureNew = SrcFeatureToWarpedFeature(poFeature); - delete poFeature; - poFeature = poFeatureNew; + poFeature = SrcFeatureToWarpedFeature(std::move(poFeature)); } - return poFeature; + return poFeature.release(); } /************************************************************************/ @@ -251,17 +237,12 @@ OGRFeature *OGRWarpedLayer::GetFeature(GIntBig nFID) OGRErr OGRWarpedLayer::ISetFeature(OGRFeature *poFeature) { - OGRErr eErr; - - OGRFeature *poFeatureNew = WarpedFeatureToSrcFeature(poFeature); - if (poFeatureNew == nullptr) + auto poFeatureNew = WarpedFeatureToSrcFeature( + std::unique_ptr(poFeature->Clone())); + if (!poFeatureNew) return OGRERR_FAILURE; - eErr = m_poDecoratedLayer->SetFeature(poFeatureNew); - - delete poFeatureNew; - - return eErr; + return m_poDecoratedLayer->SetFeature(poFeatureNew.get()); } /************************************************************************/ @@ -270,17 +251,12 @@ OGRErr OGRWarpedLayer::ISetFeature(OGRFeature *poFeature) OGRErr OGRWarpedLayer::ICreateFeature(OGRFeature *poFeature) { - OGRErr eErr; - - OGRFeature *poFeatureNew = WarpedFeatureToSrcFeature(poFeature); - if (poFeatureNew == nullptr) + auto poFeatureNew = WarpedFeatureToSrcFeature( + std::unique_ptr(poFeature->Clone())); + if (!poFeatureNew) return OGRERR_FAILURE; - eErr = m_poDecoratedLayer->CreateFeature(poFeatureNew); - - delete poFeatureNew; - - return eErr; + return m_poDecoratedLayer->CreateFeature(poFeatureNew.get()); } /************************************************************************/ @@ -289,17 +265,12 @@ OGRErr OGRWarpedLayer::ICreateFeature(OGRFeature *poFeature) OGRErr OGRWarpedLayer::IUpsertFeature(OGRFeature *poFeature) { - OGRErr eErr; - - OGRFeature *poFeatureNew = WarpedFeatureToSrcFeature(poFeature); + auto poFeatureNew = WarpedFeatureToSrcFeature( + std::unique_ptr(poFeature->Clone())); if (poFeatureNew == nullptr) return OGRERR_FAILURE; - eErr = m_poDecoratedLayer->UpsertFeature(poFeatureNew); - - delete poFeatureNew; - - return eErr; + return m_poDecoratedLayer->UpsertFeature(poFeatureNew.get()); } /************************************************************************/ @@ -313,19 +284,14 @@ OGRErr OGRWarpedLayer::IUpdateFeature(OGRFeature *poFeature, const int *panUpdatedGeomFieldsIdx, bool bUpdateStyleString) { - OGRErr eErr; - - OGRFeature *poFeatureNew = WarpedFeatureToSrcFeature(poFeature); - if (poFeatureNew == nullptr) + auto poFeatureNew = WarpedFeatureToSrcFeature( + std::unique_ptr(poFeature->Clone())); + if (!poFeatureNew) return OGRERR_FAILURE; - eErr = m_poDecoratedLayer->UpdateFeature( - poFeatureNew, nUpdatedFieldsCount, panUpdatedFieldsIdx, + return m_poDecoratedLayer->UpdateFeature( + poFeatureNew.get(), nUpdatedFieldsCount, panUpdatedFieldsIdx, nUpdatedGeomFieldsCount, panUpdatedGeomFieldsIdx, bUpdateStyleString); - - delete poFeatureNew; - - return eErr; } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h b/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h index f7f47454d40a..abe0050d41e0 100644 --- a/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h +++ b/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h @@ -33,6 +33,7 @@ #ifndef DOXYGEN_SKIP #include "ogrlayerdecorator.h" +#include /************************************************************************/ /* OGRWarpedLayer */ @@ -55,8 +56,10 @@ class OGRWarpedLayer : public OGRLayerDecorator static int ReprojectEnvelope(OGREnvelope *psEnvelope, OGRCoordinateTransformation *poCT); - OGRFeature *SrcFeatureToWarpedFeature(OGRFeature *poFeature); - OGRFeature *WarpedFeatureToSrcFeature(OGRFeature *poFeature); + std::unique_ptr + SrcFeatureToWarpedFeature(std::unique_ptr poFeature); + std::unique_ptr + WarpedFeatureToSrcFeature(std::unique_ptr poFeature); public: OGRWarpedLayer( diff --git a/ogr/ogrsf_frmts/geojson/ogr_geojson.h b/ogr/ogrsf_frmts/geojson/ogr_geojson.h index 3a4847930f16..db5c46d9a13b 100644 --- a/ogr/ogrsf_frmts/geojson/ogr_geojson.h +++ b/ogr/ogrsf_frmts/geojson/ogr_geojson.h @@ -40,6 +40,9 @@ #include "ogrgeojsonutils.h" #include "ogrgeojsonwriter.h" +constexpr const char *INVALID_CONTENT_FOR_JSON_LIKE = + "__INVALID_CONTENT_FOR_JSON_LIKE__"; + class OGRGeoJSONDataSource; GDALDataset *OGRGeoJSONDriverOpenInternal(GDALOpenInfo *poOpenInfo, diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp index 0a8b0ed23a67..426f07ef90ef 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp @@ -195,8 +195,9 @@ int OGRGeoJSONDataSource::Open(GDALOpenInfo *poOpenInfo, bool bEmitError = true; if (eGeoJSONSourceService == nSrcType) { - const CPLString osTmpFilename = CPLSPrintf( - "/vsimem/%p/%s", this, CPLGetFilename(poOpenInfo->pszFilename)); + const CPLString osTmpFilename = + VSIMemGenerateHiddenFilename(CPLSPrintf( + "geojson_%s", CPLGetFilename(poOpenInfo->pszFilename))); VSIFCloseL(VSIFileFromMemBuffer(osTmpFilename, (GByte *)pszGeoData_, nGeoDataLen_, TRUE)); pszGeoData_ = nullptr; @@ -814,10 +815,11 @@ int OGRGeoJSONDataSource::ReadFromService(GDALOpenInfo *poOpenInfo, char *pszStoredContent = OGRGeoJSONDriverStealStoredContent(pszSource); if (pszStoredContent != nullptr) { - if ((osJSonFlavor_ == "ESRIJSON" && - ESRIJSONIsObject(pszStoredContent, poOpenInfo)) || - (osJSonFlavor_ == "TopoJSON" && - TopoJSONIsObject(pszStoredContent, poOpenInfo))) + if (!EQUAL(pszStoredContent, INVALID_CONTENT_FOR_JSON_LIKE) && + ((osJSonFlavor_ == "ESRIJSON" && + ESRIJSONIsObject(pszStoredContent, poOpenInfo)) || + (osJSonFlavor_ == "TopoJSON" && + TopoJSONIsObject(pszStoredContent, poOpenInfo)))) { pszGeoData_ = pszStoredContent; nGeoDataLen_ = strlen(pszGeoData_); @@ -894,6 +896,11 @@ int OGRGeoJSONDataSource::ReadFromService(GDALOpenInfo *poOpenInfo, pszGeoData_ = nullptr; nGeoDataLen_ = 0; } + else + { + OGRGeoJSONDriverStoreContent( + pszSource, CPLStrdup(INVALID_CONTENT_FOR_JSON_LIKE)); + } return false; } } diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp index 782c3fc2bd61..b1fa548ec798 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp @@ -349,6 +349,8 @@ OGRGeoJSONSeqLayer::OGRGeoJSONSeqLayer( OGRSpatialReference::GetWGS84SRS()); m_poCT = std::move(poCT); + m_oWriteOptions.bWriteBBOX = + CPLTestBool(CSLFetchNameValueDef(papszOptions, "WRITE_BBOX", "FALSE")); m_oWriteOptions.SetRFC7946Settings(); m_oWriteOptions.SetIDOptions(papszOptions); @@ -573,7 +575,7 @@ json_object *OGRGeoJSONSeqLayer::GetNextObject(bool bLooseIdentification) (m_osFeatureBuffer.back() == '\r' || m_osFeatureBuffer.back() == '\n')) { - m_osFeatureBuffer.resize(m_osFeatureBuffer.size() - 1); + m_osFeatureBuffer.pop_back(); } if (!m_osFeatureBuffer.empty()) { @@ -816,7 +818,7 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, if (poOpenInfo->eAccess == GA_Update) return false; - m_osTmpFile = CPLSPrintf("/vsimem/geojsonseq/%p", this); + m_osTmpFile = VSIMemGenerateHiddenFilename("geojsonseq"); m_fp = VSIFileFromMemBuffer( m_osTmpFile.c_str(), reinterpret_cast(CPLStrdup(poOpenInfo->pszFilename)), @@ -831,7 +833,8 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, OGRGeoJSONDriverStealStoredContent(pszUnprefixedFilename); if (pszStoredContent) { - if (!GeoJSONSeqIsObject(pszStoredContent, poOpenInfo)) + if (EQUAL(pszStoredContent, INVALID_CONTENT_FOR_JSON_LIKE) || + !GeoJSONSeqIsObject(pszStoredContent, poOpenInfo)) { OGRGeoJSONDriverStoreContent(poOpenInfo->pszFilename, pszStoredContent); @@ -839,7 +842,7 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, } else { - m_osTmpFile = CPLSPrintf("/vsimem/geojsonseq/%p", this); + m_osTmpFile = VSIMemGenerateHiddenFilename("geojsonseq"); m_fp = VSIFileFromMemBuffer( m_osTmpFile.c_str(), reinterpret_cast(pszStoredContent), @@ -870,7 +873,7 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, return false; } - m_osTmpFile = CPLSPrintf("/vsimem/geojsonseq/%p", this); + m_osTmpFile = VSIMemGenerateHiddenFilename("geojsonseq"); m_fp = VSIFileFromMemBuffer(m_osTmpFile.c_str(), pResult->pabyData, pResult->nDataLen, true); pResult->pabyData = nullptr; @@ -1056,6 +1059,9 @@ void RegisterOGRGeoJSONSeq() " String" " Integer" " " + "