From 1bfce62c3c284166a59ce0e4041968ed1974eca7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 20 Jun 2024 18:21:52 +0200 Subject: [PATCH 01/72] Code cleanup in VRTKernelFilteredSource --- frmts/vrt/vrtdataset.h | 18 ++++----- frmts/vrt/vrtfilters.cpp | 83 +++++++++++++++------------------------- 2 files changed, 39 insertions(+), 62 deletions(-) diff --git a/frmts/vrt/vrtdataset.h b/frmts/vrt/vrtdataset.h index bbc0d60bbc46..7809c8b2211d 100644 --- a/frmts/vrt/vrtdataset.h +++ b/frmts/vrt/vrtdataset.h @@ -1601,17 +1601,14 @@ class VRTKernelFilteredSource CPL_NON_FINAL : public VRTFilteredSource CPL_DISALLOW_COPY_ASSIGN(VRTKernelFilteredSource) protected: - int m_nKernelSize; - - bool m_bSeparable; - - double *m_padfKernelCoefs; - - int m_bNormalized; + int m_nKernelSize = 0; + bool m_bSeparable = false; + // m_nKernelSize elements if m_bSeparable, m_nKernelSize * m_nKernelSize otherwise + std::vector m_adfKernelCoefs{}; + bool m_bNormalized = false; public: VRTKernelFilteredSource(); - virtual ~VRTKernelFilteredSource(); virtual CPLErr XMLInit(const CPLXMLNode *psTree, const char *, std::map &) override; @@ -1620,8 +1617,9 @@ class VRTKernelFilteredSource CPL_NON_FINAL : public VRTFilteredSource virtual CPLErr FilterData(int nXSize, int nYSize, GDALDataType eType, GByte *pabySrcData, GByte *pabyDstData) override; - CPLErr SetKernel(int nKernelSize, bool bSeparable, double *padfCoefs); - void SetNormalized(int); + CPLErr SetKernel(int nKernelSize, bool bSeparable, + const std::vector &adfNewCoefs); + void SetNormalized(bool); }; /************************************************************************/ diff --git a/frmts/vrt/vrtfilters.cpp b/frmts/vrt/vrtfilters.cpp index 6d8ba867df17..e3378cdf303e 100644 --- a/frmts/vrt/vrtfilters.cpp +++ b/frmts/vrt/vrtfilters.cpp @@ -420,28 +420,16 @@ CPLErr VRTFilteredSource::RasterIO(GDALDataType eVRTBandDataType, int nXOff, /************************************************************************/ VRTKernelFilteredSource::VRTKernelFilteredSource() - : m_nKernelSize(0), m_bSeparable(FALSE), m_padfKernelCoefs(nullptr), - m_bNormalized(FALSE) { GDALDataType aeSupTypes[] = {GDT_Float32}; SetFilteringDataTypesSupported(1, aeSupTypes); } -/************************************************************************/ -/* ~VRTKernelFilteredSource() */ -/************************************************************************/ - -VRTKernelFilteredSource::~VRTKernelFilteredSource() - -{ - CPLFree(m_padfKernelCoefs); -} - /************************************************************************/ /* SetNormalized() */ /************************************************************************/ -void VRTKernelFilteredSource::SetNormalized(int bNormalizedIn) +void VRTKernelFilteredSource::SetNormalized(bool bNormalizedIn) { m_bNormalized = bNormalizedIn; @@ -451,8 +439,9 @@ void VRTKernelFilteredSource::SetNormalized(int bNormalizedIn) /* SetKernel() */ /************************************************************************/ -CPLErr VRTKernelFilteredSource::SetKernel(int nNewKernelSize, bool bSeparable, - double *padfNewCoefs) +CPLErr +VRTKernelFilteredSource::SetKernel(int nNewKernelSize, bool bSeparable, + const std::vector &adfNewCoefs) { if (nNewKernelSize < 1 || (nNewKernelSize % 2) != 1) @@ -463,16 +452,17 @@ CPLErr VRTKernelFilteredSource::SetKernel(int nNewKernelSize, bool bSeparable, nNewKernelSize); return CE_Failure; } + if (adfNewCoefs.size() != + static_cast(nNewKernelSize) * (bSeparable ? 1 : nNewKernelSize)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "adfNewCoefs[] is not of expected size"); + return CE_Failure; + } - CPLFree(m_padfKernelCoefs); m_nKernelSize = nNewKernelSize; m_bSeparable = bSeparable; - - int nKernelBufferSize = m_nKernelSize * (m_bSeparable ? 1 : m_nKernelSize); - - m_padfKernelCoefs = - static_cast(CPLMalloc(sizeof(double) * nKernelBufferSize)); - memcpy(m_padfKernelCoefs, padfNewCoefs, sizeof(double) * nKernelBufferSize); + m_adfKernelCoefs = adfNewCoefs; SetExtraEdgePixels((nNewKernelSize - 1) / 2); @@ -565,8 +555,8 @@ CPLErr VRTKernelFilteredSource::FilterData(int nXSize, int nYSize, iJJ * nJStride; if (bHasNoData && *pfData == fNoData) continue; - dfSum += *pfData * m_padfKernelCoefs[iK]; - dfKernSum += m_padfKernelCoefs[iK]; + dfSum += *pfData * m_adfKernelCoefs[iK]; + dfKernSum += m_adfKernelCoefs[iK]; } } @@ -618,17 +608,16 @@ CPLErr VRTKernelFilteredSource::XMLInit( return CE_Failure; } - char **papszCoefItems = - CSLTokenizeString(CPLGetXMLValue(psTree, "Kernel.Coefs", "")); + const CPLStringList aosCoefItems( + CSLTokenizeString(CPLGetXMLValue(psTree, "Kernel.Coefs", ""))); - const int nCoefs = CSLCount(papszCoefItems); + const int nCoefs = aosCoefItems.size(); const bool bSquare = nCoefs == nNewKernelSize * nNewKernelSize; const bool bSeparable = nCoefs == nNewKernelSize && nCoefs != 1; if (!bSquare && !bSeparable) { - CSLDestroy(papszCoefItems); CPLError(CE_Failure, CPLE_AppDefined, "Got wrong number of filter kernel coefficients (%s). " "Expected %d or %d, got %d.", @@ -637,19 +626,17 @@ CPLErr VRTKernelFilteredSource::XMLInit( return CE_Failure; } - double *padfNewCoefs = - static_cast(CPLMalloc(sizeof(double) * nCoefs)); - + std::vector adfNewCoefs; + adfNewCoefs.reserve(nCoefs); for (int i = 0; i < nCoefs; i++) - padfNewCoefs[i] = CPLAtof(papszCoefItems[i]); - - const CPLErr eErr = SetKernel(nNewKernelSize, bSeparable, padfNewCoefs); - - CPLFree(padfNewCoefs); - CSLDestroy(papszCoefItems); - - SetNormalized(atoi(CPLGetXMLValue(psTree, "Kernel.normalized", "0"))); + adfNewCoefs.push_back(CPLAtof(aosCoefItems[i])); + const CPLErr eErr = SetKernel(nNewKernelSize, bSeparable, adfNewCoefs); + if (eErr == CE_None) + { + SetNormalized(atoi(CPLGetXMLValue(psTree, "Kernel.normalized", "0")) != + 0); + } return eErr; } @@ -673,23 +660,15 @@ CPLXMLNode *VRTKernelFilteredSource::SerializeToXML(const char *pszVRTPath) CPLXMLNode *psKernel = CPLCreateXMLNode(psSrc, CXT_Element, "Kernel"); - if (m_bNormalized) - CPLCreateXMLNode( - CPLCreateXMLNode(psKernel, CXT_Attribute, "normalized"), CXT_Text, - "1"); - else - CPLCreateXMLNode( - CPLCreateXMLNode(psKernel, CXT_Attribute, "normalized"), CXT_Text, - "0"); - - const int nCoefCount = - m_bSeparable ? m_nKernelSize : m_nKernelSize * m_nKernelSize; + CPLCreateXMLNode(CPLCreateXMLNode(psKernel, CXT_Attribute, "normalized"), + CXT_Text, m_bNormalized ? "1" : "0"); + std::string osCoefs; - for (int iCoef = 0; iCoef < nCoefCount; iCoef++) + for (auto dfVal : m_adfKernelCoefs) { if (!osCoefs.empty()) osCoefs += ' '; - osCoefs += CPLSPrintf("%.8g", m_padfKernelCoefs[iCoef]); + osCoefs += CPLSPrintf("%.8g", dfVal); } CPLSetXMLValue(psKernel, "Size", CPLSPrintf("%d", m_nKernelSize)); From 852fc9a4ee4b1e79aaa5627713309d861b03e8c5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 21 Jun 2024 14:07:33 +0200 Subject: [PATCH 02/72] DXF: add a DXF_CLOSED_LINE_AS_POLYGON=YES/NO configuration option to control whether closed POLYLINE/LWPOLYLINE should be exposed as a polygon. Defaults to NO to match current behavior. Fixes #10153 Also available in DWF driver as DWG_CLOSED_LINE_AS_POLYGON=YES/NO Also, for DXF, expose existing reader configuration options as open options: INLINE_BLOCKS, MERGE_BLOCK_GEOMETRIES, TRANSLATE_ESCAPE_SEQUENCES, INCLUDE_RAW_CODE_VALUES, 3D_EXTENSIBLE_MODE, CLOSED_LINE_AS_POLYGON, HATCH_TOLERANCE ENCODING --- autotest/ogr/ogr_dxf.py | 26 ++- doc/source/drivers/vector/dwg.rst | 8 + doc/source/drivers/vector/dxf.rst | 149 ++++++++++++------ ogr/ogrsf_frmts/dwg/ogr_dwg.h | 7 + ogr/ogrsf_frmts/dwg/ogrdwg_hatch.cpp | 2 +- ogr/ogrsf_frmts/dwg/ogrdwgdatasource.cpp | 2 + ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp | 4 +- ogr/ogrsf_frmts/dxf/ogr_dxf.h | 15 +- ogr/ogrsf_frmts/dxf/ogrdxf_hatch.cpp | 4 +- .../dxf/ogrdxf_polyline_smooth.cpp | 21 +-- ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.h | 2 +- ogr/ogrsf_frmts/dxf/ogrdxfdatasource.cpp | 46 ++++-- ogr/ogrsf_frmts/dxf/ogrdxfdriver.cpp | 32 +++- ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp | 15 +- ogr/ogrsf_frmts/dxf/ogrdxfwriterds.cpp | 2 +- 15 files changed, 244 insertions(+), 91 deletions(-) diff --git a/autotest/ogr/ogr_dxf.py b/autotest/ogr/ogr_dxf.py index 21b283e779e9..d2ecbd72d34d 100644 --- a/autotest/ogr/ogr_dxf.py +++ b/autotest/ogr/ogr_dxf.py @@ -3890,12 +3890,21 @@ def test_ogr_dxf_54(): # Test hidden objects in blocks -def test_ogr_dxf_55(): - - with gdaltest.config_option("DXF_MERGE_BLOCK_GEOMETRIES", "FALSE"): - ds = ogr.Open("data/dxf/block-hidden-entities.dxf") +@pytest.mark.parametrize("use_config_option", [True, False]) +def test_ogr_dxf_55(use_config_option): + + if use_config_option: + with gdaltest.config_option("DXF_MERGE_BLOCK_GEOMETRIES", "FALSE"): + ds = ogr.Open("data/dxf/block-hidden-entities.dxf") + else: + ds = gdal.OpenEx( + "data/dxf/block-hidden-entities.dxf", + open_options={"MERGE_BLOCK_GEOMETRIES": False}, + ) lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 6 + # Red features should be hidden, black features should be visible for number, f in enumerate(lyr): assert "#ff000000)" in f.GetStyleString() or "#000000)" in f.GetStyleString(), ( @@ -4011,3 +4020,12 @@ def test_ogr_dxf_read_closed_polyline_with_bulge(): g.ExportToWkt() == "LINESTRING (40585366.7065058 3433935.53809098,40585329.9256486 3433998.44081707,40585329.9256486 3433998.44081707,40585328.5387678 3434000.63680805,40585327.0051198 3434002.73293274,40585325.3318693 3434004.71939884,40585323.526833 3434006.58692634,40585321.5984435 3434008.32679087,40585319.5557093 3434009.93086443,40585317.4081735 3434011.39165342,40585315.1658683 3434012.70233358,40585312.8392691 3434013.85678191,40585310.4392448 3434014.84960528,40585307.9770074 3434015.67616559,40585305.4640596 3434016.33260146,40585302.9121409 3434016.81584629,40585300.3331728 3434017.12364253,40585297.7392033 3434017.25455227,40585271.1313178 3434017.68678191,40585252.1698149 3433885.99037548,40585256.74147 3433885.9161116,40585256.74147 3433885.9161116,40585266.2920614 3433886.0916242,40585275.8076317 3433886.92740148,40585285.2425893 3433888.41943902,40585294.551729 3433890.56058809,40585303.6904483 3433893.34058991,40585312.6149614 3433896.74612477,40585321.2825086 3433900.76087591,40585329.6515615 3433905.36560764,40585364.2483736 3433925.99220872,40585364.2483736 3433925.99220872,40585364.6481964 3433926.24937651,40585365.0296424 3433926.53308859,40585365.3909523 3433926.84203644,40585365.7304596 3433927.17479516,40585366.0465985 3433927.52983003,40585366.337911 3433927.90550359,40585366.6030535 3433928.30008319,40585366.840803 3433928.71174899,40585367.0500632 3433929.13860232,40585367.2298688 3433929.5786745,40585367.3793906 3433930.02993587,40585367.4979389 3433930.49030515,40585367.5849671 3433930.95765907,40585367.6400736 3433931.42984214,40585367.6630045 3433931.9046766,40585367.6536538 3433932.37997246,40585367.6120647 3433932.85353759,40585367.5384291 3433933.32318787,40585367.4330866 3433933.7867572,40585367.2965229 3433934.24210757,40585367.129368 3433934.68713883,40585366.9323928 3433935.11979846,40585366.7065058 3433935.53809098)" ) + + ds = gdal.OpenEx( + "data/dxf/closed_polyline_with_bulge.dxf", + open_options=["CLOSED_LINE_AS_POLYGON=YES"], + ) + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + g = f.GetGeometryRef() + assert g.GetGeometryType() == ogr.wkbPolygon diff --git a/doc/source/drivers/vector/dwg.rst b/doc/source/drivers/vector/dwg.rst index 5669134281a4..6e7b58ec762d 100644 --- a/doc/source/drivers/vector/dwg.rst +++ b/doc/source/drivers/vector/dwg.rst @@ -85,6 +85,14 @@ The following configuration options are available: :config:`DWG_ALL_ATTRIBUTES` to TRUE value (this is the default value). +- .. config:: DWG_CLOSED_LINE_AS_POLYGON + :choices: TRUE, FALSE + :default: FALSE + :since: 3.10 + + This option can be set to TRUE specified to ask for closed POLYLINE and + LWPOLYLINE to be exposed as OGR polygons. + Building -------- diff --git a/doc/source/drivers/vector/dxf.rst b/doc/source/drivers/vector/dxf.rst index 63ab76fe5b67..30b348621b34 100644 --- a/doc/source/drivers/vector/dxf.rst +++ b/doc/source/drivers/vector/dxf.rst @@ -34,9 +34,7 @@ fields: NULL otherwise. - SubClasses: Where available, a list of classes to which an entity belongs. -- ExtendedEntity (GDAL <= 2.2.x): The values of extended entity - attributes all appended to form a single text field, where available. -- RawCodeValues (GDAL >= 2.3.0): A string list +- RawCodeValues: A string list containing all group codes and values that are not handled by the DXF reader. Only available when the configuration option :config:`DXF_INCLUDE_RAW_CODE_VALUES=TRUE`. @@ -73,18 +71,27 @@ The following entity types are supported: polylines (those with their vertices' bulge attributes set) will be tessellated. Single-vertex polylines are translated to POINT. Polyface meshes are translated as POLYHEDRALSURFACE geometries. -- MLINE: - - (GDAL >= 2.3.0) Translated as a MULTILINESTRING. Only the geometry - is reconstructed; styling applied to individual line elements - within the MLINE is ignored. Fill colors and start/end caps are - also omitted. - - (GDAL <= 2.2.x) No support. + Starting with GDAL 3.10, the :config:`DXF_CLOSED_LINE_AS_POLYGON` + configuration option can be set to TRUE to ask for closed POLYLINE and + LWPOLYLINE to be exposed as OGR polygons. -- CIRCLE, ELLIPSE, ARC, SPLINE, (GDAL >= 2.3.0) HELIX: Translated as a + .. config:: DXF_CLOSED_LINE_AS_POLYGON + :choices: TRUE, FALSE + :default: FALSE + + +- MLINE: Translated as a MULTILINESTRING. Only the geometry + is reconstructed; styling applied to individual line elements + within the MLINE is ignored. Fill colors and start/end caps are + also omitted. + +- CIRCLE, ELLIPSE, ARC, SPLINE,HELIX: Translated as a LINESTRING, tessellating the curve into line segments. - (GDAL >= 2.3.0) CIRCLEs with nonzero "thickness" (cylinders) are + + CIRCLEs with nonzero "thickness" (cylinders) are approximated as a POLYHEDRALSURFACE. + - INSERT: By default, the block definition referenced by the INSERT will be inserted as a compound geometry (for example, a MULTILINESTRING for a block containing many lines, or a @@ -97,6 +104,7 @@ The following entity types are supported: - .. config:: DXF_MERGE_BLOCK_GEOMETRIES :choices: TRUE, FALSE + :default: FALSE To avoid merging blocks into a compound geometry the :config:`DXF_MERGE_BLOCK_GEOMETRIES` config option may @@ -115,13 +123,9 @@ The following entity types are supported: Maximum number of features inserted from a single block. Set to -1 for no limit. -- ATTDEF, ATTRIB: - - - (GDAL >= 2.3.0) Attributes (ATTRIB) are treated as TEXT entities, - and attribute definitions (ATTDEF) inside blocks are ignored. The - behavior is different when :config:`DXF_INLINE_BLOCKS` is false (see below). - - (GDAL <= 2.2.x) ATTDEF entities are treated as TEXT. ATTRIB - entities are not supported. +- ATTDEF, ATTRIB: Attributes (ATTRIB) are treated as TEXT entities, + and attribute definitions (ATTDEF) inside blocks are ignored. The + behavior is different when :config:`DXF_INLINE_BLOCKS` is false (see below). - HATCH: Line and arc boundaries are collected as a polygon geometry, but no effort is currently made to represent the fill style of HATCH @@ -134,36 +138,27 @@ The following entity types are supported: tolerance used when looking for the next component to add to the hatch boundary. - (GDAL <= 2.2.x) Only line and polyline boundary paths are translated - correctly. - - 3DFACE, SOLID, (GDAL >= 2.3.0) TRACE: Translated as POLYGON, except for SOLID and TRACE entities with only one distinct vertex (translated as POINT) or two distinct vertices (translated as LINESTRING). -- DIMENSION: - - - (GDAL >= 2.3.0) The DXF format allows each DIMENSION entity to - reference an "anonymous" block (a block whose name starts with - \*D) that contains the geometry of the DIMENSION. If present, this - anonymous block will be inlined at the required position. - Otherwise, fallback will occur to a simple DIMENSION renderer that - explodes a linear dimension as a MULTILINESTRING feature. - Arrowheads, if present, are translated as one or more additional - features. The fallback renderer will render nonlinear dimensions - as if they were linear. - - (GDAL <= 2.2.x) Dimensions are translated as a MULTILINESTRING and - a POINT for the text. - -- LEADER, MULTILEADER: - - - (GDAL >= 2.3.0) The leader line is translated as a LINESTRING - (LEADER) or MULTILINESTRING (MULTILEADER). Arrowheads, if present, - are translated as one or more additional features. Text for - MULTILEADER entities is translated into a POINT feature with a - label. Block content for MULTILEADERS is treated as for INSERT. - Spline leaders are tessellated into line segments. - - (GDAL <= 2.2.x) No support. + +- DIMENSION: The DXF format allows each DIMENSION entity to + reference an "anonymous" block (a block whose name starts with + \*D) that contains the geometry of the DIMENSION. If present, this + anonymous block will be inlined at the required position. + Otherwise, fallback will occur to a simple DIMENSION renderer that + explodes a linear dimension as a MULTILINESTRING feature. + Arrowheads, if present, are translated as one or more additional + features. The fallback renderer will render nonlinear dimensions + as if they were linear. + +- LEADER, MULTILEADER: The leader line is translated as a LINESTRING + (LEADER) or MULTILINESTRING (MULTILEADER). Arrowheads, if present, + are translated as one or more additional features. Text for + MULTILEADER entities is translated into a POINT feature with a + label. Block content for MULTILEADERS is treated as for INSERT. + Spline leaders are tessellated into line segments. - 3DSOLID, REGION, BODY, SURFACE: See below. @@ -186,6 +181,9 @@ specification, except DIMENSION, LEADER and MULTILEADER. These three entity types also currently lack support for elevations; the geometries will always be 2D. + +.. _dxf_inline_blocks: + DXF_INLINE_BLOCKS ~~~~~~~~~~~~~~~~~ @@ -272,7 +270,68 @@ override what id will be used by OGR in transcoding: name. Using a value "UTF-8" will avoid any attempt to recode the text as it is read. --------------- +Open options +------------ + +.. versionadded:: 3.10 + +|about-open-options| +The following open options are supported: + +- .. oo:: CLOSED_LINE_AS_POLYGON + :since: 3.10 + :default: NO + :choices: YES, NO + + See :config:`DXF_CLOSED_LINE_AS_POLYGON` + +- .. oo:: INLINE_BLOCKS + :since: 3.10 + :default: YES + :choices: YES, NO + + See :ref:`dxf_inline_blocks` + +- .. oo:: MERGE_BLOCK_GEOMETRIES + :since: 3.10 + :default: YES + :choices: YES, NO + + See :config:`DXF_MERGE_BLOCK_GEOMETRIES` + +- .. oo:: TRANSLATE_ESCAPE_SEQUENCES + :since: 3.10 + :default: YES + :choices: YES, NO + + See :config:`DXF_TRANSLATE_ESCAPE_SEQUENCES` + +- .. oo:: INCLUDE_RAW_CODE_VALUES + :since: 3.10 + :default: NO + :choices: YES, NO + + See :config:`DXF_INCLUDE_RAW_CODE_VALUES` + +- .. oo:: 3D_EXTENSIBLE_MODE + :since: 3.10 + :default: NO + :choices: YES, NO + + See :config:`DXF_3D_EXTENSIBLE_MODE` + +- .. oo:: HATCH_TOLERANCE + :since: 3.10 + :choices: + + See :config:`DXF_HATCH_TOLERANCE` + +- .. oo:: ENCODING + :since: 3.10 + :choices: + + See :config:`DXF_ENCODING` + DXF Writer ---------- diff --git a/ogr/ogrsf_frmts/dwg/ogr_dwg.h b/ogr/ogrsf_frmts/dwg/ogr_dwg.h index a9e74b508b49..7b29f37a6473 100644 --- a/ogr/ogrsf_frmts/dwg/ogr_dwg.h +++ b/ogr/ogrsf_frmts/dwg/ogr_dwg.h @@ -189,6 +189,8 @@ class OGRDWGDataSource final : public OGRDataSource int bAttributes; int bAllAttributes; + bool m_bClosedLineAsPolygon = false; + OGRDWGServices *poServices; OdDbDatabasePtr poDb; @@ -235,6 +237,11 @@ class OGRDWGDataSource final : public OGRDataSource return bAllAttributes; } + bool ClosedLineAsPolygon() const + { + return m_bClosedLineAsPolygon; + } + void AddStandardFields(OGRFeatureDefn *poDef); // Implemented in ogrdxf_blockmap.cpp diff --git a/ogr/ogrsf_frmts/dwg/ogrdwg_hatch.cpp b/ogr/ogrsf_frmts/dwg/ogrdwg_hatch.cpp index 5e181de37ad2..6fc6a2c0c379 100644 --- a/ogr/ogrsf_frmts/dwg/ogrdwg_hatch.cpp +++ b/ogr/ogrsf_frmts/dwg/ogrdwg_hatch.cpp @@ -144,7 +144,7 @@ static OGRErr DWGCollectBoundaryLoop(OdDbHatchPtr poHatch, int iLoop, oSmoothPolyline.Close(); - OGRLineString *poLS = oSmoothPolyline.Tessellate()->toLineString(); + OGRLineString *poLS = oSmoothPolyline.Tessellate(false)->toLineString(); poGC->addGeometryDirectly(poLS); return OGRERR_NONE; diff --git a/ogr/ogrsf_frmts/dwg/ogrdwgdatasource.cpp b/ogr/ogrsf_frmts/dwg/ogrdwgdatasource.cpp index f91dcab83553..15676b01b5e1 100644 --- a/ogr/ogrsf_frmts/dwg/ogrdwgdatasource.cpp +++ b/ogr/ogrsf_frmts/dwg/ogrdwgdatasource.cpp @@ -100,6 +100,8 @@ int OGRDWGDataSource::Open(OGRDWGServices *poServicesIn, bAttributes = CPLTestBool(CPLGetConfigOption("DWG_ATTRIBUTES", "FALSE")); bAllAttributes = CPLTestBool(CPLGetConfigOption("DWG_ALL_ATTRIBUTES", "TRUE")); + m_bClosedLineAsPolygon = + CPLTestBool(CPLGetConfigOption("DWG_CLOSED_LINE_AS_POLYGON", "FALSE")); /* -------------------------------------------------------------------- */ /* Open the file. */ diff --git a/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp b/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp index e32050f5ca43..c79476e1b2d5 100644 --- a/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp +++ b/ogr/ogrsf_frmts/dwg/ogrdwglayer.cpp @@ -692,7 +692,9 @@ OGRFeature *OGRDWGLayer::TranslateLWPOLYLINE(OdDbEntityPtr poEntity) if (poPL->isClosed()) oSmoothPolyline.Close(); - poFeature->SetGeometryDirectly(oSmoothPolyline.Tessellate()); + const bool bAsPolygon = poPL->isClosed() && poDS->ClosedLineAsPolygon(); + + poFeature->SetGeometryDirectly(oSmoothPolyline.Tessellate(bAsPolygon)); PrepareLineStyle(poFeature); diff --git a/ogr/ogrsf_frmts/dxf/ogr_dxf.h b/ogr/ogrsf_frmts/dxf/ogr_dxf.h index 7badbce4a891..31de83214961 100644 --- a/ogr/ogrsf_frmts/dxf/ogr_dxf.h +++ b/ogr/ogrsf_frmts/dxf/ogr_dxf.h @@ -675,6 +675,8 @@ class OGRDXFDataSource final : public OGRDataSource bool bMergeBlockGeometries; bool bTranslateEscapeSequences; bool bIncludeRawCodeValues; + bool m_bClosedLineAsPolygon = false; + double m_dfHatchTolerance = -1.0; bool b3DExtensibleMode; bool bHaveReadSolidData; @@ -688,7 +690,8 @@ class OGRDXFDataSource final : public OGRDataSource OGRDXFDataSource(); ~OGRDXFDataSource(); - int Open(const char *pszFilename, int bHeaderOnly = FALSE); + int Open(const char *pszFilename, bool bHeaderOnly, + CSLConstList papszOptionsIn); const char *GetName() override { @@ -731,6 +734,16 @@ class OGRDXFDataSource final : public OGRDataSource return b3DExtensibleMode; } + bool ClosedLineAsPolygon() const + { + return m_bClosedLineAsPolygon; + } + + double HatchTolerance() const + { + return m_dfHatchTolerance; + } + static void AddStandardFields(OGRFeatureDefn *poDef, const int nFieldModes); // Implemented in ogrdxf_blockmap.cpp diff --git a/ogr/ogrsf_frmts/dxf/ogrdxf_hatch.cpp b/ogr/ogrsf_frmts/dxf/ogrdxf_hatch.cpp index 6affd2e8b8d3..bf50ce8a55e8 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxf_hatch.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxf_hatch.cpp @@ -106,7 +106,7 @@ OGRDXFFeature *OGRDXFLayer::TranslateHATCH() /* -------------------------------------------------------------------- */ /* Obtain a tolerance value used when building the polygon. */ /* -------------------------------------------------------------------- */ - double dfTolerance = atof(CPLGetConfigOption("DXF_HATCH_TOLERANCE", "-1")); + double dfTolerance = poDS->HatchTolerance(); if (dfTolerance < 0) { // If the configuration variable isn't set, compute the bounding box @@ -696,7 +696,7 @@ OGRErr OGRDXFLayer::CollectPolylinePath(OGRGeometryCollection *poGC, if (nVertexCount >= 2) { oSmoothPolyline.SetUseMaxGapWhenTessellatingArcs(poDS->InlineBlocks()); - poGC->addGeometryDirectly(oSmoothPolyline.Tessellate()); + poGC->addGeometryDirectly(oSmoothPolyline.Tessellate(false)); } /* -------------------------------------------------------------------- */ diff --git a/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.cpp b/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.cpp index 17ee916fae6c..1920b3eef05a 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.cpp @@ -63,7 +63,7 @@ static double GetOGRangle(double angle) /* DXFSmoothPolyline::Tessellate() */ /************************************************************************/ -OGRGeometry *DXFSmoothPolyline::Tessellate() const +OGRGeometry *DXFSmoothPolyline::Tessellate(bool bAsPolygon) const { assert(!m_vertices.empty()); @@ -84,7 +84,8 @@ OGRGeometry *DXFSmoothPolyline::Tessellate() const /* Otherwise, presume a line string */ /* -------------------------------------------------------------------- */ - OGRLineString *poLS = new OGRLineString; + OGRLinearRing *poLR = m_bClosed && bAsPolygon ? new OGRLinearRing : nullptr; + OGRLineString *poLS = poLR ? poLR : new OGRLineString; m_blinestringstarted = false; @@ -126,28 +127,14 @@ OGRGeometry *DXFSmoothPolyline::Tessellate() const if (m_dim == 2) poLS->flattenTo2D(); -/* -------------------------------------------------------------------- */ -/* If polyline is closed, convert linestring to a linear ring */ -/* */ -/* Actually, on review I'm not convinced this is a good idea. */ -/* Note that most (all) "filled polygons" are expressed with */ -/* hatches which are now handled fairly well and they tend to */ -/* echo linear polylines. */ -/* -------------------------------------------------------------------- */ -#ifdef notdef - if (m_bClosed) + if (poLR) { - OGRLinearRing *poLR = new OGRLinearRing(); - poLR->addSubLineString(poLS, 0); - delete poLS; - // Wrap as polygon. OGRPolygon *poPoly = new OGRPolygon(); poPoly->addRingDirectly(poLR); return poPoly; } -#endif return poLS; } diff --git a/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.h b/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.h index 2ee6a08ba090..8861ccb1831d 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.h +++ b/ogr/ogrsf_frmts/dxf/ogrdxf_polyline_smooth.h @@ -114,7 +114,7 @@ class DXFSmoothPolyline { } - OGRGeometry *Tessellate() const; + OGRGeometry *Tessellate(bool bAsPolygon) const; size_t size() const { diff --git a/ogr/ogrsf_frmts/dxf/ogrdxfdatasource.cpp b/ogr/ogrsf_frmts/dxf/ogrdxfdatasource.cpp index 55f622fd3e67..b00d06e37f41 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxfdatasource.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxfdatasource.cpp @@ -100,26 +100,44 @@ OGRLayer *OGRDXFDataSource::GetLayer(int iLayer) /* Open() */ /************************************************************************/ -int OGRDXFDataSource::Open(const char *pszFilename, int bHeaderOnly) +int OGRDXFDataSource::Open(const char *pszFilename, bool bHeaderOnly, + CSLConstList papszOptionsIn) { osEncoding = CPL_ENC_ISO8859_1; osName = pszFilename; - bInlineBlocks = - CPLTestBool(CPLGetConfigOption("DXF_INLINE_BLOCKS", "TRUE")); - bMergeBlockGeometries = - CPLTestBool(CPLGetConfigOption("DXF_MERGE_BLOCK_GEOMETRIES", "TRUE")); - bTranslateEscapeSequences = CPLTestBool( - CPLGetConfigOption("DXF_TRANSLATE_ESCAPE_SEQUENCES", "TRUE")); - bIncludeRawCodeValues = - CPLTestBool(CPLGetConfigOption("DXF_INCLUDE_RAW_CODE_VALUES", "FALSE")); - b3DExtensibleMode = - CPLTestBool(CPLGetConfigOption("DXF_3D_EXTENSIBLE_MODE", "FALSE")); + bInlineBlocks = CPLTestBool( + CSLFetchNameValueDef(papszOptionsIn, "INLINE_BLOCKS", + CPLGetConfigOption("DXF_INLINE_BLOCKS", "TRUE"))); + bMergeBlockGeometries = CPLTestBool(CSLFetchNameValueDef( + papszOptionsIn, "MERGE_BLOCK_GEOMETRIES", + CPLGetConfigOption("DXF_MERGE_BLOCK_GEOMETRIES", "TRUE"))); + bTranslateEscapeSequences = CPLTestBool(CSLFetchNameValueDef( + papszOptionsIn, "TRANSLATE_ESCAPE_SEQUENCES", + CPLGetConfigOption("DXF_TRANSLATE_ESCAPE_SEQUENCES", "TRUE"))); + + bIncludeRawCodeValues = CPLTestBool(CSLFetchNameValueDef( + papszOptionsIn, "INCLUDE_RAW_CODE_VALUES", + CPLGetConfigOption("DXF_INCLUDE_RAW_CODE_VALUES", "FALSE"))); + + b3DExtensibleMode = CPLTestBool(CSLFetchNameValueDef( + papszOptionsIn, "3D_EXTENSIBLE_MODE", + CPLGetConfigOption("DXF_3D_EXTENSIBLE_MODE", "FALSE"))); + + m_bClosedLineAsPolygon = CPLTestBool(CSLFetchNameValueDef( + papszOptionsIn, "CLOSED_LINE_AS_POLYGON", + CPLGetConfigOption("DXF_CLOSED_LINE_AS_POLYGON", "FALSE"))); + + m_dfHatchTolerance = CPLAtof( + CSLFetchNameValueDef(papszOptionsIn, "HATCH_TOLERANCE", + CPLGetConfigOption("DXF_HATCH_TOLERANCE", "-1"))); + + // Only for debugging if (CPLTestBool(CPLGetConfigOption("DXF_HEADER_ONLY", "FALSE"))) - bHeaderOnly = TRUE; + bHeaderOnly = true; /* -------------------------------------------------------------------- */ /* Open the file. */ @@ -151,7 +169,9 @@ int OGRDXFDataSource::Open(const char *pszFilename, int bHeaderOnly) */ else if (EQUAL(szLineBuf, "TABLES")) { - osEncoding = CPLGetConfigOption("DXF_ENCODING", osEncoding); + osEncoding = CSLFetchNameValueDef( + papszOptionsIn, "ENCODING", + CPLGetConfigOption("DXF_ENCODING", osEncoding)); if (!ReadTablesSection()) return FALSE; diff --git a/ogr/ogrsf_frmts/dxf/ogrdxfdriver.cpp b/ogr/ogrsf_frmts/dxf/ogrdxfdriver.cpp index f109b890811d..33490946a918 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxfdriver.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxfdriver.cpp @@ -85,7 +85,8 @@ static GDALDataset *OGRDXFDriverOpen(GDALOpenInfo *poOpenInfo) OGRDXFDataSource *poDS = new OGRDXFDataSource(); - if (!poDS->Open(poOpenInfo->pszFilename)) + if (!poDS->Open(poOpenInfo->pszFilename, false, + poOpenInfo->papszOpenOptions)) { delete poDS; poDS = nullptr; @@ -146,6 +147,35 @@ void RegisterOGRDXF() "first entity'/>" ""); + poDriver->SetMetadataItem( + GDAL_DMD_OPENOPTIONLIST, + "" + " "); + poDriver->SetMetadataItem(GDAL_DS_LAYER_CREATIONOPTIONLIST, ""); diff --git a/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp b/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp index 7bacee8e844c..ba61e2a5b2ea 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxflayer.cpp @@ -1170,11 +1170,15 @@ OGRDXFFeature *OGRDXFLayer::TranslateLWPOLYLINE() /* -------------------------------------------------------------------- */ /* Close polyline if necessary. */ /* -------------------------------------------------------------------- */ - if (nPolylineFlag & 0x01) + const bool bIsClosed = (nPolylineFlag & 0x01) != 0; + if (bIsClosed) smoothPolyline.Close(); + const bool bAsPolygon = bIsClosed && poDS->ClosedLineAsPolygon(); + smoothPolyline.SetUseMaxGapWhenTessellatingArcs(poDS->InlineBlocks()); - auto poGeom = std::unique_ptr(smoothPolyline.Tessellate()); + auto poGeom = + std::unique_ptr(smoothPolyline.Tessellate(bAsPolygon)); poFeature->ApplyOCSTransformer(poGeom.get()); poFeature->SetGeometryDirectly(poGeom.release()); @@ -1419,11 +1423,14 @@ OGRDXFFeature *OGRDXFLayer::TranslatePOLYLINE() /* -------------------------------------------------------------------- */ /* Close polyline if necessary. */ /* -------------------------------------------------------------------- */ - if (nPolylineFlag & 0x01) + const bool bIsClosed = (nPolylineFlag & 0x01) != 0; + if (bIsClosed) smoothPolyline.Close(); + const bool bAsPolygon = bIsClosed && poDS->ClosedLineAsPolygon(); + smoothPolyline.SetUseMaxGapWhenTessellatingArcs(poDS->InlineBlocks()); - OGRGeometry *poGeom = smoothPolyline.Tessellate(); + OGRGeometry *poGeom = smoothPolyline.Tessellate(bAsPolygon); if ((nPolylineFlag & 8) == 0) poFeature->ApplyOCSTransformer(poGeom); diff --git a/ogr/ogrsf_frmts/dxf/ogrdxfwriterds.cpp b/ogr/ogrsf_frmts/dxf/ogrdxfwriterds.cpp index 77466d699746..8cbfc1c05b31 100644 --- a/ogr/ogrsf_frmts/dxf/ogrdxfwriterds.cpp +++ b/ogr/ogrsf_frmts/dxf/ogrdxfwriterds.cpp @@ -237,7 +237,7 @@ int OGRDXFWriterDS::Open(const char *pszFilename, char **papszOptions) /* Attempt to read the template header file so we have a list */ /* of layers, linestyles and blocks. */ /* -------------------------------------------------------------------- */ - if (!oHeaderDS.Open(osHeaderFile, TRUE)) + if (!oHeaderDS.Open(osHeaderFile, true, nullptr)) return FALSE; /* -------------------------------------------------------------------- */ From 3d8b1e07052df8ebf528a897db06bb5e892a3771 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 22 Jun 2024 23:39:30 +0200 Subject: [PATCH 03/72] CSV: allow inf, -inf and nan as numeric values --- autotest/ogr/data/csv/inf_nan.csv | 5 +++++ autotest/ogr/ogr_csv.py | 21 +++++++++++++++++++++ ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp | 20 +++++++++++++------- 3 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 autotest/ogr/data/csv/inf_nan.csv diff --git a/autotest/ogr/data/csv/inf_nan.csv b/autotest/ogr/data/csv/inf_nan.csv new file mode 100644 index 000000000000..7ab5a1f7cd1e --- /dev/null +++ b/autotest/ogr/data/csv/inf_nan.csv @@ -0,0 +1,5 @@ +id,v +1,10 +2,inf +3,-inf +4,NaN diff --git a/autotest/ogr/ogr_csv.py b/autotest/ogr/ogr_csv.py index 22d9111f4cc7..2c315d484bf5 100755 --- a/autotest/ogr/ogr_csv.py +++ b/autotest/ogr/ogr_csv.py @@ -27,6 +27,7 @@ # Boston, MA 02111-1307, USA. ############################################################################### +import math import pathlib import sys @@ -3133,6 +3134,26 @@ def test_ogr_csv_force_opening(tmp_vsimem): assert ds.GetDriver().GetDescription() == "CSV" +############################################################################### +# Test opening a CSV file with inf/nan numeric values + + +@gdaltest.enable_exceptions() +def test_ogr_csv_inf_nan(): + + ds = gdal.OpenEx("data/csv/inf_nan.csv", open_options=["AUTODETECT_TYPE=YES"]) + lyr = ds.GetLayer(0) + assert lyr.GetLayerDefn().GetFieldDefn(1).GetType() == ogr.OFTReal + f = lyr.GetNextFeature() + assert f["v"] == 10.0 + f = lyr.GetNextFeature() + assert f["v"] == float("inf") + f = lyr.GetNextFeature() + assert f["v"] == float("-inf") + f = lyr.GetNextFeature() + assert math.isnan(f["v"]) + + ############################################################################### diff --git a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp index cee5cbcd4916..14e76fae890a 100644 --- a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp +++ b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp @@ -1049,7 +1049,10 @@ char **OGRCSVLayer::AutodetectFieldTypes(CSLConstList papszOpenOptions, else eOGRFieldType = OFTInteger; } - else if (eType == CPL_VALUE_REAL) + else if (eType == CPL_VALUE_REAL || + EQUAL(papszTokens[iField], "inf") || + EQUAL(papszTokens[iField], "-inf") || + EQUAL(papszTokens[iField], "nan")) { eOGRFieldType = OFTReal; } @@ -1466,14 +1469,16 @@ OGRFeature *OGRCSVLayer::GetNextUnfilteredFeature() if (chComma) *chComma = '.'; } - CPLValueType eType = CPLGetValueType(papszTokens[iAttr]); - if (eType == CPL_VALUE_INTEGER || eType == CPL_VALUE_REAL) + char *endptr = nullptr; + const double dfVal = + CPLStrtodDelim(papszTokens[iAttr], &endptr, '.'); + if (endptr == papszTokens[iAttr] + strlen(papszTokens[iAttr])) { - poFeature->SetField(iOGRField, papszTokens[iAttr]); + poFeature->SetField(iOGRField, dfVal); if (!bWarningBadTypeOrWidth && (eFieldType == OFTInteger || eFieldType == OFTInteger64) && - eType == CPL_VALUE_REAL) + CPLGetValueType(papszTokens[iAttr]) == CPL_VALUE_REAL) { bWarningBadTypeOrWidth = true; CPLError(CE_Warning, CPLE_AppDefined, @@ -1495,8 +1500,9 @@ OGRFeature *OGRCSVLayer::GetNextUnfilteredFeature() nNextFID, poFieldDefn->GetNameRef()); } else if (!bWarningBadTypeOrWidth && - eType == CPL_VALUE_REAL && - poFieldDefn->GetWidth() > 0) + poFieldDefn->GetWidth() > 0 && + CPLGetValueType(papszTokens[iAttr]) == + CPL_VALUE_REAL) { const char *pszDot = strchr(papszTokens[iAttr], '.'); const int nPrecision = From 58dfa4a431c218e45eef53e41317a4294b61c4d7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 22 Jun 2024 23:40:30 +0200 Subject: [PATCH 04/72] Python bindings: check validity of GDALAccess flag passed to gdal.Open() --- autotest/gcore/basic_test.py | 9 +++++++++ swig/include/python/typemaps_python.i | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/autotest/gcore/basic_test.py b/autotest/gcore/basic_test.py index 2d2e2a91eb61..2719fc963dcc 100755 --- a/autotest/gcore/basic_test.py +++ b/autotest/gcore/basic_test.py @@ -68,6 +68,15 @@ def test_basic_test_1(): pytest.fail("did not get expected error message, got %s" % gdal.GetLastErrorMsg()) +def test_basic_test_invalid_open_flag(): + with pytest.raises(Exception, match="invalid value for GDALAccess"): + gdal.Open("data/byte.tif", "invalid") + + assert gdal.OF_RASTER not in (gdal.GA_ReadOnly, gdal.GA_Update) + with pytest.raises(Exception, match="invalid value for GDALAccess"): + gdal.Open("data/byte.tif", gdal.OF_RASTER) + + @pytest.mark.skipif(sys.platform != "linux", reason="Incorrect platform") def test_basic_test_strace_non_existing_file(): diff --git a/swig/include/python/typemaps_python.i b/swig/include/python/typemaps_python.i index 5ff42d4f31c3..ef5276f3cc78 100644 --- a/swig/include/python/typemaps_python.i +++ b/swig/include/python/typemaps_python.i @@ -3065,6 +3065,20 @@ OBJECT_LIST_INPUT(GDALEDTComponentHS) %#endif } +%typemap(in) GDALAccess +{ + // %typemap(in) GDALAccess + int val = 0; + int ecode = SWIG_AsVal_int($input, &val); + if (!SWIG_IsOK(ecode)) { + SWIG_exception_fail(SWIG_ArgError(ecode), "invalid value for GDALAccess"); + } + if( val != GA_ReadOnly && val != GA_Update ) + { + SWIG_exception_fail(SWIG_ValueError, "invalid value for GDALAccess"); + } + $1 = static_cast(val); +} %typemap(in) GDALRIOResampleAlg { From dc95162f236a1ffbb1c5109273cf423420d5fa38 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 22 Jun 2024 23:36:53 +0200 Subject: [PATCH 05/72] OGR SQL: use Kahan-Babuska-Neumaier algorithm for accurate SUM(), like in recent SQLite versions --- autotest/ogr/ogr_sql_test.py | 30 ++++++++++++++++++++++ ogr/ogr_swq.h | 13 +++++++++- ogr/ogrsf_frmts/generic/ogr_gensql.cpp | 11 ++++---- ogr/swq.cpp | 35 +++++++++++++++++++++++--- 4 files changed, 80 insertions(+), 9 deletions(-) diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index a328f8b36bda..4d68b46c2409 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -27,6 +27,7 @@ # Boston, MA 02111-1307, USA. ############################################################################### +import math import os import shutil @@ -2102,3 +2103,32 @@ def test_ogr_sql_identifier_hidden(): with ds.ExecuteSQL("SELECT 'foo' AS hidden FROM hidden") as sql_lyr: f = sql_lyr.GetNextFeature() assert f["hidden"] == "foo" + + +@pytest.mark.parametrize( + "input,expected_output", + [ + [(1, 1e100, 1, -1e100), 2], + [(float("inf"), 1), float("inf")], + [(1, float("-inf")), float("-inf")], + [(1, float("nan")), float("nan")], + [(float("inf"), float("-inf")), float("nan")], + ], +) +def test_ogr_sql_kahan_babuska_eumaier_summation(input, expected_output): + """Test accurate SUM() implementation using Kahan-Babuska-Neumaier algorithm""" + + ds = ogr.GetDriverByName("Memory").CreateDataSource("") + lyr = ds.CreateLayer("test") + lyr.CreateField(ogr.FieldDefn("v", ogr.OFTReal)) + for v in input: + feat = ogr.Feature(lyr.GetLayerDefn()) + feat["v"] = v + lyr.CreateFeature(feat) + + with ds.ExecuteSQL("SELECT SUM(v) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + if math.isnan(expected_output): + assert math.isnan(f["SUM_v"]) + else: + assert f["SUM_v"] == expected_output diff --git a/ogr/ogr_swq.h b/ogr/ogr_swq.h index d2461e4b0482..e8686baf728c 100644 --- a/ogr/ogr_swq.h +++ b/ogr/ogr_swq.h @@ -356,11 +356,22 @@ class CPL_UNSTABLE_API swq_summary bool operator()(const CPLString &, const CPLString &) const; }; + //! Return the sum, using Kahan-Babuska-Neumaier algorithm. + // Cf cf KahanBabushkaNeumaierSum of https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements + double sum() const + { + return sum_only_finite_terms ? sum_acc + sum_correction : sum_acc; + } + GIntBig count = 0; std::vector oVectorDistinctValues{}; std::set oSetDistinctValues{}; - double sum = 0.0; + bool sum_only_finite_terms = true; + // Sum accumulator. To get the accurate sum, use the sum() method + double sum_acc = 0.0; + // Sum correction term. + double sum_correction = 0.0; double min = 0.0; double max = 0.0; CPLString osMin{}; diff --git a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp index ac961137e95c..9c9e4972c6ae 100644 --- a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp +++ b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp @@ -992,7 +992,7 @@ bool OGRGenSQLResultsLayer::PrepareSummary() for (int iField = 0; iField < psSelectInfo->result_columns(); iField++) { - swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; + const swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; if (!psSelectInfo->column_summary.empty()) { const swq_summary &oSummary = @@ -1000,12 +1000,12 @@ bool OGRGenSQLResultsLayer::PrepareSummary() if (psColDef->col_func == SWQCF_AVG && oSummary.count > 0) { + const double dfAvg = oSummary.sum() / oSummary.count; if (psColDef->field_type == SWQ_DATE || psColDef->field_type == SWQ_TIME || psColDef->field_type == SWQ_TIMESTAMP) { struct tm brokendowntime; - double dfAvg = oSummary.sum / oSummary.count; CPLUnixTimeToYMDHMS(static_cast(dfAvg), &brokendowntime); m_poSummaryFeature->SetField( @@ -1017,8 +1017,9 @@ bool OGRGenSQLResultsLayer::PrepareSummary() 0); } else - m_poSummaryFeature->SetField( - iField, oSummary.sum / oSummary.count); + { + m_poSummaryFeature->SetField(iField, dfAvg); + } } else if (psColDef->col_func == SWQCF_MIN && oSummary.count > 0) { @@ -1045,7 +1046,7 @@ bool OGRGenSQLResultsLayer::PrepareSummary() else if (psColDef->col_func == SWQCF_COUNT) m_poSummaryFeature->SetField(iField, oSummary.count); else if (psColDef->col_func == SWQCF_SUM && oSummary.count > 0) - m_poSummaryFeature->SetField(iField, oSummary.sum); + m_poSummaryFeature->SetField(iField, oSummary.sum()); } else if (psColDef->col_func == SWQCF_COUNT) m_poSummaryFeature->SetField(iField, 0); diff --git a/ogr/swq.cpp b/ogr/swq.cpp index c4cd3d2edc84..5132f9de128d 100644 --- a/ogr/swq.cpp +++ b/ogr/swq.cpp @@ -497,15 +497,44 @@ const char *swq_select_summarize(swq_select *select_info, int dest_column, brokendowntime.tm_sec = static_cast(sField.Date.Second); summary.count++; - summary.sum += CPLYMDHMSToUnixTime(&brokendowntime); - summary.sum += + summary.sum_acc += CPLYMDHMSToUnixTime(&brokendowntime); + summary.sum_acc += fmod(static_cast(sField.Date.Second), 1.0); } } else { summary.count++; - summary.sum += CPLAtof(value); + + // Cf KahanBabushkaNeumaierSum of + // https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements + // We set a number of temporary variables as volatile, to + // prevent potential undesired compiler optimizations. + + const double dfNewVal = CPLAtof(value); + const volatile double new_sum_acc = + summary.sum_acc + dfNewVal; + if (summary.sum_only_finite_terms && + std::isfinite(dfNewVal)) + { + if (std::fabs(summary.sum_acc) >= std::fabs(dfNewVal)) + { + const volatile double diff = + (summary.sum_acc - new_sum_acc); + summary.sum_correction += (diff + dfNewVal); + } + else + { + const volatile double diff = + (dfNewVal - new_sum_acc); + summary.sum_correction += (diff + summary.sum_acc); + } + } + else + { + summary.sum_only_finite_terms = false; + } + summary.sum_acc = new_sum_acc; } } break; From f404ca3d4135942622712246584fc090c2da077c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 23 Jun 2024 15:52:44 +0200 Subject: [PATCH 06/72] OGR SQL: avoid going through string serialization for MIN(), MAX(), SUM(), AVG() on numeric fields --- ogr/ogr_swq.h | 8 +- ogr/ogrsf_frmts/generic/ogr_gensql.cpp | 124 +++++++++++++++++------- ogr/swq.cpp | 128 ++++++++++++------------- 3 files changed, 153 insertions(+), 107 deletions(-) diff --git a/ogr/ogr_swq.h b/ogr/ogr_swq.h index e8686baf728c..a61cd0cc4c18 100644 --- a/ogr/ogr_swq.h +++ b/ogr/ogr_swq.h @@ -487,9 +487,15 @@ class CPL_UNSTABLE_API swq_select std::map> m_exclude_fields{}; }; +/* This method should generally be invoked with pszValue set, except when + * called on a non-DISTINCT column definition of numeric type (SWQ_BOOLEAN, + * SWQ_INTEGER, SWQ_INTEGER64, SWQ_FLOAT), in which case pdfValue should + * rather be set. + */ const char CPL_UNSTABLE_API *swq_select_summarize(swq_select *select_info, int dest_column, - const char *value); + const char *pszValue, + const double *pdfValue); int CPL_UNSTABLE_API swq_is_reserved_keyword(const char *pszStr); diff --git a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp index 9c9e4972c6ae..3c2838a6ccab 100644 --- a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp +++ b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp @@ -827,44 +827,70 @@ bool OGRGenSQLResultsLayer::PrepareSummary() /* the where clause and no column references OGR_GEOMETRY, */ /* OGR_GEOM_WKT or OGR_GEOM_AREA special fields. */ /* -------------------------------------------------------------------- */ - int bSaveIsGeomIgnored = m_poSrcLayer->GetLayerDefn()->IsGeometryIgnored(); + + struct TempGeomIgnoredSetter + { + OGRFeatureDefn &m_oDefn; + const int m_bSaveIsGeomIgnored; + + explicit TempGeomIgnoredSetter(OGRFeatureDefn *poDefn) + : m_oDefn(*poDefn), + m_bSaveIsGeomIgnored(poDefn->IsGeometryIgnored()) + { + m_oDefn.SetGeometryIgnored(true); + } + + ~TempGeomIgnoredSetter() + { + m_oDefn.SetGeometryIgnored(m_bSaveIsGeomIgnored); + } + }; + + auto poSrcLayerDefn = m_poSrcLayer->GetLayerDefn(); + std::unique_ptr oTempGeomIgnoredSetter; + if (m_poFilterGeom == nullptr && (psSelectInfo->where_expr == nullptr || !ContainGeomSpecialField(psSelectInfo->where_expr))) { - int bFoundGeomExpr = FALSE; + bool bFoundGeomExpr = false; for (int iField = 0; iField < psSelectInfo->result_columns(); iField++) { - swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; + const swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; if (psColDef->table_index == 0 && psColDef->field_index != -1) { OGRLayer *poLayer = m_apoTableLayers[psColDef->table_index]; - int nSpecialFieldIdx = psColDef->field_index - - poLayer->GetLayerDefn()->GetFieldCount(); + const int nSpecialFieldIdx = + psColDef->field_index - + poLayer->GetLayerDefn()->GetFieldCount(); if (nSpecialFieldIdx == SPF_OGR_GEOMETRY || nSpecialFieldIdx == SPF_OGR_GEOM_WKT || nSpecialFieldIdx == SPF_OGR_GEOM_AREA) { - bFoundGeomExpr = TRUE; + bFoundGeomExpr = true; break; } if (psColDef->field_index == GEOM_FIELD_INDEX_TO_ALL_FIELD_INDEX(poLayer->GetLayerDefn(), 0)) { - bFoundGeomExpr = TRUE; + bFoundGeomExpr = true; break; } } if (psColDef->expr != nullptr && ContainGeomSpecialField(psColDef->expr)) { - bFoundGeomExpr = TRUE; + bFoundGeomExpr = true; break; } } if (!bFoundGeomExpr) - m_poSrcLayer->GetLayerDefn()->SetGeometryIgnored(TRUE); + { + // cppcheck-suppress unreadVariable + oTempGeomIgnoredSetter = + std::make_unique(poSrcLayerDefn); + } } /* -------------------------------------------------------------------- */ @@ -888,7 +914,6 @@ bool OGRGenSQLResultsLayer::PrepareSummary() m_poSummaryFeature->SetField(0, static_cast(nRes)); } - m_poSrcLayer->GetLayerDefn()->SetGeometryIgnored(bSaveIsGeomIgnored); return TRUE; } @@ -896,64 +921,89 @@ bool OGRGenSQLResultsLayer::PrepareSummary() /* Otherwise, process all source feature through the summary */ /* building facilities of SWQ. */ /* -------------------------------------------------------------------- */ - const char *pszError = nullptr; for (auto &&poSrcFeature : *m_poSrcLayer) { for (int iField = 0; iField < psSelectInfo->result_columns(); iField++) { - swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; + const swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; + const char *pszError = nullptr; if (psColDef->col_func == SWQCF_COUNT) { /* psColDef->field_index can be -1 in the case of a COUNT(*) */ if (psColDef->field_index < 0) - pszError = swq_select_summarize(psSelectInfo, iField, ""); - else if (IS_GEOM_FIELD_INDEX(m_poSrcLayer->GetLayerDefn(), + pszError = + swq_select_summarize(psSelectInfo, iField, "", nullptr); + else if (IS_GEOM_FIELD_INDEX(poSrcLayerDefn, psColDef->field_index)) { - int iSrcGeomField = ALL_FIELD_INDEX_TO_GEOM_FIELD_INDEX( - m_poSrcLayer->GetLayerDefn(), psColDef->field_index); - OGRGeometry *poGeom = + const int iSrcGeomField = + ALL_FIELD_INDEX_TO_GEOM_FIELD_INDEX( + poSrcLayerDefn, psColDef->field_index); + const OGRGeometry *poGeom = poSrcFeature->GetGeomFieldRef(iSrcGeomField); if (poGeom != nullptr) - pszError = - swq_select_summarize(psSelectInfo, iField, ""); - else - pszError = nullptr; + pszError = swq_select_summarize(psSelectInfo, iField, + "", nullptr); } else if (poSrcFeature->IsFieldSetAndNotNull( psColDef->field_index)) - pszError = swq_select_summarize( - psSelectInfo, iField, - poSrcFeature->GetFieldAsString(psColDef->field_index)); - else - pszError = nullptr; + { + if (!psColDef->distinct_flag) + { + pszError = swq_select_summarize(psSelectInfo, iField, + "", nullptr); + } + else + { + const char *pszVal = poSrcFeature->GetFieldAsString( + psColDef->field_index); + pszError = swq_select_summarize(psSelectInfo, iField, + pszVal, nullptr); + } + } } else { - const char *pszVal = nullptr; if (poSrcFeature->IsFieldSetAndNotNull(psColDef->field_index)) - pszVal = - poSrcFeature->GetFieldAsString(psColDef->field_index); - pszError = swq_select_summarize(psSelectInfo, iField, pszVal); + { + if (!psColDef->distinct_flag && + (psColDef->field_type == SWQ_BOOLEAN || + psColDef->field_type == SWQ_INTEGER || + psColDef->field_type == SWQ_INTEGER64 || + psColDef->field_type == SWQ_FLOAT)) + { + const double dfValue = poSrcFeature->GetFieldAsDouble( + psColDef->field_index); + pszError = swq_select_summarize(psSelectInfo, iField, + nullptr, &dfValue); + } + else + { + const char *pszVal = poSrcFeature->GetFieldAsString( + psColDef->field_index); + pszError = swq_select_summarize(psSelectInfo, iField, + pszVal, nullptr); + } + } + else + { + pszError = swq_select_summarize(psSelectInfo, iField, + nullptr, nullptr); + } } - if (pszError != nullptr) + if (pszError) { m_poSummaryFeature.reset(); - m_poSrcLayer->GetLayerDefn()->SetGeometryIgnored( - bSaveIsGeomIgnored); - CPLError(CE_Failure, CPLE_AppDefined, "%s", pszError); return false; } } } - m_poSrcLayer->GetLayerDefn()->SetGeometryIgnored(bSaveIsGeomIgnored); - /* -------------------------------------------------------------------- */ /* Clear away the filters we have installed till a next run through*/ /* the features. */ @@ -968,7 +1018,7 @@ bool OGRGenSQLResultsLayer::PrepareSummary() { for (int iField = 0; iField < psSelectInfo->result_columns(); iField++) { - swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; + const swq_col_def *psColDef = &psSelectInfo->column_defs[iField]; if (!psSelectInfo->column_summary.empty()) { const swq_summary &oSummary = diff --git a/ogr/swq.cpp b/ogr/swq.cpp index 5132f9de128d..123cc43198b2 100644 --- a/ogr/swq.cpp +++ b/ogr/swq.cpp @@ -323,7 +323,7 @@ int swqlex(YYSTYPE *ppNode, swq_parse_context *context) /************************************************************************/ const char *swq_select_summarize(swq_select *select_info, int dest_column, - const char *value) + const char *pszValue, const double *pdfValue) { /* -------------------------------------------------------------------- */ @@ -336,7 +336,7 @@ const char *swq_select_summarize(swq_select *select_info, int dest_column, dest_column >= static_cast(select_info->column_defs.size())) return "dest_column out of range in swq_select_summarize()."; - swq_col_def *def = &select_info->column_defs[dest_column]; + const swq_col_def *def = &select_info->column_defs[dest_column]; if (def->col_func == SWQCF_NONE && !def->distinct_flag) return nullptr; @@ -403,18 +403,18 @@ const char *swq_select_summarize(swq_select *select_info, int dest_column, if (def->distinct_flag) { - if (value == nullptr) - value = SZ_OGR_NULL; + if (pszValue == nullptr) + pszValue = SZ_OGR_NULL; try { - if (summary.oSetDistinctValues.find(value) == + if (summary.oSetDistinctValues.find(pszValue) == summary.oSetDistinctValues.end()) { - summary.oSetDistinctValues.insert(value); + summary.oSetDistinctValues.insert(pszValue); if (select_info->order_specs == 0) { // If not sorted, keep values in their original order - summary.oVectorDistinctValues.emplace_back(value); + summary.oVectorDistinctValues.emplace_back(pszValue); } summary.count++; } @@ -434,59 +434,78 @@ const char *swq_select_summarize(swq_select *select_info, int dest_column, switch (def->col_func) { case SWQCF_MIN: - if (value != nullptr && value[0] != '\0') + if (pdfValue) { - if (def->field_type == SWQ_DATE || - def->field_type == SWQ_TIME || - def->field_type == SWQ_TIMESTAMP || - def->field_type == SWQ_STRING) + if (*pdfValue < summary.min) + summary.min = *pdfValue; + summary.count++; + } + else if (pszValue && pszValue[0] != '\0') + { + if (summary.count == 0 || strcmp(pszValue, summary.osMin) < 0) { - if (summary.count == 0 || strcmp(value, summary.osMin) < 0) - { - summary.osMin = value; - } + summary.osMin = pszValue; } - else + summary.count++; + } + break; + case SWQCF_MAX: + if (pdfValue) + { + if (*pdfValue > summary.max) + summary.max = *pdfValue; + summary.count++; + } + else if (pszValue && pszValue[0] != '\0') + { + if (summary.count == 0 || strcmp(pszValue, summary.osMax) > 0) { - double df_val = CPLAtof(value); - if (df_val < summary.min) - summary.min = df_val; + summary.osMax = pszValue; } summary.count++; } break; - case SWQCF_MAX: - if (value != nullptr && value[0] != '\0') + case SWQCF_AVG: + case SWQCF_SUM: + if (pdfValue) { - if (def->field_type == SWQ_DATE || - def->field_type == SWQ_TIME || - def->field_type == SWQ_TIMESTAMP || - def->field_type == SWQ_STRING) + summary.count++; + + // Cf KahanBabushkaNeumaierSum of + // https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements + // We set a number of temporary variables as volatile, to + // prevent potential undesired compiler optimizations. + + const double dfNewVal = *pdfValue; + const volatile double new_sum_acc = summary.sum_acc + dfNewVal; + if (summary.sum_only_finite_terms && std::isfinite(dfNewVal)) { - if (summary.count == 0 || strcmp(value, summary.osMax) > 0) + if (std::fabs(summary.sum_acc) >= std::fabs(dfNewVal)) + { + const volatile double diff = + (summary.sum_acc - new_sum_acc); + summary.sum_correction += (diff + dfNewVal); + } + else { - summary.osMax = value; + const volatile double diff = (dfNewVal - new_sum_acc); + summary.sum_correction += (diff + summary.sum_acc); } } else { - double df_val = CPLAtof(value); - if (df_val > summary.max) - summary.max = df_val; + summary.sum_only_finite_terms = false; } - summary.count++; + summary.sum_acc = new_sum_acc; } - break; - case SWQCF_AVG: - case SWQCF_SUM: - if (value != nullptr && value[0] != '\0') + else if (pszValue && pszValue[0] != '\0') { if (def->field_type == SWQ_DATE || def->field_type == SWQ_TIME || def->field_type == SWQ_TIMESTAMP) { OGRField sField; - if (OGRParseDate(value, &sField, 0)) + if (OGRParseDate(pszValue, &sField, 0)) { struct tm brokendowntime; brokendowntime.tm_year = sField.Date.Year - 1900; @@ -504,43 +523,14 @@ const char *swq_select_summarize(swq_select *select_info, int dest_column, } else { - summary.count++; - - // Cf KahanBabushkaNeumaierSum of - // https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements - // We set a number of temporary variables as volatile, to - // prevent potential undesired compiler optimizations. - - const double dfNewVal = CPLAtof(value); - const volatile double new_sum_acc = - summary.sum_acc + dfNewVal; - if (summary.sum_only_finite_terms && - std::isfinite(dfNewVal)) - { - if (std::fabs(summary.sum_acc) >= std::fabs(dfNewVal)) - { - const volatile double diff = - (summary.sum_acc - new_sum_acc); - summary.sum_correction += (diff + dfNewVal); - } - else - { - const volatile double diff = - (dfNewVal - new_sum_acc); - summary.sum_correction += (diff + summary.sum_acc); - } - } - else - { - summary.sum_only_finite_terms = false; - } - summary.sum_acc = new_sum_acc; + return "swq_select_summarize() - AVG()/SUM() called on " + "unexpected field type"; } } break; case SWQCF_COUNT: - if (value != nullptr) + if (pdfValue || pszValue) summary.count++; break; From 48daa9f068fff1f42a3cfc4f632d7f86048740fb Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 23 Jun 2024 17:43:39 +0200 Subject: [PATCH 07/72] OGRSQL: add STDDEV_POP() and STDDEV_SAMP() aggregate functions --- autotest/cpp/test_ogr_swq.cpp | 3 +- autotest/ogr/ogr_sql_rfc28.py | 19 +-- autotest/ogr/ogr_sql_test.py | 12 +- doc/source/user/ogr_sql_dialect.rst | 15 ++- ogr/ogr_swq.h | 14 +++ ogr/ogrsf_frmts/generic/ogr_gensql.cpp | 165 ++++++++++++++++++------- ogr/swq.cpp | 53 +++++++- ogr/swq_op_registrar.cpp | 2 + ogr/swq_select.cpp | 80 +++++++----- 9 files changed, 265 insertions(+), 98 deletions(-) diff --git a/autotest/cpp/test_ogr_swq.cpp b/autotest/cpp/test_ogr_swq.cpp index 520a5846ba86..73074b9e13c6 100644 --- a/autotest/cpp/test_ogr_swq.cpp +++ b/autotest/cpp/test_ogr_swq.cpp @@ -216,7 +216,8 @@ TEST_F(test_ogr_swq, select_unparse) swq_select select; const char *pszSQL = "SELECT DISTINCT a, \"a b\" AS renamed, AVG(x.a) AS avg, MIN(a), " - "MAX(\"a b\"), SUM(a), AVG(a), COUNT(a), COUNT(DISTINCT a) " + "MAX(\"a b\"), SUM(a), AVG(a), COUNT(a), COUNT(DISTINCT a), " + "STDDEV_POP(a), STDDEV_SAMP(a) " "FROM 'foo'.\"FOO BAR\" AS x " "JOIN 'bar'.BAR AS y ON FOO.x = BAR.y " "WHERE 1 ORDER BY a, \"a b\" DESC " diff --git a/autotest/ogr/ogr_sql_rfc28.py b/autotest/ogr/ogr_sql_rfc28.py index d5aa09bdac81..6b54af821e2b 100755 --- a/autotest/ogr/ogr_sql_rfc28.py +++ b/autotest/ogr/ogr_sql_rfc28.py @@ -881,7 +881,7 @@ def test_ogr_rfc28_39(data_ds): ############################################################################### -# Test MIN(), MAX() and AVG() on a date (#5333) +# Test MIN(), MAX(), AVG(), STDDEV_POP(), STDDEV_SAMP() on a date (#5333) def test_ogr_rfc28_40(): @@ -896,15 +896,16 @@ def test_ogr_rfc28_40(): feat.SetField(0, "2013/01/01 00:00:00") lyr.CreateFeature(feat) - with ds.ExecuteSQL("SELECT MIN(DATE), MAX(DATE), AVG(DATE) from test") as lyr: + with ds.ExecuteSQL( + "SELECT MIN(DATE), MAX(DATE), AVG(DATE), STDDEV_POP(DATE), STDDEV_SAMP(DATE) from test" + ) as sql_lyr: - ogrtest.check_features_against_list(lyr, "MIN_DATE", ["2013/01/01 00:00:00"]) - lyr.ResetReading() - ogrtest.check_features_against_list(lyr, "MAX_DATE", ["2013/12/31 23:59:59"]) - lyr.ResetReading() - ogrtest.check_features_against_list( - lyr, "AVG_DATE", ["2013/07/02 11:59:59.500"] - ) + f = sql_lyr.GetNextFeature() + assert f["MIN_DATE"] == "2013/01/01 00:00:00" + assert f["MAX_DATE"] == "2013/12/31 23:59:59" + assert f["AVG_DATE"] == "2013/07/02 11:59:59.500" + assert f["STDDEV_POP_DATE"] == pytest.approx(15767999.5, rel=1e-15) + assert f["STDDEV_SAMP_DATE"] == pytest.approx(22299318.744392183, rel=1e-15) ############################################################################### diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index 4d68b46c2409..cba0f8a0ad33 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -255,12 +255,14 @@ def test_ogr_sql_4(data_ds): def test_ogr_sql_5(data_ds): with data_ds.ExecuteSQL( - "select max(eas_id), min(eas_id), avg(eas_id), sum(eas_id), count(eas_id) from idlink" + "select max(eas_id), min(eas_id), avg(eas_id), STDDEV_POP(eas_id), STDDEV_SAMP(eas_id), sum(eas_id), count(eas_id) from idlink" ) as sql_lyr: feat = sql_lyr.GetNextFeature() assert feat["max_eas_id"] == 179 assert feat["min_eas_id"] == 158 assert feat["avg_eas_id"] == pytest.approx(168.142857142857, abs=1e-12) + assert feat["STDDEV_POP_eas_id"] == pytest.approx(5.9384599116647205, rel=1e-15) + assert feat["STDDEV_SAMP_eas_id"] == pytest.approx(6.414269805898183, rel=1e-15) assert feat["count_eas_id"] == 7 assert feat["sum_eas_id"] == 1177 @@ -789,10 +791,15 @@ def ds_for_invalid_statements(): "SELECT MAX(foo) FROM my_layer", "SELECT SUM(foo) FROM my_layer", "SELECT AVG(foo) FROM my_layer", + "SELECT STDDEV_POP(foo) FROM my_layer", + "SELECT STDDEV_SAMP(foo) FROM my_layer", "SELECT SUM(strfield) FROM my_layer", "SELECT AVG(strfield) FROM my_layer", "SELECT AVG(intfield, intfield) FROM my_layer", + "SELECT STDDEV_POP(strfield) FROM my_layer", + "SELECT STDDEV_SAMP(strfield) FROM my_layer", "SELECT * FROM my_layer WHERE AVG(intfield) = 1", + "SELECT * FROM my_layer WHERE STDDEV_POP(intfield) = 1", "SELECT * FROM 'foo' foo", "SELECT * FROM my_layer WHERE strfield =", "SELECT * FROM my_layer WHERE strfield = foo", @@ -1101,10 +1108,11 @@ def test_ogr_sql_count_and_null(): assert feat.GetFieldAsInteger(2) == 4, fieldname with ds.ExecuteSQL( - "select avg(intfield) from layer where intfield is null" + "select avg(intfield), STDDEV_POP(intfield) from layer where intfield is null" ) as sql_lyr: feat = sql_lyr.GetNextFeature() assert feat.IsFieldSetAndNotNull(0) == 0 + assert feat.IsFieldSetAndNotNull(1) == 0 # Fix crash when first values is null (#4509) with ds.ExecuteSQL("select distinct strfield_first_null from layer") as sql_lyr: diff --git a/doc/source/user/ogr_sql_dialect.rst b/doc/source/user/ogr_sql_dialect.rst index 207d934bc869..605441add76b 100644 --- a/doc/source/user/ogr_sql_dialect.rst +++ b/doc/source/user/ogr_sql_dialect.rst @@ -107,16 +107,23 @@ memory may be used for datasets with a large number of distinct values. There are also several summarization operators that may be applied to columns. When a summarization operator is applied to any field, then all fields must -have summarization operators applied. The summarization operators are -COUNT (a count of instances), AVG (numerical average), SUM (numerical sum), -MIN (lexical or numerical minimum), and MAX (lexical or numerical maximum). +have summarization operators applied. The summarization operators are: + +- COUNT: count of instances +- AVG: numerical average: +- SUM: numerical sum +- MIN: lexical or numerical minimum +- MAX: lexical or numerical maximum +- STDDEV_POP: (GDAL >= 3.10) numerical population standard deviation. Applied on Date/DateTime/Time fields, this returns a value in seconds. +- STDDEV_SAMP: (GDAL >= 3.10) numerical `sample standard deviation `__. Applied on Date/DateTime/Time fields, this returns a value in seconds. + This example produces a variety of summarization information on parcel property values: .. code-block:: SELECT MIN(prop_value), MAX(prop_value), AVG(prop_value), SUM(prop_value), - COUNT(prop_value) FROM polylayer WHERE prov_name = 'Ontario' + COUNT(prop_value), STDDEV_POP(prop_value) FROM polylayer WHERE prov_name = 'Ontario' It is also possible to apply the COUNT() operator to a DISTINCT SELECT to get a count of distinct values, for instance: diff --git a/ogr/ogr_swq.h b/ogr/ogr_swq.h index a61cd0cc4c18..71f4f45e6b6b 100644 --- a/ogr/ogr_swq.h +++ b/ogr/ogr_swq.h @@ -63,11 +63,17 @@ typedef enum SWQ_CONCAT, SWQ_SUBSTR, SWQ_HSTORE_GET_VALUE, + SWQ_AVG, + SWQ_AGGREGATE_BEGIN = SWQ_AVG, SWQ_MIN, SWQ_MAX, SWQ_COUNT, SWQ_SUM, + SWQ_STDDEV_POP, + SWQ_STDDEV_SAMP, + SWQ_AGGREGATE_END = SWQ_STDDEV_SAMP, + SWQ_CAST, SWQ_CUSTOM_FUNC, /* only if parsing done in bAcceptCustomFuncs mode */ SWQ_ARGUMENT_LIST /* temporary value only set during parsing and replaced by @@ -318,6 +324,8 @@ typedef enum SWQCF_MAX = SWQ_MAX, SWQCF_COUNT = SWQ_COUNT, SWQCF_SUM = SWQ_SUM, + SWQCF_STDDEV_POP = SWQ_STDDEV_POP, + SWQCF_STDDEV_SAMP = SWQ_STDDEV_SAMP, SWQCF_CUSTOM } swq_col_func; @@ -374,6 +382,12 @@ class CPL_UNSTABLE_API swq_summary double sum_correction = 0.0; double min = 0.0; double max = 0.0; + + // Welford's online algorithm for variance: + // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + double mean_for_variance = 0.0; + double sq_dist_from_mean_acc = 0.0; // "M2" + CPLString osMin{}; CPLString osMax{}; }; diff --git a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp index 3c2838a6ccab..3b2e7af7b269 100644 --- a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp +++ b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp @@ -247,10 +247,15 @@ OGRGenSQLResultsLayer::OGRGenSQLResultsLayer( oFDefn.SetType(OFTInteger64); else if (poSrcFDefn != nullptr) { - if (psColDef->col_func != SWQCF_AVG || - psColDef->field_type == SWQ_DATE || - psColDef->field_type == SWQ_TIME || - psColDef->field_type == SWQ_TIMESTAMP) + if (psColDef->col_func == SWQCF_STDDEV_POP || + psColDef->col_func == SWQCF_STDDEV_SAMP) + { + oFDefn.SetType(OFTReal); + } + else if (psColDef->col_func != SWQCF_AVG || + psColDef->field_type == SWQ_DATE || + psColDef->field_type == SWQ_TIME || + psColDef->field_type == SWQ_TIMESTAMP) { oFDefn.SetType(poSrcFDefn->GetType()); if (psColDef->col_func == SWQCF_NONE || @@ -261,8 +266,13 @@ OGRGenSQLResultsLayer::OGRGenSQLResultsLayer( } } else + { oFDefn.SetType(OFTReal); + } + if (psColDef->col_func != SWQCF_AVG && + psColDef->col_func != SWQCF_STDDEV_POP && + psColDef->col_func != SWQCF_STDDEV_SAMP && psColDef->col_func != SWQCF_SUM) { oFDefn.SetWidth(poSrcFDefn->GetWidth()); @@ -1048,55 +1058,116 @@ bool OGRGenSQLResultsLayer::PrepareSummary() const swq_summary &oSummary = psSelectInfo->column_summary[iField]; - if (psColDef->col_func == SWQCF_AVG && oSummary.count > 0) + switch (psColDef->col_func) { - const double dfAvg = oSummary.sum() / oSummary.count; - if (psColDef->field_type == SWQ_DATE || - psColDef->field_type == SWQ_TIME || - psColDef->field_type == SWQ_TIMESTAMP) + case SWQCF_NONE: + case SWQCF_CUSTOM: + break; + + case SWQCF_AVG: { - struct tm brokendowntime; - CPLUnixTimeToYMDHMS(static_cast(dfAvg), - &brokendowntime); - m_poSummaryFeature->SetField( - iField, brokendowntime.tm_year + 1900, - brokendowntime.tm_mon + 1, brokendowntime.tm_mday, - brokendowntime.tm_hour, brokendowntime.tm_min, - static_cast(brokendowntime.tm_sec + - fmod(dfAvg, 1)), - 0); + if (oSummary.count > 0) + { + const double dfAvg = + oSummary.sum() / oSummary.count; + if (psColDef->field_type == SWQ_DATE || + psColDef->field_type == SWQ_TIME || + psColDef->field_type == SWQ_TIMESTAMP) + { + struct tm brokendowntime; + CPLUnixTimeToYMDHMS(static_cast(dfAvg), + &brokendowntime); + m_poSummaryFeature->SetField( + iField, brokendowntime.tm_year + 1900, + brokendowntime.tm_mon + 1, + brokendowntime.tm_mday, + brokendowntime.tm_hour, + brokendowntime.tm_min, + static_cast(brokendowntime.tm_sec + + fmod(dfAvg, 1)), + 0); + } + else + { + m_poSummaryFeature->SetField(iField, dfAvg); + } + } + break; } - else + + case SWQCF_MIN: { - m_poSummaryFeature->SetField(iField, dfAvg); + if (oSummary.count > 0) + { + if (psColDef->field_type == SWQ_DATE || + psColDef->field_type == SWQ_TIME || + psColDef->field_type == SWQ_TIMESTAMP || + psColDef->field_type == SWQ_STRING) + m_poSummaryFeature->SetField( + iField, oSummary.osMin.c_str()); + else + m_poSummaryFeature->SetField(iField, + oSummary.min); + } + break; + } + + case SWQCF_MAX: + { + if (oSummary.count > 0) + { + if (psColDef->field_type == SWQ_DATE || + psColDef->field_type == SWQ_TIME || + psColDef->field_type == SWQ_TIMESTAMP || + psColDef->field_type == SWQ_STRING) + m_poSummaryFeature->SetField( + iField, oSummary.osMax.c_str()); + else + m_poSummaryFeature->SetField(iField, + oSummary.max); + } + break; + } + + case SWQCF_COUNT: + { + m_poSummaryFeature->SetField(iField, oSummary.count); + break; + } + + case SWQCF_SUM: + { + if (oSummary.count > 0) + m_poSummaryFeature->SetField(iField, + oSummary.sum()); + break; + } + + case SWQCF_STDDEV_POP: + { + if (oSummary.count > 0) + { + const double dfVariance = + oSummary.sq_dist_from_mean_acc / oSummary.count; + m_poSummaryFeature->SetField(iField, + sqrt(dfVariance)); + } + break; + } + + case SWQCF_STDDEV_SAMP: + { + if (oSummary.count > 1) + { + const double dfSampleVariance = + oSummary.sq_dist_from_mean_acc / + (oSummary.count - 1); + m_poSummaryFeature->SetField( + iField, sqrt(dfSampleVariance)); + } + break; } } - else if (psColDef->col_func == SWQCF_MIN && oSummary.count > 0) - { - if (psColDef->field_type == SWQ_DATE || - psColDef->field_type == SWQ_TIME || - psColDef->field_type == SWQ_TIMESTAMP || - psColDef->field_type == SWQ_STRING) - m_poSummaryFeature->SetField(iField, - oSummary.osMin.c_str()); - else - m_poSummaryFeature->SetField(iField, oSummary.min); - } - else if (psColDef->col_func == SWQCF_MAX && oSummary.count > 0) - { - if (psColDef->field_type == SWQ_DATE || - psColDef->field_type == SWQ_TIME || - psColDef->field_type == SWQ_TIMESTAMP || - psColDef->field_type == SWQ_STRING) - m_poSummaryFeature->SetField(iField, - oSummary.osMax.c_str()); - else - m_poSummaryFeature->SetField(iField, oSummary.max); - } - else if (psColDef->col_func == SWQCF_COUNT) - m_poSummaryFeature->SetField(iField, oSummary.count); - else if (psColDef->col_func == SWQCF_SUM && oSummary.count > 0) - m_poSummaryFeature->SetField(iField, oSummary.sum()); } else if (psColDef->col_func == SWQCF_COUNT) m_poSummaryFeature->SetField(iField, 0); diff --git a/ogr/swq.cpp b/ogr/swq.cpp index 123cc43198b2..38d983d42489 100644 --- a/ogr/swq.cpp +++ b/ogr/swq.cpp @@ -534,14 +534,61 @@ const char *swq_select_summarize(swq_select *select_info, int dest_column, summary.count++; break; + case SWQCF_STDDEV_POP: + case SWQCF_STDDEV_SAMP: + { + const auto UpdateVariance = [&summary](double dfValue) + { + // Welford's online algorithm for variance: + // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + summary.count++; + const double dfDelta = dfValue - summary.mean_for_variance; + summary.mean_for_variance += dfDelta / summary.count; + const double dfDelta2 = dfValue - summary.mean_for_variance; + summary.sq_dist_from_mean_acc += dfDelta * dfDelta2; + }; + + if (pdfValue) + { + UpdateVariance(*pdfValue); + } + else if (pszValue && pszValue[0] != '\0') + { + if (def->field_type == SWQ_DATE || + def->field_type == SWQ_TIME || + def->field_type == SWQ_TIMESTAMP) + { + OGRField sField; + if (OGRParseDate(pszValue, &sField, 0)) + { + struct tm brokendowntime; + brokendowntime.tm_year = sField.Date.Year - 1900; + brokendowntime.tm_mon = sField.Date.Month - 1; + brokendowntime.tm_mday = sField.Date.Day; + brokendowntime.tm_hour = sField.Date.Hour; + brokendowntime.tm_min = sField.Date.Minute; + brokendowntime.tm_sec = + static_cast(sField.Date.Second); + + UpdateVariance(static_cast( + CPLYMDHMSToUnixTime(&brokendowntime))); + } + } + else + { + return "swq_select_summarize() - STDDEV() called on " + "unexpected field type"; + } + } + + break; + } + case SWQCF_NONE: break; case SWQCF_CUSTOM: return "swq_select_summarize() called on custom field function."; - - default: - return "swq_select_summarize() - unexpected col_func"; } return nullptr; diff --git a/ogr/swq_op_registrar.cpp b/ogr/swq_op_registrar.cpp index 8b82a11da0f9..6b63d5b8b31b 100644 --- a/ogr/swq_op_registrar.cpp +++ b/ogr/swq_op_registrar.cpp @@ -71,6 +71,8 @@ static const swq_operation swq_apsOperations[] = { {"MAX", SWQ_MAX, SWQGeneralEvaluator, SWQColumnFuncChecker}, {"COUNT", SWQ_COUNT, SWQGeneralEvaluator, SWQColumnFuncChecker}, {"SUM", SWQ_SUM, SWQGeneralEvaluator, SWQColumnFuncChecker}, + {"STDDEV_POP", SWQ_STDDEV_POP, SWQGeneralEvaluator, SWQColumnFuncChecker}, + {"STDDEV_SAMP", SWQ_STDDEV_SAMP, SWQGeneralEvaluator, SWQColumnFuncChecker}, {"CAST", SWQ_CAST, SWQCastEvaluator, SWQCastChecker}}; diff --git a/ogr/swq_select.cpp b/ogr/swq_select.cpp index db91efe48552..9e9d84cb8727 100644 --- a/ogr/swq_select.cpp +++ b/ogr/swq_select.cpp @@ -203,16 +203,34 @@ char *swq_select::Unparse() } else { - if (def->col_func == SWQCF_AVG) - osSelect += "AVG("; - else if (def->col_func == SWQCF_MIN) - osSelect += "MIN("; - else if (def->col_func == SWQCF_MAX) - osSelect += "MAX("; - else if (def->col_func == SWQCF_COUNT) - osSelect += "COUNT("; - else if (def->col_func == SWQCF_SUM) - osSelect += "SUM("; + switch (def->col_func) + { + case SWQCF_NONE: + break; + case SWQCF_AVG: + osSelect += "AVG("; + break; + case SWQCF_MIN: + osSelect += "MIN("; + break; + case SWQCF_MAX: + osSelect += "MAX("; + break; + case SWQCF_COUNT: + osSelect += "COUNT("; + break; + case SWQCF_SUM: + osSelect += "SUM("; + break; + case SWQCF_STDDEV_POP: + osSelect += "STDDEV_POP("; + break; + case SWQCF_STDDEV_SAMP: + osSelect += "STDDEV_SAMP("; + break; + case SWQCF_CUSTOM: + break; + } if (def->distinct_flag && def->col_func == SWQCF_COUNT) osSelect += "DISTINCT "; @@ -368,8 +386,8 @@ int swq_select::PushField(swq_expr_node *poExpr, const char *pszAlias, } else if (poExpr->eNodeType == SNT_OPERATION && (poExpr->nOperation == SWQ_CAST || - (poExpr->nOperation >= SWQ_AVG && - poExpr->nOperation <= SWQ_SUM)) && + (poExpr->nOperation >= SWQ_AGGREGATE_BEGIN && + poExpr->nOperation <= SWQ_AGGREGATE_END)) && poExpr->nSubExprCount >= 1 && poExpr->papoSubExpr[0]->eNodeType == SNT_COLUMN) { @@ -573,8 +591,8 @@ int swq_select::PushField(swq_expr_node *poExpr, const char *pszAlias, /* Do we have a special column function in play? */ /* -------------------------------------------------------------------- */ if (poExpr->eNodeType == SNT_OPERATION && - static_cast(poExpr->nOperation) >= SWQ_AVG && - static_cast(poExpr->nOperation) <= SWQ_SUM) + static_cast(poExpr->nOperation) >= SWQ_AGGREGATE_BEGIN && + static_cast(poExpr->nOperation) <= SWQ_AGGREGATE_END) { if (poExpr->nSubExprCount != 1) { @@ -1015,11 +1033,11 @@ CPLErr swq_select::parse(swq_field_list *field_list, } // Identify column function if present. - if (((def->col_func == SWQCF_MIN || def->col_func == SWQCF_MAX || - def->col_func == SWQCF_AVG || def->col_func == SWQCF_SUM) && - def->field_type == SWQ_GEOMETRY) || - ((def->col_func == SWQCF_AVG || def->col_func == SWQCF_SUM) && - def->field_type == SWQ_STRING)) + if (def->col_func != SWQCF_NONE && def->col_func != SWQCF_CUSTOM && + def->col_func != SWQCF_COUNT && + (def->field_type == SWQ_GEOMETRY || + (def->field_type == SWQ_STRING && def->col_func != SWQCF_MIN && + def->col_func != SWQCF_MAX))) { // Possibly this is already enforced by the checker? const swq_operation *op = swq_op_registrar::GetOperator( @@ -1065,9 +1083,17 @@ CPLErr swq_select::parse(swq_field_list *field_list, } } - if (def->col_func == SWQCF_MIN || def->col_func == SWQCF_MAX || - def->col_func == SWQCF_AVG || def->col_func == SWQCF_SUM || - def->col_func == SWQCF_COUNT) + if (def->col_func == SWQCF_NONE) + { + if (query_mode == SWQM_DISTINCT_LIST) + { + def->distinct_flag = TRUE; + this_indicator = SWQM_DISTINCT_LIST; + } + else + this_indicator = SWQM_RECORDSET; + } + else if (def->col_func != SWQCF_CUSTOM) { this_indicator = SWQM_SUMMARY_RECORD; if (def->col_func == SWQCF_COUNT && def->distinct_flag && @@ -1078,16 +1104,6 @@ CPLErr swq_select::parse(swq_field_list *field_list, return CE_Failure; } } - else if (def->col_func == SWQCF_NONE) - { - if (query_mode == SWQM_DISTINCT_LIST) - { - def->distinct_flag = TRUE; - this_indicator = SWQM_DISTINCT_LIST; - } - else - this_indicator = SWQM_RECORDSET; - } if (this_indicator != query_mode && this_indicator != -1 && query_mode != 0) From b012d9a3a885414d3a017d1fa8f22ad5e3518bbd Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 23 Jun 2024 17:45:22 +0200 Subject: [PATCH 08/72] SQLite/GPKG: add STDDEV_POP() and STDDEV_SAMP() aggregate functions --- autotest/ogr/ogr_sqlite.py | 12 ++++ doc/source/user/sql_sqlite_dialect.rst | 9 +++ .../sqlite/ogrsqlitesqlfunctionscommon.cpp | 71 +++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/autotest/ogr/ogr_sqlite.py b/autotest/ogr/ogr_sqlite.py index a85b64511b75..f395e9592fa5 100755 --- a/autotest/ogr/ogr_sqlite.py +++ b/autotest/ogr/ogr_sqlite.py @@ -4095,3 +4095,15 @@ def test_ogr_sql_ST_Area_on_ellipsoid(tmp_vsimem, require_spatialite): with ds.ExecuteSQL("SELECT ST_Area(null, 1) FROM my_layer") as sql_lyr: f = sql_lyr.GetNextFeature() assert f[0] is None + + +def test_ogr_sqlite_stddev(): + """Test STDDEV_POP() and STDDEV_SAMP""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v REAL)") + ds.ExecuteSQL("INSERT INTO test VALUES (4),(NULL),('invalid'),(5)") + with ds.ExecuteSQL("SELECT STDDEV_POP(v), STDDEV_SAMP(v) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == pytest.approx(0.5, rel=1e-15) + assert f.GetField(1) == pytest.approx(0.5**0.5, rel=1e-15) diff --git a/doc/source/user/sql_sqlite_dialect.rst b/doc/source/user/sql_sqlite_dialect.rst index 20b18076b250..042cc0c5425a 100644 --- a/doc/source/user/sql_sqlite_dialect.rst +++ b/doc/source/user/sql_sqlite_dialect.rst @@ -201,6 +201,15 @@ For example we can select the annotation features as: SELECT * FROM nation WHERE OGR_STYLE LIKE 'LABEL%' +Statistics functions +++++++++++++++++++++ + +In addition to standard COUNT(), SUM(), AVG(), MIN(), MAX(), the following +aggregate functions are available: + +- STDDEV_POP: (GDAL >= 3.10) numerical population standard deviation. +- STDDEV_SAMP: (GDAL >= 3.10) numerical `sample standard deviation `__ + Spatialite SQL functions ++++++++++++++++++++++++ diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp b/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp index fe5f0d515669..fd886800542c 100644 --- a/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp +++ b/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp @@ -336,6 +336,69 @@ static void OGRSQLITE_LIKE(sqlite3_context *pContext, int argc, insensitive, bUTF8Strings)); } +/************************************************************************/ +/* OGRSQLITE_STDDEV_Step() */ +/************************************************************************/ + +// Welford's online algorithm for variance: +// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm +struct OGRSQLITE_STDDEV_Context +{ + int64_t nValues; + double dfMean; + double dfM2; // Accumulator for squared distance from the mean +}; + +static void OGRSQLITE_STDDEV_Step(sqlite3_context *pContext, int /* argc*/, + sqlite3_value **argv) +{ + auto pAggCtxt = + static_cast(sqlite3_aggregate_context( + pContext, static_cast(sizeof(OGRSQLITE_STDDEV_Context)))); + const auto eType = sqlite3_value_type(argv[0]); + if (eType != SQLITE_INTEGER && eType != SQLITE_FLOAT) + return; + + const double dfValue = sqlite3_value_double(argv[0]); + pAggCtxt->nValues++; + const double dfDelta = dfValue - pAggCtxt->dfMean; + pAggCtxt->dfMean += dfDelta / pAggCtxt->nValues; + const double dfDelta2 = dfValue - pAggCtxt->dfMean; + pAggCtxt->dfM2 += dfDelta * dfDelta2; +} + +/************************************************************************/ +/* OGRSQLITE_STDDEV_POP_Finalize() */ +/************************************************************************/ + +static void OGRSQLITE_STDDEV_POP_Finalize(sqlite3_context *pContext) +{ + auto pAggCtxt = + static_cast(sqlite3_aggregate_context( + pContext, static_cast(sizeof(OGRSQLITE_STDDEV_Context)))); + if (pAggCtxt->nValues > 0) + { + sqlite3_result_double(pContext, + sqrt(pAggCtxt->dfM2 / pAggCtxt->nValues)); + } +} + +/************************************************************************/ +/* OGRSQLITE_STDDEV_SAMP_Finalize() */ +/************************************************************************/ + +static void OGRSQLITE_STDDEV_SAMP_Finalize(sqlite3_context *pContext) +{ + auto pAggCtxt = + static_cast(sqlite3_aggregate_context( + pContext, static_cast(sizeof(OGRSQLITE_STDDEV_Context)))); + if (pAggCtxt->nValues > 1) + { + sqlite3_result_double(pContext, + sqrt(pAggCtxt->dfM2 / (pAggCtxt->nValues - 1))); + } +} + /************************************************************************/ /* OGRSQLiteRegisterSQLFunctionsCommon() */ /************************************************************************/ @@ -365,6 +428,14 @@ static OGRSQLiteExtensionData *OGRSQLiteRegisterSQLFunctionsCommon(sqlite3 *hDB) OGRSQLITE_LIKE, nullptr, nullptr); } + sqlite3_create_function(hDB, "STDDEV_POP", 1, UTF8_INNOCUOUS, nullptr, + nullptr, OGRSQLITE_STDDEV_Step, + OGRSQLITE_STDDEV_POP_Finalize); + + sqlite3_create_function(hDB, "STDDEV_SAMP", 1, UTF8_INNOCUOUS, nullptr, + nullptr, OGRSQLITE_STDDEV_Step, + OGRSQLITE_STDDEV_SAMP_Finalize); + pData->SetRegExpCache(OGRSQLiteRegisterRegExpFunction(hDB)); return pData; From 8a11460049da10ea83b8c59e568e8e5bf36eaeae Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 23 Jun 2024 20:42:49 +0200 Subject: [PATCH 09/72] [Lint] OSM: more usage of std::vector, std::unique_ptr, and C++ cast operator --- ogr/ogrsf_frmts/osm/CMakeLists.txt | 3 +- ogr/ogrsf_frmts/osm/gpb.h | 18 +- ogr/ogrsf_frmts/osm/ogr_osm.h | 125 ++--- ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp | 631 +++++++++++------------ ogr/ogrsf_frmts/osm/ogrosmdriver.cpp | 3 +- ogr/ogrsf_frmts/osm/ogrosmlayer.cpp | 136 +++-- ogr/ogrsf_frmts/osm/osm_parser.cpp | 162 +++--- 7 files changed, 542 insertions(+), 536 deletions(-) diff --git a/ogr/ogrsf_frmts/osm/CMakeLists.txt b/ogr/ogrsf_frmts/osm/CMakeLists.txt index fe87d02d5444..7b78fbc2309c 100644 --- a/ogr/ogrsf_frmts/osm/CMakeLists.txt +++ b/ogr/ogrsf_frmts/osm/CMakeLists.txt @@ -2,7 +2,8 @@ add_gdal_driver( TARGET ogr_OSM SOURCES ogrosmdatasource.cpp ogrosmdriver.cpp ogrosmlayer.cpp osm_parser.cpp - BUILTIN) + BUILTIN + STRONG_CXX_WFLAGS) gdal_standard_includes(ogr_OSM) set(GDAL_DATA_FILES diff --git a/ogr/ogrsf_frmts/osm/gpb.h b/ogr/ogrsf_frmts/osm/gpb.h index 763d7e8fa0b3..e49097cdfef8 100644 --- a/ogr/ogrsf_frmts/osm/gpb.h +++ b/ogr/ogrsf_frmts/osm/gpb.h @@ -91,7 +91,7 @@ inline int ReadVarUInt32(const GByte **ppabyData) if (!(nByte & 0x80)) { *ppabyData = pabyData + 1; - return nVal | ((unsigned)nByte << nShift); + return nVal | (static_cast(nByte) << nShift); } nVal |= (nByte & 0x7f) << nShift; pabyData++; @@ -102,7 +102,7 @@ inline int ReadVarUInt32(const GByte **ppabyData) if (!(nByte & 0x80)) { *ppabyData = pabyData + 1; - return nVal | (((unsigned)nByte & 0xf) << nShift); + return nVal | ((static_cast(nByte) & 0xf) << nShift); } *ppabyData = pabyData; return nVal; @@ -120,7 +120,8 @@ inline int ReadVarUInt32(const GByte **ppabyData) #define READ_SIZE(pabyData, pabyDataLimit, nSize) \ { \ READ_VARUINT32(pabyData, pabyDataLimit, nSize); \ - if (CHECK_OOB && nSize > (unsigned int)(pabyDataLimit - pabyData)) \ + if (CHECK_OOB && \ + nSize > static_cast(pabyDataLimit - pabyData)) \ THROW_GPB_EXCEPTION; \ } @@ -140,9 +141,9 @@ inline GUIntBig ReadVarUInt64(const GByte **ppabyData) if (!(nByte & 0x80)) { *ppabyData = pabyData + 1; - return nVal | ((GUIntBig)nByte << nShift); + return nVal | (static_cast(nByte) << nShift); } - nVal |= ((GUIntBig)(nByte & 0x7f)) << nShift; + nVal |= (static_cast(nByte & 0x7f)) << nShift; pabyData++; nShift += 7; if (nShift == 63) @@ -151,7 +152,7 @@ inline GUIntBig ReadVarUInt64(const GByte **ppabyData) if (!(nByte & 0x80)) { *ppabyData = pabyData + 1; - return nVal | (((GUIntBig)nByte & 1) << nShift); + return nVal | ((static_cast(nByte) & 1) << nShift); } *ppabyData = pabyData; return nVal; @@ -169,7 +170,8 @@ inline GUIntBig ReadVarUInt64(const GByte **ppabyData) #define READ_SIZE64(pabyData, pabyDataLimit, nSize) \ { \ READ_VARUINT64(pabyData, pabyDataLimit, nSize); \ - if (CHECK_OOB && nSize > (unsigned int)(pabyDataLimit - pabyData)) \ + if (CHECK_OOB && \ + nSize > static_cast(pabyDataLimit - pabyData)) \ THROW_GPB_EXCEPTION; \ } @@ -314,7 +316,7 @@ inline void SkipVarInt(const GByte **ppabyData) do \ { \ READ_SIZE(pabyData, pabyDataLimit, l_nDataLength); \ - pszTxt = (char *)VSI_MALLOC_VERBOSE(l_nDataLength + 1); \ + pszTxt = static_cast(VSI_MALLOC_VERBOSE(l_nDataLength + 1)); \ if (pszTxt == nullptr) \ THROW_GPB_EXCEPTION; \ memcpy(pszTxt, pabyData, l_nDataLength); \ diff --git a/ogr/ogrsf_frmts/osm/ogr_osm.h b/ogr/ogrsf_frmts/osm/ogr_osm.h index 4ffe52d4b343..c4e29cafda93 100644 --- a/ogr/ogrsf_frmts/osm/ogr_osm.h +++ b/ogr/ogrsf_frmts/osm/ogr_osm.h @@ -46,7 +46,7 @@ #include "ogrsqlitevfs.h" -class ConstCharComp +class OGROSMConstCharComp { public: bool operator()(const char *a, const char *b) const @@ -58,25 +58,28 @@ class ConstCharComp class OGROSMComputedAttribute { public: - CPLString osName; - int nIndex; - OGRFieldType eType; - CPLString osSQL; - sqlite3_stmt *hStmt; - std::vector aosAttrToBind; - std::vector anIndexToBind; - bool bHardcodedZOrder; - - OGROSMComputedAttribute() - : nIndex(-1), eType(OFTString), hStmt(nullptr), bHardcodedZOrder(false) - { - } + CPLString osName{}; + int nIndex = -1; + OGRFieldType eType = OFTString; + CPLString osSQL{}; + sqlite3_stmt *hStmt = nullptr; + std::vector aosAttrToBind{}; + std::vector anIndexToBind{}; + bool bHardcodedZOrder = false; + + OGROSMComputedAttribute() = default; - explicit OGROSMComputedAttribute(const char *pszName) - : osName(pszName), nIndex(-1), eType(OFTString), hStmt(nullptr), - bHardcodedZOrder(false) + explicit OGROSMComputedAttribute(const char *pszName) : osName(pszName) { } + + OGROSMComputedAttribute(OGROSMComputedAttribute &&) = default; + OGROSMComputedAttribute &operator=(OGROSMComputedAttribute &&) = default; + + private: + OGROSMComputedAttribute(const OGROSMComputedAttribute &) = delete; + OGROSMComputedAttribute & + operator=(const OGROSMComputedAttribute &) = delete; }; /************************************************************************/ @@ -93,21 +96,18 @@ class OGROSMLayer final : public OGRLayer int m_nIdxLayer = 0; OGRFeatureDefn *m_poFeatureDefn = nullptr; OGRSpatialReference *m_poSRS = nullptr; - long m_nFeatureCount = 0; std::vector m_apszNames{}; /* Needed to keep a "reference" to the string inserted into oMapFieldNameToIndex */ - std::map m_oMapFieldNameToIndex{}; + std::map m_oMapFieldNameToIndex{}; std::vector m_oComputedAttributes{}; bool m_bResetReadingAllowed = false; - int m_nFeatureArraySize = 0; - int m_nFeatureArrayMaxSize = 0; - int m_nFeatureArrayIndex = 0; - OGRFeature **m_papoFeatures = nullptr; + size_t m_nFeatureArrayIndex = 0; + std::vector> m_apoFeatures{}; bool m_bHasOSMId = false; int m_nIndexOSMId = -1; @@ -128,20 +128,23 @@ class OGROSMLayer final : public OGRLayer bool m_bUserInterested = true; - bool AddToArray(OGRFeature *poFeature, int bCheckFeatureThreshold); + bool AddToArray(std::unique_ptr, bool bCheckFeatureThreshold); int AddInOtherOrAllTags(const char *pszK); char szLaunderedFieldName[256]; const char *GetLaunderedFieldName(const char *pszName); - std::vector apszInsignificantKeys; - std::map aoSetInsignificantKeys; + std::vector apszInsignificantKeys{}; + std::map aoSetInsignificantKeys{}; + + std::vector apszIgnoreKeys{}; + std::map aoSetIgnoreKeys{}; - std::vector apszIgnoreKeys; - std::map aoSetIgnoreKeys; + std::set aoSetWarnKeys{}; - std::set aoSetWarnKeys; + OGROSMLayer(const OGROSMLayer &) = delete; + OGROSMLayer &operator=(const OGROSMLayer &) = delete; public: OGROSMLayer(OGROSMDataSource *m_poDS, int m_nIdxLayer, const char *pszName); @@ -175,9 +178,10 @@ class OGROSMLayer final : public OGRLayer const OGREnvelope *GetSpatialFilterEnvelope(); - int AddFeature(OGRFeature *poFeature, int bAttrFilterAlreadyEvaluated, - int *pbFilteredOut = nullptr, - int bCheckFeatureThreshold = TRUE); + bool AddFeature(std::unique_ptr poFeature, + bool bAttrFilterAlreadyEvaluated, + bool *pbFilteredOut = nullptr, + bool bCheckFeatureThreshold = true); void ForceResetReading(); void AddField(const char *pszName, OGRFieldType eFieldType, @@ -304,15 +308,15 @@ class OGROSMLayer final : public OGRLayer /* OGROSMDataSource */ /************************************************************************/ -typedef struct +struct KeyDesc { - char *pszK; - int nKeyIndex; - int nOccurrences; - std::vector asValues; - std::map - anMapV; /* map that is the reverse of asValues */ -} KeyDesc; + char *pszK = nullptr; + int nKeyIndex = 0; + int nOccurrences = 0; + std::vector apszValues{}; + //! map that is the reverse of apszValues + std::map anMapV{}; +}; typedef struct { @@ -331,7 +335,7 @@ typedef struct union { - int nValueIndex; /* index of KeyDesc.asValues */ + int nValueIndex; /* index of KeyDesc.apszValues */ int nOffsetInpabyNonRedundantValues; /* offset in OGROSMDataSource.pabyNonRedundantValues */ @@ -359,20 +363,20 @@ typedef struct int nLat; } LonLat; -typedef struct +struct WayFeaturePair { - GIntBig nWayID; - GIntBig - *panNodeRefs; /* point to a sub-array of OGROSMDataSource.anReqIds */ - unsigned int nRefs; - unsigned int nTags; - IndexedKVP *pasTags; /* point to a sub-array of + GIntBig nWayID = 0; + /* point to a sub-array of OGROSMDataSource.anReqIds */ + GIntBig *panNodeRefs = nullptr; + unsigned int nRefs = 0; + unsigned int nTags = 0; + IndexedKVP *pasTags = nullptr; /* point to a sub-array of OGROSMDataSource.pasAccumulatedTags */ - OSMInfo sInfo; - OGRFeature *poFeature; - bool bIsArea : 1; - bool bAttrFilterAlreadyEvaluated : 1; -} WayFeaturePair; + OSMInfo sInfo{}; + std::unique_ptr poFeature{}; + bool bIsArea = false; + bool bAttrFilterAlreadyEvaluated = false; +}; #ifdef ENABLE_NODE_LOOKUP_BY_HASHING typedef struct @@ -387,8 +391,7 @@ class OGROSMDataSource final : public OGRDataSource { friend class OGROSMLayer; - int m_nLayers = 0; - OGROSMLayer **m_papoLayers = nullptr; + std::vector> m_apoLayers{}; char *m_pszName = nullptr; OGREnvelope m_sExtent{}; @@ -492,11 +495,10 @@ class OGROSMDataSource final : public OGRDataSource unsigned int MAX_INDEXED_VALUES_PER_KEY = 0; GByte *pabyNonRedundantValues = nullptr; int nNonRedundantValuesLen = 0; - WayFeaturePair *m_pasWayFeaturePairs = nullptr; - int m_nWayFeaturePairs = 0; + std::vector m_asWayFeaturePairs{}; - std::vector m_asKeys{}; - std::map + std::vector m_apsKeys{}; + std::map m_aoMapIndexedKeys{}; /* map that is the reverse of asKeys */ CPLString m_osNodesFilename{}; @@ -574,6 +576,9 @@ class OGROSMDataSource final : public OGRDataSource int iCurLayer, const std::vector &oAttributes); bool IsClosedWayTaggedAsPolygon(unsigned int nTags, const OSMTag *pasTags); + OGROSMDataSource(const OGROSMDataSource &) = delete; + OGROSMDataSource &operator=(const OGROSMDataSource &) = delete; + public: OGROSMDataSource(); virtual ~OGROSMDataSource(); @@ -585,7 +590,7 @@ class OGROSMDataSource final : public OGRDataSource virtual int GetLayerCount() override { - return m_nLayers; + return static_cast(m_apoLayers.size()); } virtual OGRLayer *GetLayer(int) override; diff --git a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp index 53226246e321..9e7a6aaef4b6 100644 --- a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp +++ b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp @@ -69,7 +69,7 @@ #include "sqlite3.h" #undef SQLITE_STATIC -#define SQLITE_STATIC ((sqlite3_destructor_type) nullptr) +#define SQLITE_STATIC (static_cast(nullptr)) constexpr int LIMIT_IDS_PER_REQUEST = 200; @@ -159,7 +159,7 @@ constexpr int HASHED_INDEXES_ARRAY_SIZE = 3145739; constexpr int COLLISION_BUCKET_ARRAY_SIZE = (MAX_ACCUMULATED_NODES / 100) * 40; // hash function = identity -#define HASH_ID_FUNC(x) ((GUIntBig)(x)) +#define HASH_ID_FUNC(x) (static_cast(x)) #endif // ENABLE_NODE_LOOKUP_BY_HASHING // #define FAKE_LOOKUP_NODES @@ -174,9 +174,9 @@ static void WriteVarSInt64(GIntBig nSVal, GByte **ppabyData); class DSToBeOpened { public: - GIntBig nPID; - CPLString osDSName; - CPLString osInterestLayers; + GIntBig nPID{}; + CPLString osDSName{}; + CPLString osInterestLayers{}; }; static CPLMutex *hMutex = nullptr; @@ -205,13 +205,14 @@ static CPLString GetInterestLayersForDSName(const CPLString &osDSName) { CPLMutexHolder oMutexHolder(&hMutex); GIntBig nPID = CPLGetPID(); - for (int i = 0; i < (int)oListDSToBeOpened.size(); i++) + for (auto oIter = oListDSToBeOpened.begin(); + oIter < oListDSToBeOpened.end(); ++oIter) { - if (oListDSToBeOpened[i].nPID == nPID && - oListDSToBeOpened[i].osDSName == osDSName) + const auto &ds = *oIter; + if (ds.nPID == nPID && ds.osDSName == osDSName) { - CPLString osInterestLayers = oListDSToBeOpened[i].osInterestLayers; - oListDSToBeOpened.erase(oListDSToBeOpened.begin() + i); + CPLString osInterestLayers = ds.osInterestLayers; + oListDSToBeOpened.erase(oIter); return osInterestLayers; } } @@ -224,7 +225,7 @@ static CPLString GetInterestLayersForDSName(const CPLString &osDSName) OGROSMDataSource::OGROSMDataSource() { - m_asKeys.push_back(nullptr); // guard to avoid index 0 to be used + m_apsKeys.push_back(nullptr); // guard to avoid index 0 to be used MAX_INDEXED_KEYS = static_cast( atoi(CPLGetConfigOption("OSM_MAX_INDEXED_KEYS", "32768"))); @@ -239,9 +240,7 @@ OGROSMDataSource::OGROSMDataSource() OGROSMDataSource::~OGROSMDataSource() { - for (int i = 0; i < m_nLayers; i++) - delete m_papoLayers[i]; - CPLFree(m_papoLayers); + m_apoLayers.clear(); CPLFree(m_pszName); @@ -278,11 +277,6 @@ OGROSMDataSource::~OGROSMDataSource() CPLFree(m_pasLonLatArray); CPLFree(m_panUnsortedReqIds); - for (int i = 0; i < m_nWayFeaturePairs; i++) - { - delete m_pasWayFeaturePairs[i].poFeature; - } - CPLFree(m_pasWayFeaturePairs); CPLFree(m_pasAccumulatedTags); CPLFree(pabyNonRedundantKeys); CPLFree(pabyNonRedundantValues); @@ -301,14 +295,14 @@ OGROSMDataSource::~OGROSMDataSource() fclose(f); #endif - for (int i = 1; i < static_cast(m_asKeys.size()); i++) + for (int i = 1; i < static_cast(m_apsKeys.size()); i++) { - KeyDesc *psKD = m_asKeys[i]; + KeyDesc *psKD = m_apsKeys[i]; if (psKD) { CPLFree(psKD->pszK); - for (int j = 0; j < static_cast(psKD->asValues.size()); j++) - CPLFree(psKD->asValues[j]); + for (int j = 0; j < static_cast(psKD->apszValues.size()); j++) + CPLFree(psKD->apszValues[j]); delete psKD; } } @@ -545,7 +539,7 @@ bool OGROSMDataSource::FlushCurrentSectorCompressedCase() { GByte abyOutBuffer[2 * SECTOR_SIZE]; GByte *pabyOut = abyOutBuffer; - LonLat *pasLonLatIn = (LonLat *)m_pabySector; + LonLat *pasLonLatIn = reinterpret_cast(m_pabySector); int nLastLon = 0; int nLastLat = 0; bool bLastValid = false; @@ -729,7 +723,7 @@ bool OGROSMDataSource::IndexPointCustom(OSMNode *psNode) void OGROSMDataSource::NotifyNodes(unsigned int nNodes, OSMNode *pasNodes) { const OGREnvelope *psEnvelope = - m_papoLayers[IDX_LYR_POINTS]->GetSpatialFilterEnvelope(); + m_apoLayers[IDX_LYR_POINTS]->GetSpatialFilterEnvelope(); for (unsigned int i = 0; i < nNodes; i++) { @@ -744,7 +738,7 @@ void OGROSMDataSource::NotifyNodes(unsigned int nNodes, OSMNode *pasNodes) if (!IndexPoint(&pasNodes[i])) break; - if (!m_papoLayers[IDX_LYR_POINTS]->IsUserInterested()) + if (!m_apoLayers[IDX_LYR_POINTS]->IsUserInterested()) continue; bool bInterestingTag = m_bReportAllNodes; @@ -755,7 +749,7 @@ void OGROSMDataSource::NotifyNodes(unsigned int nNodes, OSMNode *pasNodes) for (unsigned int j = 0; j < pasNodes[i].nTags; j++) { const char *pszK = pasTags[j].pszK; - if (m_papoLayers[IDX_LYR_POINTS]->IsSignificantKey(pszK)) + if (m_apoLayers[IDX_LYR_POINTS]->IsSignificantKey(pszK)) { bInterestingTag = true; break; @@ -765,19 +759,20 @@ void OGROSMDataSource::NotifyNodes(unsigned int nNodes, OSMNode *pasNodes) if (bInterestingTag) { - OGRFeature *poFeature = - new OGRFeature(m_papoLayers[IDX_LYR_POINTS]->GetLayerDefn()); + auto poFeature = std::make_unique( + m_apoLayers[IDX_LYR_POINTS]->GetLayerDefn()); poFeature->SetGeometryDirectly( new OGRPoint(pasNodes[i].dfLon, pasNodes[i].dfLat)); - m_papoLayers[IDX_LYR_POINTS]->SetFieldsFromTags( - poFeature, pasNodes[i].nID, false, pasNodes[i].nTags, pasTags, - &pasNodes[i].sInfo); + m_apoLayers[IDX_LYR_POINTS]->SetFieldsFromTags( + poFeature.get(), pasNodes[i].nID, false, pasNodes[i].nTags, + pasTags, &pasNodes[i].sInfo); - int bFilteredOut = FALSE; - if (!m_papoLayers[IDX_LYR_POINTS]->AddFeature( - poFeature, FALSE, &bFilteredOut, !m_bFeatureAdded)) + bool bFilteredOut = false; + if (!m_apoLayers[IDX_LYR_POINTS]->AddFeature(std::move(poFeature), + false, &bFilteredOut, + !m_bFeatureAdded)) { m_bStopParsing = true; break; @@ -935,7 +930,8 @@ void OGROSMDataSource::LookupNodesSQLite() while (sqlite3_step(hStmt) == SQLITE_ROW) { const GIntBig id = sqlite3_column_int64(hStmt, 0); - LonLat *psLonLat = (LonLat *)sqlite3_column_blob(hStmt, 1); + const LonLat *psLonLat = + reinterpret_cast(sqlite3_column_blob(hStmt, 1)); m_panReqIds[j] = id; m_pasLonLatArray[j].nLon = psLonLat->nLon; @@ -956,7 +952,7 @@ static bool DecompressSector(const GByte *pabyIn, int nSectorSize, GByte *pabyOut) { const GByte *pabyPtr = pabyIn; - LonLat *pasLonLatOut = (LonLat *)pabyOut; + LonLat *pasLonLatOut = reinterpret_cast(pabyOut); int nLastLon = 0; int nLastLat = 0; bool bLastValid = false; @@ -969,9 +965,9 @@ static bool DecompressSector(const GByte *pabyIn, int nSectorSize, if (bLastValid) { pasLonLatOut[i].nLon = - (int)(nLastLon + ReadVarSInt64(&pabyPtr)); + static_cast(nLastLon + ReadVarSInt64(&pabyPtr)); pasLonLatOut[i].nLat = - (int)(nLastLat + ReadVarSInt64(&pabyPtr)); + static_cast(nLastLat + ReadVarSInt64(&pabyPtr)); } else { @@ -990,7 +986,7 @@ static bool DecompressSector(const GByte *pabyIn, int nSectorSize, } } - int nRead = (int)(pabyPtr - pabyIn); + int nRead = static_cast(pabyPtr - pabyIn); nRead = ROUND_COMPRESS_SIZE(nRead); return nRead == nSectorSize; } @@ -1308,11 +1304,11 @@ static void WriteVarInt(unsigned int nVal, std::vector &abyData) { if ((nVal & (~0x7fU)) == 0) { - abyData.push_back((GByte)nVal); + abyData.push_back(static_cast(nVal)); return; } - abyData.push_back(0x80 | (GByte)(nVal & 0x7f)); + abyData.push_back(0x80 | static_cast(nVal & 0x7f)); nVal >>= 7; } } @@ -1325,13 +1321,13 @@ static void WriteVarInt64(GUIntBig nVal, std::vector &abyData) { while (true) { - if ((((GUInt32)nVal) & (~0x7fU)) == 0) + if ((static_cast(nVal) & (~0x7fU)) == 0) { - abyData.push_back((GByte)nVal); + abyData.push_back(static_cast(nVal)); return; } - abyData.push_back(0x80 | (GByte)(nVal & 0x7f)); + abyData.push_back(0x80 | static_cast(nVal & 0x7f)); nVal >>= 7; } } @@ -1348,11 +1344,11 @@ static void WriteVarSInt64(GIntBig nSVal, std::vector &abyData) { if ((nVal & (~0x7f)) == 0) { - abyData.push_back((GByte)nVal); + abyData.push_back(static_cast(nVal)); return; } - abyData.push_back(0x80 | (GByte)(nVal & 0x7f)); + abyData.push_back(0x80 | static_cast(nVal & 0x7f)); nVal >>= 7; } } @@ -1370,12 +1366,12 @@ static void WriteVarSInt64(GIntBig nSVal, GByte **ppabyData) { if ((nVal & (~0x7f)) == 0) { - *pabyData = (GByte)nVal; + *pabyData = static_cast(nVal); *ppabyData = pabyData + 1; return; } - *pabyData = 0x80 | (GByte)(nVal & 0x7f); + *pabyData = 0x80 | static_cast(nVal & 0x7f); nVal >>= 7; pabyData++; } @@ -1403,8 +1399,9 @@ void OGROSMDataSource::CompressWay(bool bIsArea, unsigned int nTags, } else { - const char *pszK = (const char *)pabyNonRedundantKeys + - pasTags[iTag].uKey.nOffsetInpabyNonRedundantKeys; + const char *pszK = + reinterpret_cast(pabyNonRedundantKeys) + + pasTags[iTag].uKey.nOffsetInpabyNonRedundantKeys; abyCompressedWay.push_back(0); @@ -1420,7 +1417,7 @@ void OGROSMDataSource::CompressWay(bool bIsArea, unsigned int nTags, else { const char *pszV = - (const char *)pabyNonRedundantValues + + reinterpret_cast(pabyNonRedundantValues) + pasTags[iTag].uVal.nOffsetInpabyNonRedundantValues; if (pasTags[iTag].bKIsIndex) @@ -1455,8 +1452,8 @@ void OGROSMDataSource::CompressWay(bool bIsArea, unsigned int nTags, reinterpret_cast(&(pasLonLatPairs[0])) + sizeof(LonLat)); for (int i = 1; i < nPoints; i++) { - GIntBig nDiff64 = (GIntBig)pasLonLatPairs[i].nLon - - (GIntBig)pasLonLatPairs[i - 1].nLon; + GIntBig nDiff64 = static_cast(pasLonLatPairs[i].nLon) - + static_cast(pasLonLatPairs[i - 1].nLon); WriteVarSInt64(nDiff64, abyCompressedWay); nDiff64 = pasLonLatPairs[i].nLat - pasLonLatPairs[i - 1].nLat; @@ -1510,14 +1507,15 @@ void OGROSMDataSource::UncompressWay(int nBytes, const GByte *pabyCompressedWay, if (pasTags) { - CPLAssert(nK >= 0 && nK < (int)m_asKeys.size()); + CPLAssert(nK >= 0 && static_cast(nK) < m_apsKeys.size()); pasTags[iTag].pszK = - nK ? m_asKeys[nK]->pszK : reinterpret_cast(pszK); + nK ? m_apsKeys[nK]->pszK : reinterpret_cast(pszK); CPLAssert(nK == 0 || - (nV >= 0 && nV < (int)m_asKeys[nK]->asValues.size())); - pasTags[iTag].pszV = - nV ? m_asKeys[nK]->asValues[nV] : (const char *)pszV; + (nV >= 0 && static_cast(nV) < + m_apsKeys[nK]->apszValues.size())); + pasTags[iTag].pszV = nV ? m_apsKeys[nK]->apszValues[nV] + : reinterpret_cast(pszV); } } @@ -1550,8 +1548,8 @@ void OGROSMDataSource::UncompressWay(int nBytes, const GByte *pabyCompressedWay, pabyPtr += 2 * sizeof(int); do { - lonLat.nLon = (int)(lonLat.nLon + ReadVarSInt64(&pabyPtr)); - lonLat.nLat = (int)(lonLat.nLat + ReadVarSInt64(&pabyPtr)); + lonLat.nLon = static_cast(lonLat.nLon + ReadVarSInt64(&pabyPtr)); + lonLat.nLat = static_cast(lonLat.nLat + ReadVarSInt64(&pabyPtr)); asCoords.emplace_back(lonLat); } while (pabyPtr < pabyCompressedWay + nBytes); } @@ -1622,26 +1620,24 @@ int OGROSMDataSource::FindNode(GIntBig nID) void OGROSMDataSource::ProcessWaysBatch() { - if (m_nWayFeaturePairs == 0) + if (m_asWayFeaturePairs.empty()) return; - // printf("nodes = %d, features = %d\n", nUnsortedReqIds, nWayFeaturePairs); + // printf("nodes = %d, features = %d\n", nUnsortedReqIds, int(m_asWayFeaturePairs.size())); LookupNodes(); - for (int iPair = 0; iPair < m_nWayFeaturePairs; iPair++) + for (WayFeaturePair &sWayFeaturePairs : m_asWayFeaturePairs) { - WayFeaturePair *psWayFeaturePairs = &m_pasWayFeaturePairs[iPair]; - - const bool bIsArea = psWayFeaturePairs->bIsArea; + const bool bIsArea = sWayFeaturePairs.bIsArea; m_asLonLatCache.clear(); #ifdef ENABLE_NODE_LOOKUP_BY_HASHING if (m_bHashedIndexValid) { - for (unsigned int i = 0; i < psWayFeaturePairs->nRefs; i++) + for (unsigned int i = 0; i < sWayFeaturePairs.nRefs; i++) { int nIndInHashArray = static_cast( - HASH_ID_FUNC(psWayFeaturePairs->panNodeRefs[i]) % + HASH_ID_FUNC(sWayFeaturePairs.panNodeRefs[i]) % HASHED_INDEXES_ARRAY_SIZE); int nIdx = m_panHashedIndexes[nIndInHashArray]; if (nIdx < -1) @@ -1651,7 +1647,7 @@ void OGROSMDataSource::ProcessWaysBatch() { nIdx = m_psCollisionBuckets[iBucket].nInd; if (m_panReqIds[nIdx] == - psWayFeaturePairs->panNodeRefs[i]) + sWayFeaturePairs.panNodeRefs[i]) break; iBucket = m_psCollisionBuckets[iBucket].nNext; if (iBucket < 0) @@ -1662,7 +1658,7 @@ void OGROSMDataSource::ProcessWaysBatch() } } else if (nIdx >= 0 && - m_panReqIds[nIdx] != psWayFeaturePairs->panNodeRefs[i]) + m_panReqIds[nIdx] != sWayFeaturePairs.panNodeRefs[i]) nIdx = -1; if (nIdx >= 0) @@ -1675,20 +1671,20 @@ void OGROSMDataSource::ProcessWaysBatch() #endif // ENABLE_NODE_LOOKUP_BY_HASHING { int nIdx = -1; - for (unsigned int i = 0; i < psWayFeaturePairs->nRefs; i++) + for (unsigned int i = 0; i < sWayFeaturePairs.nRefs; i++) { - if (nIdx >= 0 && psWayFeaturePairs->panNodeRefs[i] == - psWayFeaturePairs->panNodeRefs[i - 1] + 1) + if (nIdx >= 0 && sWayFeaturePairs.panNodeRefs[i] == + sWayFeaturePairs.panNodeRefs[i - 1] + 1) { - if (nIdx + 1 < (int)m_nReqIds && + if (static_cast(nIdx + 1) < m_nReqIds && m_panReqIds[nIdx + 1] == - psWayFeaturePairs->panNodeRefs[i]) + sWayFeaturePairs.panNodeRefs[i]) nIdx++; else nIdx = -1; } else - nIdx = FindNode(psWayFeaturePairs->panNodeRefs[i]); + nIdx = FindNode(sWayFeaturePairs.panNodeRefs[i]); if (nIdx >= 0) { m_asLonLatCache.push_back(m_pasLonLatArray[nIdx]); @@ -1706,28 +1702,27 @@ void OGROSMDataSource::ProcessWaysBatch() CPLDebug("OSM", "Way " CPL_FRMT_GIB " with %d nodes that could be found. Discarding it", - psWayFeaturePairs->nWayID, + sWayFeaturePairs.nWayID, static_cast(m_asLonLatCache.size())); - delete psWayFeaturePairs->poFeature; - psWayFeaturePairs->poFeature = nullptr; - psWayFeaturePairs->bIsArea = false; + sWayFeaturePairs.poFeature.reset(); + sWayFeaturePairs.bIsArea = false; continue; } - if (bIsArea && m_papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested()) + if (bIsArea && m_apoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested()) { - IndexWay(psWayFeaturePairs->nWayID, /*bIsArea = */ true, - psWayFeaturePairs->nTags, psWayFeaturePairs->pasTags, + IndexWay(sWayFeaturePairs.nWayID, /*bIsArea = */ true, + sWayFeaturePairs.nTags, sWayFeaturePairs.pasTags, m_asLonLatCache.data(), static_cast(m_asLonLatCache.size()), - &psWayFeaturePairs->sInfo); + &sWayFeaturePairs.sInfo); } else - IndexWay(psWayFeaturePairs->nWayID, bIsArea, 0, nullptr, + IndexWay(sWayFeaturePairs.nWayID, bIsArea, 0, nullptr, m_asLonLatCache.data(), static_cast(m_asLonLatCache.size()), nullptr); - if (psWayFeaturePairs->poFeature == nullptr) + if (sWayFeaturePairs.poFeature == nullptr) { continue; } @@ -1743,35 +1738,33 @@ void OGROSMDataSource::ProcessWaysBatch() INT_TO_DBL(m_asLonLatCache[i].nLat)); } - psWayFeaturePairs->poFeature->SetGeometryDirectly(poGeom); + sWayFeaturePairs.poFeature->SetGeometryDirectly(poGeom); - if (m_asLonLatCache.size() != psWayFeaturePairs->nRefs) - CPLDebug( - "OSM", - "For way " CPL_FRMT_GIB ", got only %d nodes instead of %d", - psWayFeaturePairs->nWayID, nPoints, psWayFeaturePairs->nRefs); - - int bFilteredOut = FALSE; - if (!m_papoLayers[IDX_LYR_LINES]->AddFeature( - psWayFeaturePairs->poFeature, - psWayFeaturePairs->bAttrFilterAlreadyEvaluated, &bFilteredOut, + if (m_asLonLatCache.size() != sWayFeaturePairs.nRefs) + CPLDebug("OSM", + "For way " CPL_FRMT_GIB + ", got only %d nodes instead of %d", + sWayFeaturePairs.nWayID, nPoints, sWayFeaturePairs.nRefs); + + bool bFilteredOut = false; + if (!m_apoLayers[IDX_LYR_LINES]->AddFeature( + std::move(sWayFeaturePairs.poFeature), + sWayFeaturePairs.bAttrFilterAlreadyEvaluated, &bFilteredOut, !m_bFeatureAdded)) m_bStopParsing = true; else if (!bFilteredOut) m_bFeatureAdded = true; } - if (m_papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested()) + if (m_apoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested()) { - for (int iPair = 0; iPair < m_nWayFeaturePairs; iPair++) + for (WayFeaturePair &sWayFeaturePairs : m_asWayFeaturePairs) { - WayFeaturePair *psWayFeaturePairs = &m_pasWayFeaturePairs[iPair]; - - if (psWayFeaturePairs->bIsArea && - (psWayFeaturePairs->nTags || m_bReportAllWays)) + if (sWayFeaturePairs.bIsArea && + (sWayFeaturePairs.nTags || m_bReportAllWays)) { sqlite3_bind_int64(m_hInsertPolygonsStandaloneStmt, 1, - psWayFeaturePairs->nWayID); + sWayFeaturePairs.nWayID); int rc = sqlite3_step(m_hInsertPolygonsStandaloneStmt); sqlite3_reset(m_hInsertPolygonsStandaloneStmt); @@ -1780,13 +1773,13 @@ void OGROSMDataSource::ProcessWaysBatch() CPLError(CE_Failure, CPLE_AppDefined, "Failed inserting into " "polygons_standalone " CPL_FRMT_GIB ": %s", - psWayFeaturePairs->nWayID, sqlite3_errmsg(m_hDB)); + sWayFeaturePairs.nWayID, sqlite3_errmsg(m_hDB)); } } } } - m_nWayFeaturePairs = 0; + m_asWayFeaturePairs.clear(); m_nUnsortedReqIds = 0; m_nAccumulatedTags = 0; @@ -1903,7 +1896,7 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) for (unsigned int i = 0; i < psWay->nTags; i++) { const char *pszK = psWay->pasTags[i].pszK; - if (m_papoLayers[IDX_LYR_LINES]->IsSignificantKey(pszK)) + if (m_apoLayers[IDX_LYR_LINES]->IsSignificantKey(pszK)) { bInterestingTag = true; break; @@ -1911,29 +1904,29 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) } } - OGRFeature *poFeature = nullptr; + std::unique_ptr poFeature; bool bAttrFilterAlreadyEvaluated = false; - if (!bIsArea && m_papoLayers[IDX_LYR_LINES]->IsUserInterested() && + if (!bIsArea && m_apoLayers[IDX_LYR_LINES]->IsUserInterested() && bInterestingTag) { - poFeature = new OGRFeature(m_papoLayers[IDX_LYR_LINES]->GetLayerDefn()); + poFeature = std::make_unique( + m_apoLayers[IDX_LYR_LINES]->GetLayerDefn()); - m_papoLayers[IDX_LYR_LINES]->SetFieldsFromTags( - poFeature, psWay->nID, false, psWay->nTags, psWay->pasTags, + m_apoLayers[IDX_LYR_LINES]->SetFieldsFromTags( + poFeature.get(), psWay->nID, false, psWay->nTags, psWay->pasTags, &psWay->sInfo); // Optimization: if we have an attribute filter, that does not require // geometry, and if we don't need to index ways, then we can just // evaluate the attribute filter without the geometry. - if (m_papoLayers[IDX_LYR_LINES]->HasAttributeFilter() && - !m_papoLayers[IDX_LYR_LINES] + if (m_apoLayers[IDX_LYR_LINES]->HasAttributeFilter() && + !m_apoLayers[IDX_LYR_LINES] ->AttributeFilterEvaluationNeedsGeometry() && !m_bIndexWays) { - if (!m_papoLayers[IDX_LYR_LINES]->EvaluateAttributeFilter( - poFeature)) + if (!m_apoLayers[IDX_LYR_LINES]->EvaluateAttributeFilter( + poFeature.get())) { - delete poFeature; return; } bAttrFilterAlreadyEvaluated = true; @@ -1946,7 +1939,8 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) if (m_nUnsortedReqIds + psWay->nRefs > static_cast(MAX_ACCUMULATED_NODES) || - m_nWayFeaturePairs == MAX_DELAYED_FEATURES || + m_asWayFeaturePairs.size() == + static_cast(MAX_DELAYED_FEATURES) || m_nAccumulatedTags + psWay->nTags > static_cast(MAX_ACCUMULATED_TAGS) || nNonRedundantKeysLen + 1024 > MAX_NON_REDUNDANT_KEYS || @@ -1955,25 +1949,24 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) ProcessWaysBatch(); } - WayFeaturePair *psWayFeaturePairs = - &m_pasWayFeaturePairs[m_nWayFeaturePairs]; + m_asWayFeaturePairs.push_back(WayFeaturePair()); + WayFeaturePair &sWayFeaturePairs = m_asWayFeaturePairs.back(); - psWayFeaturePairs->nWayID = psWay->nID; - psWayFeaturePairs->nRefs = psWay->nRefs - (bIsArea ? 1 : 0); - psWayFeaturePairs->panNodeRefs = m_panUnsortedReqIds + m_nUnsortedReqIds; - psWayFeaturePairs->poFeature = poFeature; - psWayFeaturePairs->bIsArea = bIsArea; - psWayFeaturePairs->bAttrFilterAlreadyEvaluated = - bAttrFilterAlreadyEvaluated; + sWayFeaturePairs.nWayID = psWay->nID; + sWayFeaturePairs.nRefs = psWay->nRefs - (bIsArea ? 1 : 0); + sWayFeaturePairs.panNodeRefs = m_panUnsortedReqIds + m_nUnsortedReqIds; + sWayFeaturePairs.poFeature = std::move(poFeature); + sWayFeaturePairs.bIsArea = bIsArea; + sWayFeaturePairs.bAttrFilterAlreadyEvaluated = bAttrFilterAlreadyEvaluated; - if (bIsArea && m_papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested()) + if (bIsArea && m_apoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested()) { unsigned int nTagCount = 0; if (m_bNeedsToSaveWayInfo) { if (!psWay->sInfo.bTimeStampIsStr) - psWayFeaturePairs->sInfo.ts.nTimeStamp = + sWayFeaturePairs.sInfo.ts.nTimeStamp = psWay->sInfo.ts.nTimeStamp; else { @@ -1987,30 +1980,31 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) brokendown.tm_mday = sField.Date.Day; brokendown.tm_hour = sField.Date.Hour; brokendown.tm_min = sField.Date.Minute; - brokendown.tm_sec = (int)(sField.Date.Second + .5); - psWayFeaturePairs->sInfo.ts.nTimeStamp = + brokendown.tm_sec = + static_cast(sField.Date.Second + .5); + sWayFeaturePairs.sInfo.ts.nTimeStamp = CPLYMDHMSToUnixTime(&brokendown); } else - psWayFeaturePairs->sInfo.ts.nTimeStamp = 0; + sWayFeaturePairs.sInfo.ts.nTimeStamp = 0; } - psWayFeaturePairs->sInfo.nChangeset = psWay->sInfo.nChangeset; - psWayFeaturePairs->sInfo.nVersion = psWay->sInfo.nVersion; - psWayFeaturePairs->sInfo.nUID = psWay->sInfo.nUID; - psWayFeaturePairs->sInfo.bTimeStampIsStr = false; - psWayFeaturePairs->sInfo.pszUserSID = ""; // FIXME + sWayFeaturePairs.sInfo.nChangeset = psWay->sInfo.nChangeset; + sWayFeaturePairs.sInfo.nVersion = psWay->sInfo.nVersion; + sWayFeaturePairs.sInfo.nUID = psWay->sInfo.nUID; + sWayFeaturePairs.sInfo.bTimeStampIsStr = false; + sWayFeaturePairs.sInfo.pszUserSID = ""; // FIXME } else { - psWayFeaturePairs->sInfo.ts.nTimeStamp = 0; - psWayFeaturePairs->sInfo.nChangeset = 0; - psWayFeaturePairs->sInfo.nVersion = 0; - psWayFeaturePairs->sInfo.nUID = 0; - psWayFeaturePairs->sInfo.bTimeStampIsStr = false; - psWayFeaturePairs->sInfo.pszUserSID = ""; + sWayFeaturePairs.sInfo.ts.nTimeStamp = 0; + sWayFeaturePairs.sInfo.nChangeset = 0; + sWayFeaturePairs.sInfo.nVersion = 0; + sWayFeaturePairs.sInfo.nUID = 0; + sWayFeaturePairs.sInfo.bTimeStampIsStr = false; + sWayFeaturePairs.sInfo.pszUserSID = ""; } - psWayFeaturePairs->pasTags = m_pasAccumulatedTags + m_nAccumulatedTags; + sWayFeaturePairs.pasTags = m_pasAccumulatedTags + m_nAccumulatedTags; for (unsigned int iTag = 0; iTag < psWay->nTags; iTag++) { @@ -2028,14 +2022,14 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) KeyDesc *psKD = nullptr; if (oIterK == m_aoMapIndexedKeys.end()) { - if (m_asKeys.size() >= 1 + MAX_INDEXED_KEYS) + if (m_apsKeys.size() >= 1 + MAX_INDEXED_KEYS) { - if (m_asKeys.size() == 1 + MAX_INDEXED_KEYS) + if (m_apsKeys.size() == 1 + MAX_INDEXED_KEYS) { CPLDebug("OSM", "More than %d different keys found", MAX_INDEXED_KEYS); // To avoid next warnings. - m_asKeys.push_back(nullptr); + m_apsKeys.push_back(nullptr); } const int nLenK = static_cast(strlen(pszK)) + 1; @@ -2057,12 +2051,12 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) { psKD = new KeyDesc(); psKD->pszK = CPLStrdup(pszK); - psKD->nKeyIndex = static_cast(m_asKeys.size()); + psKD->nKeyIndex = static_cast(m_apsKeys.size()); psKD->nOccurrences = 0; - psKD->asValues.push_back(CPLStrdup( + psKD->apszValues.push_back(CPLStrdup( "")); // guard value to avoid index 0 to be used m_aoMapIndexedKeys[psKD->pszK] = psKD; - m_asKeys.push_back(psKD); + m_apsKeys.push_back(psKD); } } else @@ -2079,16 +2073,16 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) } if (psKD != nullptr && - psKD->asValues.size() < 1 + MAX_INDEXED_VALUES_PER_KEY) + psKD->apszValues.size() < 1 + MAX_INDEXED_VALUES_PER_KEY) { int nValueIndex = 0; auto oIterV = psKD->anMapV.find(pszV); if (oIterV == psKD->anMapV.end()) { char *pszVDup = CPLStrdup(pszV); - nValueIndex = static_cast(psKD->asValues.size()); + nValueIndex = static_cast(psKD->apszValues.size()); psKD->anMapV[pszVDup] = nValueIndex; - psKD->asValues.push_back(pszVDup); + psKD->apszValues.push_back(pszVDup); } else nValueIndex = oIterV->second; @@ -2102,12 +2096,12 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) const int nLenV = static_cast(strlen(pszV)) + 1; if (psKD != nullptr && - psKD->asValues.size() == 1 + MAX_INDEXED_VALUES_PER_KEY) + psKD->apszValues.size() == 1 + MAX_INDEXED_VALUES_PER_KEY) { CPLDebug("OSM", "More than %d different values for tag %s", MAX_INDEXED_VALUES_PER_KEY, pszK); // To avoid next warnings. - psKD->asValues.push_back(CPLStrdup("")); + psKD->apszValues.push_back(CPLStrdup("")); } if (nNonRedundantValuesLen + nLenV > MAX_NON_REDUNDANT_VALUES) @@ -2131,23 +2125,21 @@ void OGROSMDataSource::NotifyWay(OSMWay *psWay) break; } - psWayFeaturePairs->nTags = nTagCount; + sWayFeaturePairs.nTags = nTagCount; } else { - psWayFeaturePairs->sInfo.ts.nTimeStamp = 0; - psWayFeaturePairs->sInfo.nChangeset = 0; - psWayFeaturePairs->sInfo.nVersion = 0; - psWayFeaturePairs->sInfo.nUID = 0; - psWayFeaturePairs->sInfo.bTimeStampIsStr = false; - psWayFeaturePairs->sInfo.pszUserSID = ""; + sWayFeaturePairs.sInfo.ts.nTimeStamp = 0; + sWayFeaturePairs.sInfo.nChangeset = 0; + sWayFeaturePairs.sInfo.nVersion = 0; + sWayFeaturePairs.sInfo.nUID = 0; + sWayFeaturePairs.sInfo.bTimeStampIsStr = false; + sWayFeaturePairs.sInfo.pszUserSID = ""; - psWayFeaturePairs->nTags = 0; - psWayFeaturePairs->pasTags = nullptr; + sWayFeaturePairs.nTags = 0; + sWayFeaturePairs.pasTags = nullptr; } - m_nWayFeaturePairs++; - memcpy(m_panUnsortedReqIds + m_nUnsortedReqIds, psWay->panNodeRefs, sizeof(GIntBig) * (psWay->nRefs - (bIsArea ? 1 : 0))); m_nUnsortedReqIds += (psWay->nRefs - (bIsArea ? 1 : 0)); @@ -2342,14 +2334,15 @@ OGRGeometry *OGROSMDataSource::BuildMultiPolygon(OSMRelation *psRelation, if (poMLS->getNumGeometries() > 0) { - OGRGeometryH hPoly = OGRBuildPolygonFromEdges((OGRGeometryH)poMLS, TRUE, - FALSE, 0, nullptr); + OGRGeometryH hPoly = OGRBuildPolygonFromEdges( + OGRGeometry::ToHandle(poMLS), TRUE, FALSE, 0, nullptr); if (hPoly != nullptr && OGR_G_GetGeometryType(hPoly) == wkbPolygon) { OGRPolygon *poSuperPoly = OGRGeometry::FromHandle(hPoly)->toPolygon(); - for (unsigned int i = 0; - i < 1 + (unsigned int)poSuperPoly->getNumInteriorRings(); i++) + const unsigned nRings = 1 + static_cast( + poSuperPoly->getNumInteriorRings()); + for (unsigned int i = 0; i < nRings; i++) { OGRLinearRing *poRing = (i == 0) ? poSuperPoly->getExteriorRing() @@ -2494,7 +2487,7 @@ OGRGeometry *OGROSMDataSource::BuildGeometryCollection(OSMRelation *psRelation, void OGROSMDataSource::NotifyRelation(OSMRelation *psRelation) { - if (m_nWayFeaturePairs != 0) + if (!m_asWayFeaturePairs.empty()) ProcessWaysBatch(); m_nRelationsProcessed++; @@ -2542,26 +2535,26 @@ void OGROSMDataSource::NotifyRelation(OSMRelation *psRelation) const int iCurLayer = bMultiPolygon ? IDX_LYR_MULTIPOLYGONS : bMultiLineString ? IDX_LYR_MULTILINESTRINGS : IDX_LYR_OTHER_RELATIONS; - if (!m_papoLayers[iCurLayer]->IsUserInterested()) + if (!m_apoLayers[iCurLayer]->IsUserInterested()) return; - OGRFeature *poFeature = nullptr; + std::unique_ptr poFeature; if (!(bMultiPolygon && !bInterestingTagFound) && // We cannot do early filtering for multipolygon that has no // interesting tag, since we may fetch attributes from ways. - m_papoLayers[iCurLayer]->HasAttributeFilter() && - !m_papoLayers[iCurLayer]->AttributeFilterEvaluationNeedsGeometry()) + m_apoLayers[iCurLayer]->HasAttributeFilter() && + !m_apoLayers[iCurLayer]->AttributeFilterEvaluationNeedsGeometry()) { - poFeature = new OGRFeature(m_papoLayers[iCurLayer]->GetLayerDefn()); + poFeature = std::make_unique( + m_apoLayers[iCurLayer]->GetLayerDefn()); - m_papoLayers[iCurLayer]->SetFieldsFromTags( - poFeature, psRelation->nID, false, psRelation->nTags, + m_apoLayers[iCurLayer]->SetFieldsFromTags( + poFeature.get(), psRelation->nID, false, psRelation->nTags, psRelation->pasTags, &psRelation->sInfo); - if (!m_papoLayers[iCurLayer]->EvaluateAttributeFilter(poFeature)) + if (!m_apoLayers[iCurLayer]->EvaluateAttributeFilter(poFeature.get())) { - delete poFeature; return; } } @@ -2592,10 +2585,11 @@ void OGROSMDataSource::NotifyRelation(OSMRelation *psRelation) bool bAttrFilterAlreadyEvaluated = true; if (poFeature == nullptr) { - poFeature = new OGRFeature(m_papoLayers[iCurLayer]->GetLayerDefn()); + poFeature = std::make_unique( + m_apoLayers[iCurLayer]->GetLayerDefn()); - m_papoLayers[iCurLayer]->SetFieldsFromTags( - poFeature, psRelation->nID, false, + m_apoLayers[iCurLayer]->SetFieldsFromTags( + poFeature.get(), psRelation->nID, false, nExtraTags ? nExtraTags : psRelation->nTags, nExtraTags ? pasExtraTags : psRelation->pasTags, &psRelation->sInfo); @@ -2605,18 +2599,14 @@ void OGROSMDataSource::NotifyRelation(OSMRelation *psRelation) poFeature->SetGeometryDirectly(poGeom); - int bFilteredOut = FALSE; - if (!m_papoLayers[iCurLayer]->AddFeature( - poFeature, bAttrFilterAlreadyEvaluated, &bFilteredOut, - !m_bFeatureAdded)) + bool bFilteredOut = FALSE; + if (!m_apoLayers[iCurLayer]->AddFeature( + std::move(poFeature), bAttrFilterAlreadyEvaluated, + &bFilteredOut, !m_bFeatureAdded)) m_bStopParsing = true; else if (!bFilteredOut) m_bFeatureAdded = true; } - else - { - delete poFeature; - } } static void OGROSMNotifyRelation(OSMRelation *psRelation, @@ -2650,7 +2640,7 @@ void OGROSMDataSource::ProcessPolygonsStandalone() bool bFirst = true; while (m_bHasRowInPolygonsStandalone && - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->m_nFeatureArraySize < 10000) + m_apoLayers[IDX_LYR_MULTIPOLYGONS]->m_apoFeatures.size() < 10000) { if (bFirst) { @@ -2685,17 +2675,18 @@ void OGROSMDataSource::ProcessPolygonsStandalone() INT_TO_DBL(m_asLonLatCache[j].nLat)); } - OGRFeature *poFeature = new OGRFeature( - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->GetLayerDefn()); + auto poFeature = std::make_unique( + m_apoLayers[IDX_LYR_MULTIPOLYGONS]->GetLayerDefn()); - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->SetFieldsFromTags( - poFeature, id, true, nTags, pasTags, &sInfo); + m_apoLayers[IDX_LYR_MULTIPOLYGONS]->SetFieldsFromTags( + poFeature.get(), id, true, nTags, pasTags, &sInfo); poFeature->SetGeometryDirectly(poMulti); - int bFilteredOut = FALSE; - if (!m_papoLayers[IDX_LYR_MULTIPOLYGONS]->AddFeature( - poFeature, FALSE, &bFilteredOut, !m_bFeatureAdded)) + bool bFilteredOut = false; + if (!m_apoLayers[IDX_LYR_MULTIPOLYGONS]->AddFeature( + std::move(poFeature), FALSE, &bFilteredOut, + !m_bFeatureAdded)) { m_bStopParsing = true; break; @@ -2778,31 +2769,26 @@ int OGROSMDataSource::Open(const char *pszFilename, char **papszOpenOptionsIn) if (m_bCompressNodes) CPLDebug("OSM", "Using compression for nodes DB"); - m_nLayers = 5; - m_papoLayers = static_cast( - CPLMalloc(m_nLayers * sizeof(OGROSMLayer *))); - - m_papoLayers[IDX_LYR_POINTS] = - new OGROSMLayer(this, IDX_LYR_POINTS, "points"); - m_papoLayers[IDX_LYR_POINTS]->GetLayerDefn()->SetGeomType(wkbPoint); + // Do not change the below order without updating the IDX_LYR_ constants! + m_apoLayers.emplace_back( + std::make_unique(this, IDX_LYR_POINTS, "points")); + m_apoLayers.back()->GetLayerDefn()->SetGeomType(wkbPoint); - m_papoLayers[IDX_LYR_LINES] = new OGROSMLayer(this, IDX_LYR_LINES, "lines"); - m_papoLayers[IDX_LYR_LINES]->GetLayerDefn()->SetGeomType(wkbLineString); + m_apoLayers.emplace_back( + std::make_unique(this, IDX_LYR_LINES, "lines")); + m_apoLayers.back()->GetLayerDefn()->SetGeomType(wkbLineString); - m_papoLayers[IDX_LYR_MULTILINESTRINGS] = - new OGROSMLayer(this, IDX_LYR_MULTILINESTRINGS, "multilinestrings"); - m_papoLayers[IDX_LYR_MULTILINESTRINGS]->GetLayerDefn()->SetGeomType( - wkbMultiLineString); + m_apoLayers.emplace_back(std::make_unique( + this, IDX_LYR_MULTILINESTRINGS, "multilinestrings")); + m_apoLayers.back()->GetLayerDefn()->SetGeomType(wkbMultiLineString); - m_papoLayers[IDX_LYR_MULTIPOLYGONS] = - new OGROSMLayer(this, IDX_LYR_MULTIPOLYGONS, "multipolygons"); - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->GetLayerDefn()->SetGeomType( - wkbMultiPolygon); + m_apoLayers.emplace_back(std::make_unique( + this, IDX_LYR_MULTIPOLYGONS, "multipolygons")); + m_apoLayers.back()->GetLayerDefn()->SetGeomType(wkbMultiPolygon); - m_papoLayers[IDX_LYR_OTHER_RELATIONS] = - new OGROSMLayer(this, IDX_LYR_OTHER_RELATIONS, "other_relations"); - m_papoLayers[IDX_LYR_OTHER_RELATIONS]->GetLayerDefn()->SetGeomType( - wkbGeometryCollection); + m_apoLayers.emplace_back(std::make_unique( + this, IDX_LYR_OTHER_RELATIONS, "other_relations")); + m_apoLayers.back()->GetLayerDefn()->SetGeomType(wkbGeometryCollection); if (!ParseConf(papszOpenOptionsIn)) { @@ -2828,26 +2814,26 @@ int OGROSMDataSource::Open(const char *pszFilename, char **papszOpenOptionsIn) } const auto eTagsSubType = m_bTagsAsHSTORE ? OFSTNone : OFSTJSON; - for (int i = 0; i < m_nLayers; i++) + for (auto &&poLayer : m_apoLayers) { - if (m_papoLayers[i]->HasAllTags()) + if (poLayer->HasAllTags()) { - m_papoLayers[i]->AddField("all_tags", OFTString, eTagsSubType); - if (m_papoLayers[i]->HasOtherTags()) + poLayer->AddField("all_tags", OFTString, eTagsSubType); + if (poLayer->HasOtherTags()) { - m_papoLayers[i]->SetHasOtherTags(false); + poLayer->SetHasOtherTags(false); } } - else if (m_papoLayers[i]->HasOtherTags()) - m_papoLayers[i]->AddField("other_tags", OFTString, eTagsSubType); + else if (poLayer->HasOtherTags()) + poLayer->AddField("other_tags", OFTString, eTagsSubType); } m_bNeedsToSaveWayInfo = - (m_papoLayers[IDX_LYR_MULTIPOLYGONS]->HasTimestamp() || - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->HasChangeset() || - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->HasVersion() || - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->HasUID() || - m_papoLayers[IDX_LYR_MULTIPOLYGONS]->HasUser()); + (m_apoLayers[IDX_LYR_MULTIPOLYGONS]->HasTimestamp() || + m_apoLayers[IDX_LYR_MULTIPOLYGONS]->HasChangeset() || + m_apoLayers[IDX_LYR_MULTIPOLYGONS]->HasVersion() || + m_apoLayers[IDX_LYR_MULTIPOLYGONS]->HasUID() || + m_apoLayers[IDX_LYR_MULTIPOLYGONS]->HasUser()); m_panReqIds = static_cast( VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_NODES * sizeof(GIntBig))); @@ -2861,8 +2847,16 @@ int OGROSMDataSource::Open(const char *pszFilename, char **papszOpenOptionsIn) VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_NODES * sizeof(LonLat))); m_panUnsortedReqIds = static_cast( VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_NODES * sizeof(GIntBig))); - m_pasWayFeaturePairs = static_cast( - VSI_MALLOC_VERBOSE(MAX_DELAYED_FEATURES * sizeof(WayFeaturePair))); + try + { + m_asWayFeaturePairs.resize(MAX_DELAYED_FEATURES); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "OGROSMDataSource::Open(): out of memory"); + return FALSE; + } m_pasAccumulatedTags = static_cast( VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_TAGS * sizeof(IndexedKVP))); pabyNonRedundantValues = @@ -2870,9 +2864,8 @@ int OGROSMDataSource::Open(const char *pszFilename, char **papszOpenOptionsIn) pabyNonRedundantKeys = static_cast(VSI_MALLOC_VERBOSE(MAX_NON_REDUNDANT_KEYS)); if (m_panReqIds == nullptr || m_pasLonLatArray == nullptr || - m_panUnsortedReqIds == nullptr || m_pasWayFeaturePairs == nullptr || - m_pasAccumulatedTags == nullptr || pabyNonRedundantValues == nullptr || - pabyNonRedundantKeys == nullptr) + m_panUnsortedReqIds == nullptr || m_pasAccumulatedTags == nullptr || + pabyNonRedundantValues == nullptr || pabyNonRedundantKeys == nullptr) { return FALSE; } @@ -2911,7 +2904,8 @@ int OGROSMDataSource::Open(const char *pszFilename, char **papszOpenOptionsIn) CPLPushErrorHandler(CPLQuietErrorHandler); bool bSuccess = - VSIFSeekL(m_fpNodes, (vsi_l_offset)(nSize * 3 / 4), SEEK_SET) == 0; + VSIFSeekL(m_fpNodes, static_cast(nSize * 3 / 4), + SEEK_SET) == 0; CPLPopErrorHandler(); if (bSuccess) @@ -3380,9 +3374,9 @@ void OGROSMDataSource::AddComputedAttributes( { if (!oAttributes[i].osSQL.empty()) { - m_papoLayers[iCurLayer]->AddComputedAttribute(oAttributes[i].osName, - oAttributes[i].eType, - oAttributes[i].osSQL); + m_apoLayers[iCurLayer]->AddComputedAttribute(oAttributes[i].osName, + oAttributes[i].eType, + oAttributes[i].osSQL); } } } @@ -3425,14 +3419,17 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) iCurLayer = -1; pszLine++; - ((char *)pszLine)[strlen(pszLine) - 1] = '\0'; /* Evil but OK */ - for (int i = 0; i < m_nLayers; i++) + const_cast(pszLine)[strlen(pszLine) - 1] = + '\0'; /* Evil but OK */ + int i = 0; + for (auto &&poLayer : m_apoLayers) { - if (strcmp(pszLine, m_papoLayers[i]->GetName()) == 0) + if (strcmp(pszLine, poLayer->GetName()) == 0) { iCurLayer = i; break; } + ++i; } if (iCurLayer < 0) { @@ -3533,89 +3530,88 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) strcmp(papszTokens[0], "other_tags") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasOtherTags(false); + m_apoLayers[iCurLayer]->SetHasOtherTags(false); else if (strcmp(papszTokens[1], "yes") == 0) - m_papoLayers[iCurLayer]->SetHasOtherTags(true); + m_apoLayers[iCurLayer]->SetHasOtherTags(true); } else if (CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "all_tags") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasAllTags(false); + m_apoLayers[iCurLayer]->SetHasAllTags(false); else if (strcmp(papszTokens[1], "yes") == 0) - m_papoLayers[iCurLayer]->SetHasAllTags(true); + m_apoLayers[iCurLayer]->SetHasAllTags(true); } else if (CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_id") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasOSMId(false); + m_apoLayers[iCurLayer]->SetHasOSMId(false); else if (strcmp(papszTokens[1], "yes") == 0) { - m_papoLayers[iCurLayer]->SetHasOSMId(true); - m_papoLayers[iCurLayer]->AddField("osm_id", OFTString); + m_apoLayers[iCurLayer]->SetHasOSMId(true); + m_apoLayers[iCurLayer]->AddField("osm_id", OFTString); if (iCurLayer == IDX_LYR_MULTIPOLYGONS) - m_papoLayers[iCurLayer]->AddField("osm_way_id", - OFTString); + m_apoLayers[iCurLayer]->AddField("osm_way_id", + OFTString); } } else if (CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_version") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasVersion(false); + m_apoLayers[iCurLayer]->SetHasVersion(false); else if (strcmp(papszTokens[1], "yes") == 0) { - m_papoLayers[iCurLayer]->SetHasVersion(true); - m_papoLayers[iCurLayer]->AddField("osm_version", - OFTInteger); + m_apoLayers[iCurLayer]->SetHasVersion(true); + m_apoLayers[iCurLayer]->AddField("osm_version", OFTInteger); } } else if (CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_timestamp") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasTimestamp(false); + m_apoLayers[iCurLayer]->SetHasTimestamp(false); else if (strcmp(papszTokens[1], "yes") == 0) { - m_papoLayers[iCurLayer]->SetHasTimestamp(true); - m_papoLayers[iCurLayer]->AddField("osm_timestamp", - OFTDateTime); + m_apoLayers[iCurLayer]->SetHasTimestamp(true); + m_apoLayers[iCurLayer]->AddField("osm_timestamp", + OFTDateTime); } } else if (CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_uid") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasUID(false); + m_apoLayers[iCurLayer]->SetHasUID(false); else if (strcmp(papszTokens[1], "yes") == 0) { - m_papoLayers[iCurLayer]->SetHasUID(true); - m_papoLayers[iCurLayer]->AddField("osm_uid", OFTInteger); + m_apoLayers[iCurLayer]->SetHasUID(true); + m_apoLayers[iCurLayer]->AddField("osm_uid", OFTInteger); } } else if (CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_user") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasUser(false); + m_apoLayers[iCurLayer]->SetHasUser(false); else if (strcmp(papszTokens[1], "yes") == 0) { - m_papoLayers[iCurLayer]->SetHasUser(true); - m_papoLayers[iCurLayer]->AddField("osm_user", OFTString); + m_apoLayers[iCurLayer]->SetHasUser(true); + m_apoLayers[iCurLayer]->AddField("osm_user", OFTString); } } else if (CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_changeset") == 0) { if (strcmp(papszTokens[1], "no") == 0) - m_papoLayers[iCurLayer]->SetHasChangeset(false); + m_apoLayers[iCurLayer]->SetHasChangeset(false); else if (strcmp(papszTokens[1], "yes") == 0) { - m_papoLayers[iCurLayer]->SetHasChangeset(true); - m_papoLayers[iCurLayer]->AddField("osm_changeset", - OFTInteger); + m_apoLayers[iCurLayer]->SetHasChangeset(true); + m_apoLayers[iCurLayer]->AddField("osm_changeset", + OFTInteger); } } else if (CSLCount(papszTokens) == 2 && @@ -3625,8 +3621,8 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) CSLTokenizeString2(papszTokens[1], ",", 0); for (int i = 0; papszTokens2[i] != nullptr; i++) { - m_papoLayers[iCurLayer]->AddField(papszTokens2[i], - OFTString); + m_apoLayers[iCurLayer]->AddField(papszTokens2[i], + OFTString); for (const char *&pszIgnoredKey : m_ignoredKeys) { if (strcmp(papszTokens2[i], pszIgnoredKey) == 0) @@ -3643,7 +3639,7 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) CSLTokenizeString2(papszTokens[1], ",", 0); for (int i = 0; papszTokens2[i] != nullptr; i++) { - m_papoLayers[iCurLayer]->AddInsignificantKey( + m_apoLayers[iCurLayer]->AddInsignificantKey( papszTokens2[i]); } CSLDestroy(papszTokens2); @@ -3655,8 +3651,8 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) CSLTokenizeString2(papszTokens[1], ",", 0); for (int i = 0; papszTokens2[i] != nullptr; i++) { - m_papoLayers[iCurLayer]->AddIgnoreKey(papszTokens2[i]); - m_papoLayers[iCurLayer]->AddWarnKey(papszTokens2[i]); + m_apoLayers[iCurLayer]->AddIgnoreKey(papszTokens2[i]); + m_apoLayers[iCurLayer]->AddWarnKey(papszTokens2[i]); } CSLDestroy(papszTokens2); } @@ -3709,11 +3705,11 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) if (!bFound) { const int idx = - m_papoLayers[iCurLayer]->GetLayerDefn()->GetFieldIndex( + m_apoLayers[iCurLayer]->GetLayerDefn()->GetFieldIndex( osName); if (idx >= 0) { - m_papoLayers[iCurLayer] + m_apoLayers[iCurLayer] ->GetLayerDefn() ->GetFieldDefn(idx) ->SetType(eType); @@ -3821,29 +3817,24 @@ int OGROSMDataSource::MyResetReading() sqlite3_reset(m_hSelectPolygonsStandaloneStmt); { - for (int i = 0; i < m_nWayFeaturePairs; i++) - { - delete m_pasWayFeaturePairs[i].poFeature; - } - m_nWayFeaturePairs = 0; + m_asWayFeaturePairs.clear(); m_nUnsortedReqIds = 0; m_nReqIds = 0; m_nAccumulatedTags = 0; nNonRedundantKeysLen = 0; nNonRedundantValuesLen = 0; - for (int i = 1; i < static_cast(m_asKeys.size()); i++) + for (KeyDesc *psKD : m_apsKeys) { - KeyDesc *psKD = m_asKeys[i]; if (psKD) { CPLFree(psKD->pszK); - for (int j = 0; j < (int)psKD->asValues.size(); j++) - CPLFree(psKD->asValues[j]); + for (auto *pszValue : psKD->apszValues) + CPLFree(pszValue); delete psKD; } } - m_asKeys.resize(1); // keep guard to avoid index 0 to be used + m_apsKeys.resize(1); // keep guard to avoid index 0 to be used m_aoMapIndexedKeys.clear(); } @@ -3878,9 +3869,9 @@ int OGROSMDataSource::MyResetReading() } } - for (int i = 0; i < m_nLayers; i++) + for (auto &&poLayer : m_apoLayers) { - m_papoLayers[i]->ForceResetReading(); + poLayer->ForceResetReading(); } m_bStopParsing = false; @@ -3911,7 +3902,7 @@ OGRFeature *OGROSMDataSource::GetNextFeature(OGRLayer **ppoBelongingLayer, if (m_poCurrentLayer == nullptr) { - m_poCurrentLayer = m_papoLayers[0]; + m_poCurrentLayer = m_apoLayers[0].get(); } if (pdfProgressPct != nullptr || pfnProgress != nullptr) { @@ -3999,9 +3990,9 @@ bool OGROSMDataSource::ParseNextChunk(int nIdxLayer, if (!pfnProgress(dfPct, "", pProgressData)) { m_bStopParsing = true; - for (int i = 0; i < m_nLayers; i++) + for (auto &&poLayer : m_apoLayers) { - m_papoLayers[i]->ForceResetReading(); + poLayer->ForceResetReading(); } return false; } @@ -4011,7 +4002,7 @@ bool OGROSMDataSource::ParseNextChunk(int nIdxLayer, { if (eRet == OSM_EOF) { - if (m_nWayFeaturePairs != 0) + if (!m_asWayFeaturePairs.empty()) ProcessWaysBatch(); ProcessPolygonsStandalone(); @@ -4098,12 +4089,12 @@ bool OGROSMDataSource::TransferToDiskIfNecesserary() { VSIFSeekL(fp, 0, SEEK_END); vsi_l_offset nCurSize = VSIFTellL(fp); - GIntBig nNewSize = - static_cast(m_nMaxSizeForInMemoryDBInMB) * + vsi_l_offset nNewSize = + static_cast(m_nMaxSizeForInMemoryDBInMB) * 1024 * 1024; CPLPushErrorHandler(CPLQuietErrorHandler); const bool bSuccess = - VSIFSeekL(fp, (vsi_l_offset)nNewSize, SEEK_SET) == 0; + VSIFSeekL(fp, nNewSize, SEEK_SET) == 0; CPLPopErrorHandler(); if (bSuccess) @@ -4224,10 +4215,10 @@ int OGROSMDataSource::TestCapability(const char *pszCap) OGRLayer *OGROSMDataSource::GetLayer(int iLayer) { - if (iLayer < 0 || iLayer >= m_nLayers) + if (iLayer < 0 || static_cast(iLayer) >= m_apoLayers.size()) return nullptr; - return m_papoLayers[iLayer]; + return m_apoLayers[iLayer].get(); } /************************************************************************/ @@ -4263,6 +4254,10 @@ class OGROSMSingleFeatureLayer final : public OGRLayer OGRFeatureDefn *poFeatureDefn; int iNextShapeId; + OGROSMSingleFeatureLayer(const OGROSMSingleFeatureLayer &) = delete; + OGROSMSingleFeatureLayer & + operator=(const OGROSMSingleFeatureLayer &) = delete; + public: OGROSMSingleFeatureLayer(const char *pszLayerName, int nVal); OGROSMSingleFeatureLayer(const char *pszLayerName, const char *pszVal); @@ -4404,26 +4399,26 @@ OGRLayer *OGROSMDataSource::ExecuteSQL(const char *pszSQLCommand, char **papszTokens = CSLTokenizeString2(pszSQLCommand + 21, ",", CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES); - for (int i = 0; i < m_nLayers; i++) + for (auto &&poLayer : m_apoLayers) { - m_papoLayers[i]->SetDeclareInterest(FALSE); + poLayer->SetDeclareInterest(FALSE); } for (int i = 0; papszTokens[i] != nullptr; i++) { OGROSMLayer *poLayer = - reinterpret_cast(GetLayerByName(papszTokens[i])); + dynamic_cast(GetLayerByName(papszTokens[i])); if (poLayer != nullptr) { poLayer->SetDeclareInterest(TRUE); } } - if (m_papoLayers[IDX_LYR_POINTS]->IsUserInterested() && - !m_papoLayers[IDX_LYR_LINES]->IsUserInterested() && - !m_papoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() && - !m_papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() && - !m_papoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested()) + if (m_apoLayers[IDX_LYR_POINTS]->IsUserInterested() && + !m_apoLayers[IDX_LYR_LINES]->IsUserInterested() && + !m_apoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() && + !m_apoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() && + !m_apoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested()) { if (CPLGetConfigOption("OSM_INDEX_POINTS", nullptr) == nullptr) { @@ -4444,10 +4439,10 @@ OGRLayer *OGROSMDataSource::ExecuteSQL(const char *pszSQLCommand, m_bUseWaysIndex = false; } } - else if (m_papoLayers[IDX_LYR_LINES]->IsUserInterested() && - !m_papoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() && - !m_papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() && - !m_papoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested()) + else if (m_apoLayers[IDX_LYR_LINES]->IsUserInterested() && + !m_apoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() && + !m_apoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() && + !m_apoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested()) { if (CPLGetConfigOption("OSM_INDEX_WAYS", nullptr) == nullptr) { @@ -4526,10 +4521,10 @@ OGRLayer *OGROSMDataSource::ExecuteSQL(const char *pszSQLCommand, { /* Backup current optimization parameters */ m_abSavedDeclaredInterest.resize(0); - for (int i = 0; i < m_nLayers; i++) + for (auto &&poLayer : m_apoLayers) { m_abSavedDeclaredInterest.push_back( - m_papoLayers[i]->IsUserInterested()); + poLayer->IsUserInterested()); } m_bIndexPointsBackup = m_bIndexPoints; m_bUsePointsIndexBackup = m_bUsePointsIndex; @@ -4578,9 +4573,11 @@ void OGROSMDataSource::ReleaseResultSet(OGRLayer *poLayer) m_bIsFeatureCountEnabled = false; /* Restore backup'ed optimization parameters */ - for (int i = 0; i < m_nLayers; i++) + int i = 0; + for (auto &&poIterLayer : m_apoLayers) { - m_papoLayers[i]->SetDeclareInterest(m_abSavedDeclaredInterest[i]); + poIterLayer->SetDeclareInterest(m_abSavedDeclaredInterest[i]); + ++i; } if (m_bIndexPointsBackup && !m_bIndexPoints) CPLDebug("OSM", "Re-enabling indexing of nodes"); @@ -4590,7 +4587,7 @@ void OGROSMDataSource::ReleaseResultSet(OGRLayer *poLayer) CPLDebug("OSM", "Re-enabling indexing of ways"); m_bIndexWays = m_bIndexWaysBackup; m_bUseWaysIndex = m_bUseWaysIndexBackup; - m_abSavedDeclaredInterest.resize(0); + m_abSavedDeclaredInterest.clear(); } delete poLayer; diff --git a/ogr/ogrsf_frmts/osm/ogrosmdriver.cpp b/ogr/ogrsf_frmts/osm/ogrosmdriver.cpp index 14903e128d69..7414ee5d6132 100644 --- a/ogr/ogrsf_frmts/osm/ogrosmdriver.cpp +++ b/ogr/ogrsf_frmts/osm/ogrosmdriver.cpp @@ -51,7 +51,8 @@ static int OGROSMDriverIdentify(GDALOpenInfo *poOpenInfo) if (poOpenInfo->fpL == nullptr || poOpenInfo->nHeaderBytes == 0) return GDAL_IDENTIFY_FALSE; - if (strstr((const char *)poOpenInfo->pabyHeader, "(poOpenInfo->pabyHeader), + " #include #include #include @@ -56,8 +57,11 @@ #include "osm_parser.h" #include "sqlite3.h" -constexpr int SWITCH_THRESHOLD = 10000; -constexpr int MAX_THRESHOLD = 100000; +#undef SQLITE_TRANSIENT +#define SQLITE_TRANSIENT reinterpret_cast(-1) + +constexpr size_t SWITCH_THRESHOLD = 10000; +constexpr size_t MAX_THRESHOLD = 100000; /************************************************************************/ /* OGROSMLayer() */ @@ -90,12 +94,6 @@ OGROSMLayer::~OGROSMLayer() if (m_poSRS) m_poSRS->Release(); - for (int i = 0; i < m_nFeatureArraySize; i++) - { - if (m_papoFeatures[i]) - delete m_papoFeatures[i]; - } - for (int i = 0; i < static_cast(m_apszNames.size()); i++) CPLFree(m_apszNames[i]); @@ -109,8 +107,6 @@ OGROSMLayer::~OGROSMLayer() { sqlite3_finalize(m_oComputedAttributes[i].hStmt); } - - CPLFree(m_papoFeatures); } /************************************************************************/ @@ -131,14 +127,8 @@ void OGROSMLayer::ResetReading() void OGROSMLayer::ForceResetReading() { - for (int i = 0; i < m_nFeatureArraySize; i++) - { - if (m_papoFeatures[i]) - delete m_papoFeatures[i]; - } + m_apoFeatures.clear(); m_nFeatureArrayIndex = 0; - m_nFeatureArraySize = 0; - m_nFeatureCount = 0; m_bResetReadingAllowed = false; } @@ -208,7 +198,7 @@ OGRFeature *OGROSMLayer::MyGetNextFeature(OGROSMLayer **ppoNewCurLayer, *ppoNewCurLayer = m_poDS->GetCurrentLayer(); m_bResetReadingAllowed = true; - if (m_nFeatureArraySize == 0) + if (m_apoFeatures.empty()) { if (m_poDS->IsInterleavedReading()) { @@ -225,15 +215,15 @@ OGRFeature *OGROSMLayer::MyGetNextFeature(OGROSMLayer **ppoNewCurLayer, // force a switch to that layer, so that it gets emptied. for (int i = 0; i < m_poDS->GetLayerCount(); i++) { - if (m_poDS->m_papoLayers[i] != this && - m_poDS->m_papoLayers[i]->m_nFeatureArraySize > + if (m_poDS->m_apoLayers[i].get() != this && + m_poDS->m_apoLayers[i]->m_apoFeatures.size() > SWITCH_THRESHOLD) { - *ppoNewCurLayer = m_poDS->m_papoLayers[i]; + *ppoNewCurLayer = m_poDS->m_apoLayers[i].get(); CPLDebug("OSM", "Switching to '%s' as they are too many " "features in '%s'", - m_poDS->m_papoLayers[i]->GetName(), GetName()); + m_poDS->m_apoLayers[i]->GetName(), GetName()); return nullptr; } } @@ -241,21 +231,21 @@ OGRFeature *OGROSMLayer::MyGetNextFeature(OGROSMLayer **ppoNewCurLayer, // Read some more data and accumulate features. m_poDS->ParseNextChunk(m_nIdxLayer, pfnProgress, pProgressData); - if (m_nFeatureArraySize == 0) + if (m_apoFeatures.empty()) { // If there are really no more features to read in the // current layer, force a switch to another non-empty layer. for (int i = 0; i < m_poDS->GetLayerCount(); i++) { - if (m_poDS->m_papoLayers[i] != this && - m_poDS->m_papoLayers[i]->m_nFeatureArraySize > 0) + if (m_poDS->m_apoLayers[i].get() != this && + !m_poDS->m_apoLayers[i]->m_apoFeatures.empty()) { - *ppoNewCurLayer = m_poDS->m_papoLayers[i]; + *ppoNewCurLayer = m_poDS->m_apoLayers[i].get(); CPLDebug("OSM", "Switching to '%s' as they are " "no more feature in '%s'", - m_poDS->m_papoLayers[i]->GetName(), GetName()); + m_poDS->m_apoLayers[i]->GetName(), GetName()); return nullptr; } } @@ -272,7 +262,7 @@ OGRFeature *OGROSMLayer::MyGetNextFeature(OGROSMLayer **ppoNewCurLayer, int bRet = m_poDS->ParseNextChunk(m_nIdxLayer, nullptr, nullptr); // cppcheck-suppress knownConditionTrueFalse - if (m_nFeatureArraySize != 0) + if (!m_apoFeatures.empty()) break; if (bRet == FALSE) return nullptr; @@ -280,15 +270,16 @@ OGRFeature *OGROSMLayer::MyGetNextFeature(OGROSMLayer **ppoNewCurLayer, } } - OGRFeature *poFeature = m_papoFeatures[m_nFeatureArrayIndex]; - - m_papoFeatures[m_nFeatureArrayIndex] = nullptr; + auto poFeature = std::move(m_apoFeatures[m_nFeatureArrayIndex]); m_nFeatureArrayIndex++; - if (m_nFeatureArrayIndex == m_nFeatureArraySize) - m_nFeatureArrayIndex = m_nFeatureArraySize = 0; + if (m_nFeatureArrayIndex == m_apoFeatures.size()) + { + m_nFeatureArrayIndex = 0; + m_apoFeatures.clear(); + } - return poFeature; + return poFeature.release(); } /************************************************************************/ @@ -311,9 +302,10 @@ int OGROSMLayer::TestCapability(const char *pszCap) /* AddToArray() */ /************************************************************************/ -bool OGROSMLayer::AddToArray(OGRFeature *poFeature, int bCheckFeatureThreshold) +bool OGROSMLayer::AddToArray(std::unique_ptr poFeature, + bool bCheckFeatureThreshold) { - if (bCheckFeatureThreshold && m_nFeatureArraySize > MAX_THRESHOLD) + if (bCheckFeatureThreshold && m_apoFeatures.size() > MAX_THRESHOLD) { if (!m_bHasWarnedTooManyFeatures) { @@ -330,25 +322,18 @@ bool OGROSMLayer::AddToArray(OGRFeature *poFeature, int bCheckFeatureThreshold) return false; } - if (m_nFeatureArraySize == m_nFeatureArrayMaxSize) + try { - m_nFeatureArrayMaxSize = - m_nFeatureArrayMaxSize + m_nFeatureArrayMaxSize / 2 + 128; - CPLDebug("OSM", "For layer %s, new max size is %d", GetName(), - m_nFeatureArrayMaxSize); - OGRFeature **papoNewFeatures = - static_cast(VSI_REALLOC_VERBOSE( - m_papoFeatures, m_nFeatureArrayMaxSize * sizeof(OGRFeature *))); - if (papoNewFeatures == nullptr) - { - CPLError(CE_Failure, CPLE_AppDefined, - "For layer %s, cannot resize feature array to %d features", - GetName(), m_nFeatureArrayMaxSize); - return false; - } - m_papoFeatures = papoNewFeatures; + m_apoFeatures.push_back(std::move(poFeature)); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "For layer %s, cannot resize feature array to %" PRIu64 + " features", + GetName(), static_cast(m_apoFeatures.size()) + 1); + return false; } - m_papoFeatures[m_nFeatureArraySize++] = poFeature; return true; } @@ -366,44 +351,40 @@ int OGROSMLayer::EvaluateAttributeFilter(OGRFeature *poFeature) /* AddFeature() */ /************************************************************************/ -int OGROSMLayer::AddFeature(OGRFeature *poFeature, - int bAttrFilterAlreadyEvaluated, int *pbFilteredOut, - int bCheckFeatureThreshold) +bool OGROSMLayer::AddFeature(std::unique_ptr poFeature, + bool bAttrFilterAlreadyEvaluated, + bool *pbFilteredOut, bool bCheckFeatureThreshold) { if (!m_bUserInterested) { if (pbFilteredOut) - *pbFilteredOut = TRUE; - delete poFeature; - return TRUE; + *pbFilteredOut = true; + return true; } OGRGeometry *poGeom = poFeature->GetGeometryRef(); if (poGeom) poGeom->assignSpatialReference(m_poSRS); - if ((m_poFilterGeom == nullptr || - FilterGeometry(poFeature->GetGeometryRef())) && + if ((m_poFilterGeom == nullptr || FilterGeometry(poGeom)) && (m_poAttrQuery == nullptr || bAttrFilterAlreadyEvaluated || - m_poAttrQuery->Evaluate(poFeature))) + m_poAttrQuery->Evaluate(poFeature.get()))) { - if (!AddToArray(poFeature, bCheckFeatureThreshold)) + if (!AddToArray(std::move(poFeature), bCheckFeatureThreshold)) { - delete poFeature; - return FALSE; + return false; } } else { if (pbFilteredOut) - *pbFilteredOut = TRUE; - delete poFeature; - return TRUE; + *pbFilteredOut = true; + return true; } if (pbFilteredOut) - *pbFilteredOut = FALSE; - return TRUE; + *pbFilteredOut = false; + return true; } /************************************************************************/ @@ -479,8 +460,7 @@ void OGROSMLayer::AddField(const char *pszName, OGRFieldType eFieldType, int OGROSMLayer::GetFieldIndex(const char *pszName) { - std::map::iterator oIter = - m_oMapFieldNameToIndex.find(pszName); + const auto oIter = m_oMapFieldNameToIndex.find(pszName); if (oIter != m_oMapFieldNameToIndex.end()) return oIter->second; @@ -497,7 +477,7 @@ int OGROSMLayer::AddInOtherOrAllTags(const char *pszK) if (aoSetIgnoreKeys.find(pszK) == aoSetIgnoreKeys.end()) { - char *pszColon = strchr((char *)pszK, ':'); + char *pszColon = strchr(const_cast(pszK), ':'); if (pszColon) { char chBackup = pszColon[1]; @@ -655,7 +635,7 @@ void OGROSMLayer::SetFieldsFromTags(OGRFeature *poFeature, GIntBig nID, } if (m_bHasChangeset) { - poFeature->SetField("osm_changeset", (int)psInfo->nChangeset); + poFeature->SetField("osm_changeset", psInfo->nChangeset); } m_osAllTagsBuffer.clear(); @@ -929,8 +909,8 @@ void OGROSMLayer::SetFieldsFromTags(OGRFeature *poFeature, GIntBig nID, { case SQLITE_INTEGER: poFeature->SetField( - oAttr.nIndex, - (GIntBig)sqlite3_column_int64(oAttr.hStmt, 0)); + oAttr.nIndex, static_cast(sqlite3_column_int64( + oAttr.hStmt, 0))); break; case SQLITE_FLOAT: poFeature->SetField(oAttr.nIndex, @@ -938,8 +918,8 @@ void OGROSMLayer::SetFieldsFromTags(OGRFeature *poFeature, GIntBig nID, break; case SQLITE_TEXT: poFeature->SetField( - oAttr.nIndex, - (const char *)sqlite3_column_text(oAttr.hStmt, 0)); + oAttr.nIndex, reinterpret_cast( + sqlite3_column_text(oAttr.hStmt, 0))); break; default: break; diff --git a/ogr/ogrsf_frmts/osm/osm_parser.cpp b/ogr/ogrsf_frmts/osm/osm_parser.cpp index cb333d27fe8b..ec44919bfa59 100644 --- a/ogr/ogrsf_frmts/osm/osm_parser.cpp +++ b/ogr/ogrsf_frmts/osm/osm_parser.cpp @@ -461,22 +461,23 @@ constexpr int READSTRINGTABLE_IDX_STRING = 1; static bool ReadStringTable(const GByte *pabyData, const GByte *pabyDataLimit, OSMContext *psCtxt) { - char *pszStrBuf = (char *)pabyData; + const GByte *const pabyDataStart = pabyData; unsigned int nStrCount = 0; int *panStrOff = psCtxt->panStrOff; - psCtxt->pszStrBuf = pszStrBuf; + psCtxt->pszStrBuf = reinterpret_cast(const_cast(pabyData)); try { - if ((unsigned int)(pabyDataLimit - pabyData) > psCtxt->nStrAllocated) + if (static_cast(pabyDataLimit - pabyData) > + psCtxt->nStrAllocated) { psCtxt->nStrAllocated = std::max(psCtxt->nStrAllocated * 2, - (unsigned int)(pabyDataLimit - pabyData)); - int *panStrOffNew = (int *)VSI_REALLOC_VERBOSE( - panStrOff, psCtxt->nStrAllocated * sizeof(int)); + static_cast(pabyDataLimit - pabyData)); + int *panStrOffNew = static_cast(VSI_REALLOC_VERBOSE( + panStrOff, psCtxt->nStrAllocated * sizeof(int))); if (panStrOffNew == nullptr) THROW_OSM_PARSING_EXCEPTION; panStrOff = panStrOffNew; @@ -493,7 +494,7 @@ static bool ReadStringTable(const GByte *pabyData, const GByte *pabyDataLimit, READ_SIZE(pabyData, pabyDataLimit, nDataLength); panStrOff[nStrCount++] = - static_cast(pabyData - (GByte *)pszStrBuf); + static_cast(pabyData - pabyDataStart); GByte *pbSaved = const_cast(&pabyData[nDataLength]); pabyData += nDataLength; @@ -614,9 +615,10 @@ static bool ReadDenseNodes(const GByte *pabyData, const GByte *pabyDataLimit, { psCtxt->nNodesAllocated = std::max(psCtxt->nNodesAllocated * 2, nSize); - OSMNode *pasNodesNew = (OSMNode *)VSI_REALLOC_VERBOSE( - psCtxt->pasNodes, - psCtxt->nNodesAllocated * sizeof(OSMNode)); + OSMNode *pasNodesNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasNodes, + psCtxt->nNodesAllocated * sizeof(OSMNode))); if (pasNodesNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->pasNodes = pasNodesNew; @@ -693,9 +695,10 @@ static bool ReadDenseNodes(const GByte *pabyData, const GByte *pabyDataLimit, psCtxt->nTagsAllocated = std::max(psCtxt->nTagsAllocated * 2, nMaxTags); - OSMTag *pasTagsNew = (OSMTag *)VSI_REALLOC_VERBOSE( - psCtxt->pasTags, - psCtxt->nTagsAllocated * sizeof(OSMTag)); + OSMTag *pasTagsNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasTags, + psCtxt->nTagsAllocated * sizeof(OSMTag))); if (pasTagsNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->pasTags = pasTagsNew; @@ -820,11 +823,13 @@ static bool ReadDenseNodes(const GByte *pabyData, const GByte *pabyDataLimit, pasNodes[nNodes].nID = nID; pasNodes[nNodes].dfLat = - .000000001 * (psCtxt->nLatOffset + - ((double)psCtxt->nGranularity * nLat)); + .000000001 * + (psCtxt->nLatOffset + + (static_cast(psCtxt->nGranularity) * nLat)); pasNodes[nNodes].dfLon = - .000000001 * (psCtxt->nLonOffset + - ((double)psCtxt->nGranularity * nLon)); + .000000001 * + (psCtxt->nLonOffset + + (static_cast(psCtxt->nGranularity) * nLon)); if (pasNodes[nNodes].dfLon < -180 || pasNodes[nNodes].dfLon > 180 || pasNodes[nNodes].dfLat < -90 || pasNodes[nNodes].dfLat > 90) @@ -974,16 +979,18 @@ static bool ReadNode(const GByte *pabyData, const GByte *pabyDataLimit, GIntBig nLat = 0; READ_VARSINT64_NOCHECK(pabyData, pabyDataLimit, nLat); sNode.dfLat = - 0.000000001 * (psCtxt->nLatOffset + - ((double)psCtxt->nGranularity * nLat)); + 0.000000001 * + (psCtxt->nLatOffset + + (static_cast(psCtxt->nGranularity) * nLat)); } else if (nKey == MAKE_KEY(NODE_IDX_LON, WT_VARINT)) { GIntBig nLon = 0; READ_VARSINT64_NOCHECK(pabyData, pabyDataLimit, nLon); sNode.dfLon = - 0.000000001 * (psCtxt->nLonOffset + - ((double)psCtxt->nGranularity * nLon)); + 0.000000001 * + (psCtxt->nLonOffset + + (static_cast(psCtxt->nGranularity) * nLon)); } else if (nKey == MAKE_KEY(NODE_IDX_KEYS, WT_DATA)) { @@ -997,9 +1004,10 @@ static bool ReadNode(const GByte *pabyData, const GByte *pabyDataLimit, { psCtxt->nTagsAllocated = std::max(psCtxt->nTagsAllocated * 2, nSize); - OSMTag *pasTagsNew = (OSMTag *)VSI_REALLOC_VERBOSE( - psCtxt->pasTags, - psCtxt->nTagsAllocated * sizeof(OSMTag)); + OSMTag *pasTagsNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasTags, + psCtxt->nTagsAllocated * sizeof(OSMTag))); if (pasTagsNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->pasTags = pasTagsNew; @@ -1124,9 +1132,10 @@ static bool ReadWay(const GByte *pabyData, const GByte *pabyDataLimit, { psCtxt->nTagsAllocated = std::max(psCtxt->nTagsAllocated * 2, nSize); - OSMTag *pasTagsNew = (OSMTag *)VSI_REALLOC_VERBOSE( - psCtxt->pasTags, - psCtxt->nTagsAllocated * sizeof(OSMTag)); + OSMTag *pasTagsNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasTags, + psCtxt->nTagsAllocated * sizeof(OSMTag))); if (pasTagsNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->pasTags = pasTagsNew; @@ -1194,9 +1203,10 @@ static bool ReadWay(const GByte *pabyData, const GByte *pabyDataLimit, { psCtxt->nNodeRefsAllocated = std::max(psCtxt->nNodeRefsAllocated * 2, nSize); - GIntBig *panNodeRefsNew = (GIntBig *)VSI_REALLOC_VERBOSE( - psCtxt->panNodeRefs, - psCtxt->nNodeRefsAllocated * sizeof(GIntBig)); + GIntBig *panNodeRefsNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->panNodeRefs, + psCtxt->nNodeRefsAllocated * sizeof(GIntBig))); if (panNodeRefsNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->panNodeRefs = panNodeRefsNew; @@ -1286,9 +1296,10 @@ static bool ReadRelation(const GByte *pabyData, const GByte *pabyDataLimit, { psCtxt->nTagsAllocated = std::max(psCtxt->nTagsAllocated * 2, nSize); - OSMTag *pasTagsNew = (OSMTag *)VSI_REALLOC_VERBOSE( - psCtxt->pasTags, - psCtxt->nTagsAllocated * sizeof(OSMTag)); + OSMTag *pasTagsNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasTags, + psCtxt->nTagsAllocated * sizeof(OSMTag))); if (pasTagsNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->pasTags = pasTagsNew; @@ -1355,9 +1366,10 @@ static bool ReadRelation(const GByte *pabyData, const GByte *pabyDataLimit, { psCtxt->nMembersAllocated = std::max(psCtxt->nMembersAllocated * 2, nSize); - OSMMember *pasMembersNew = (OSMMember *)VSI_REALLOC_VERBOSE( - psCtxt->pasMembers, - psCtxt->nMembersAllocated * sizeof(OSMMember)); + OSMMember *pasMembersNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasMembers, + psCtxt->nMembersAllocated * sizeof(OSMMember))); if (pasMembersNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->pasMembers = pasMembersNew; @@ -1416,7 +1428,8 @@ static bool ReadRelation(const GByte *pabyData, const GByte *pabyDataLimit, if (nType > MEMBER_RELATION) THROW_OSM_PARSING_EXCEPTION; - psCtxt->pasMembers[nIter].eType = (OSMMemberType)nType; + psCtxt->pasMembers[nIter].eType = + static_cast(nType); } pabyData += nSize; } @@ -1832,9 +1845,10 @@ static bool ReadBlob(OSMContext *psCtxt, BlobType eType) if (psCtxt->nUncompressedAllocated > UINT_MAX - EXTRA_BYTES) THROW_OSM_PARSING_EXCEPTION; - pabyUncompressedNew = (GByte *)VSI_REALLOC_VERBOSE( - psCtxt->pabyUncompressed, - psCtxt->nUncompressedAllocated + EXTRA_BYTES); + pabyUncompressedNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pabyUncompressed, + psCtxt->nUncompressedAllocated + EXTRA_BYTES)); if (pabyUncompressedNew == nullptr) THROW_OSM_PARSING_EXCEPTION; psCtxt->pabyUncompressed = pabyUncompressedNew; @@ -2101,7 +2115,7 @@ static void EmptyNotifyBoundsFunc(double /* dfXMin */, double /* dfYMin */, static const char *OSM_AddString(OSMContext *psCtxt, const char *pszStr) { - int nLen = (int)strlen(pszStr); + const auto nLen = strlen(pszStr); if (psCtxt->nStrLength + nLen + 1 > psCtxt->nStrAllocated) { CPLError(CE_Failure, CPLE_AppDefined, "String buffer too small"); @@ -2110,7 +2124,7 @@ static const char *OSM_AddString(OSMContext *psCtxt, const char *pszStr) char *pszRet = psCtxt->pszStrBuf + psCtxt->nStrLength; memcpy(pszRet, pszStr, nLen); pszRet[nLen] = '\0'; - psCtxt->nStrLength += nLen + 1; + psCtxt->nStrLength += static_cast(nLen) + 1; return pszRet; } @@ -2131,7 +2145,7 @@ static void XMLCALL OSM_XML_startElementCbk(void *pUserData, const char *pszName, const char **ppszAttr) { - OSMContext *psCtxt = (OSMContext *)pUserData; + OSMContext *psCtxt = static_cast(pUserData); const char **ppszIter = ppszAttr; if (psCtxt->bStopParsing) @@ -2384,8 +2398,9 @@ static void XMLCALL OSM_XML_startElementCbk(void *pUserData, { int nMembersAllocated = std::max(psCtxt->nMembersAllocated * 2, psCtxt->sRelation.nMembers + 1); - OSMMember *pasMembersNew = (OSMMember *)VSI_REALLOC_VERBOSE( - psCtxt->pasMembers, nMembersAllocated * sizeof(OSMMember)); + OSMMember *pasMembersNew = + static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasMembers, nMembersAllocated * sizeof(OSMMember))); if (pasMembersNew == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, @@ -2436,8 +2451,8 @@ static void XMLCALL OSM_XML_startElementCbk(void *pUserData, if (psCtxt->nTags == psCtxt->nTagsAllocated) { psCtxt->nTagsAllocated = psCtxt->nTagsAllocated * 2; - OSMTag *pasTagsNew = (OSMTag *)VSI_REALLOC_VERBOSE( - psCtxt->pasTags, psCtxt->nTagsAllocated * sizeof(OSMTag)); + OSMTag *pasTagsNew = static_cast(VSI_REALLOC_VERBOSE( + psCtxt->pasTags, psCtxt->nTagsAllocated * sizeof(OSMTag))); if (pasTagsNew == nullptr) { if (psCtxt->bInNode) @@ -2487,7 +2502,7 @@ static void XMLCALL OSM_XML_startElementCbk(void *pUserData, static void XMLCALL OSM_XML_endElementCbk(void *pUserData, const char *pszName) { - OSMContext *psCtxt = (OSMContext *)pUserData; + OSMContext *psCtxt = static_cast(pUserData); if (psCtxt->bStopParsing) return; @@ -2568,7 +2583,7 @@ static void XMLCALL OSM_XML_dataHandlerCbk(void *pUserData, { CPLError(CE_Failure, CPLE_AppDefined, "File probably corrupted (million laugh pattern)"); - XML_StopParser(psCtxt->hXMLParser, XML_FALSE); + XML_StopParser(psCtxt->hXMLParser, false); psCtxt->bStopParsing = true; return; } @@ -2592,24 +2607,27 @@ static OSMRetCode XML_ProcessBlock(OSMContext *psCtxt) { psCtxt->nDataHandlerCounter = 0; - const unsigned int nLen = (unsigned int)VSIFReadL( - psCtxt->pabyBlob, 1, XML_BUFSIZE, psCtxt->fp); + const unsigned int nLen = static_cast( + VSIFReadL(psCtxt->pabyBlob, 1, XML_BUFSIZE, psCtxt->fp)); psCtxt->nBytesRead += nLen; psCtxt->bEOF = nLen < XML_BUFSIZE; const int eErr = - XML_Parse(psCtxt->hXMLParser, (const char *)psCtxt->pabyBlob, nLen, + XML_Parse(psCtxt->hXMLParser, + reinterpret_cast(psCtxt->pabyBlob), nLen, psCtxt->bEOF); if (eErr == XML_STATUS_ERROR) { - CPLError(CE_Failure, CPLE_AppDefined, - "XML parsing of OSM file failed : %s " - "at line %d, column %d", - XML_ErrorString(XML_GetErrorCode(psCtxt->hXMLParser)), - (int)XML_GetCurrentLineNumber(psCtxt->hXMLParser), - (int)XML_GetCurrentColumnNumber(psCtxt->hXMLParser)); + CPLError( + CE_Failure, CPLE_AppDefined, + "XML parsing of OSM file failed : %s " + "at line %d, column %d", + XML_ErrorString(XML_GetErrorCode(psCtxt->hXMLParser)), + static_cast(XML_GetCurrentLineNumber(psCtxt->hXMLParser)), + static_cast( + XML_GetCurrentColumnNumber(psCtxt->hXMLParser))); psCtxt->bStopParsing = true; } psCtxt->nWithoutEventCounter++; @@ -2649,7 +2667,7 @@ OSMContext *OSM_Open(const char *pszFilename, NotifyNodesFunc pfnNotifyNodes, bool bPBF = false; - if (strstr((const char *)abyHeader, "(abyHeader), "nBlobSizeAllocated = XML_BUFSIZE; psCtxt->nStrAllocated = 1024 * 1024; - psCtxt->pszStrBuf = (char *)VSI_MALLOC_VERBOSE(psCtxt->nStrAllocated); + psCtxt->pszStrBuf = + static_cast(VSI_MALLOC_VERBOSE(psCtxt->nStrAllocated)); if (psCtxt->pszStrBuf) psCtxt->pszStrBuf[0] = '\0'; @@ -2726,23 +2745,23 @@ OSMContext *OSM_Open(const char *pszFilename, NotifyNodesFunc pfnNotifyNodes, psCtxt->bTryToFetchBounds = true; psCtxt->nNodesAllocated = 1; - psCtxt->pasNodes = (OSMNode *)VSI_MALLOC_VERBOSE( - sizeof(OSMNode) * psCtxt->nNodesAllocated); + psCtxt->pasNodes = static_cast( + VSI_MALLOC_VERBOSE(sizeof(OSMNode) * psCtxt->nNodesAllocated)); psCtxt->nTagsAllocated = 256; - psCtxt->pasTags = (OSMTag *)VSI_MALLOC_VERBOSE(sizeof(OSMTag) * - psCtxt->nTagsAllocated); + psCtxt->pasTags = static_cast( + VSI_MALLOC_VERBOSE(sizeof(OSMTag) * psCtxt->nTagsAllocated)); /* 300 is the recommended value, but there are files with more than 2000 * so we should be able */ /* to realloc over that value */ psCtxt->nMembersAllocated = 2000; - psCtxt->pasMembers = (OSMMember *)VSI_MALLOC_VERBOSE( - sizeof(OSMMember) * psCtxt->nMembersAllocated); + psCtxt->pasMembers = static_cast( + VSI_MALLOC_VERBOSE(sizeof(OSMMember) * psCtxt->nMembersAllocated)); psCtxt->nNodeRefsAllocated = 10000; - psCtxt->panNodeRefs = (GIntBig *)VSI_MALLOC_VERBOSE( - sizeof(GIntBig) * psCtxt->nNodeRefsAllocated); + psCtxt->panNodeRefs = static_cast( + VSI_MALLOC_VERBOSE(sizeof(GIntBig) * psCtxt->nNodeRefsAllocated)); if (psCtxt->pszStrBuf == nullptr || psCtxt->pasNodes == nullptr || psCtxt->pasTags == nullptr || psCtxt->pasMembers == nullptr || @@ -2754,14 +2773,15 @@ OSMContext *OSM_Open(const char *pszFilename, NotifyNodesFunc pfnNotifyNodes, } #endif - psCtxt->pabyBlob = (GByte *)VSI_MALLOC_VERBOSE(psCtxt->nBlobSizeAllocated); + psCtxt->pabyBlob = + static_cast(VSI_MALLOC_VERBOSE(psCtxt->nBlobSizeAllocated)); if (psCtxt->pabyBlob == nullptr) { OSM_Close(psCtxt); return nullptr; } - psCtxt->pabyBlobHeader = - (GByte *)VSI_MALLOC_VERBOSE(MAX_BLOB_HEADER_SIZE + EXTRA_BYTES); + psCtxt->pabyBlobHeader = static_cast( + VSI_MALLOC_VERBOSE(MAX_BLOB_HEADER_SIZE + EXTRA_BYTES)); if (psCtxt->pabyBlobHeader == nullptr) { OSM_Close(psCtxt); From 62fc992460537a55a4aa512c7fcc18f7fe522417 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 23 Jun 2024 23:50:40 +0200 Subject: [PATCH 10/72] Fix number of new Coverity Scan warnings, likely due to new version of the tool --- alg/gdalchecksum.cpp | 7 +- alg/gdalrasterize.cpp | 7 +- alg/gdalwarpkernel.cpp | 5 +- alg/internal_libqhull/poly_r.c | 17 ++- apps/argparse/argparse.hpp | 6 +- apps/gdalargumentparser.cpp | 8 +- autotest/cpp/test_gdal.cpp | 2 +- frmts/grib/degrib/degrib/clock.c | 1 + frmts/gsg/gsbgdataset.cpp | 32 ++--- frmts/gtiff/gt_overview.cpp | 7 +- frmts/gtiff/gtiffdataset_read.cpp | 3 +- frmts/gtiff/gtiffdataset_write.cpp | 7 +- frmts/gtiff/gtiffrasterband_read.cpp | 4 + frmts/mrf/marfa.h | 6 +- frmts/netcdf/netcdfdataset.cpp | 122 +++++++++++------- frmts/northwood/northwood.cpp | 2 + frmts/pdf/pdfdataset.cpp | 16 ++- gcore/gdaljp2structure.cpp | 4 +- gcore/gdalmultidim.cpp | 1 + ogr/ogr_p.h | 3 +- ogr/ogrlinestring.cpp | 28 ++-- .../arrow_common/ograrrowlayer.hpp | 2 +- ogr/ogrsf_frmts/geojson/libjson/json_util.c | 4 +- ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp | 2 +- .../gpkg/ogrgeopackagetablelayer.cpp | 11 +- ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp | 45 ++++--- ogr/ogrsf_frmts/ntf/ntfrecord.cpp | 1 + ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp | 20 ++- .../openfilegdb/filegdbtable_freelist.cpp | 3 + ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp | 1 + ogr/ogrsf_frmts/pmtiles/pmtiles/pmtiles.hpp | 3 + ogr/ogrsf_frmts/shape/ogrshape.h | 1 + ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp | 7 +- ogr/ogrsf_frmts/vfk/vfkreader.cpp | 4 +- ogr/ogrsf_frmts/vfk/vfkreader.h | 2 +- ogr/ogrsf_frmts/vfk/vfkreaderp.h | 4 +- ogr/ogrsf_frmts/vfk/vfkreadersqlite.cpp | 6 +- ogr/ogrsf_frmts/vrt/ogrvrtdriver.cpp | 11 +- ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp | 1 + port/cpl_md5.cpp | 1 + port/cpl_spawn.cpp | 8 +- port/cpl_vsi_mem.cpp | 101 +++++++++------ port/cpl_vsil_s3.cpp | 1 + port/cpl_vsisimple.cpp | 6 +- port/cpl_worker_thread_pool.cpp | 39 +++++- port/cpl_worker_thread_pool.h | 7 +- 46 files changed, 374 insertions(+), 205 deletions(-) diff --git a/alg/gdalchecksum.cpp b/alg/gdalchecksum.cpp index e0f0dcafacf8..64f6158433b8 100644 --- a/alg/gdalchecksum.cpp +++ b/alg/gdalchecksum.cpp @@ -80,12 +80,9 @@ int CPL_STDCALL GDALChecksumImage(GDALRasterBandH hBand, int nXOff, int nYOff, const auto IntFromDouble = [](double dfVal) { int nVal; - if (CPLIsNan(dfVal) || CPLIsInf(dfVal)) + if (!std::isfinite(dfVal)) { - // Most compilers seem to cast NaN or Inf to 0x80000000. - // but VC7 is an exception. So we force the result - // of such a cast. - nVal = 0x80000000; + nVal = INT_MIN; } else { diff --git a/alg/gdalrasterize.cpp b/alg/gdalrasterize.cpp index 16be0bac7c55..6aa6efc998e3 100644 --- a/alg/gdalrasterize.cpp +++ b/alg/gdalrasterize.cpp @@ -1559,11 +1559,8 @@ CPLErr GDALRasterizeLayers(GDALDatasetH hDS, int nBandCount, int *panBandList, if (!(pszYChunkSize && ((nYChunkSize = atoi(pszYChunkSize))) != 0)) { const GIntBig nYChunkSize64 = GDALGetCacheMax64() / nScanlineBytes; - const int knIntMax = std::numeric_limits::max(); - if (nYChunkSize64 > knIntMax) - nYChunkSize = knIntMax; - else - nYChunkSize = static_cast(nYChunkSize64); + nYChunkSize = static_cast( + std::min(nYChunkSize64, std::numeric_limits::max())); } if (nYChunkSize < 1) diff --git a/alg/gdalwarpkernel.cpp b/alg/gdalwarpkernel.cpp index 98b1555c4a1b..51361a45d12c 100644 --- a/alg/gdalwarpkernel.cpp +++ b/alg/gdalwarpkernel.cpp @@ -513,6 +513,7 @@ static CPLErr GWKRun(GDALWarpKernel *poWK, const char *pszFuncName, job.pfnFunc = pfnFunc; } + bool bStopFlag; { std::unique_lock lock(psThreadData->mutex); @@ -550,6 +551,8 @@ static CPLErr GWKRun(GDALWarpKernel *poWK, const char *pszFuncName, } } } + + bStopFlag = psThreadData->stopFlag; } /* -------------------------------------------------------------------- */ @@ -557,7 +560,7 @@ static CPLErr GWKRun(GDALWarpKernel *poWK, const char *pszFuncName, /* -------------------------------------------------------------------- */ psThreadData->poJobQueue->WaitCompletion(); - return psThreadData->stopFlag ? CE_Failure : CE_None; + return bStopFlag ? CE_Failure : CE_None; } /************************************************************************/ diff --git a/alg/internal_libqhull/poly_r.c b/alg/internal_libqhull/poly_r.c index c68596a12bf9..b7e09b54fc8d 100644 --- a/alg/internal_libqhull/poly_r.c +++ b/alg/internal_libqhull/poly_r.c @@ -1137,10 +1137,13 @@ ridgeT *qh_newridge(qhT *qh) { qh_memalloc_(qh, (int)sizeof(ridgeT), freelistp, ridge, ridgeT); memset((char *)ridge, (size_t)0, sizeof(ridgeT)); zinc_(Ztotridges); + ridge->id= qh->ridge_id; if (qh->ridge_id == UINT_MAX) { qh_fprintf(qh, qh->ferr, 7074, "qhull warning: more than 2^32 ridges. Qhull results are OK. Since the ridge ID wraps around to 0, two ridges may have the same identifier.\n"); + qh->ridge_id = 0; + } else { + qh->ridge_id++; } - ridge->id= qh->ridge_id++; trace4((qh, qh->ferr, 4056, "qh_newridge: created ridge r%d\n", ridge->id)); return(ridge); } /* newridge */ @@ -1176,10 +1179,14 @@ int qh_pointid(qhT *qh, pointT *point) { offset= (ptr_intT)(point - qh->first_point); /* coverity[divide_arg] */ id= offset / qh->hull_dim; - }else if ((id= qh_setindex(qh->other_points, point)) != -1) - id += qh->num_points; - else - return qh_IDunknown; + } else { + id = qh_setindex(qh->other_points, point); + if (id >= 0) { + id += qh->num_points; + } else { + return qh_IDunknown; + } + } return (int)id; } /* pointid */ diff --git a/apps/argparse/argparse.hpp b/apps/argparse/argparse.hpp index a29c120e00ab..7b589e7c79c8 100644 --- a/apps/argparse/argparse.hpp +++ b/apps/argparse/argparse.hpp @@ -2041,8 +2041,10 @@ class ArgumentParser { } stream << std::setw(2) << " "; - stream << std::setw(static_cast(longest_arg_length - 2)) - << command; + if (longest_arg_length >= 2) { + stream << std::setw(static_cast(longest_arg_length - 2)) + << command; + } stream << " " << subparser->get().m_description << "\n"; } } diff --git a/apps/gdalargumentparser.cpp b/apps/gdalargumentparser.cpp index c70a939be599..f9992cea7317 100644 --- a/apps/gdalargumentparser.cpp +++ b/apps/gdalargumentparser.cpp @@ -50,10 +50,10 @@ GDALArgumentParser::GDALArgumentParser(const std::string &program_name, add_argument("-h", "--help") .flag() .action( - [this, program_name](const auto &) + [this](const auto &) { std::cout << usage() << std::endl << std::endl; - std::cout << _("Note: ") << program_name + std::cout << _("Note: ") << m_program_name << _(" --long-usage for full help.") << std::endl; std::exit(0); }) @@ -77,11 +77,11 @@ GDALArgumentParser::GDALArgumentParser(const std::string &program_name, .flag() .hidden() .action( - [program_name](const auto &) + [this](const auto &) { printf("%s was compiled against GDAL %s and " "is running against GDAL %s\n", - program_name.c_str(), GDAL_RELEASE_NAME, + m_program_name.c_str(), GDAL_RELEASE_NAME, GDALVersionInfo("RELEASE_NAME")); std::exit(0); }) diff --git a/autotest/cpp/test_gdal.cpp b/autotest/cpp/test_gdal.cpp index 459105bdb9ec..8e8af2c85dd3 100644 --- a/autotest/cpp/test_gdal.cpp +++ b/autotest/cpp/test_gdal.cpp @@ -3413,7 +3413,7 @@ TEST_F(test_gdal, gtiff_ReadCompressedData) CE_None); EXPECT_EQ(nGotSize, nNeededSize); EXPECT_NE(pBuffer, nullptr); - if (pBuffer != nullptr && nGotSize == nNeededSize) + if (pBuffer != nullptr && nGotSize == nNeededSize && nNeededSize >= 2) { const GByte *pabyBuffer = static_cast(pBuffer); EXPECT_EQ(pabyBuffer[0], 0xFF); diff --git a/frmts/grib/degrib/degrib/clock.c b/frmts/grib/degrib/degrib/clock.c index 0f1ea08fcdab..a38543c8df45 100644 --- a/frmts/grib/degrib/degrib/clock.c +++ b/frmts/grib/degrib/degrib/clock.c @@ -729,6 +729,7 @@ sChar Clock_GetTimeZone () #else const struct tm *gmTimePtr = gmtime (&ansTime); #endif + timeZone = 0; if (gmTimePtr) { timeZone = gmTimePtr->tm_hour; diff --git a/frmts/gsg/gsbgdataset.cpp b/frmts/gsg/gsbgdataset.cpp index 2a234236fbb8..41b9d3d25a54 100644 --- a/frmts/gsg/gsbgdataset.cpp +++ b/frmts/gsg/gsbgdataset.cpp @@ -53,7 +53,7 @@ class GSBGDataset final : public GDALPamDataset static const float fNODATA_VALUE; static const size_t nHEADER_SIZE; - static CPLErr WriteHeader(VSILFILE *fp, GInt16 nXSize, GInt16 nYSize, + static CPLErr WriteHeader(VSILFILE *fp, int nXSize, int nYSize, double dfMinX, double dfMaxX, double dfMinY, double dfMaxY, double dfMinZ, double dfMaxZ); @@ -417,9 +417,9 @@ CPLErr GSBGRasterBand::IWriteBlock(int nBlockXOff, int nBlockYOff, void *pImage) if (bHeaderNeedsUpdate && dfMaxZ > dfMinZ) { - CPLErr eErr = poGDS->WriteHeader(poGDS->fp, (GInt16)nRasterXSize, - (GInt16)nRasterYSize, dfMinX, dfMaxX, - dfMinY, dfMaxY, dfMinZ, dfMaxZ); + CPLErr eErr = + poGDS->WriteHeader(poGDS->fp, nRasterXSize, nRasterYSize, dfMinX, + dfMaxX, dfMinY, dfMaxY, dfMinZ, dfMaxZ); return eErr; } @@ -686,9 +686,9 @@ CPLErr GSBGDataset::SetGeoTransform(double *padfGeoTransform) padfGeoTransform[5] * (nRasterYSize - 0.5) + padfGeoTransform[3]; double dfMaxY = padfGeoTransform[3] + padfGeoTransform[5] / 2; - CPLErr eErr = WriteHeader(fp, (GInt16)poGRB->nRasterXSize, - (GInt16)poGRB->nRasterYSize, dfMinX, dfMaxX, - dfMinY, dfMaxY, poGRB->dfMinZ, poGRB->dfMaxZ); + CPLErr eErr = + WriteHeader(fp, poGRB->nRasterXSize, poGRB->nRasterYSize, dfMinX, + dfMaxX, dfMinY, dfMaxY, poGRB->dfMinZ, poGRB->dfMaxZ); if (eErr == CE_None) { @@ -705,7 +705,7 @@ CPLErr GSBGDataset::SetGeoTransform(double *padfGeoTransform) /* WriteHeader() */ /************************************************************************/ -CPLErr GSBGDataset::WriteHeader(VSILFILE *fp, GInt16 nXSize, GInt16 nYSize, +CPLErr GSBGDataset::WriteHeader(VSILFILE *fp, int nXSize, int nYSize, double dfMinX, double dfMaxX, double dfMinY, double dfMaxY, double dfMinZ, double dfMaxZ) @@ -724,7 +724,8 @@ CPLErr GSBGDataset::WriteHeader(VSILFILE *fp, GInt16 nXSize, GInt16 nYSize, return CE_Failure; } - GInt16 nTemp = CPL_LSBWORD16(nXSize); + assert(nXSize >= 0 && nXSize <= std::numeric_limits::max()); + GInt16 nTemp = CPL_LSBWORD16(static_cast(nXSize)); if (VSIFWriteL((void *)&nTemp, 2, 1, fp) != 1) { CPLError(CE_Failure, CPLE_FileIO, @@ -732,7 +733,8 @@ CPLErr GSBGDataset::WriteHeader(VSILFILE *fp, GInt16 nXSize, GInt16 nYSize, return CE_Failure; } - nTemp = CPL_LSBWORD16(nYSize); + assert(nYSize >= 0 && nYSize <= std::numeric_limits::max()); + nTemp = CPL_LSBWORD16(static_cast(nYSize)); if (VSIFWriteL((void *)&nTemp, 2, 1, fp) != 1) { CPLError(CE_Failure, CPLE_FileIO, @@ -847,8 +849,8 @@ GDALDataset *GSBGDataset::Create(const char *pszFilename, int nXSize, return nullptr; } - CPLErr eErr = WriteHeader(fp, (GInt16)nXSize, (GInt16)nYSize, 0.0, nXSize, - 0.0, nYSize, 0.0, 0.0); + CPLErr eErr = + WriteHeader(fp, nXSize, nYSize, 0.0, nXSize, 0.0, nYSize, 0.0, 0.0); if (eErr != CE_None) { VSIFCloseL(fp); @@ -941,8 +943,8 @@ GDALDataset *GSBGDataset::CreateCopy(const char *pszFilename, return nullptr; } - GInt16 nXSize = (GInt16)poSrcBand->GetXSize(); - GInt16 nYSize = (GInt16)poSrcBand->GetYSize(); + const int nXSize = poSrcBand->GetXSize(); + const int nYSize = poSrcBand->GetYSize(); double adfGeoTransform[6]; poSrcDS->GetGeoTransform(adfGeoTransform); @@ -974,7 +976,7 @@ GDALDataset *GSBGDataset::CreateCopy(const char *pszFilename, float fSrcNoDataValue = (float)poSrcBand->GetNoDataValue(&bSrcHasNDValue); double dfMinZ = std::numeric_limits::max(); double dfMaxZ = std::numeric_limits::lowest(); - for (GInt16 iRow = nYSize - 1; iRow >= 0; iRow--) + for (int iRow = nYSize - 1; iRow >= 0; iRow--) { eErr = poSrcBand->RasterIO(GF_Read, 0, iRow, nXSize, 1, pfData, nXSize, 1, GDT_Float32, 0, 0, nullptr); diff --git a/frmts/gtiff/gt_overview.cpp b/frmts/gtiff/gt_overview.cpp index 38ca89e434c2..6405f32261ad 100644 --- a/frmts/gtiff/gt_overview.cpp +++ b/frmts/gtiff/gt_overview.cpp @@ -188,8 +188,11 @@ toff_t GTIFFWriteDirectory(TIFF *hTIFF, int nSubfileType, int nXSize, } TIFFWriteDirectory(hTIFF); - TIFFSetDirectory(hTIFF, - static_cast(TIFFNumberOfDirectories(hTIFF) - 1)); + const tdir_t nNumberOfDirs = TIFFNumberOfDirectories(hTIFF); + if (nNumberOfDirs > 0) // always true, but to please Coverity + { + TIFFSetDirectory(hTIFF, static_cast(nNumberOfDirs - 1)); + } const toff_t nOffset = TIFFCurrentDirOffset(hTIFF); diff --git a/frmts/gtiff/gtiffdataset_read.cpp b/frmts/gtiff/gtiffdataset_read.cpp index 9e602fa2619d..59da8e19c738 100644 --- a/frmts/gtiff/gtiffdataset_read.cpp +++ b/frmts/gtiff/gtiffdataset_read.cpp @@ -106,7 +106,8 @@ int GTiffDataset::GetJPEGOverviewCount() GByte abyFFD8[] = {0xFF, 0xD8}; if (TIFFGetField(m_hTIFF, TIFFTAG_JPEGTABLES, &nJPEGTableSize, &pJPEGTable)) { - if (pJPEGTable == nullptr || nJPEGTableSize > INT_MAX || + if (pJPEGTable == nullptr || nJPEGTableSize < 2 || + nJPEGTableSize > INT_MAX || static_cast(pJPEGTable)[nJPEGTableSize - 1] != 0xD9) { m_nJPEGOverviewCount = 0; diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 062ba5cd1bfc..601ce94e5685 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -2117,8 +2117,11 @@ void GTiffDataset::Crystalize() } else { - TIFFSetDirectory( - m_hTIFF, static_cast(TIFFNumberOfDirectories(m_hTIFF) - 1)); + const tdir_t nNumberOfDirs = TIFFNumberOfDirectories(m_hTIFF); + if (nNumberOfDirs > 0) + { + TIFFSetDirectory(m_hTIFF, static_cast(nNumberOfDirs - 1)); + } } RestoreVolatileParameters(m_hTIFF); diff --git a/frmts/gtiff/gtiffrasterband_read.cpp b/frmts/gtiff/gtiffrasterband_read.cpp index b16d57b4436a..22b90e51e259 100644 --- a/frmts/gtiff/gtiffrasterband_read.cpp +++ b/frmts/gtiff/gtiffrasterband_read.cpp @@ -32,6 +32,7 @@ #include "gtiffjpegoverviewds.h" #include +#include #include #include #include @@ -509,6 +510,9 @@ CPLVirtualMem *GTiffRasterBand::GetVirtualMemAutoInternal(GDALRWFlag eRWFlag, CPLAssert(panByteCounts[0] == static_cast(nBlockSize)); // Now simulate the writing of other blocks. + assert(nBlocks > 0); + assert(static_cast(nBlockSize) < + std::numeric_limits::max() / nBlocks); const vsi_l_offset nDataSize = static_cast(nBlockSize) * nBlocks; if (VSIFTruncateL(fp, nBaseOffset + nDataSize) != 0) diff --git a/frmts/mrf/marfa.h b/frmts/mrf/marfa.h index 6e0cff3171d2..20c908fff4fc 100644 --- a/frmts/mrf/marfa.h +++ b/frmts/mrf/marfa.h @@ -966,11 +966,11 @@ class LERC_Band final : public MRFRasterBand protected: virtual CPLErr Decompress(buf_mgr &dst, buf_mgr &src) override; virtual CPLErr Compress(buf_mgr &dst, buf_mgr &src) override; - double precision; + double precision = 0; // L1 or L2 - int version; + int version = 0; // L2 version - int l2ver; + int l2ver = 0; // Build a MRF header for a single LERC tile static CPLXMLNode *GetMRFConfig(GDALOpenInfo *poOpenInfo); diff --git a/frmts/netcdf/netcdfdataset.cpp b/frmts/netcdf/netcdfdataset.cpp index 2960870dccf4..ae3dbf3f9e2e 100644 --- a/frmts/netcdf/netcdfdataset.cpp +++ b/frmts/netcdf/netcdfdataset.cpp @@ -7043,7 +7043,9 @@ bool netCDFDataset::CloneGrp(int nOldGrpId, int nNewGrpId, bool bIsNC4, int nDimCount = -1; int status = nc_inq_ndims(nOldGrpId, &nDimCount); NCDF_ERR(status); - int *panDimIds = static_cast(CPLMalloc(sizeof(int) * nDimCount)); + if (nDimCount < 0 || nDimCount > NC_MAX_DIMS) + return false; + int anDimIds[NC_MAX_DIMS]; int nUnlimiDimID = -1; status = nc_inq_unlimdim(nOldGrpId, &nUnlimiDimID); NCDF_ERR(status); @@ -7052,21 +7054,21 @@ bool netCDFDataset::CloneGrp(int nOldGrpId, int nNewGrpId, bool bIsNC4, // In NC4, the dimension ids of a group are not necessarily in // [0,nDimCount-1] range int nDimCount2 = -1; - status = nc_inq_dimids(nOldGrpId, &nDimCount2, panDimIds, FALSE); + status = nc_inq_dimids(nOldGrpId, &nDimCount2, anDimIds, FALSE); NCDF_ERR(status); CPLAssert(nDimCount == nDimCount2); } else { for (int i = 0; i < nDimCount; i++) - panDimIds[i] = i; + anDimIds[i] = i; } for (int i = 0; i < nDimCount; i++) { char szDimName[NC_MAX_NAME + 1]; szDimName[0] = 0; size_t nLen = 0; - const int nDimId = panDimIds[i]; + const int nDimId = anDimIds[i]; status = nc_inq_dim(nOldGrpId, nDimId, szDimName, &nLen); NCDF_ERR(status); if (NCDFIsUnlimitedDim(bIsNC4, nOldGrpId, nDimId)) @@ -7079,11 +7081,9 @@ bool netCDFDataset::CloneGrp(int nOldGrpId, int nNewGrpId, bool bIsNC4, CPLAssert(nDimId == nNewDimId); if (status != NC_NOERR) { - CPLFree(panDimIds); return false; } } - CPLFree(panDimIds); // Clone main attributes if (!CloneAttributes(nOldGrpId, nNewGrpId, NC_GLOBAL, NC_GLOBAL)) @@ -7108,7 +7108,6 @@ bool netCDFDataset::CloneGrp(int nOldGrpId, int nNewGrpId, bool bIsNC4, int nVarDimCount = -1; status = nc_inq_varndims(nOldGrpId, i, &nVarDimCount); NCDF_ERR(status); - int anDimIds[NC_MAX_DIMS]; status = nc_inq_vardimid(nOldGrpId, i, anDimIds); NCDF_ERR(status); int nNewVarId = -1; @@ -10379,7 +10378,7 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, NCDFSafeStrcat(&pszAttrValue, "{", &nAttrValueSize); double dfValue = 0.0; - size_t m; + size_t m = 0; char szTemp[256]; bool bSetDoubleFromStr = false; @@ -10398,10 +10397,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, CPLCalloc(nAttrLen, sizeof(signed char))); nc_get_att_schar(nCdfId, nVarId, pszAttrName, pscTemp); dfValue = static_cast(pscTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - snprintf(szTemp, sizeof(szTemp), "%d,", pscTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + snprintf(szTemp, sizeof(szTemp), "%d,", pscTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } snprintf(szTemp, sizeof(szTemp), "%d", pscTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10414,10 +10416,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, static_cast(CPLCalloc(nAttrLen, sizeof(short))); nc_get_att_short(nCdfId, nVarId, pszAttrName, psTemp); dfValue = static_cast(psTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - snprintf(szTemp, sizeof(szTemp), "%d,", psTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + snprintf(szTemp, sizeof(szTemp), "%d,", psTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } snprintf(szTemp, sizeof(szTemp), "%d", psTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10429,10 +10434,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, int *pnTemp = static_cast(CPLCalloc(nAttrLen, sizeof(int))); nc_get_att_int(nCdfId, nVarId, pszAttrName, pnTemp); dfValue = static_cast(pnTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - snprintf(szTemp, sizeof(szTemp), "%d,", pnTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + snprintf(szTemp, sizeof(szTemp), "%d,", pnTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } snprintf(szTemp, sizeof(szTemp), "%d", pnTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10445,10 +10453,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, static_cast(CPLCalloc(nAttrLen, sizeof(float))); nc_get_att_float(nCdfId, nVarId, pszAttrName, pfTemp); dfValue = static_cast(pfTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - CPLsnprintf(szTemp, sizeof(szTemp), "%.8g,", pfTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + CPLsnprintf(szTemp, sizeof(szTemp), "%.8g,", pfTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } CPLsnprintf(szTemp, sizeof(szTemp), "%.8g", pfTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10461,10 +10472,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, static_cast(CPLCalloc(nAttrLen, sizeof(double))); nc_get_att_double(nCdfId, nVarId, pszAttrName, pdfTemp); dfValue = pdfTemp[0]; - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - CPLsnprintf(szTemp, sizeof(szTemp), "%.16g,", pdfTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + CPLsnprintf(szTemp, sizeof(szTemp), "%.16g,", pdfTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } CPLsnprintf(szTemp, sizeof(szTemp), "%.16g", pdfTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10478,12 +10492,15 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, nc_get_att_string(nCdfId, nVarId, pszAttrName, ppszTemp); bSetDoubleFromStr = true; dfValue = 0.0; - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - NCDFSafeStrcat(&pszAttrValue, - ppszTemp[m] ? ppszTemp[m] : "{NULL}", - &nAttrValueSize); - NCDFSafeStrcat(&pszAttrValue, ",", &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + NCDFSafeStrcat(&pszAttrValue, + ppszTemp[m] ? ppszTemp[m] : "{NULL}", + &nAttrValueSize); + NCDFSafeStrcat(&pszAttrValue, ",", &nAttrValueSize); + } } NCDFSafeStrcat(&pszAttrValue, ppszTemp[m] ? ppszTemp[m] : "{NULL}", &nAttrValueSize); @@ -10497,10 +10514,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, CPLCalloc(nAttrLen, sizeof(unsigned char))); nc_get_att_uchar(nCdfId, nVarId, pszAttrName, pucTemp); dfValue = static_cast(pucTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - CPLsnprintf(szTemp, sizeof(szTemp), "%u,", pucTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + CPLsnprintf(szTemp, sizeof(szTemp), "%u,", pucTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } CPLsnprintf(szTemp, sizeof(szTemp), "%u", pucTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10514,10 +10534,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, CPLCalloc(nAttrLen, sizeof(unsigned short))); nc_get_att_ushort(nCdfId, nVarId, pszAttrName, pusTemp); dfValue = static_cast(pusTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - CPLsnprintf(szTemp, sizeof(szTemp), "%u,", pusTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + CPLsnprintf(szTemp, sizeof(szTemp), "%u,", pusTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } CPLsnprintf(szTemp, sizeof(szTemp), "%u", pusTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10530,10 +10553,13 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, static_cast(CPLCalloc(nAttrLen, sizeof(int))); nc_get_att_uint(nCdfId, nVarId, pszAttrName, punTemp); dfValue = static_cast(punTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - CPLsnprintf(szTemp, sizeof(szTemp), "%u,", punTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + CPLsnprintf(szTemp, sizeof(szTemp), "%u,", punTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } CPLsnprintf(szTemp, sizeof(szTemp), "%u", punTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10546,11 +10572,14 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, static_cast(CPLCalloc(nAttrLen, sizeof(GIntBig))); nc_get_att_longlong(nCdfId, nVarId, pszAttrName, panTemp); dfValue = static_cast(panTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - CPLsnprintf(szTemp, sizeof(szTemp), CPL_FRMT_GIB ",", - panTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + CPLsnprintf(szTemp, sizeof(szTemp), CPL_FRMT_GIB ",", + panTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } CPLsnprintf(szTemp, sizeof(szTemp), CPL_FRMT_GIB, panTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); @@ -10563,11 +10592,14 @@ static CPLErr NCDFGetAttr1(int nCdfId, int nVarId, const char *pszAttrName, static_cast(CPLCalloc(nAttrLen, sizeof(GUIntBig))); nc_get_att_ulonglong(nCdfId, nVarId, pszAttrName, panTemp); dfValue = static_cast(panTemp[0]); - for (m = 0; m < nAttrLen - 1; m++) + if (nAttrLen > 1) { - CPLsnprintf(szTemp, sizeof(szTemp), CPL_FRMT_GUIB ",", - panTemp[m]); - NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + for (m = 0; m < nAttrLen - 1; m++) + { + CPLsnprintf(szTemp, sizeof(szTemp), CPL_FRMT_GUIB ",", + panTemp[m]); + NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); + } } CPLsnprintf(szTemp, sizeof(szTemp), CPL_FRMT_GUIB, panTemp[m]); NCDFSafeStrcat(&pszAttrValue, szTemp, &nAttrValueSize); diff --git a/frmts/northwood/northwood.cpp b/frmts/northwood/northwood.cpp index 1917e154f2fd..a12d8a849a8e 100644 --- a/frmts/northwood/northwood.cpp +++ b/frmts/northwood/northwood.cpp @@ -32,6 +32,7 @@ #include "northwood.h" #include +#include #include #include @@ -79,6 +80,7 @@ int nwt_ParseHeader(NWT_GRID *pGrd, const unsigned char *nwtHeader) memcpy(&pGrd->dfMaxY, &nwtHeader[37], sizeof(pGrd->dfMaxY)); CPL_LSBPTR64(&pGrd->dfMaxY); + assert(pGrd->nXSide > 1); pGrd->dfStepSize = (pGrd->dfMaxX - pGrd->dfMinX) / (pGrd->nXSide - 1); /* dfTmp = (pGrd->dfMaxY - pGrd->dfMinY) / (pGrd->nYSide - 1); */ diff --git a/frmts/pdf/pdfdataset.cpp b/frmts/pdf/pdfdataset.cpp index 33bc89b0f49d..1f6b63961887 100644 --- a/frmts/pdf/pdfdataset.cpp +++ b/frmts/pdf/pdfdataset.cpp @@ -2480,7 +2480,7 @@ GDALPDFObject *PDFDataset::GetCatalog() return m_poCatalogObject; #ifdef HAVE_POPPLER - if (m_bUseLib.test(PDFLIB_POPPLER)) + if (m_bUseLib.test(PDFLIB_POPPLER) && m_poDocPoppler) { m_poCatalogObjectPoppler = std::make_unique(m_poDocPoppler->getXRef()->getCatalog()); @@ -2491,7 +2491,7 @@ GDALPDFObject *PDFDataset::GetCatalog() #endif #ifdef HAVE_PODOFO - if (m_bUseLib.test(PDFLIB_PODOFO)) + if (m_bUseLib.test(PDFLIB_PODOFO) && m_poDocPodofo) { int nCatalogNum = 0; int nCatalogGen = 0; @@ -2517,7 +2517,7 @@ GDALPDFObject *PDFDataset::GetCatalog() #endif #ifdef HAVE_PDFIUM - if (m_bUseLib.test(PDFLIB_PDFIUM)) + if (m_bUseLib.test(PDFLIB_PDFIUM) && m_poDocPdfium) { const CPDF_Dictionary *catalog = m_poDocPdfium->doc->GetRoot(); if (catalog) @@ -4957,11 +4957,11 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) return nullptr; } poPageObj = GDALPDFObjectPdfium::Build(pageObj); - if (poPageObj == nullptr) - return nullptr; } #endif // ~ HAVE_PDFIUM + if (poPageObj == nullptr) + return nullptr; GDALPDFDictionary *poPageDict = poPageObj->GetDictionary(); if (poPageDict == nullptr) { @@ -5032,7 +5032,9 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) if (pszDumpCatalog != nullptr) { GDALPDFDumper oDumper(pszFilename, pszDumpCatalog); - oDumper.Dump(poDS->GetCatalog()); + auto poCatalog = poDS->GetCatalog(); + if (poCatalog) + oDumper.Dump(poCatalog); } int nBandsGuessed = 0; @@ -5090,6 +5092,7 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) #ifdef HAVE_PDFIUM if (bUseLib.test(PDFLIB_PDFIUM)) { + CPLAssert(poPagePdfium); CFX_FloatRect rect = poPagePdfium->page->GetBBox(); dfX1 = rect.left; dfX2 = rect.right; @@ -5134,6 +5137,7 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) #ifdef HAVE_PDFIUM if (bUseLib.test(PDFLIB_PDFIUM)) { + CPLAssert(poPagePdfium); dfRotation = poPagePdfium->page->GetPageRotation() * 90; } #endif diff --git a/gcore/gdaljp2structure.cpp b/gcore/gdaljp2structure.cpp index b3ae58eba05f..e55f0787b93a 100644 --- a/gcore/gdaljp2structure.cpp +++ b/gcore/gdaljp2structure.cpp @@ -29,6 +29,7 @@ #include "cpl_port.h" #include "gdaljp2metadata.h" +#include #include #include #if HAVE_FCNTL_H @@ -2177,7 +2178,8 @@ static void GDALGetJPEG2000StructureInternal(CPLXMLNode *psParent, VSILFILE *fp, CPLXMLNode *psBinaryContent = CPLCreateXMLNode(nullptr, CXT_Element, "BinaryContent"); GByte *pabyBoxData = oBox.ReadBoxData(); - int nBoxLength = static_cast(nBoxDataLength); + const int nBoxLength = static_cast( + std::min(nBoxDataLength, INT_MAX / 2 - 1)); char *pszBinaryContent = static_cast(VSIMalloc(2 * nBoxLength + 1)); if (pabyBoxData && pszBinaryContent) diff --git a/gcore/gdalmultidim.cpp b/gcore/gdalmultidim.cpp index 2e380faaafd1..ab7ece1b95f1 100644 --- a/gcore/gdalmultidim.cpp +++ b/gcore/gdalmultidim.cpp @@ -3116,6 +3116,7 @@ bool GDALAbstractMDArray::ProcessPerChunk(const GUInt64 *arrayStartIdx, goto end; } + assert(dimIdx > 0); dimIdx--; // cppcheck-suppress negativeContainerIndex switch (stack[dimIdx].return_point) diff --git a/ogr/ogr_p.h b/ogr/ogr_p.h index 6fde155e9c1f..f4ee626786d1 100644 --- a/ogr/ogr_p.h +++ b/ogr/ogr_p.h @@ -276,7 +276,8 @@ inline uint64_t OGRRoundValueIEEE754(uint64_t nVal, int nBitsPrecision) return nVal; if (nNullifiedBits >= MANTISSA_SIZE) nNullifiedBits = MANTISSA_SIZE; - nVal &= std::numeric_limits::max() << nNullifiedBits; + nVal >>= nNullifiedBits; + nVal <<= nNullifiedBits; return nVal; } diff --git a/ogr/ogrlinestring.cpp b/ogr/ogrlinestring.cpp index 8d99615b4cec..9d59218fb9a4 100644 --- a/ogr/ogrlinestring.cpp +++ b/ogr/ogrlinestring.cpp @@ -2528,6 +2528,8 @@ void OGRSimpleCurve::segmentize(double dfMaxLength) const double dfSquareMaxLength = dfMaxLength * dfMaxLength; // First pass to compute new number of points + constexpr double REL_EPSILON_LENGTH_SQUARE = 1e-5; + constexpr double REL_EPSILON_ROUND = 1e-2; for (int i = 0; i < nPointCount; i++) { nNewPointCount++; @@ -2539,10 +2541,11 @@ void OGRSimpleCurve::segmentize(double dfMaxLength) const double dfX = paoPoints[i + 1].x - paoPoints[i].x; const double dfY = paoPoints[i + 1].y - paoPoints[i].y; const double dfSquareDist = dfX * dfX + dfY * dfY; - if (dfSquareDist - dfSquareMaxLength > 1e-5 * dfSquareMaxLength) + if (dfSquareDist - dfSquareMaxLength > + REL_EPSILON_LENGTH_SQUARE * dfSquareMaxLength) { - const double dfIntermediatePoints = - floor(sqrt(dfSquareDist / dfSquareMaxLength) - 1e-2); + const double dfIntermediatePoints = floor( + sqrt(dfSquareDist / dfSquareMaxLength) - REL_EPSILON_ROUND); const int nIntermediatePoints = DoubleToIntClamp(dfIntermediatePoints); @@ -2620,28 +2623,33 @@ void OGRSimpleCurve::segmentize(double dfMaxLength) const double dfX = paoPoints[i + 1].x - paoPoints[i].x; const double dfY = paoPoints[i + 1].y - paoPoints[i].y; const double dfSquareDist = dfX * dfX + dfY * dfY; - if (dfSquareDist - dfSquareMaxLength > 1e-5 * dfSquareMaxLength) + + // Must be kept in sync with the initial pass loop + if (dfSquareDist - dfSquareMaxLength > + REL_EPSILON_LENGTH_SQUARE * dfSquareMaxLength) { - const double dfIntermediatePoints = - floor(sqrt(dfSquareDist / dfSquareMaxLength) - 1e-2); + const double dfIntermediatePoints = floor( + sqrt(dfSquareDist / dfSquareMaxLength) - REL_EPSILON_ROUND); const int nIntermediatePoints = DoubleToIntClamp(dfIntermediatePoints); for (int j = 1; j <= nIntermediatePoints; j++) { - paoNewPoints[nNewPointCount + j - 1].x = + // coverity[overflow_const] + const int newI = nNewPointCount + j - 1; + paoNewPoints[newI].x = paoPoints[i].x + j * dfX / (nIntermediatePoints + 1); - paoNewPoints[nNewPointCount + j - 1].y = + paoNewPoints[newI].y = paoPoints[i].y + j * dfY / (nIntermediatePoints + 1); if (padfZ != nullptr) { // No interpolation. - padfNewZ[nNewPointCount + j - 1] = padfZ[i]; + padfNewZ[newI] = padfZ[i]; } if (padfM != nullptr) { // No interpolation. - padfNewM[nNewPointCount + j - 1] = padfM[i]; + padfNewM[newI] = padfM[i]; } } diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp index ac5d7d96b256..b364925ea9a9 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp @@ -2385,7 +2385,7 @@ inline OGRFeature *OGRArrowLayer::ReadFeature( arrow::LargeBinaryArray::offset_type out_length = 0; const uint8_t *data = castArray->GetValue(nIdxInBatch, &out_length); - if (out_length <= INT_MAX) + if (out_length <= INT_MAX - 1) { poFeature->SetField(i, static_cast(out_length), data); } diff --git a/ogr/ogrsf_frmts/geojson/libjson/json_util.c b/ogr/ogrsf_frmts/geojson/libjson/json_util.c index 0fcb0d4298d3..505834d83ce6 100644 --- a/ogr/ogrsf_frmts/geojson/libjson/json_util.c +++ b/ogr/ogrsf_frmts/geojson/libjson/json_util.c @@ -188,7 +188,7 @@ int json_object_to_fd(int fd, struct json_object *obj, int flags) } static int _json_object_to_fd(int fd, struct json_object *obj, int flags, const char *filename) { - int ret; + ssize_t ret; const char *json_str; unsigned int wpos, wsize; @@ -204,7 +204,7 @@ static int _json_object_to_fd(int fd, struct json_object *obj, int flags, const wpos = 0; while (wpos < wsize) { - if ((ret = (int)write(fd, json_str + wpos, wsize - wpos)) < 0) + if ((ret = write(fd, json_str + wpos, wsize - wpos)) < 0) { _json_c_set_last_err("json_object_to_file: error writing file %s: %s\n", filename, strerror(errno)); diff --git a/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp b/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp index 4635659c873c..dccaf32d23d1 100644 --- a/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp +++ b/ogr/ogrsf_frmts/gmlas/ogrgmlaslayer.cpp @@ -1128,7 +1128,7 @@ void OGRGMLASLayer::InsertNewField(int nInsertPos, if (strcmp(poFeature->GetFieldAsString(szLAYER_NAME), GetName()) == 0) { int nFieldIndex = poFeature->GetFieldAsInteger(szFIELD_INDEX); - if (nFieldIndex >= nInsertPos) + if (nFieldIndex >= nInsertPos && nFieldIndex < INT_MAX) { poFeature->SetField(szFIELD_INDEX, nFieldIndex + 1); CPL_IGNORE_RET_VAL( diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp index ad16578e40d5..be706dbebb4a 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp @@ -8231,7 +8231,10 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, } if (iField == psHelper->m_nFieldCount) + { + std::unique_lock oLock(psFillArrowArray->oMutex); psFillArrowArray->nCountRows++; + } return; error: @@ -8655,7 +8658,13 @@ int OGRGeoPackageTableLayer::GetNextArrowArray(struct ArrowArrayStream *stream, sizeof(struct ArrowArray)); memset(task->m_psArrowArray.get(), 0, sizeof(struct ArrowArray)); - if (task->m_bMemoryLimitReached) + const bool bMemoryLimitReached = [&task]() + { + std::unique_lock oLock(task->m_oMutex); + return task->m_bMemoryLimitReached; + }(); + + if (bMemoryLimitReached) { m_nIsCompatOfOptimizedGetNextArrowArray = false; stopThread(); diff --git a/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp b/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp index 3302b5d938d9..fadf1938b908 100644 --- a/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp +++ b/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp @@ -4265,24 +4265,35 @@ OGRErr OGRMVTWriterDataset::PreGenerateForTileReal( oBuffer.assign(static_cast(pCompressed), nCompressedSize); CPLFree(pCompressed); - std::unique_ptr> poLockGuard; + const auto InsertIntoDb = [&]() + { + m_nTempTiles++; + sqlite3_bind_int(m_hInsertStmt, 1, nZ); + sqlite3_bind_int(m_hInsertStmt, 2, nTileX); + sqlite3_bind_int(m_hInsertStmt, 3, nTileY); + sqlite3_bind_text(m_hInsertStmt, 4, osTargetName.c_str(), -1, + SQLITE_STATIC); + sqlite3_bind_int64(m_hInsertStmt, 5, nSerial); + sqlite3_bind_blob(m_hInsertStmt, 6, oBuffer.data(), + static_cast(oBuffer.size()), SQLITE_STATIC); + sqlite3_bind_int(m_hInsertStmt, 7, + static_cast(poGPBFeature->getType())); + sqlite3_bind_double(m_hInsertStmt, 8, dfAreaOrLength); + int rc = sqlite3_step(m_hInsertStmt); + sqlite3_reset(m_hInsertStmt); + return rc; + }; + + int rc; if (m_bThreadPoolOK) - poLockGuard = std::make_unique>(m_oDBMutex); - - m_nTempTiles++; - sqlite3_bind_int(m_hInsertStmt, 1, nZ); - sqlite3_bind_int(m_hInsertStmt, 2, nTileX); - sqlite3_bind_int(m_hInsertStmt, 3, nTileY); - sqlite3_bind_text(m_hInsertStmt, 4, osTargetName.c_str(), -1, - SQLITE_STATIC); - sqlite3_bind_int64(m_hInsertStmt, 5, nSerial); - sqlite3_bind_blob(m_hInsertStmt, 6, oBuffer.data(), - static_cast(oBuffer.size()), SQLITE_STATIC); - sqlite3_bind_int(m_hInsertStmt, 7, - static_cast(poGPBFeature->getType())); - sqlite3_bind_double(m_hInsertStmt, 8, dfAreaOrLength); - int rc = sqlite3_step(m_hInsertStmt); - sqlite3_reset(m_hInsertStmt); + { + std::lock_guard oLock(m_oDBMutex); + rc = InsertIntoDb(); + } + else + { + rc = InsertIntoDb(); + } if (!(rc == SQLITE_OK || rc == SQLITE_DONE)) { diff --git a/ogr/ogrsf_frmts/ntf/ntfrecord.cpp b/ogr/ogrsf_frmts/ntf/ntfrecord.cpp index 0bf6bebec369..1fe02fb7ed63 100644 --- a/ogr/ogrsf_frmts/ntf/ntfrecord.cpp +++ b/ogr/ogrsf_frmts/ntf/ntfrecord.cpp @@ -74,6 +74,7 @@ NTFRecord::NTFRecord(VSILFILE *fp) : nType(99), nLength(0), pszData(nullptr) if (pszData == nullptr) { nLength = nNewLength - 2; + // coverity[overflow_sink] pszData = static_cast(VSI_MALLOC_VERBOSE(nLength + 1)); if (pszData == nullptr) { diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp index 20d715ea3eef..18936e188acb 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp @@ -30,6 +30,7 @@ #include "filegdbtable.h" #include +#include #include #include #include @@ -3664,6 +3665,7 @@ OGRGeometry *FileGDBOGRGeometryConverterImpl::CreateCurveGeometry( returnError(); } const int nMaxSize = static_cast(nMaxSize64); + // coverity[overflow_sink] GByte *pabyExtShapeBuffer = static_cast(VSI_MALLOC_VERBOSE(nMaxSize)); if (pabyExtShapeBuffer == nullptr) @@ -3860,11 +3862,19 @@ FileGDBOGRGeometryConverterImpl::GetAsGeometry(const OGRField *psField) ReadVarUInt64NoCheck(pabyCur, m); const double dfMScale = SanitizeScale(poGeomField->GetMScale()); - const double dfM = - m == 0 - ? std::numeric_limits::quiet_NaN() - : (m - 1U) / dfMScale + poGeomField->GetMOrigin(); - return new OGRPoint(dfX, dfY, dfZ, dfM); + if (m == 0) + { + return new OGRPoint( + dfX, dfY, dfZ, + std::numeric_limits::quiet_NaN()); + } + else + { + assert(m >= 1U); + const double dfM = + (m - 1U) / dfMScale + poGeomField->GetMOrigin(); + return new OGRPoint(dfX, dfY, dfZ, dfM); + } } return new OGRPoint(dfX, dfY, dfZ); } diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_freelist.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_freelist.cpp index c666c1bbce44..56768752d458 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable_freelist.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable_freelist.cpp @@ -32,6 +32,7 @@ #include "filegdbtable_priv.h" #include +#include #include #include @@ -140,6 +141,7 @@ void FileGDBTable::AddEntryToFreelist(uint64_t nOffset, uint32_t nSize) VSIFCloseL(fp); return; } + assert(iSlot < 100); // Read the last page index of the identified slot uint32_t nPageIdx = @@ -284,6 +286,7 @@ uint64_t FileGDBTable::GetOffsetOfFreeAreaFromFreeList(uint32_t nSize) VSIFCloseL(fp); return OFFSET_MINUS_ONE; } + assert(iSlot < 100); // Read the last page index of the identified slot uint32_t nPageIdx = diff --git a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp index 53226246e321..895574462710 100644 --- a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp +++ b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp @@ -925,6 +925,7 @@ void OGROSMDataSource::LookupNodesSQLite() if (nToQuery > static_cast(LIMIT_IDS_PER_REQUEST)) nToQuery = static_cast(LIMIT_IDS_PER_REQUEST); + assert(nToQuery > 0); sqlite3_stmt *hStmt = m_pahSelectNodeStmt[nToQuery - 1]; for (unsigned int i = iCur; i < iCur + nToQuery; i++) { diff --git a/ogr/ogrsf_frmts/pmtiles/pmtiles/pmtiles.hpp b/ogr/ogrsf_frmts/pmtiles/pmtiles/pmtiles.hpp index 82ed8c56f5d0..c09f44d02020 100644 --- a/ogr/ogrsf_frmts/pmtiles/pmtiles/pmtiles.hpp +++ b/ogr/ogrsf_frmts/pmtiles/pmtiles/pmtiles.hpp @@ -576,6 +576,9 @@ inline std::tuple make_root_leaves(const std::fun if (root_bytes.length() < 16384 - 127) { return std::make_tuple(root_bytes, leaves_bytes, num_leaves); } + if (leaf_size > std::numeric_limits::max() / 2) { + return std::make_tuple(compressed, "", 0); + } leaf_size *= 2; } } diff --git a/ogr/ogrsf_frmts/shape/ogrshape.h b/ogr/ogrsf_frmts/shape/ogrshape.h index 471471f5f1a7..b907abff3bab 100644 --- a/ogr/ogrsf_frmts/shape/ogrshape.h +++ b/ogr/ogrsf_frmts/shape/ogrshape.h @@ -339,6 +339,7 @@ class OGRShapeDataSource final : public OGRDataSource VSILFILE *m_psLockFile = nullptr; CPLJoinableThread *m_hRefreshLockFileThread = nullptr; bool m_bExitRefreshLockFileThread = false; + bool m_bRefreshLockFileThreadStarted = false; double m_dfRefreshLockDelay = 0; std::vector GetLayerNames() const; diff --git a/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp b/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp index 8b8d432e1373..69e45b7d4b50 100644 --- a/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp +++ b/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp @@ -1403,6 +1403,7 @@ void OGRShapeDataSource::RefreshLockFile(void *_self) OGRShapeDataSource *self = static_cast(_self); CPLAssert(self->m_psLockFile); CPLAcquireMutex(self->m_poRefreshLockFileMutex, 1000); + self->m_bRefreshLockFileThreadStarted = true; CPLCondSignal(self->m_poRefreshLockFileCond); unsigned int nInc = 0; while (!(self->m_bExitRefreshLockFileThread)) @@ -1500,6 +1501,7 @@ bool OGRShapeDataSource::UncompressIfNeeded() } m_psLockFile = f; m_bExitRefreshLockFileThread = false; + m_bRefreshLockFileThreadStarted = false; // Config option mostly for testing purposes // coverity[tainted_data] m_dfRefreshLockDelay = CPLAtof(CPLGetConfigOption( @@ -1516,7 +1518,10 @@ bool OGRShapeDataSource::UncompressIfNeeded() else { CPLAcquireMutex(m_poRefreshLockFileMutex, 1000); - CPLCondWait(m_poRefreshLockFileCond, m_poRefreshLockFileMutex); + while (!m_bRefreshLockFileThreadStarted) + { + CPLCondWait(m_poRefreshLockFileCond, m_poRefreshLockFileMutex); + } CPLReleaseMutex(m_poRefreshLockFileMutex); } } diff --git a/ogr/ogrsf_frmts/vfk/vfkreader.cpp b/ogr/ogrsf_frmts/vfk/vfkreader.cpp index 16fbdc85e0ff..05f61e78ce08 100644 --- a/ogr/ogrsf_frmts/vfk/vfkreader.cpp +++ b/ogr/ogrsf_frmts/vfk/vfkreader.cpp @@ -314,7 +314,7 @@ int VFKReader::ReadDataBlocks(bool bSuppressGeometry) \return number of data records or -1 on error */ -int VFKReader::ReadDataRecords(IVFKDataBlock *poDataBlock) +int64_t VFKReader::ReadDataRecords(IVFKDataBlock *poDataBlock) { const char *pszName = nullptr; IVFKDataBlock *poDataBlockCurrent = nullptr; @@ -342,7 +342,7 @@ int VFKReader::ReadDataRecords(IVFKDataBlock *poDataBlock) int iLine = 0; int nSkipped = 0; int nDupl = 0; - int nRecords = 0; + int64_t nRecords = 0; bool bInHeader = true; CPLString osBlockNameLast; char *pszLine = nullptr; diff --git a/ogr/ogrsf_frmts/vfk/vfkreader.h b/ogr/ogrsf_frmts/vfk/vfkreader.h index 6d3898350017..7ce3fbfe2399 100644 --- a/ogr/ogrsf_frmts/vfk/vfkreader.h +++ b/ogr/ogrsf_frmts/vfk/vfkreader.h @@ -470,7 +470,7 @@ class IVFKReader virtual bool IsValid() const = 0; virtual bool HasFileField() const = 0; virtual int ReadDataBlocks(bool = false) = 0; - virtual int ReadDataRecords(IVFKDataBlock * = nullptr) = 0; + virtual int64_t ReadDataRecords(IVFKDataBlock * = nullptr) = 0; virtual int LoadGeometry() = 0; virtual int GetDataBlockCount() const = 0; diff --git a/ogr/ogrsf_frmts/vfk/vfkreaderp.h b/ogr/ogrsf_frmts/vfk/vfkreaderp.h index 735ecece239c..8891158fa392 100644 --- a/ogr/ogrsf_frmts/vfk/vfkreaderp.h +++ b/ogr/ogrsf_frmts/vfk/vfkreaderp.h @@ -108,7 +108,7 @@ class VFKReader : public IVFKReader } int ReadDataBlocks(bool = false) override; - int ReadDataRecords(IVFKDataBlock * = nullptr) override; + int64_t ReadDataRecords(IVFKDataBlock * = nullptr) override; int LoadGeometry() override; int GetDataBlockCount() const override @@ -166,7 +166,7 @@ class VFKReaderSQLite : public VFKReader } int ReadDataBlocks(bool = false) override; - int ReadDataRecords(IVFKDataBlock * = nullptr) override; + int64_t ReadDataRecords(IVFKDataBlock * = nullptr) override; sqlite3_stmt *PrepareStatement(const char *); OGRErr ExecuteSQL(const char *, CPLErr = CE_Failure); diff --git a/ogr/ogrsf_frmts/vfk/vfkreadersqlite.cpp b/ogr/ogrsf_frmts/vfk/vfkreadersqlite.cpp index 84c71e5326a5..6a49db943a5e 100644 --- a/ogr/ogrsf_frmts/vfk/vfkreadersqlite.cpp +++ b/ogr/ogrsf_frmts/vfk/vfkreadersqlite.cpp @@ -338,13 +338,13 @@ int VFKReaderSQLite::ReadDataBlocks(bool bSuppressGeometry) \return number of data records or -1 on error */ -int VFKReaderSQLite::ReadDataRecords(IVFKDataBlock *poDataBlock) +int64_t VFKReaderSQLite::ReadDataRecords(IVFKDataBlock *poDataBlock) { CPLString osSQL; IVFKDataBlock *poDataBlockCurrent = nullptr; sqlite3_stmt *hStmt = nullptr; const char *pszName = nullptr; - int nDataRecords = 0; + int64_t nDataRecords = 0; bool bReadVfk = !m_bDbSource; bool bReadDb = false; @@ -360,7 +360,7 @@ int VFKReaderSQLite::ReadDataRecords(IVFKDataBlock *poDataBlock) hStmt = PrepareStatement(osSQL.c_str()); if (ExecuteSQL(hStmt) == OGRERR_NONE) { - nDataRecords = sqlite3_column_int(hStmt, 0); + nDataRecords = sqlite3_column_int64(hStmt, 0); if (nDataRecords > 0) bReadDb = true; /* -> read from DB */ else diff --git a/ogr/ogrsf_frmts/vrt/ogrvrtdriver.cpp b/ogr/ogrsf_frmts/vrt/ogrvrtdriver.cpp index 65bfa072e3a1..f86a32a817e8 100644 --- a/ogr/ogrsf_frmts/vrt/ogrvrtdriver.cpp +++ b/ogr/ogrsf_frmts/vrt/ogrvrtdriver.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -130,9 +131,14 @@ static GDALDataset *OGRVRTDriverOpen(GDALOpenInfo *poOpenInfo) "configuration option"); return nullptr; } + if (static_cast(sStatBuf.st_size) > + std::numeric_limits::max() - 1) + { + return nullptr; + } // It is the right file, now load the full XML definition. - const int nLen = static_cast(sStatBuf.st_size); + const size_t nLen = static_cast(sStatBuf.st_size); pszXML = static_cast(VSI_MALLOC_VERBOSE(nLen + 1)); if (pszXML == nullptr) @@ -140,8 +146,7 @@ static GDALDataset *OGRVRTDriverOpen(GDALOpenInfo *poOpenInfo) pszXML[nLen] = '\0'; VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET); - if (static_cast(VSIFReadL(pszXML, 1, nLen, poOpenInfo->fpL)) != - nLen) + if (VSIFReadL(pszXML, 1, nLen, poOpenInfo->fpL) != nLen) { CPLFree(pszXML); return nullptr; diff --git a/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp b/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp index 3bb8ef68f0b2..cbfcec134998 100644 --- a/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp +++ b/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp @@ -1376,6 +1376,7 @@ int OGRWFSDataSource::Open(const char *pszFilename, int bUpdateIn, OGRLayer::GetSupportedSRSListRetType apoSupportedCRSList; if (psOtherSRS) { + if (pszDefaultSRS) { auto poSRS = std::unique_ptrbits[0]; + // coverity[overflow_const] if ((context->bits[0] = (t + (static_cast(len) << 3)) & 0xffffffff) < t) context->bits[1]++; /* Carry from low to high */ diff --git a/port/cpl_spawn.cpp b/port/cpl_spawn.cpp index 3fb75d7274ea..64b7a3bd1a38 100644 --- a/port/cpl_spawn.cpp +++ b/port/cpl_spawn.cpp @@ -474,7 +474,7 @@ int CPLPipeRead(CPL_FILE_HANDLE fin, void *data, int length) { while (true) { - const int n = static_cast(read(fin, pabyData, nRemain)); + const auto n = read(fin, pabyData, nRemain); if (n < 0) { if (errno == EINTR) @@ -485,7 +485,7 @@ int CPLPipeRead(CPL_FILE_HANDLE fin, void *data, int length) else if (n == 0) return FALSE; pabyData += n; - nRemain -= n; + nRemain -= static_cast(n); break; } } @@ -515,7 +515,7 @@ int CPLPipeWrite(CPL_FILE_HANDLE fout, const void *data, int length) { while (true) { - const int n = static_cast(write(fout, pabyData, nRemain)); + const auto n = write(fout, pabyData, nRemain); if (n < 0) { if (errno == EINTR) @@ -524,7 +524,7 @@ int CPLPipeWrite(CPL_FILE_HANDLE fout, const void *data, int length) return FALSE; } pabyData += n; - nRemain -= n; + nRemain -= static_cast(n); break; } } diff --git a/port/cpl_vsi_mem.cpp b/port/cpl_vsi_mem.cpp index caf3a5436c22..17f57ef9fa88 100644 --- a/port/cpl_vsi_mem.cpp +++ b/port/cpl_vsi_mem.cpp @@ -344,7 +344,11 @@ int VSIMemHandle::Close() int VSIMemHandle::Seek(vsi_l_offset nOffset, int nWhence) { - CPL_SHARED_LOCK oLock(poFile->m_oMutex); + vsi_l_offset nLength; + { + CPL_SHARED_LOCK oLock(poFile->m_oMutex); + nLength = poFile->nLength; + } bExtendFileAtNextWrite = false; if (nWhence == SEEK_CUR) @@ -361,7 +365,7 @@ int VSIMemHandle::Seek(vsi_l_offset nOffset, int nWhence) } else if (nWhence == SEEK_END) { - m_nOffset = poFile->nLength + nOffset; + m_nOffset = nLength + nOffset; } else { @@ -371,7 +375,7 @@ int VSIMemHandle::Seek(vsi_l_offset nOffset, int nWhence) bEOF = false; - if (m_nOffset > poFile->nLength) + if (m_nOffset > nLength) { if (bUpdate) // Writable files are zero-extended by seek past end. { @@ -399,13 +403,7 @@ vsi_l_offset VSIMemHandle::Tell() size_t VSIMemHandle::Read(void *pBuffer, size_t nSize, size_t nCount) { - CPL_SHARED_LOCK oLock(poFile->m_oMutex); - - if (!m_bReadAllowed) - { - m_bError = true; - return 0; - } + const vsi_l_offset nOffset = m_nOffset; size_t nBytesToRead = nSize * nCount; if (nBytesToRead == 0) @@ -417,21 +415,32 @@ size_t VSIMemHandle::Read(void *pBuffer, size_t nSize, size_t nCount) return 0; } - if (poFile->nLength <= m_nOffset || nBytesToRead + m_nOffset < nBytesToRead) + if (!m_bReadAllowed) { - bEOF = true; + m_bError = true; return 0; } - if (nBytesToRead + m_nOffset > poFile->nLength) + { - nBytesToRead = static_cast(poFile->nLength - m_nOffset); - nCount = nBytesToRead / nSize; - bEOF = true; + CPL_SHARED_LOCK oLock(poFile->m_oMutex); + + if (poFile->nLength <= nOffset || nBytesToRead + nOffset < nBytesToRead) + { + bEOF = true; + return 0; + } + if (nBytesToRead + nOffset > poFile->nLength) + { + nBytesToRead = static_cast(poFile->nLength - nOffset); + nCount = nBytesToRead / nSize; + bEOF = true; + } + + if (nBytesToRead) + memcpy(pBuffer, poFile->pabyData + nOffset, + static_cast(nBytesToRead)); } - if (nBytesToRead) - memcpy(pBuffer, poFile->pabyData + m_nOffset, - static_cast(nBytesToRead)); m_nOffset += nBytesToRead; return nCount; @@ -465,46 +474,50 @@ size_t VSIMemHandle::PRead(void *pBuffer, size_t nSize, size_t VSIMemHandle::Write(const void *pBuffer, size_t nSize, size_t nCount) { - CPL_EXCLUSIVE_LOCK oLock(poFile->m_oMutex); + const vsi_l_offset nOffset = m_nOffset; if (!bUpdate) { errno = EACCES; return 0; } + + const size_t nBytesToWrite = nSize * nCount; + if (bExtendFileAtNextWrite) { bExtendFileAtNextWrite = false; - if (!poFile->SetLength(m_nOffset)) + CPL_EXCLUSIVE_LOCK oLock(poFile->m_oMutex); + if (!poFile->SetLength(nOffset)) return 0; } - const size_t nBytesToWrite = nSize * nCount; - if (nCount > 0 && nBytesToWrite / nCount != nSize) - { - return 0; - } - if (nBytesToWrite + m_nOffset < nBytesToWrite) { - return 0; - } + CPL_EXCLUSIVE_LOCK oLock(poFile->m_oMutex); - if (nBytesToWrite + m_nOffset > poFile->nLength) - { - if (!poFile->SetLength(nBytesToWrite + m_nOffset)) + if (nCount > 0 && nBytesToWrite / nCount != nSize) + { return 0; + } + if (nBytesToWrite + nOffset < nBytesToWrite) + { + return 0; + } + + if (nBytesToWrite + nOffset > poFile->nLength) + { + if (!poFile->SetLength(nBytesToWrite + nOffset)) + return 0; + } + + if (nBytesToWrite) + memcpy(poFile->pabyData + nOffset, pBuffer, nBytesToWrite); + + time(&poFile->mTime); } - if (nBytesToWrite) - memcpy(poFile->pabyData + m_nOffset, pBuffer, nBytesToWrite); - // Coverity seems to be confused by the fact that we access m_nOffset - // under a shared lock in most places, except here under an exclusive lock - // which is fine - // coverity[missing_lock] m_nOffset += nBytesToWrite; - time(&poFile->mTime); - return nCount; } @@ -684,8 +697,12 @@ VSIVirtualHandle *VSIMemFilesystemHandler::Open(const char *pszFilename, #endif if (strchr(pszAccess, 'a')) { - CPL_SHARED_LOCK oLock(poFile->m_oMutex); - poHandle->m_nOffset = poFile->nLength; + vsi_l_offset nOffset; + { + CPL_SHARED_LOCK oLock(poFile->m_oMutex); + nOffset = poFile->nLength; + } + poHandle->m_nOffset = nOffset; } return poHandle; diff --git a/port/cpl_vsil_s3.cpp b/port/cpl_vsil_s3.cpp index 6be5177127e0..5225e45b9f1e 100644 --- a/port/cpl_vsil_s3.cpp +++ b/port/cpl_vsil_s3.cpp @@ -3814,6 +3814,7 @@ int IVSIS3LikeFSHandlerWithMultipartUpload::CopyFileRestartable( while (!bStop) { oCV.wait(oLock); + // coverity[ uninit_use_in_call] oLock.unlock(); const bool bInterrupt = !pProgressFunc(double(iCurChunk) / nChunkCount, diff --git a/port/cpl_vsisimple.cpp b/port/cpl_vsisimple.cpp index 744b540a08cd..a8d545ab2bb8 100644 --- a/port/cpl_vsisimple.cpp +++ b/port/cpl_vsisimple.cpp @@ -56,6 +56,7 @@ #include #include #include +#include #if HAVE_SYS_STAT_H #include #endif @@ -1401,8 +1402,11 @@ GIntBig CPLGetPhysicalRAM(void) { const long nPhysPages = sysconf(_SC_PHYS_PAGES); const long nPageSize = sysconf(_SC_PAGESIZE); - if (nPhysPages < 0 || nPageSize < 0) + if (nPhysPages <= 0 || nPageSize <= 0 || + nPhysPages > std::numeric_limits::max() / nPageSize) + { return 0; + } GIntBig nVal = static_cast(nPhysPages) * nPageSize; #ifdef __linux diff --git a/port/cpl_worker_thread_pool.cpp b/port/cpl_worker_thread_pool.cpp index 9087cf0270f7..444055a1879b 100644 --- a/port/cpl_worker_thread_pool.cpp +++ b/port/cpl_worker_thread_pool.cpp @@ -86,6 +86,16 @@ CPLWorkerThreadPool::~CPLWorkerThreadPool() CPLListDestroy(psWaitingWorkerThreadsList); } +/************************************************************************/ +/* GetThreadCount() */ +/************************************************************************/ + +int CPLWorkerThreadPool::GetThreadCount() const +{ + std::unique_lock oGuard(m_mutex); + return m_nMaxThreads; +} + /************************************************************************/ /* WorkerThreadFunction() */ /************************************************************************/ @@ -130,7 +140,12 @@ void CPLWorkerThreadPool::WorkerThreadFunction(void *user_data) */ bool CPLWorkerThreadPool::SubmitJob(CPLThreadFunc pfnFunc, void *pData) { - CPLAssert(m_nMaxThreads > 0); +#ifdef DEBUG + { + std::unique_lock oGuard(m_mutex); + CPLAssert(m_nMaxThreads > 0); + } +#endif bool bMustIncrementWaitingWorkerThreadsAfterSubmission = false; if (threadLocalCurrentThreadPool == this) @@ -234,6 +249,7 @@ bool CPLWorkerThreadPool::SubmitJob(CPLThreadFunc pfnFunc, void *pData) { std::lock_guard oGuardWT(psWorkerThread->m_mutex); + // coverity[ uninit_use_in_call] oGuard.unlock(); psWorkerThread->m_cv.notify_one(); } @@ -257,7 +273,12 @@ bool CPLWorkerThreadPool::SubmitJob(CPLThreadFunc pfnFunc, void *pData) bool CPLWorkerThreadPool::SubmitJobs(CPLThreadFunc pfnFunc, const std::vector &apData) { - CPLAssert(m_nMaxThreads > 0); +#ifdef DEBUG + { + std::unique_lock oGuard(m_mutex); + CPLAssert(m_nMaxThreads > 0); + } +#endif if (threadLocalCurrentThreadPool == this) { @@ -361,6 +382,7 @@ bool CPLWorkerThreadPool::SubmitJobs(CPLThreadFunc pfnFunc, #endif { std::lock_guard oGuardWT(psWorkerThread->m_mutex); + // coverity[ uninit_use_in_call] oGuard.unlock(); psWorkerThread->m_cv.notify_one(); } @@ -467,11 +489,14 @@ bool CPLWorkerThreadPool::Setup(int nThreads, CPLThreadFunc pfnInitFunc, bool bRet = true; for (int i = static_cast(aWT.size()); i < nThreads; i++) { - std::unique_ptr wt(new CPLWorkerThread); + auto wt = std::make_unique(); wt->pfnInitFunc = pfnInitFunc; wt->pInitData = pasInitData ? pasInitData[i] : nullptr; wt->poTP = this; - wt->bMarkedAsWaiting = false; + { + std::lock_guard oGuard(wt->m_mutex); + wt->bMarkedAsWaiting = false; + } wt->hThread = CPLCreateJoinableThread(WorkerThreadFunction, wt.get()); if (wt->hThread == nullptr) { @@ -575,8 +600,12 @@ CPLWorkerThreadPool::GetNextJob(CPLWorkerThread *psWorkerThread) #endif std::unique_lock oGuardThisThread(psWorkerThread->m_mutex); + // coverity[uninit_use_in_call] oGuard.unlock(); - psWorkerThread->m_cv.wait(oGuardThisThread); + while (psWorkerThread->bMarkedAsWaiting && eState != CPLWTS_STOP) + { + psWorkerThread->m_cv.wait(oGuardThisThread); + } } } diff --git a/port/cpl_worker_thread_pool.h b/port/cpl_worker_thread_pool.h index a6a0519220f2..77ecdcb2518c 100644 --- a/port/cpl_worker_thread_pool.h +++ b/port/cpl_worker_thread_pool.h @@ -80,7 +80,7 @@ class CPL_DLL CPLWorkerThreadPool CPL_DISALLOW_COPY_ASSIGN(CPLWorkerThreadPool) std::vector> aWT{}; - std::mutex m_mutex{}; + mutable std::mutex m_mutex{}; std::condition_variable m_cv{}; volatile CPLWorkerThreadState eState = CPLWTS_OK; CPLList *psJobQueue = nullptr; @@ -112,10 +112,7 @@ class CPL_DLL CPLWorkerThreadPool void WaitEvent(); /** Return the number of threads setup */ - int GetThreadCount() const - { - return m_nMaxThreads; - } + int GetThreadCount() const; }; /** Job queue */ From 54f2209e2cb975318f0c7cc5814b8230c75c2006 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 25 Jun 2024 22:44:19 +0200 Subject: [PATCH 11/72] Add OGRCurve::reversePoints(), and deprecated OGRLinearRing::reverseWindingOrder() --- autotest/cpp/test_ogr.cpp | 25 +++++++++++++++++++ ogr/ogr_geometry.h | 13 ++++++++-- ogr/ogrcompoundcurve.cpp | 18 +++++++++++++ ogr/ogrcurve.cpp | 12 +++++++++ ogr/ogrcurvecollection.cpp | 25 +++++++++++++++++++ ogr/ogrlinearring.cpp | 15 +++-------- ogr/ogrpgeogeometry.cpp | 8 +++--- .../parquet/ogrparquetwriterlayer.cpp | 2 +- 8 files changed, 100 insertions(+), 18 deletions(-) diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp index 90806d2e0da7..498ba17f627f 100644 --- a/autotest/cpp/test_ogr.cpp +++ b/autotest/cpp/test_ogr.cpp @@ -4175,4 +4175,29 @@ TEST_F(test_ogr, OGRGeometry_removeEmptyParts) } } +// Test OGRCurve::reversePoints() +TEST_F(test_ogr, OGRCurve_reversePoints) +{ + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "COMPOUNDCURVE ZM (CIRCULARSTRING ZM (0 0 10 20,1 1 11 21,2 0 12 " + "22),(2 0 12 22,3 0 13 2))", + nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + poGeom->toCurve()->reversePoints(); + char *pszWKT = nullptr; + poGeom->exportToWkt(&pszWKT, wkbVariantIso); + EXPECT_TRUE(pszWKT != nullptr); + if (pszWKT) + { + EXPECT_STREQ( + pszWKT, "COMPOUNDCURVE ZM ((3 0 13 2,2 0 12 22),CIRCULARSTRING " + "ZM (2 0 12 22,1 1 11 21,0 0 10 20))"); + } + CPLFree(pszWKT); + delete poGeom; + } +} + } // namespace diff --git a/ogr/ogr_geometry.h b/ogr/ogr_geometry.h index 0c46c2fa6a89..3a957c5e8e08 100644 --- a/ogr/ogr_geometry.h +++ b/ogr/ogr_geometry.h @@ -1408,6 +1408,7 @@ class CPL_DLL OGRCurve : public OGRGeometry virtual double get_GeodesicArea( const OGRSpatialReference *poSRSOverride = nullptr) const = 0; virtual int isClockwise() const; + virtual void reversePoints() = 0; /** Down-cast to OGRSimpleCurve*. * Implies prior checking that wkbFlatten(getGeometryType()) == @@ -1716,7 +1717,7 @@ class CPL_DLL OGRSimpleCurve : public OGRCurve void addSubLineString(const OGRLineString *, int nStartVertex = 0, int nEndVertex = -1); - void reversePoints(void); + void reversePoints() override; virtual OGRPointIterator *getPointIterator() const override; // non-standard from OGRGeometry @@ -1901,7 +1902,12 @@ class CPL_DLL OGRLinearRing : public OGRLineString // Non standard. virtual const char *getGeometryName() const override; virtual OGRLinearRing *clone() const override; - virtual void reverseWindingOrder(); + + //! @cond Doxygen_Suppress + void reverseWindingOrder() + CPL_WARN_DEPRECATED("Use reversePoints() instead"); + //! @endcond + virtual void closeRings() override; OGRBoolean isPointInRing(const OGRPoint *pt, int bTestEnvelope = TRUE) const; @@ -2153,6 +2159,8 @@ class CPL_DLL OGRCurveCollection bool hasEmptyParts() const; void removeEmptyParts(); + void reversePoints(); + OGRErr transform(OGRGeometry *poGeom, OGRCoordinateTransformation *poCT); void flattenTo2D(OGRGeometry *poGeom); void segmentize(double dfMaxLength); @@ -2317,6 +2325,7 @@ class CPL_DLL OGRCompoundCurve : public OGRCurve double dfToleranceEps = DEFAULT_TOLERANCE_EPSILON); OGRCurve *stealCurve(int); virtual OGRPointIterator *getPointIterator() const override; + void reversePoints() override; // Non-standard from OGRGeometry. virtual OGRwkbGeometryType getGeometryType() const override; diff --git a/ogr/ogrcompoundcurve.cpp b/ogr/ogrcompoundcurve.cpp index c3728dc7dd1e..0b2448f241c1 100644 --- a/ogr/ogrcompoundcurve.cpp +++ b/ogr/ogrcompoundcurve.cpp @@ -988,3 +988,21 @@ void OGRCompoundCurve::removeEmptyParts() { oCC.removeEmptyParts(); } + +/************************************************************************/ +/* reversePoints() */ +/************************************************************************/ + +/** + * \brief Reverse point order. + * + * This method updates the points in this curve in place + * reversing the point ordering (first for last, etc) and component ordering. + * + * @since 3.10 + */ +void OGRCompoundCurve::reversePoints() + +{ + oCC.reversePoints(); +} diff --git a/ogr/ogrcurve.cpp b/ogr/ogrcurve.cpp index b12b146cec1b..73ad9befdcd5 100644 --- a/ogr/ogrcurve.cpp +++ b/ogr/ogrcurve.cpp @@ -854,3 +854,15 @@ int OGRCurve::isClockwise() const return dfSum < 0; } + +/** + * \fn void OGRCurve::reversePoints(); + * + * \brief Reverse point order. + * + * This method updates the points in this curve in place + * reversing the point ordering (first for last, etc) and component ordering + * for a compound curve. + * + * @since 3.10 + */ diff --git a/ogr/ogrcurvecollection.cpp b/ogr/ogrcurvecollection.cpp index cde915c87a04..e17429c45d6a 100644 --- a/ogr/ogrcurvecollection.cpp +++ b/ogr/ogrcurvecollection.cpp @@ -791,4 +791,29 @@ void OGRCurveCollection::removeEmptyParts() } } +/************************************************************************/ +/* reversePoints() */ +/************************************************************************/ + +/** + * \brief Reverse point order. + * + * This method updates the points in this curve in place + * reversing the point ordering (first for last, etc) and component ordering. + * + * @since 3.10 + */ +void OGRCurveCollection::reversePoints() + +{ + for (int i = 0; i < nCurveCount / 2; ++i) + { + std::swap(papoCurves[i], papoCurves[nCurveCount - 1 - i]); + } + for (int i = 0; i < nCurveCount; ++i) + { + papoCurves[i]->reversePoints(); + } +} + //! @endcond diff --git a/ogr/ogrlinearring.cpp b/ogr/ogrlinearring.cpp index e98f92a62a3a..f9a11cf70528 100644 --- a/ogr/ogrlinearring.cpp +++ b/ogr/ogrlinearring.cpp @@ -448,24 +448,17 @@ OGRLinearRing *OGRLinearRing::clone() const /* reverseWindingOrder() */ /************************************************************************/ +//! @cond Doxygen_Suppress /** Reverse order of points. */ void OGRLinearRing::reverseWindingOrder() { - OGRPoint pointA; - OGRPoint pointB; - - for (int i = 0; i < nPointCount / 2; i++) - { - getPoint(i, &pointA); - const int pos = nPointCount - i - 1; - getPoint(pos, &pointB); - setPoint(i, &pointB); - setPoint(pos, &pointA); - } + reversePoints(); } +//! @endcond + /************************************************************************/ /* closeRing() */ /************************************************************************/ diff --git a/ogr/ogrpgeogeometry.cpp b/ogr/ogrpgeogeometry.cpp index ccb5387b7a7f..57a781519ea8 100644 --- a/ogr/ogrpgeogeometry.cpp +++ b/ogr/ogrpgeogeometry.cpp @@ -1004,7 +1004,7 @@ id,WKT assert(poRing); // Outer ring must be clockwise. if (!poRing->isClockwise()) - poRing->reverseWindingOrder(); + poRing->reversePoints(); } else { @@ -1012,7 +1012,7 @@ id,WKT assert(poRing); // Inner rings should be anti-clockwise. if (poRing->isClockwise()) - poRing->reverseWindingOrder(); + poRing->reversePoints(); } int nRingNumPoints = poRing->getNumPoints(); @@ -1260,7 +1260,7 @@ id,WKT assert(poRing != nullptr); // Outer ring must be clockwise. if (!poRing->isClockwise()) - poRing->reverseWindingOrder(); + poRing->reversePoints(); } else { @@ -1268,7 +1268,7 @@ id,WKT assert(poRing != nullptr); // Inner rings should be anti-clockwise. if (poRing->isClockwise()) - poRing->reverseWindingOrder(); + poRing->reversePoints(); } int nRingNumPoints = poRing->getNumPoints(); diff --git a/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp b/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp index 5241ac22228a..05203ca985f9 100644 --- a/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp +++ b/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp @@ -1148,7 +1148,7 @@ void OGRParquetWriterLayer::FixupGeometryBeforeWriting(OGRGeometry *poGeom) if ((bFirstRing && poRing->isClockwise()) || (!bFirstRing && !poRing->isClockwise())) { - poRing->reverseWindingOrder(); + poRing->reversePoints(); } bFirstRing = false; } From e20037f47b0e173e18bc61bb9b08d45c9cdc6157 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 25 Jun 2024 22:44:40 +0200 Subject: [PATCH 12/72] GML geometry reader: add support for gml:OrientableCurve Fixes #10301 --- autotest/ogr/ogr_gml_geom.py | 30 +++++++++++++++++++++++ ogr/gml2ogrgeometry.cpp | 46 ++++++++++++++++++++++++++---------- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/autotest/ogr/ogr_gml_geom.py b/autotest/ogr/ogr_gml_geom.py index 46ddda5f0e60..2343e8f7bf3b 100755 --- a/autotest/ogr/ogr_gml_geom.py +++ b/autotest/ogr/ogr_gml_geom.py @@ -1579,6 +1579,17 @@ def test_gml_invalid_geoms(): '0 0 4 0 4 4 0 4 0 0', "POLYGON ((0 0,4 0,4 4,0 4,0 0))", ), + ("", None), + ("", None), + ("", None), + ( + "", + None, + ), + ( + "0 0", + None, + ), ("", None), ("", None), ("", None), @@ -3003,3 +3014,22 @@ def test_gml_read_gml_ArcByCenterPoint_projected_crs_northing_easting(): """ ) assert g is not None + + +############################################################################### +# Test reading an OrientableCurve + + +def test_gml_OrientableCurve(): + + g = ogr.CreateGeometryFromGML( + """ 0 1 2 3 """ + ) + assert g is not None + assert g.ExportToWkt() == "LINESTRING (0 1,2 3)" + + g = ogr.CreateGeometryFromGML( + """ 0 1 2 3 """ + ) + assert g is not None + assert g.ExportToWkt() == "LINESTRING (2 3,0 1)" diff --git a/ogr/gml2ogrgeometry.cpp b/ogr/gml2ogrgeometry.cpp index 1b5714777939..70cbe1706b84 100644 --- a/ogr/gml2ogrgeometry.cpp +++ b/ogr/gml2ogrgeometry.cpp @@ -2898,19 +2898,7 @@ static std::unique_ptr GML2OGRGeometry_XMLNode_Internal( // correct orientation of the line string if (bEdgeOrientation != bOrientation) { - int iStartCoord = 0; - int iEndCoord = poLineString->getNumPoints() - 1; - OGRPoint oTempStartPoint; - OGRPoint oTempEndPoint; - while (iStartCoord < iEndCoord) - { - poLineString->getPoint(iStartCoord, &oTempStartPoint); - poLineString->getPoint(iEndCoord, &oTempEndPoint); - poLineString->setPoint(iStartCoord, &oTempEndPoint); - poLineString->setPoint(iEndCoord, &oTempStartPoint); - iStartCoord++; - iEndCoord--; - } + poLineString->reversePoints(); } return poLineString; } @@ -3642,6 +3630,38 @@ static std::unique_ptr GML2OGRGeometry_XMLNode_Internal( return poGeom; } + /* -------------------------------------------------------------------- */ + /* OrientableCurve */ + /* -------------------------------------------------------------------- */ + if (EQUAL(pszBaseGeometry, "OrientableCurve")) + { + // Find baseCurve. + const CPLXMLNode *psChild = FindBareXMLChild(psNode, "baseCurve"); + + psChild = GetChildElement(psChild); + if (psChild == nullptr) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Missing for OrientableCurve."); + return nullptr; + } + + auto poGeom = GML2OGRGeometry_XMLNode_Internal( + psChild, nPseudoBoolGetSecondaryGeometryOption, nRecLevel + 1, + nSRSDimension, pszSRSName); + if (!poGeom || !OGR_GT_IsCurve(poGeom->getGeometryType())) + { + CPLError(CE_Failure, CPLE_AppDefined, + "baseCurve of OrientableCurve is not a curve."); + return nullptr; + } + if (!GetElementOrientation(psNode)) + { + poGeom->toCurve()->reversePoints(); + } + return poGeom; + } + /* -------------------------------------------------------------------- */ /* OrientableSurface */ /* -------------------------------------------------------------------- */ From b9d2971bce0a74acda16fe9a9b9ae6273d9243fc Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 26 Jun 2024 20:20:26 +0200 Subject: [PATCH 13/72] GTiff: make SetNoDataValue(double) work on a Int64/UInt64 band Refs / fixes #10306 --- autotest/cpp/test_gdal_gtiff.cpp | 28 ++++++++++++++++++++ frmts/gtiff/gtiffrasterband_write.cpp | 37 ++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/autotest/cpp/test_gdal_gtiff.cpp b/autotest/cpp/test_gdal_gtiff.cpp index 1ea518295fb6..b9a24c5a59b3 100644 --- a/autotest/cpp/test_gdal_gtiff.cpp +++ b/autotest/cpp/test_gdal_gtiff.cpp @@ -240,4 +240,32 @@ TEST_F(test_gdal_gtiff, raster_min_max) GDALClose(ds); } +// Test setting a nodata value with SetNoDataValue(double) on a int64 dataset +TEST_F(test_gdal_gtiff, set_nodata_value_on_int64) +{ + std::string osTmpFile = "/vsimem/temp.tif"; + auto poDS = + std::unique_ptr(GDALDriver::FromHandle(drv_)->Create( + osTmpFile.c_str(), 1, 1, 1, GDT_Int64, nullptr)); + EXPECT_EQ(poDS->GetRasterBand(1)->SetNoDataValue(1), CE_None); + { + int bGotNoData = false; + EXPECT_EQ(poDS->GetRasterBand(1)->GetNoDataValue(&bGotNoData), 1.0); + EXPECT_TRUE(bGotNoData); + } + { + int bGotNoData = false; + EXPECT_EQ(poDS->GetRasterBand(1)->GetNoDataValueAsInt64(&bGotNoData), + 1.0); + EXPECT_TRUE(bGotNoData); + } + int64_t nVal = 0; + EXPECT_EQ(poDS->GetRasterBand(1)->RasterIO(GF_Read, 0, 0, 1, 1, &nVal, 1, 1, + GDT_Int64, 0, 0, nullptr), + CE_None); + EXPECT_EQ(nVal, 1); + poDS.reset(); + VSIUnlink(osTmpFile.c_str()); +} + } // namespace diff --git a/frmts/gtiff/gtiffrasterband_write.cpp b/frmts/gtiff/gtiffrasterband_write.cpp index de8d09e6934e..0b7ee7dc8663 100644 --- a/frmts/gtiff/gtiffrasterband_write.cpp +++ b/frmts/gtiff/gtiffrasterband_write.cpp @@ -34,6 +34,7 @@ #include #include "cpl_vsi_virtual.h" +#include "gdal_priv_templates.hpp" #include "gtiff.h" #include "tifvsi.h" @@ -705,6 +706,33 @@ CPLErr GTiffRasterBand::SetColorTable(GDALColorTable *poCT) CPLErr GTiffRasterBand::SetNoDataValue(double dfNoData) { + const auto SetNoDataMembers = [this, dfNoData]() + { + m_bNoDataSet = true; + m_dfNoDataValue = dfNoData; + + m_poGDS->m_bNoDataSet = true; + m_poGDS->m_dfNoDataValue = dfNoData; + + if (eDataType == GDT_Int64 && GDALIsValueExactAs(dfNoData)) + { + m_bNoDataSetAsInt64 = true; + m_nNoDataValueInt64 = static_cast(dfNoData); + + m_poGDS->m_bNoDataSetAsInt64 = true; + m_poGDS->m_nNoDataValueInt64 = static_cast(dfNoData); + } + else if (eDataType == GDT_UInt64 && + GDALIsValueExactAs(dfNoData)) + { + m_bNoDataSetAsUInt64 = true; + m_nNoDataValueUInt64 = static_cast(dfNoData); + + m_poGDS->m_bNoDataSetAsUInt64 = true; + m_poGDS->m_nNoDataValueUInt64 = static_cast(dfNoData); + } + }; + m_poGDS->LoadGeoreferencingAndPamIfNeeded(); if (m_poGDS->m_bNoDataSet && @@ -713,8 +741,7 @@ CPLErr GTiffRasterBand::SetNoDataValue(double dfNoData) { ResetNoDataValues(false); - m_bNoDataSet = true; - m_dfNoDataValue = dfNoData; + SetNoDataMembers(); return CE_None; } @@ -767,11 +794,7 @@ CPLErr GTiffRasterBand::SetNoDataValue(double dfNoData) { ResetNoDataValues(true); - m_poGDS->m_bNoDataSet = true; - m_poGDS->m_dfNoDataValue = dfNoData; - - m_bNoDataSet = true; - m_dfNoDataValue = dfNoData; + SetNoDataMembers(); } return eErr; From 7f800a6f640699af9fa5e0568e40ef24e4a29d30 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 26 Jun 2024 20:21:19 +0200 Subject: [PATCH 14/72] gdal_rasterize: on a int64 band, set nodata value as int64 Fixes #10306 --- apps/gdal_rasterize_lib.cpp | 25 +++++++++++-------- autotest/utilities/test_gdal_rasterize_lib.py | 4 ++- doc/source/programs/gdal_rasterize.rst | 4 ++- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/apps/gdal_rasterize_lib.cpp b/apps/gdal_rasterize_lib.cpp index 39913ad306cb..12ebd7cb2466 100644 --- a/apps/gdal_rasterize_lib.cpp +++ b/apps/gdal_rasterize_lib.cpp @@ -482,7 +482,7 @@ static GDALDatasetH CreateOutputDataset( OGREnvelope sEnvelop, GDALDriverH hDriver, const char *pszDest, int nXSize, int nYSize, double dfXRes, double dfYRes, bool bTargetAlignedPixels, int nBandCount, GDALDataType eOutputType, char **papszCreationOptions, - const std::vector &adfInitVals, int bNoDataSet, double dfNoData) + const std::vector &adfInitVals, const char *pszNoData) { bool bFirstLayer = true; char *pszWKT = nullptr; @@ -598,12 +598,16 @@ static GDALDatasetH CreateOutputDataset( } }*/ - if (bNoDataSet) + if (pszNoData) { for (int iBand = 0; iBand < nBandCount; iBand++) { GDALRasterBandH hBand = GDALGetRasterBand(hDstDS, iBand + 1); - GDALSetRasterNoDataValue(hBand, dfNoData); + if (GDALGetRasterDataType(hBand) == GDT_Int64) + GDALSetRasterNoDataValueAsInt64(hBand, + CPLAtoGIntBig(pszNoData)); + else + GDALSetRasterNoDataValue(hBand, CPLAtof(pszNoData)); } } @@ -649,8 +653,7 @@ struct GDALRasterizeOptions char **papszCreationOptions; GDALDataType eOutputType; std::vector adfInitVals; - int bNoDataSet; - double dfNoData; + char *pszNoData; OGREnvelope sEnvelop; int nXSize, nYSize; OGRSpatialReferenceH hSRS; @@ -832,7 +835,7 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, psOptions->bTargetAlignedPixels, static_cast(psOptions->anBandList.size()), eOutputType, psOptions->papszCreationOptions, psOptions->adfInitVals, - psOptions->bNoDataSet, psOptions->dfNoData); + psOptions->pszNoData); if (hDstDS == nullptr) { GDALDatasetReleaseResultSet(hSrcDataset, hLayer); @@ -910,7 +913,7 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, psOptions->dfYRes, psOptions->bTargetAlignedPixels, static_cast(psOptions->anBandList.size()), eOutputType, psOptions->papszCreationOptions, psOptions->adfInitVals, - psOptions->bNoDataSet, psOptions->dfNoData); + psOptions->pszNoData); if (hDstDS == nullptr) { GDALRasterizeOptionsFree(psOptionsToFree); @@ -1020,8 +1023,7 @@ GDALRasterizeOptionsNew(char **papszArgv, psOptions->dfXRes = 0; psOptions->dfYRes = 0; psOptions->eOutputType = GDT_Unknown; - psOptions->bNoDataSet = FALSE; - psOptions->dfNoData = 0; + psOptions->pszNoData = nullptr; psOptions->nXSize = 0; psOptions->nYSize = 0; psOptions->hSRS = nullptr; @@ -1195,8 +1197,8 @@ GDALRasterizeOptionsNew(char **papszArgv, } else if (i < argc - 1 && EQUAL(papszArgv[i], "-a_nodata")) { - psOptions->dfNoData = CPLAtof(papszArgv[i + 1]); - psOptions->bNoDataSet = TRUE; + CPLFree(psOptions->pszNoData); + psOptions->pszNoData = CPLStrdup(papszArgv[i + 1]); i += 1; psOptions->bCreateOutput = true; } @@ -1472,6 +1474,7 @@ void GDALRasterizeOptionsFree(GDALRasterizeOptions *psOptions) CPLFree(psOptions->pszDialect); CPLFree(psOptions->pszBurnAttribute); CPLFree(psOptions->pszWHERE); + CPLFree(psOptions->pszNoData); OSRDestroySpatialReference(psOptions->hSRS); delete psOptions; diff --git a/autotest/utilities/test_gdal_rasterize_lib.py b/autotest/utilities/test_gdal_rasterize_lib.py index b2669d966a87..67b2ba8d6d42 100755 --- a/autotest/utilities/test_gdal_rasterize_lib.py +++ b/autotest/utilities/test_gdal_rasterize_lib.py @@ -712,11 +712,13 @@ def test_gdal_rasterize_lib_int64_attribute(): feature["val"] = val layer.CreateFeature(feature) + noData = -(1 << 63) target_ds = gdal.Rasterize( - "", vector_ds, format="MEM", attribute="val", width=2, height=2 + "", vector_ds, format="MEM", attribute="val", width=2, height=2, noData=noData ) assert target_ds is not None assert target_ds.GetRasterBand(1).DataType == gdal.GDT_Int64 + assert target_ds.GetRasterBand(1).GetNoDataValue() == noData assert struct.unpack("Q" * 4, target_ds.ReadRaster())[0] == val diff --git a/doc/source/programs/gdal_rasterize.rst b/doc/source/programs/gdal_rasterize.rst index 43744d1fa967..6067de7a1d9d 100644 --- a/doc/source/programs/gdal_rasterize.rst +++ b/doc/source/programs/gdal_rasterize.rst @@ -173,7 +173,9 @@ raster data is only supported since GDAL 2.1.0. .. option:: -ot - Force the output bands to be of the indicated data type. Defaults to ``Float64`` + Force the output bands to be of the indicated data type. Defaults to ``Float64``, + unless the attribute field to burn is of type ``Int64``, in which case ``Int64`` + is used for the output raster data type. .. option:: -optim {AUTO|VECTOR|RASTER} From badc324442d722ff6c173583217fb64ca3d011b3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 26 Jun 2024 21:11:55 +0200 Subject: [PATCH 15/72] gdal_rasterize: restrict to defaulting to Int64 raster data type only if the output driver supports it (and the burned field is Int64) --- apps/gdal_rasterize_lib.cpp | 62 ++++++++++++++------------ doc/source/programs/gdal_rasterize.rst | 2 +- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/apps/gdal_rasterize_lib.cpp b/apps/gdal_rasterize_lib.cpp index 12ebd7cb2466..493922ee1068 100644 --- a/apps/gdal_rasterize_lib.cpp +++ b/apps/gdal_rasterize_lib.cpp @@ -793,6 +793,36 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, } } + const auto GetOutputDataType = [&](OGRLayerH hLayer) + { + CPLAssert(bCreateOutput); + CPLAssert(hDriver); + GDALDataType eOutputType = psOptions->eOutputType; + if (eOutputType == GDT_Unknown && + psOptions->pszBurnAttribute != nullptr) + { + OGRFeatureDefnH hLayerDefn = OGR_L_GetLayerDefn(hLayer); + const int iBurnField = + OGR_FD_GetFieldIndex(hLayerDefn, psOptions->pszBurnAttribute); + if (iBurnField >= 0 && OGR_Fld_GetType(OGR_FD_GetFieldDefn( + hLayerDefn, iBurnField)) == OFTInteger64) + { + const char *pszMD = GDALGetMetadataItem( + hDriver, GDAL_DMD_CREATIONDATATYPES, nullptr); + if (pszMD && CPLStringList(CSLTokenizeString2(pszMD, " ", 0)) + .FindString("Int64") >= 0) + { + eOutputType = GDT_Int64; + } + } + } + if (eOutputType == GDT_Unknown) + { + eOutputType = GDT_Float64; + } + return eOutputType; + }; + /* -------------------------------------------------------------------- */ /* Process SQL request. */ /* -------------------------------------------------------------------- */ @@ -809,25 +839,7 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, std::vector ahLayers; ahLayers.push_back(hLayer); - GDALDataType eOutputType = psOptions->eOutputType; - if (eOutputType == GDT_Unknown && - psOptions->pszBurnAttribute != nullptr) - { - OGRFeatureDefnH hLayerDefn = OGR_L_GetLayerDefn(hLayer); - int iBurnField = OGR_FD_GetFieldIndex( - hLayerDefn, psOptions->pszBurnAttribute); - if (iBurnField >= 0 && - OGR_Fld_GetType(OGR_FD_GetFieldDefn( - hLayerDefn, iBurnField)) == OFTInteger64) - { - eOutputType = GDT_Int64; - } - } - if (eOutputType == GDT_Unknown) - { - eOutputType = GDT_Float64; - } - + const GDALDataType eOutputType = GetOutputDataType(hLayer); hDstDS = CreateOutputDataset( ahLayers, psOptions->hSRS, psOptions->sEnvelop, hDriver, pszDest, psOptions->nXSize, psOptions->nYSize, @@ -885,18 +897,10 @@ GDALDatasetH GDALRasterize(const char *pszDest, GDALDatasetH hDstDS, GDALRasterizeOptionsFree(psOptionsToFree); return nullptr; } - if (eOutputType == GDT_Unknown && - psOptions->pszBurnAttribute != nullptr) + if (eOutputType == GDT_Unknown) { - OGRFeatureDefnH hLayerDefn = OGR_L_GetLayerDefn(hLayer); - int iBurnField = OGR_FD_GetFieldIndex( - hLayerDefn, psOptions->pszBurnAttribute); - if (iBurnField >= 0 && - OGR_Fld_GetType(OGR_FD_GetFieldDefn( - hLayerDefn, iBurnField)) == OFTInteger64) - { + if (GetOutputDataType(hLayer) == GDT_Int64) eOutputType = GDT_Int64; - } } ahLayers.push_back(hLayer); diff --git a/doc/source/programs/gdal_rasterize.rst b/doc/source/programs/gdal_rasterize.rst index 6067de7a1d9d..c382c30b7eee 100644 --- a/doc/source/programs/gdal_rasterize.rst +++ b/doc/source/programs/gdal_rasterize.rst @@ -175,7 +175,7 @@ raster data is only supported since GDAL 2.1.0. Force the output bands to be of the indicated data type. Defaults to ``Float64``, unless the attribute field to burn is of type ``Int64``, in which case ``Int64`` - is used for the output raster data type. + is used for the output raster data type if the output driver supports it. .. option:: -optim {AUTO|VECTOR|RASTER} From 01d25df6f1cae57a6b83c1bd88e63e78a239730a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 12:22:59 +0200 Subject: [PATCH 16/72] PG: fix ogr2ogr scenarios to PostgreSQL when there are several input layer names like schema_name.layer_name (3.9.0 regression) Fixes #10311 --- autotest/ogr/ogr_pg.py | 35 ++++++++++++++++++++++++++ ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp | 2 ++ 2 files changed, 37 insertions(+) diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index 8a8ad9a75357..a00a8f496dda 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -6121,3 +6121,38 @@ def test_ogr_pg_skip_conflicts(pg_ds): feat["beginnt"] = "2020-07-10T04:48:14Z" assert lyr.CreateFeature(feat) == ogr.OGRERR_NONE assert lyr.GetFeatureCount() == 2 + + +############################################################################### +# Test scenario of https://github.com/OSGeo/gdal/issues/10311 + + +@only_without_postgis +@gdaltest.enable_exceptions() +def test_ogr_pg_ogr2ogr_with_multiple_dotted_table_name(pg_ds): + + tmp_schema = "tmp_schema_issue_10311" + pg_ds.ExecuteSQL(f'CREATE SCHEMA "{tmp_schema}"') + try: + src_ds = gdal.GetDriverByName("Memory").Create("", 0, 0, 0, gdal.GDT_Unknown) + lyr = src_ds.CreateLayer(tmp_schema + ".table1", geom_type=ogr.wkbNone) + lyr.CreateField(ogr.FieldDefn("str")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["str"] = "foo" + lyr.CreateFeature(f) + lyr = src_ds.CreateLayer(tmp_schema + ".table2", geom_type=ogr.wkbNone) + lyr.CreateField(ogr.FieldDefn("str")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["str"] = "bar" + lyr.CreateFeature(f) + + gdal.VectorTranslate(pg_ds.GetDescription(), src_ds) + + pg_ds = reconnect(pg_ds) + lyr = pg_ds.GetLayerByName(tmp_schema + ".table1") + assert lyr.GetFeatureCount() == 1 + lyr = pg_ds.GetLayerByName(tmp_schema + ".table2") + assert lyr.GetFeatureCount() == 1 + + finally: + pg_ds.ExecuteSQL(f'DROP SCHEMA "{tmp_schema}" CASCADE') diff --git a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp index 62415a574683..1c2c6a93b8e0 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp @@ -2114,6 +2114,8 @@ OGRPGDataSource::FindSchema(const char *pszSchemaNameIn) return pszSchemaNameIn; } + EndCopy(); + std::string osSchemaName; std::string osCommand( "SELECT nspname FROM pg_catalog.pg_namespace WHERE nspname ILIKE "); From e6ff99385a4b6d6dae5cba57a04c692edeebcff3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 17:53:43 +0200 Subject: [PATCH 17/72] GTiff: handle TIFF color map that uses a 256 multiplication factor and add open option: .. oo:: COLOR_TABLE_MULTIPLIER :choices: AUTO, 1, 256, 257 :since: 3.10.0 Specifies the value by which to multiply GDAL color table entry values, usually in [0,255] range, to obtain a TIFF color map value, or on reading by which to divide TIFF color map values to get a GDAL color table entry. Since GDAL 2.3.0, GDAL consistently uses 257, but it might be necessary to use 256 for compatibility with files generated by other software. In AUTO mode, GDAL 3.10 or later can automatically detect the 256 multiplication factor when all values in the TIFF color map are multiple of that value. and same as creation option. Fixes #10310 --- autotest/gcore/tiff_write.py | 41 +++++++- doc/source/drivers/raster/gtiff.rst | 32 ++++++- frmts/gtiff/geotiff.cpp | 19 +++- frmts/gtiff/gt_overview.cpp | 22 ++++- frmts/gtiff/gtiffdataset.cpp | 1 + frmts/gtiff/gtiffdataset.h | 17 +++- frmts/gtiff/gtiffdataset_read.cpp | 109 +++++++++++++++------- frmts/gtiff/gtiffdataset_write.cpp | 129 +++++++++++++++++++------- frmts/gtiff/gtiffrasterband_write.cpp | 13 ++- 9 files changed, 299 insertions(+), 84 deletions(-) diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index 11a2dc8e3e0b..d2195a60a845 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -1068,6 +1068,7 @@ def test_tiff_write_26(): ct.SetColorEntry(1, (255, 255, 0, 255)) ct.SetColorEntry(2, (255, 0, 255, 255)) ct.SetColorEntry(3, (0, 255, 255, 255)) + ct.SetColorEntry(3, (0, 255, 255, 255)) ds.GetRasterBand(1).SetRasterColorTable(ct) @@ -1088,8 +1089,6 @@ def test_tiff_write_26(): ct = None ds = None - gdaltest.tiff_drv.Delete("tmp/ct8.tif") - ############################################################################### # Test color table in a 16 bit image @@ -11559,3 +11558,41 @@ def test_tiff_write_too_many_gcps(tmp_vsimem, with_initial_gcps): ds = gdal.Open(filename) assert ds.GetGCPCount() == 0 ds = None + + +############################################################################### +# Test writing/reading a TIFF color map using 256 as the multiplication factor +# https://github.com/OSGeo/gdal/issues/10310 + + +def test_tiff_write_colormap_256_mult_factor(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + ds = gdal.GetDriverByName("GTiff").Create( + filename, 1, 1, 1, gdal.GDT_Byte, ["COLOR_TABLE_MULTIPLIER=256"] + ) + ds.GetRasterBand(1).SetRasterColorInterpretation(gdal.GCI_PaletteIndex) + ct = gdal.ColorTable() + ct.SetColorEntry(0, (0, 0, 0, 255)) + ct.SetColorEntry(1, (1, 2, 3, 255)) + ct.SetColorEntry(2, (255, 255, 255, 255)) + ds.GetRasterBand(1).SetRasterColorTable(ct) + ds = None + + # Check we auto-guess correctly the 256 multiplication factor + ds = gdal.Open(filename) + ct = ds.GetRasterBand(1).GetRasterColorTable() + assert ( + ct.GetColorEntry(0) == (0, 0, 0, 255) + and ct.GetColorEntry(1) == (1, 2, 3, 255) + and ct.GetColorEntry(2) == (255, 255, 255, 255) + ), "Wrong color table entry." + + # Check we get wrong values when not specifying the appropriate multiplier + ds = gdal.OpenEx(filename, open_options=["COLOR_TABLE_MULTIPLIER=257"]) + ct = ds.GetRasterBand(1).GetRasterColorTable() + assert ( + ct.GetColorEntry(0) == (0, 0, 0, 255) + and ct.GetColorEntry(1) == (0, 1, 2, 255) + and ct.GetColorEntry(2) == (254, 254, 254, 255) + ), "Wrong color table entry." diff --git a/doc/source/drivers/raster/gtiff.rst b/doc/source/drivers/raster/gtiff.rst index 1f62e2f17eab..f338e3d77db3 100644 --- a/doc/source/drivers/raster/gtiff.rst +++ b/doc/source/drivers/raster/gtiff.rst @@ -341,15 +341,31 @@ This driver supports the following open options: blocks never written and save space; however, most non-GDAL packages cannot read such files. -- **IGNORE_COG_LAYOUT_BREAK=YES/NO** (GDAL >= 3.8): Updating a COG - (Cloud Optimized GeoTIFF) file generally breaks part of the optimizations, - but still produces a valid GeoTIFF file. +.. oo:: IGNORE_COG_LAYOUT_BREAK + :choices: YES, NO + :since: 3.8 + :default: NO + + Updating a COG (Cloud Optimized GeoTIFF) file generally breaks part of the + optimizations, but still produces a valid GeoTIFF file. Starting with GDAL 3.8, to avoid undesired loss of the COG characteristics, opening such a file in update mode will be rejected, unless this option is also set to YES (default is NO). This option has only effect on COG files and when opening in update mode, and is ignored on regular (Geo)TIFF files. +.. oo:: COLOR_TABLE_MULTIPLIER + :choices: AUTO, 1, 256, 257 + :since: 3.10.0 + + Specifies the value by which to multiply GDAL color table entry values, usually + in [0,255] range, to obtain a TIFF color map value, or on reading by which + to divide TIFF color map values to get a GDAL color table entry. Since GDAL 2.3.0, + GDAL consistently uses 257, but it might be necessary to use 256 for + compatibility with files generated by other software. + In AUTO mode, GDAL 3.10 or later can automatically detect the 256 multiplication + factor when all values in the TIFF color map are multiple of that value. + Creation Issues --------------- @@ -773,6 +789,16 @@ This driver supports the following creation options: .. note:: Write support for GeoTIFF 1.1 requires libgeotiff 1.6.0 or later. +- .. co:: COLOR_TABLE_MULTIPLIER + :choices: 1, 256, 257 + :since: 3.10.0 + :default: 257 + + Specifies the value by which to multiply GDAL color table entry values, usually + in [0,255] range, to obtain a TIFF color map value. Since GDAL 2.3.0, + GDAL consistently uses 257, but it might be necessary to use 256 for + compatibility with files generated by other software. + Subdatasets ~~~~~~~~~~~ diff --git a/frmts/gtiff/geotiff.cpp b/frmts/gtiff/geotiff.cpp index 7102d1adb2d5..21406e1dc551 100644 --- a/frmts/gtiff/geotiff.cpp +++ b/frmts/gtiff/geotiff.cpp @@ -727,7 +727,7 @@ void GTiffWriteJPEGTables(TIFF *hTIFF, const char *pszPhotometric, GTiffDataset::CreateLL(osTmpFilenameIn, nInMemImageWidth, nInMemImageHeight, (nBands <= 4) ? nBands : 1, (l_nBitsPerSample <= 8) ? GDT_Byte : GDT_UInt16, - 0.0, papszLocalParameters, &fpTmp, osTmp); + 0.0, 0, papszLocalParameters, &fpTmp, osTmp); CSLDestroy(papszLocalParameters); if (hTIFFTmp) { @@ -1541,6 +1541,14 @@ void GDALRegister_GTiff() " 1.1" " " #endif + " " ""; /* -------------------------------------------------------------------- */ @@ -1578,6 +1586,15 @@ void GDALRegister_GTiff() " " ""); poDriver->SetMetadataItem(GDAL_DMD_SUBDATASETS, "YES"); poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); diff --git a/frmts/gtiff/gt_overview.cpp b/frmts/gtiff/gt_overview.cpp index 38ca89e434c2..644a1c9bc670 100644 --- a/frmts/gtiff/gt_overview.cpp +++ b/frmts/gtiff/gt_overview.cpp @@ -46,6 +46,7 @@ #include "gdal.h" #include "gdal_priv.h" #include "gtiff.h" +#include "gtiffdataset.h" #include "tiff.h" #include "tiffvers.h" #include "tifvsi.h" @@ -749,17 +750,28 @@ CPLErr GTIFFBuildOverviewsEx(const char *pszFilename, int nBands, panBlue = static_cast( CPLCalloc(nColorCount, sizeof(unsigned short))); + const int nColorTableMultiplier = std::max( + 1, + std::min( + 257, + atoi(CSLFetchNameValueDef( + papszOptions, "COLOR_TABLE_MULTIPLIER", + CPLSPrintf( + "%d", + GTiffDataset::DEFAULT_COLOR_TABLE_MULTIPLIER_257))))); + for (int iColor = 0; iColor < nColorCount; iColor++) { GDALColorEntry sRGB = {0, 0, 0, 0}; if (poCT->GetColorEntryAsRGB(iColor, &sRGB)) { - // TODO(schwehr): Check for underflow. - // Going from signed short to unsigned short. - panRed[iColor] = static_cast(257 * sRGB.c1); - panGreen[iColor] = static_cast(257 * sRGB.c2); - panBlue[iColor] = static_cast(257 * sRGB.c3); + panRed[iColor] = GTiffDataset::ClampCTEntry( + iColor, 1, sRGB.c1, nColorTableMultiplier); + panGreen[iColor] = GTiffDataset::ClampCTEntry( + iColor, 2, sRGB.c2, nColorTableMultiplier); + panBlue[iColor] = GTiffDataset::ClampCTEntry( + iColor, 3, sRGB.c3, nColorTableMultiplier); } } } diff --git a/frmts/gtiff/gtiffdataset.cpp b/frmts/gtiff/gtiffdataset.cpp index 40cd55c94bd7..bfa27d35e522 100644 --- a/frmts/gtiff/gtiffdataset.cpp +++ b/frmts/gtiff/gtiffdataset.cpp @@ -1117,6 +1117,7 @@ void GTiffDataset::ScanDirectories() poODS->ShareLockWithParentDataset(this); poODS->SetStructuralMDFromParent(this); poODS->m_pszFilename = CPLStrdup(m_pszFilename); + poODS->m_nColorTableMultiplier = m_nColorTableMultiplier; if (poODS->OpenOffset(VSI_TIFFOpenChild(m_hTIFF), nThisDir, eAccess) != CE_None || poODS->GetRasterCount() != GetRasterCount()) diff --git a/frmts/gtiff/gtiffdataset.h b/frmts/gtiff/gtiffdataset.h index 18b0827caf3b..a59e4efc1a89 100644 --- a/frmts/gtiff/gtiffdataset.h +++ b/frmts/gtiff/gtiffdataset.h @@ -205,6 +205,16 @@ class GTiffDataset final : public GDALPamDataset int m_nRefBaseMapping = 0; int m_nDisableMultiThreadedRead = 0; + public: + static constexpr int DEFAULT_COLOR_TABLE_MULTIPLIER_257 = 257; + + private: + //! Multiplication factor to go from GDAL [0,255] color table range to + // TIFF [0,65535] color map one. + // 0 is not a valid value, and means not specified by user through the + // COLOR_TABLE_MULTIPLIER open / creation option. + int m_nColorTableMultiplier = 0; + GTIFFKeysFlavorEnum m_eGeoTIFFKeysFlavor = GEOTIFF_KEYS_STANDARD; GeoTIFFVersionEnum m_eGeoTIFFVersion = GEOTIFF_VERSION_AUTO; @@ -550,8 +560,8 @@ class GTiffDataset final : public GDALPamDataset static TIFF *CreateLL(const char *pszFilename, int nXSize, int nYSize, int nBands, GDALDataType eType, double dfExtraSpaceForOverviews, - char **papszParamList, VSILFILE **pfpL, - CPLString &osTmpFilename); + int nColorTableMultiplier, char **papszParamList, + VSILFILE **pfpL, CPLString &osTmpFilename); CPLErr WriteEncodedTileOrStrip(uint32_t tile_or_strip, void *data, int bPreserveDataBuffer); @@ -560,6 +570,9 @@ class GTiffDataset final : public GDALPamDataset char **papszParamList, uint32_t nBitsPerSample); static const GTIFFTag *GetTIFFTags(); + + static unsigned short ClampCTEntry(int iColor, int iComp, int nCTEntryVal, + int nMultFactor); }; GTIFFKeysFlavorEnum GetGTIFFKeysFlavor(CSLConstList papszOptions); diff --git a/frmts/gtiff/gtiffdataset_read.cpp b/frmts/gtiff/gtiffdataset_read.cpp index 9e602fa2619d..b90694f8a654 100644 --- a/frmts/gtiff/gtiffdataset_read.cpp +++ b/frmts/gtiff/gtiffdataset_read.cpp @@ -3961,6 +3961,12 @@ GDALDataset *GTiffDataset::Open(GDALOpenInfo *poOpenInfo) poDS->m_bHasGotSiblingFiles = true; } + // Should be capped by 257, to avoid 65535 / m_nColorTableMultiplier to overflow 255 + poDS->m_nColorTableMultiplier = std::max( + 0, std::min(257, + atoi(CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, + "COLOR_TABLE_MULTIPLIER", "0")))); + if (poDS->OpenOffset(l_hTIFF, TIFFCurrentDirOffset(l_hTIFF), poOpenInfo->eAccess, bAllowRGBAInterface, true) != CE_None) @@ -5230,54 +5236,87 @@ CPLErr GTiffDataset::OpenOffset(TIFF *hTIFFIn, toff_t nDirOffsetIn, } else { - unsigned short nMaxColor = 0; - m_poColorTable = new GDALColorTable(); const int nColorCount = 1 << m_nBitsPerSample; + if (m_nColorTableMultiplier == 0) + { + // TIFF color maps are in the [0, 65535] range, so some remapping must + // be done to get values in the [0, 255] range, but it is not clear + // how to do that exactly. Since GDAL 2.3.0 we have standardized on + // using a 257 multiplication factor (https://github.com/OSGeo/gdal/commit/eeec5b62e385d53e7f2edaba7b73c7c74bc2af39) + // but other software uses 256 (cf https://github.com/OSGeo/gdal/issues/10310) + // Do a first pass to check if all values are multiples of 256 or 257. + bool bFoundNonZeroEntry = false; + bool bAllValuesMultipleOf256 = true; + bool bAllValuesMultipleOf257 = true; + unsigned short nMaxColor = 0; + for (int iColor = 0; iColor < nColorCount; ++iColor) + { + if (panRed[iColor] > 0 || panGreen[iColor] > 0 || + panBlue[iColor] > 0) + { + bFoundNonZeroEntry = true; + } + if ((panRed[iColor] % 256) != 0 || + (panGreen[iColor] % 256) != 0 || + (panBlue[iColor] % 256) != 0) + { + bAllValuesMultipleOf256 = false; + } + if ((panRed[iColor] % 257) != 0 || + (panGreen[iColor] % 257) != 0 || + (panBlue[iColor] % 257) != 0) + { + bAllValuesMultipleOf257 = false; + } + + nMaxColor = std::max(nMaxColor, panRed[iColor]); + nMaxColor = std::max(nMaxColor, panGreen[iColor]); + nMaxColor = std::max(nMaxColor, panBlue[iColor]); + } + + if (nMaxColor > 0 && nMaxColor < 256) + { + // Bug 1384 - Some TIFF files are generated with color map entry + // values in range 0-255 instead of 0-65535 - try to handle these + // gracefully. + m_nColorTableMultiplier = 1; + CPLDebug("GTiff", + "TIFF ColorTable seems to be improperly scaled with " + "values all in [0,255] range, fixing up."); + } + else + { + if (!bAllValuesMultipleOf256 && !bAllValuesMultipleOf257) + { + CPLDebug("GTiff", + "The color map contains entries which are not " + "multiple of 256 or 257, so we don't know for " + "sure how to remap them to [0, 255]. Default to " + "using a 257 multiplication factor"); + } + m_nColorTableMultiplier = + (bFoundNonZeroEntry && bAllValuesMultipleOf256) + ? 256 + : DEFAULT_COLOR_TABLE_MULTIPLIER_257; + } + } + CPLAssert(m_nColorTableMultiplier > 0); + CPLAssert(m_nColorTableMultiplier <= 257); for (int iColor = nColorCount - 1; iColor >= 0; iColor--) { - // TODO(schwehr): Ensure the color entries are never negative? - const unsigned short divisor = 257; const GDALColorEntry oEntry = { - static_cast(panRed[iColor] / divisor), - static_cast(panGreen[iColor] / divisor), - static_cast(panBlue[iColor] / divisor), + static_cast(panRed[iColor] / m_nColorTableMultiplier), + static_cast(panGreen[iColor] / m_nColorTableMultiplier), + static_cast(panBlue[iColor] / m_nColorTableMultiplier), static_cast( m_bNoDataSet && static_cast(m_dfNoDataValue) == iColor ? 0 : 255)}; m_poColorTable->SetColorEntry(iColor, &oEntry); - - nMaxColor = std::max(nMaxColor, panRed[iColor]); - nMaxColor = std::max(nMaxColor, panGreen[iColor]); - nMaxColor = std::max(nMaxColor, panBlue[iColor]); - } - - // Bug 1384 - Some TIFF files are generated with color map entry - // values in range 0-255 instead of 0-65535 - try to handle these - // gracefully. - if (nMaxColor > 0 && nMaxColor < 256) - { - CPLDebug( - "GTiff", - "TIFF ColorTable seems to be improperly scaled, fixing up."); - - for (int iColor = nColorCount - 1; iColor >= 0; iColor--) - { - // TODO(schwehr): Ensure the color entries are never negative? - const GDALColorEntry oEntry = { - static_cast(panRed[iColor]), - static_cast(panGreen[iColor]), - static_cast(panBlue[iColor]), - m_bNoDataSet && static_cast(m_dfNoDataValue) == iColor - ? static_cast(0) - : static_cast(255)}; - - m_poColorTable->SetColorEntry(iColor, &oEntry); - } } } diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 062ba5cd1bfc..3e0696bec063 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -2579,13 +2579,11 @@ CPLErr GTiffDataset::RegisterNewOverviewDataset(toff_t nOverviewOffset, /* CreateTIFFColorTable() */ /************************************************************************/ -static void CreateTIFFColorTable(GDALColorTable *poColorTable, int nBits, - std::vector &anTRed, - std::vector &anTGreen, - std::vector &anTBlue, - unsigned short *&panRed, - unsigned short *&panGreen, - unsigned short *&panBlue) +static void CreateTIFFColorTable( + GDALColorTable *poColorTable, int nBits, int nColorTableMultiplier, + std::vector &anTRed, std::vector &anTGreen, + std::vector &anTBlue, unsigned short *&panRed, + unsigned short *&panGreen, unsigned short *&panBlue) { int nColors; @@ -2608,9 +2606,12 @@ static void CreateTIFFColorTable(GDALColorTable *poColorTable, int nBits, poColorTable->GetColorEntryAsRGB(iColor, &sRGB); - anTRed[iColor] = static_cast(257 * sRGB.c1); - anTGreen[iColor] = static_cast(257 * sRGB.c2); - anTBlue[iColor] = static_cast(257 * sRGB.c3); + anTRed[iColor] = GTiffDataset::ClampCTEntry(iColor, 1, sRGB.c1, + nColorTableMultiplier); + anTGreen[iColor] = GTiffDataset::ClampCTEntry( + iColor, 2, sRGB.c2, nColorTableMultiplier); + anTBlue[iColor] = GTiffDataset::ClampCTEntry(iColor, 3, sRGB.c3, + nColorTableMultiplier); } else { @@ -2829,8 +2830,12 @@ CPLErr GTiffDataset::CreateOverviewsFromSrcOverviews(GDALDataset *poSrcDS, if (nPhotometric == PHOTOMETRIC_PALETTE && m_poColorTable != nullptr) { - CreateTIFFColorTable(m_poColorTable, nOvBitsPerSample, anTRed, anTGreen, - anTBlue, panRed, panGreen, panBlue); + if (m_nColorTableMultiplier == 0) + m_nColorTableMultiplier = DEFAULT_COLOR_TABLE_MULTIPLIER_257; + + CreateTIFFColorTable(m_poColorTable, nOvBitsPerSample, + m_nColorTableMultiplier, anTRed, anTGreen, anTBlue, + panRed, panGreen, panBlue); } int nOvrBlockXSize = 0; @@ -3116,8 +3121,12 @@ CPLErr GTiffDataset::IBuildOverviews(const char *pszResampling, int nOverviews, if (nPhotometric == PHOTOMETRIC_PALETTE && m_poColorTable != nullptr) { - CreateTIFFColorTable(m_poColorTable, nOvBitsPerSample, anTRed, anTGreen, - anTBlue, panRed, panGreen, panBlue); + if (m_nColorTableMultiplier == 0) + m_nColorTableMultiplier = DEFAULT_COLOR_TABLE_MULTIPLIER_257; + + CreateTIFFColorTable(m_poColorTable, nOvBitsPerSample, + m_nColorTableMultiplier, anTRed, anTGreen, anTBlue, + panRed, panGreen, panBlue); } /* -------------------------------------------------------------------- */ @@ -5002,8 +5011,8 @@ static GTiffProfile GetProfile(const char *pszProfile) TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, int l_nBands, GDALDataType eType, double dfExtraSpaceForOverviews, - char **papszParamList, VSILFILE **pfpL, - CPLString &l_osTmpFilename) + int nColorTableMultiplier, char **papszParamList, + VSILFILE **pfpL, CPLString &l_osTmpFilename) { GTiffOneTimeInit(); @@ -5833,9 +5842,12 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, { if (eType == GDT_Byte) { - panTRed[iColor] = static_cast(257 * iColor); - panTGreen[iColor] = static_cast(257 * iColor); - panTBlue[iColor] = static_cast(257 * iColor); + panTRed[iColor] = GTiffDataset::ClampCTEntry( + iColor, 1, iColor, nColorTableMultiplier); + panTGreen[iColor] = GTiffDataset::ClampCTEntry( + iColor, 2, iColor, nColorTableMultiplier); + panTBlue[iColor] = GTiffDataset::ClampCTEntry( + iColor, 3, iColor, nColorTableMultiplier); } else { @@ -6071,7 +6083,7 @@ int GTiffDataset::GuessJPEGQuality(bool &bOutHasQuantizationTable, CPLString osTmp; TIFF *hTIFFTmp = CreateLL(osTmpFilenameIn, 16, 16, (nBands <= 4) ? nBands : 1, - GetRasterBand(1)->GetRasterDataType(), 0.0, + GetRasterBand(1)->GetRasterDataType(), 0.0, 0, papszLocalParameters, &fpTmp, osTmp); CPLPopErrorHandler(); if (!hTIFFTmp) @@ -6221,11 +6233,19 @@ GDALDataset *GTiffDataset::Create(const char *pszFilename, int nXSize, VSILFILE *l_fpL = nullptr; CPLString l_osTmpFilename; + const int nColorTableMultiplier = std::max( + 1, + std::min(257, + atoi(CSLFetchNameValueDef( + papszParamList, "COLOR_TABLE_MULTIPLIER", + CPLSPrintf("%d", DEFAULT_COLOR_TABLE_MULTIPLIER_257))))); + /* -------------------------------------------------------------------- */ /* Create the underlying TIFF file. */ /* -------------------------------------------------------------------- */ TIFF *l_hTIFF = CreateLL(pszFilename, nXSize, nYSize, l_nBands, eType, 0, - papszParamList, &l_fpL, l_osTmpFilename); + nColorTableMultiplier, papszParamList, &l_fpL, + l_osTmpFilename); const bool bStreaming = !l_osTmpFilename.empty(); if (l_hTIFF == nullptr) @@ -6252,6 +6272,9 @@ GDALDataset *GTiffDataset::Create(const char *pszFilename, int nXSize, poDS->nRasterXSize = nXSize; poDS->nRasterYSize = nYSize; poDS->eAccess = GA_Update; + + poDS->m_nColorTableMultiplier = nColorTableMultiplier; + poDS->m_bCrystalized = false; poDS->m_nSamplesPerPixel = static_cast(l_nBands); poDS->m_pszFilename = CPLStrdup(pszFilename); @@ -6344,11 +6367,10 @@ GDALDataset *GTiffDataset::Create(const char *pszFilename, int nXSize, for (int iColor = nColorCount - 1; iColor >= 0; iColor--) { - const unsigned short divisor = 257; const GDALColorEntry oEntry = { - static_cast(panRed[iColor] / divisor), - static_cast(panGreen[iColor] / divisor), - static_cast(panBlue[iColor] / divisor), + static_cast(panRed[iColor] / nColorTableMultiplier), + static_cast(panGreen[iColor] / nColorTableMultiplier), + static_cast(panBlue[iColor] / nColorTableMultiplier), static_cast(255)}; poDS->m_poColorTable->SetColorEntry(iColor, &oEntry); @@ -6919,9 +6941,17 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename, const int nXSize = poSrcDS->GetRasterXSize(); const int nYSize = poSrcDS->GetRasterYSize(); + + const int nColorTableMultiplier = std::max( + 1, + std::min(257, + atoi(CSLFetchNameValueDef( + papszOptions, "COLOR_TABLE_MULTIPLIER", + CPLSPrintf("%d", DEFAULT_COLOR_TABLE_MULTIPLIER_257))))); + TIFF *l_hTIFF = CreateLL(pszFilename, nXSize, nYSize, l_nBands, eType, - dfExtraSpaceForOverviews, papszCreateOptions, - &l_fpL, l_osTmpFilename); + dfExtraSpaceForOverviews, nColorTableMultiplier, + papszCreateOptions, &l_fpL, l_osTmpFilename); const bool bStreaming = !l_osTmpFilename.empty(); CSLDestroy(papszCreateOptions); @@ -7024,9 +7054,12 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename, poCT->GetColorEntryAsRGB(iColor, &sRGB); - anTRed[iColor] = static_cast(257 * sRGB.c1); - anTGreen[iColor] = static_cast(257 * sRGB.c2); - anTBlue[iColor] = static_cast(257 * sRGB.c3); + anTRed[iColor] = GTiffDataset::ClampCTEntry( + iColor, 1, sRGB.c1, nColorTableMultiplier); + anTGreen[iColor] = GTiffDataset::ClampCTEntry( + iColor, 2, sRGB.c2, nColorTableMultiplier); + anTBlue[iColor] = GTiffDataset::ClampCTEntry( + iColor, 3, sRGB.c3, nColorTableMultiplier); } else { @@ -7061,9 +7094,12 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename, poCT->GetColorEntryAsRGB(iColor, &sRGB); - panTRed[iColor] = static_cast(257 * sRGB.c1); - panTGreen[iColor] = static_cast(257 * sRGB.c2); - panTBlue[iColor] = static_cast(257 * sRGB.c3); + panTRed[iColor] = GTiffDataset::ClampCTEntry( + iColor, 1, sRGB.c1, nColorTableMultiplier); + panTGreen[iColor] = GTiffDataset::ClampCTEntry( + iColor, 2, sRGB.c2, nColorTableMultiplier); + panTBlue[iColor] = GTiffDataset::ClampCTEntry( + iColor, 3, sRGB.c3, nColorTableMultiplier); } else { @@ -7485,6 +7521,7 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename, poDS->m_pszFilename = CPLStrdup(pszFilename); poDS->m_fpL = l_fpL; poDS->m_bIMDRPCMetadataLoaded = true; + poDS->m_nColorTableMultiplier = nColorTableMultiplier; const bool bAppend = CPLFetchBool(papszOptions, "APPEND_SUBDATASET", false); if (poDS->OpenOffset(l_hTIFF, @@ -8785,3 +8822,29 @@ CPLErr GTiffRasterBand::CreateMaskBand(int nFlagsIn) return GDALPamRasterBand::CreateMaskBand(nFlagsIn); } + +/************************************************************************/ +/* ClampCTEntry() */ +/************************************************************************/ + +/* static */ unsigned short GTiffDataset::ClampCTEntry(int iColor, int iComp, + int nCTEntryVal, + int nMultFactor) +{ + const int nVal = nCTEntryVal * nMultFactor; + if (nVal < 0) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Color table entry [%d][%d] = %d, clamped to 0", iColor, iComp, + nCTEntryVal); + return 0; + } + if (nVal > 65535) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Color table entry [%d][%d] = %d, clamped to 65535", iColor, + iComp, nCTEntryVal); + return 65535; + } + return static_cast(nVal); +} diff --git a/frmts/gtiff/gtiffrasterband_write.cpp b/frmts/gtiff/gtiffrasterband_write.cpp index de8d09e6934e..2cdbd126e4dd 100644 --- a/frmts/gtiff/gtiffrasterband_write.cpp +++ b/frmts/gtiff/gtiffrasterband_write.cpp @@ -651,6 +651,10 @@ CPLErr GTiffRasterBand::SetColorTable(GDALColorTable *poCT) unsigned short *panTBlue = static_cast( CPLMalloc(sizeof(unsigned short) * nColors)); + if (m_poGDS->m_nColorTableMultiplier == 0) + m_poGDS->m_nColorTableMultiplier = + GTiffDataset::DEFAULT_COLOR_TABLE_MULTIPLIER_257; + for (int iColor = 0; iColor < nColors; ++iColor) { if (iColor < poCT->GetColorEntryCount()) @@ -658,9 +662,12 @@ CPLErr GTiffRasterBand::SetColorTable(GDALColorTable *poCT) GDALColorEntry sRGB; poCT->GetColorEntryAsRGB(iColor, &sRGB); - panTRed[iColor] = static_cast(257 * sRGB.c1); - panTGreen[iColor] = static_cast(257 * sRGB.c2); - panTBlue[iColor] = static_cast(257 * sRGB.c3); + panTRed[iColor] = GTiffDataset::ClampCTEntry( + iColor, 1, sRGB.c1, m_poGDS->m_nColorTableMultiplier); + panTGreen[iColor] = GTiffDataset::ClampCTEntry( + iColor, 2, sRGB.c2, m_poGDS->m_nColorTableMultiplier); + panTBlue[iColor] = GTiffDataset::ClampCTEntry( + iColor, 3, sRGB.c3, m_poGDS->m_nColorTableMultiplier); } else { From b9a770af73d16f25daba1e415b9d7afa4a4949c7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:01:56 +0200 Subject: [PATCH 18/72] GTiff: use std::unique_ptr instead of raw pointer --- frmts/gtiff/gtiffdataset.cpp | 4 +--- frmts/gtiff/gtiffdataset.h | 2 +- frmts/gtiff/gtiffdataset_read.cpp | 11 +++++------ frmts/gtiff/gtiffdataset_write.cpp | 6 +++--- frmts/gtiff/gtiffrasterband_read.cpp | 2 +- frmts/gtiff/gtiffrasterband_write.cpp | 11 ++--------- 6 files changed, 13 insertions(+), 23 deletions(-) diff --git a/frmts/gtiff/gtiffdataset.cpp b/frmts/gtiff/gtiffdataset.cpp index bfa27d35e522..3e6f28c6dfc5 100644 --- a/frmts/gtiff/gtiffdataset.cpp +++ b/frmts/gtiff/gtiffdataset.cpp @@ -308,9 +308,7 @@ std::tuple GTiffDataset::Finalize() bDroppedRef = true; } - if (m_poColorTable != nullptr) - delete m_poColorTable; - m_poColorTable = nullptr; + m_poColorTable.reset(); if (m_hTIFF) { diff --git a/frmts/gtiff/gtiffdataset.h b/frmts/gtiff/gtiffdataset.h index a59e4efc1a89..3065cc39bec0 100644 --- a/frmts/gtiff/gtiffdataset.h +++ b/frmts/gtiff/gtiffdataset.h @@ -155,7 +155,7 @@ class GTiffDataset final : public GDALPamDataset m_poMaskExtOvrDS{}; // Used with MASK_OVERVIEW_DATASET open option GTiffJPEGOverviewDS **m_papoJPEGOverviewDS = nullptr; std::vector m_aoGCPs{}; - GDALColorTable *m_poColorTable = nullptr; + std::unique_ptr m_poColorTable{}; char **m_papszMetadataFiles = nullptr; GByte *m_pabyBlockBuf = nullptr; char **m_papszCreationOptions = nullptr; diff --git a/frmts/gtiff/gtiffdataset_read.cpp b/frmts/gtiff/gtiffdataset_read.cpp index b90694f8a654..cd4d5aac82d0 100644 --- a/frmts/gtiff/gtiffdataset_read.cpp +++ b/frmts/gtiff/gtiffdataset_read.cpp @@ -4569,11 +4569,10 @@ void GTiffDataset::ApplyPamInfo() if (i == 1) { - auto poCT = poBand->GDALPamRasterBand::GetColorTable(); + const auto poCT = poBand->GDALPamRasterBand::GetColorTable(); if (poCT) { - delete m_poColorTable; - m_poColorTable = poCT->Clone(); + m_poColorTable.reset(poCT->Clone()); } } } @@ -5215,7 +5214,7 @@ CPLErr GTiffDataset::OpenOffset(TIFF *hTIFFIn, toff_t nDirOffsetIn, // data types (per #1882) if (m_nBitsPerSample <= 16 && m_nPhotometric == PHOTOMETRIC_MINISWHITE) { - m_poColorTable = new GDALColorTable(); + m_poColorTable = std::make_unique(); const int nColorCount = 1 << m_nBitsPerSample; for (int iColor = 0; iColor < nColorCount; ++iColor) @@ -5231,12 +5230,12 @@ CPLErr GTiffDataset::OpenOffset(TIFF *hTIFFIn, toff_t nDirOffsetIn, } else { - m_poColorTable = nullptr; + m_poColorTable.reset(); } } else { - m_poColorTable = new GDALColorTable(); + m_poColorTable = std::make_unique(); const int nColorCount = 1 << m_nBitsPerSample; diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 3e0696bec063..ef03725782cd 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -2833,7 +2833,7 @@ CPLErr GTiffDataset::CreateOverviewsFromSrcOverviews(GDALDataset *poSrcDS, if (m_nColorTableMultiplier == 0) m_nColorTableMultiplier = DEFAULT_COLOR_TABLE_MULTIPLIER_257; - CreateTIFFColorTable(m_poColorTable, nOvBitsPerSample, + CreateTIFFColorTable(m_poColorTable.get(), nOvBitsPerSample, m_nColorTableMultiplier, anTRed, anTGreen, anTBlue, panRed, panGreen, panBlue); } @@ -3124,7 +3124,7 @@ CPLErr GTiffDataset::IBuildOverviews(const char *pszResampling, int nOverviews, if (m_nColorTableMultiplier == 0) m_nColorTableMultiplier = DEFAULT_COLOR_TABLE_MULTIPLIER_257; - CreateTIFFColorTable(m_poColorTable, nOvBitsPerSample, + CreateTIFFColorTable(m_poColorTable.get(), nOvBitsPerSample, m_nColorTableMultiplier, anTRed, anTGreen, anTBlue, panRed, panGreen, panBlue); } @@ -6361,7 +6361,7 @@ GDALDataset *GTiffDataset::Create(const char *pszFilename, int nXSize, TIFFGetField(l_hTIFF, TIFFTAG_COLORMAP, &panRed, &panGreen, &panBlue)) { - poDS->m_poColorTable = new GDALColorTable(); + poDS->m_poColorTable = std::make_unique(); const int nColorCount = 1 << poDS->m_nBitsPerSample; diff --git a/frmts/gtiff/gtiffrasterband_read.cpp b/frmts/gtiff/gtiffrasterband_read.cpp index b16d57b4436a..6dcc6e7c6604 100644 --- a/frmts/gtiff/gtiffrasterband_read.cpp +++ b/frmts/gtiff/gtiffrasterband_read.cpp @@ -1633,7 +1633,7 @@ GDALColorTable *GTiffRasterBand::GetColorTable() m_poGDS->LoadGeoreferencingAndPamIfNeeded(); if (nBand == 1) - return m_poGDS->m_poColorTable; + return m_poGDS->m_poColorTable.get(); return nullptr; } diff --git a/frmts/gtiff/gtiffrasterband_write.cpp b/frmts/gtiff/gtiffrasterband_write.cpp index 2cdbd126e4dd..5798e5bbec81 100644 --- a/frmts/gtiff/gtiffrasterband_write.cpp +++ b/frmts/gtiff/gtiffrasterband_write.cpp @@ -624,11 +624,7 @@ CPLErr GTiffRasterBand::SetColorTable(GDALColorTable *poCT) TIFFUnsetField(m_poGDS->m_hTIFF, TIFFTAG_COLORMAP); } - if (m_poGDS->m_poColorTable) - { - delete m_poGDS->m_poColorTable; - m_poGDS->m_poColorTable = nullptr; - } + m_poGDS->m_poColorTable.reset(); return CE_None; } @@ -696,10 +692,7 @@ CPLErr GTiffRasterBand::SetColorTable(GDALColorTable *poCT) eErr = GDALPamRasterBand::SetColorTable(poCT); } - if (m_poGDS->m_poColorTable) - delete m_poGDS->m_poColorTable; - - m_poGDS->m_poColorTable = poCT->Clone(); + m_poGDS->m_poColorTable.reset(poCT->Clone()); m_eBandInterp = GCI_PaletteIndex; return eErr; From 992e862c3b4ffcd918662227c0b136ab01179282 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:17:25 +0200 Subject: [PATCH 19/72] CMake: move setting C/C++ standards from CMakeLists.txt to cmake/helpers/GdalCAndCXXStandards.cmake --- CMakeLists.txt | 12 +----------- cmake/helpers/GdalCAndCXXStandards.cmake | 10 ++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 cmake/helpers/GdalCAndCXXStandards.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index f6f81c1c929d..bc27733f8199 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,18 +35,8 @@ define_property( PROPERTY PLUGIN_OUTPUT_DIR BRIEF_DOCS "Plugin modules build directories" FULL_DOCS "Plugin modules build directories") -# -# check compiler and set preferences. -if (NOT CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_STANDARD_REQUIRED ON) -endif() - -if (NOT CMAKE_C_STANDARD) - set(CMAKE_C_STANDARD 99) - set(CMAKE_C_STANDARD_REQUIRED ON) -endif() +include(GdalCAndCXXStandards) # if (MSVC) add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) diff --git a/cmake/helpers/GdalCAndCXXStandards.cmake b/cmake/helpers/GdalCAndCXXStandards.cmake new file mode 100644 index 000000000000..6e2480cab02c --- /dev/null +++ b/cmake/helpers/GdalCAndCXXStandards.cmake @@ -0,0 +1,10 @@ + +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) +endif() + +if (NOT CMAKE_C_STANDARD) + set(CMAKE_C_STANDARD 99) + set(CMAKE_C_STANDARD_REQUIRED ON) +endif() From 59c0d1b979d0cb718b4786c35b1ef4f729308445 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:20:56 +0200 Subject: [PATCH 20/72] CMake: move things related to setting compilation flags to cmake/helpers/GdalCompilationFlags.cmake --- CMakeLists.txt | 6 - cmake/helpers/GdalCompilationFlags.cmake | 219 +++++++++++++++++++++++ gdal.cmake | 211 +--------------------- 3 files changed, 220 insertions(+), 216 deletions(-) create mode 100644 cmake/helpers/GdalCompilationFlags.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index bc27733f8199..32b0011b45d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,12 +37,6 @@ define_property( FULL_DOCS "Plugin modules build directories") include(GdalCAndCXXStandards) -# -if (MSVC) - add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) - add_definitions(-DNOMINMAX) -endif () -# include(CheckCompilerMachineOption) include(CheckCompilerSIMDFeature) include(Ccache) diff --git a/cmake/helpers/GdalCompilationFlags.cmake b/cmake/helpers/GdalCompilationFlags.cmake new file mode 100644 index 000000000000..3501de613802 --- /dev/null +++ b/cmake/helpers/GdalCompilationFlags.cmake @@ -0,0 +1,219 @@ + +# ###################################################################################################################### +# Detect available warning flags + +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) + +# Do that check now, since we need the result of HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT for cpl_config.h + +set(GDAL_C_WARNING_FLAGS) +set(GDAL_CXX_WARNING_FLAGS) + +if (MSVC) + # 1. conditional expression is constant + # 2. 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' + # 3. non DLL-interface classkey 'identifier' used as base for DLL-interface classkey 'identifier' + # 4. ?????????? + # 5. 'identifier' : unreferenced formal parameter + # 6. 'conversion' : conversion from 'type1' to 'type2', signed/unsigned mismatch + # 7. nonstandard extension used : translation unit is empty (only applies to C source code) + # 8. new behavior: elements of array 'array' will be default initialized (needed for + # https://trac.osgeo.org/gdal/changeset/35593) + # 9. interaction between '_setjmp' and C++ object destruction is non-portable + # + set(GDAL_C_WARNING_FLAGS + /W4 + /wd4127 + /wd4251 + /wd4275 + /wd4786 + /wd4100 + /wd4245 + /wd4206 + /wd4351 + /wd4611) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS}) + add_compile_options(/EHsc) + + # The following are extra disables that can be applied to external source not under our control that we wish to use + # less stringent warnings with. + set(GDAL_SOFTWARNFLAGS + /wd4244 + /wd4702 + /wd4701 + /wd4013 + /wd4706 + /wd4057 + /wd4210 + /wd4305) + +else () + + set(GDAL_SOFTWARNFLAGS "") + + macro (detect_and_set_c_warning_flag flag_name) + string(TOUPPER ${flag_name} flag_name_upper) + string(REPLACE "-" "_" flag_name_upper "${flag_name_upper}") + string(REPLACE "=" "_" flag_name_upper "${flag_name_upper}") + check_c_compiler_flag(-W${flag_name} "HAVE_WFLAG_${flag_name_upper}") + if (HAVE_WFLAG_${flag_name_upper}) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -W${flag_name}) + endif () + endmacro () + + macro (detect_and_set_cxx_warning_flag flag_name) + string(TOUPPER ${flag_name} flag_name_upper) + string(REPLACE "-" "_" flag_name_upper "${flag_name_upper}") + string(REPLACE "=" "_" flag_name_upper "${flag_name_upper}") + check_cxx_compiler_flag(-W${flag_name} "HAVE_WFLAG_${flag_name_upper}") + if (HAVE_WFLAG_${flag_name_upper}) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -W${flag_name}) + endif () + endmacro () + + macro (detect_and_set_c_and_cxx_warning_flag flag_name) + string(TOUPPER ${flag_name} flag_name_upper) + string(REPLACE "-" "_" flag_name_upper "${flag_name_upper}") + string(REPLACE "=" "_" flag_name_upper "${flag_name_upper}") + check_c_compiler_flag(-W${flag_name} "HAVE_WFLAG_${flag_name_upper}") + if (HAVE_WFLAG_${flag_name_upper}) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -W${flag_name}) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -W${flag_name}) + endif () + endmacro () + + detect_and_set_c_and_cxx_warning_flag(all) + detect_and_set_c_and_cxx_warning_flag(extra) + detect_and_set_c_and_cxx_warning_flag(init-self) + detect_and_set_c_and_cxx_warning_flag(unused-parameter) + detect_and_set_c_warning_flag(missing-prototypes) + detect_and_set_c_and_cxx_warning_flag(missing-declarations) + detect_and_set_c_and_cxx_warning_flag(shorten-64-to-32) + detect_and_set_c_and_cxx_warning_flag(logical-op) + detect_and_set_c_and_cxx_warning_flag(shadow) + detect_and_set_cxx_warning_flag(shadow-field) # CLang only for now + detect_and_set_c_and_cxx_warning_flag(missing-include-dirs) + check_c_compiler_flag("-Wformat -Werror=format-security -Wno-format-nonliteral" HAVE_WFLAG_FORMAT_SECURITY) + if (HAVE_WFLAG_FORMAT_SECURITY) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -Wformat -Werror=format-security -Wno-format-nonliteral) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -Wformat -Werror=format-security -Wno-format-nonliteral) + else () + detect_and_set_c_and_cxx_warning_flag(format) + endif () + detect_and_set_c_and_cxx_warning_flag(error=vla) + detect_and_set_c_and_cxx_warning_flag(no-clobbered) + detect_and_set_c_and_cxx_warning_flag(date-time) + detect_and_set_c_and_cxx_warning_flag(null-dereference) + detect_and_set_c_and_cxx_warning_flag(duplicate-cond) + detect_and_set_cxx_warning_flag(extra-semi) + detect_and_set_c_and_cxx_warning_flag(comma) + detect_and_set_c_and_cxx_warning_flag(float-conversion) + check_c_compiler_flag("-Wdocumentation -Wno-documentation-deprecated-sync" HAVE_WFLAG_DOCUMENTATION_AND_NO_DEPRECATED) + if (HAVE_WFLAG_DOCUMENTATION_AND_NO_DEPRECATED) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -Wdocumentation -Wno-documentation-deprecated-sync) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -Wdocumentation -Wno-documentation-deprecated-sync) + endif () + detect_and_set_cxx_warning_flag(unused-private-field) + detect_and_set_cxx_warning_flag(non-virtual-dtor) + detect_and_set_cxx_warning_flag(overloaded-virtual) + detect_and_set_cxx_warning_flag(suggest-override) + + check_cxx_compiler_flag(-fno-operator-names HAVE_FLAG_NO_OPERATOR_NAMES) + if (HAVE_FLAG_NO_OPERATOR_NAMES) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -fno-operator-names) + endif () + + check_cxx_compiler_flag(-Wzero-as-null-pointer-constant HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT) + if (HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -Wzero-as-null-pointer-constant) + endif () + + # Detect -Wold-style-cast but do not add it by default, as not all targets support it + check_cxx_compiler_flag(-Wold-style-cast HAVE_WFLAG_OLD_STYLE_CAST) + if (HAVE_WFLAG_OLD_STYLE_CAST) + set(WFLAG_OLD_STYLE_CAST -Wold-style-cast) + endif () + + # Detect Weffc++ but do not add it by default, as not all targets support it + check_cxx_compiler_flag(-Weffc++ HAVE_WFLAG_EFFCXX) + if (HAVE_WFLAG_EFFCXX) + set(WFLAG_EFFCXX -Weffc++) + endif () + + if (CMAKE_BUILD_TYPE MATCHES Debug) + check_c_compiler_flag(-ftrapv HAVE_FTRAPV) + if (HAVE_FTRAPV) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -ftrapv) + set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -ftrapv) + endif () + endif () + +endif () + +add_compile_definitions($<$:DEBUG>) + +# message(STATUS "GDAL_C_WARNING_FLAGS: ${GDAL_C_WARNING_FLAGS}") message(STATUS "GDAL_CXX_WARNING_FLAGS: ${GDAL_CXX_WARNING_FLAGS}") + +if (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + check_cxx_compiler_flag(-fno-finite-math-only HAVE_FLAG_NO_FINITE_MATH_ONLY) + if (HAVE_FLAG_NO_FINITE_MATH_ONLY) + # Intel CXX compiler based on clang defaults to -ffinite-math-only, which breaks std::isinf(), std::isnan(), etc. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-finite-math-only") + endif () + + set(TEST_LINK_STDCPP_SOURCE_CODE + "#include + int main(){ + std::string s; + s += \"x\"; + return 0; + }") + check_cxx_source_compiles("${TEST_LINK_STDCPP_SOURCE_CODE}" _TEST_LINK_STDCPP) + if( NOT _TEST_LINK_STDCPP ) + message(WARNING "Cannot link code using standard C++ library. Automatically adding -lstdc++ to CMAKE_EXE_LINKER_FLAGS, CMAKE_SHARED_LINKER_FLAGS and CMAKE_MODULE_LINKER_FLAGS") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lstdc++") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -lstdc++") + + check_cxx_source_compiles("${TEST_LINK_STDCPP_SOURCE_CODE}" _TEST_LINK_STDCPP_AGAIN) + if( NOT _TEST_LINK_STDCPP_AGAIN ) + message(FATAL_ERROR "Cannot link C++ program") + endif() + endif() + + check_c_compiler_flag(-wd188 HAVE_WD188) # enumerated type mixed with another type + if( HAVE_WD188 ) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -wd188) + endif() + check_c_compiler_flag(-wd2259 HAVE_WD2259) # non-pointer conversion from ... may lose significant bits + if( HAVE_WD2259 ) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -wd2259) + endif() + check_c_compiler_flag(-wd2312 HAVE_WD2312) # pointer cast involving 64-bit pointed-to type + if( HAVE_WD2259 ) + set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -wd2312) + endif() +endif () + +# Default definitions during build +add_definitions(-DGDAL_COMPILATION) + +if (MSVC) + add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) + add_definitions(-DNOMINMAX) +endif () + +if (MINGW) + if (TARGET_CPU MATCHES "x86_64") + add_definitions(-m64) + endif () + # Workaround for export too large error - force problematic large file to be optimized to prevent string table + # overflow error Used -Os instead of -O2 as previous issues had mentioned, since -Os is roughly speaking -O2, + # excluding any optimizations that take up extra space. Given that the issue is a string table overflowing, -Os seemed + # appropriate. Solves issue of https://github.com/OSGeo/gdal/issues/4706 with for example x86_64-w64-mingw32-gcc-posix + # (GCC) 9.3-posix 20200320 + if (CMAKE_BUILD_TYPE MATCHES Debug OR CMAKE_BUILD_TYPE STREQUAL "") + add_compile_options(-Os) + endif () +endif () diff --git a/gdal.cmake b/gdal.cmake index e390f8e41041..f806dad4a74d 100644 --- a/gdal.cmake +++ b/gdal.cmake @@ -44,199 +44,7 @@ option(CSHARP_MONO "Whether to force the C# compiler to be Mono" OFF) # this file is populated only be scripts/install_bash_completions.cmake.in install(CODE "file(REMOVE \"${PROJECT_BINARY_DIR}/install_manifest_extra.txt\")") -# ###################################################################################################################### -# Detect available warning flags - -# Do that check now, since we need the result of HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT for cpl_config.h - -set(GDAL_C_WARNING_FLAGS) -set(GDAL_CXX_WARNING_FLAGS) - -if (MSVC) - # 1. conditional expression is constant - # 2. 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' - # 3. non DLL-interface classkey 'identifier' used as base for DLL-interface classkey 'identifier' - # 4. ?????????? - # 5. 'identifier' : unreferenced formal parameter - # 6. 'conversion' : conversion from 'type1' to 'type2', signed/unsigned mismatch - # 7. nonstandard extension used : translation unit is empty (only applies to C source code) - # 8. new behavior: elements of array 'array' will be default initialized (needed for - # https://trac.osgeo.org/gdal/changeset/35593) - # 9. interaction between '_setjmp' and C++ object destruction is non-portable - # - set(GDAL_C_WARNING_FLAGS - /W4 - /wd4127 - /wd4251 - /wd4275 - /wd4786 - /wd4100 - /wd4245 - /wd4206 - /wd4351 - /wd4611) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS}) - add_compile_options(/EHsc) - - # The following are extra disables that can be applied to external source not under our control that we wish to use - # less stringent warnings with. - set(GDAL_SOFTWARNFLAGS - /wd4244 - /wd4702 - /wd4701 - /wd4013 - /wd4706 - /wd4057 - /wd4210 - /wd4305) - -else () - - set(GDAL_SOFTWARNFLAGS "") - - macro (detect_and_set_c_warning_flag flag_name) - string(TOUPPER ${flag_name} flag_name_upper) - string(REPLACE "-" "_" flag_name_upper "${flag_name_upper}") - string(REPLACE "=" "_" flag_name_upper "${flag_name_upper}") - check_c_compiler_flag(-W${flag_name} "HAVE_WFLAG_${flag_name_upper}") - if (HAVE_WFLAG_${flag_name_upper}) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -W${flag_name}) - endif () - endmacro () - - macro (detect_and_set_cxx_warning_flag flag_name) - string(TOUPPER ${flag_name} flag_name_upper) - string(REPLACE "-" "_" flag_name_upper "${flag_name_upper}") - string(REPLACE "=" "_" flag_name_upper "${flag_name_upper}") - check_cxx_compiler_flag(-W${flag_name} "HAVE_WFLAG_${flag_name_upper}") - if (HAVE_WFLAG_${flag_name_upper}) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -W${flag_name}) - endif () - endmacro () - - macro (detect_and_set_c_and_cxx_warning_flag flag_name) - string(TOUPPER ${flag_name} flag_name_upper) - string(REPLACE "-" "_" flag_name_upper "${flag_name_upper}") - string(REPLACE "=" "_" flag_name_upper "${flag_name_upper}") - check_c_compiler_flag(-W${flag_name} "HAVE_WFLAG_${flag_name_upper}") - if (HAVE_WFLAG_${flag_name_upper}) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -W${flag_name}) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -W${flag_name}) - endif () - endmacro () - - detect_and_set_c_and_cxx_warning_flag(all) - detect_and_set_c_and_cxx_warning_flag(extra) - detect_and_set_c_and_cxx_warning_flag(init-self) - detect_and_set_c_and_cxx_warning_flag(unused-parameter) - detect_and_set_c_warning_flag(missing-prototypes) - detect_and_set_c_and_cxx_warning_flag(missing-declarations) - detect_and_set_c_and_cxx_warning_flag(shorten-64-to-32) - detect_and_set_c_and_cxx_warning_flag(logical-op) - detect_and_set_c_and_cxx_warning_flag(shadow) - detect_and_set_cxx_warning_flag(shadow-field) # CLang only for now - detect_and_set_c_and_cxx_warning_flag(missing-include-dirs) - check_c_compiler_flag("-Wformat -Werror=format-security -Wno-format-nonliteral" HAVE_WFLAG_FORMAT_SECURITY) - if (HAVE_WFLAG_FORMAT_SECURITY) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -Wformat -Werror=format-security -Wno-format-nonliteral) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -Wformat -Werror=format-security -Wno-format-nonliteral) - else () - detect_and_set_c_and_cxx_warning_flag(format) - endif () - detect_and_set_c_and_cxx_warning_flag(error=vla) - detect_and_set_c_and_cxx_warning_flag(no-clobbered) - detect_and_set_c_and_cxx_warning_flag(date-time) - detect_and_set_c_and_cxx_warning_flag(null-dereference) - detect_and_set_c_and_cxx_warning_flag(duplicate-cond) - detect_and_set_cxx_warning_flag(extra-semi) - detect_and_set_c_and_cxx_warning_flag(comma) - detect_and_set_c_and_cxx_warning_flag(float-conversion) - check_c_compiler_flag("-Wdocumentation -Wno-documentation-deprecated-sync" HAVE_WFLAG_DOCUMENTATION_AND_NO_DEPRECATED) - if (HAVE_WFLAG_DOCUMENTATION_AND_NO_DEPRECATED) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -Wdocumentation -Wno-documentation-deprecated-sync) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -Wdocumentation -Wno-documentation-deprecated-sync) - endif () - detect_and_set_cxx_warning_flag(unused-private-field) - detect_and_set_cxx_warning_flag(non-virtual-dtor) - detect_and_set_cxx_warning_flag(overloaded-virtual) - detect_and_set_cxx_warning_flag(suggest-override) - - check_cxx_compiler_flag(-fno-operator-names HAVE_FLAG_NO_OPERATOR_NAMES) - if (HAVE_FLAG_NO_OPERATOR_NAMES) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -fno-operator-names) - endif () - - check_cxx_compiler_flag(-Wzero-as-null-pointer-constant HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT) - if (HAVE_GCC_WARNING_ZERO_AS_NULL_POINTER_CONSTANT) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -Wzero-as-null-pointer-constant) - endif () - - # Detect -Wold-style-cast but do not add it by default, as not all targets support it - check_cxx_compiler_flag(-Wold-style-cast HAVE_WFLAG_OLD_STYLE_CAST) - if (HAVE_WFLAG_OLD_STYLE_CAST) - set(WFLAG_OLD_STYLE_CAST -Wold-style-cast) - endif () - - # Detect Weffc++ but do not add it by default, as not all targets support it - check_cxx_compiler_flag(-Weffc++ HAVE_WFLAG_EFFCXX) - if (HAVE_WFLAG_EFFCXX) - set(WFLAG_EFFCXX -Weffc++) - endif () - - if (CMAKE_BUILD_TYPE MATCHES Debug) - check_c_compiler_flag(-ftrapv HAVE_FTRAPV) - if (HAVE_FTRAPV) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -ftrapv) - set(GDAL_CXX_WARNING_FLAGS ${GDAL_CXX_WARNING_FLAGS} -ftrapv) - endif () - endif () - -endif () - -add_compile_definitions($<$:DEBUG>) - -# message(STATUS "GDAL_C_WARNING_FLAGS: ${GDAL_C_WARNING_FLAGS}") message(STATUS "GDAL_CXX_WARNING_FLAGS: ${GDAL_CXX_WARNING_FLAGS}") - -if (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - check_cxx_compiler_flag(-fno-finite-math-only HAVE_FLAG_NO_FINITE_MATH_ONLY) - if (HAVE_FLAG_NO_FINITE_MATH_ONLY) - # Intel CXX compiler based on clang defaults to -ffinite-math-only, which breaks std::isinf(), std::isnan(), etc. - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-finite-math-only") - endif () - - set(TEST_LINK_STDCPP_SOURCE_CODE - "#include - int main(){ - std::string s; - s += \"x\"; - return 0; - }") - check_cxx_source_compiles("${TEST_LINK_STDCPP_SOURCE_CODE}" _TEST_LINK_STDCPP) - if( NOT _TEST_LINK_STDCPP ) - message(WARNING "Cannot link code using standard C++ library. Automatically adding -lstdc++ to CMAKE_EXE_LINKER_FLAGS, CMAKE_SHARED_LINKER_FLAGS and CMAKE_MODULE_LINKER_FLAGS") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lstdc++") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -lstdc++") - - check_cxx_source_compiles("${TEST_LINK_STDCPP_SOURCE_CODE}" _TEST_LINK_STDCPP_AGAIN) - if( NOT _TEST_LINK_STDCPP_AGAIN ) - message(FATAL_ERROR "Cannot link C++ program") - endif() - endif() - - check_c_compiler_flag(-wd188 HAVE_WD188) # enumerated type mixed with another type - if( HAVE_WD188 ) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -wd188) - endif() - check_c_compiler_flag(-wd2259 HAVE_WD2259) # non-pointer conversion from ... may lose significant bits - if( HAVE_WD2259 ) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -wd2259) - endif() - check_c_compiler_flag(-wd2312 HAVE_WD2312) # pointer cast involving 64-bit pointed-to type - if( HAVE_WD2259 ) - set(GDAL_C_WARNING_FLAGS ${GDAL_C_WARNING_FLAGS} -wd2312) - endif() -endif () +include(GdalCompilationFlags) # ###################################################################################################################### # generate ${CMAKE_CURRENT_BINARY_DIR}/port/cpl_config.h @@ -325,9 +133,6 @@ if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST CACHE INTERNAL "Previous value of USE_ALTERNATE_LINKER") endif() -# Default definitions during build -add_definitions(-DGDAL_COMPILATION) - if (ENABLE_IPO) include(CheckIPOSupported) check_ipo_supported(RESULT result) @@ -384,20 +189,6 @@ if (MINGW AND BUILD_SHARED_LIBS) set_target_properties(${GDAL_LIB_TARGET_NAME} PROPERTIES SUFFIX "-${GDAL_SOVERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}") endif () -if (MINGW) - if (TARGET_CPU MATCHES "x86_64") - add_definitions(-m64) - endif () - # Workaround for export too large error - force problematic large file to be optimized to prevent string table - # overflow error Used -Os instead of -O2 as previous issues had mentioned, since -Os is roughly speaking -O2, - # excluding any optimizations that take up extra space. Given that the issue is a string table overflowing, -Os seemed - # appropriate. Solves issue of https://github.com/OSGeo/gdal/issues/4706 with for example x86_64-w64-mingw32-gcc-posix - # (GCC) 9.3-posix 20200320 - if (CMAKE_BUILD_TYPE MATCHES Debug OR CMAKE_BUILD_TYPE STREQUAL "") - add_compile_options(-Os) - endif () -endif () - # Install properties if (GDAL_ENABLE_MACOSX_FRAMEWORK) set(FRAMEWORK_VERSION ${GDAL_VERSION_MAJOR}.${GDAL_VERSION_MINOR}) From 9e3eba5637b8250baaf14810a53d57f0904a680d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:21:46 +0200 Subject: [PATCH 21/72] GdalStandardIncludes.cmake: make it work in STANDALONE mode --- cmake/helpers/GdalStandardIncludes.cmake | 26 ++++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cmake/helpers/GdalStandardIncludes.cmake b/cmake/helpers/GdalStandardIncludes.cmake index e138d2175838..b4f98e6562f2 100644 --- a/cmake/helpers/GdalStandardIncludes.cmake +++ b/cmake/helpers/GdalStandardIncludes.cmake @@ -8,15 +8,19 @@ GdalStandardIncludes #]=======================================================================] function(gdal_standard_includes _TARGET) - target_include_directories(${_TARGET} PRIVATE - $ - $ - $ - $ - $ # port - $ - $ - $ # ogr/ogrsf_frmts - $ # frmts - ) + if (STANDALONE) + target_include_directories(${_TARGET} PRIVATE $) + else() + target_include_directories(${_TARGET} PRIVATE + $ + $ + $ + $ + $ # port + $ + $ + $ # ogr/ogrsf_frmts + $ # frmts + ) + endif() endfunction() From e9f80e745759c65c6ad492a5c2c8cf418ec92a7a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:21:57 +0200 Subject: [PATCH 22/72] GdalVersion.cmake: make it work in STANDALONE mode --- cmake/helpers/GdalVersion.cmake | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cmake/helpers/GdalVersion.cmake b/cmake/helpers/GdalVersion.cmake index b466522da06d..7961085ba325 100644 --- a/cmake/helpers/GdalVersion.cmake +++ b/cmake/helpers/GdalVersion.cmake @@ -16,8 +16,10 @@ GdalVersion #]=======================================================================] +set(GDAL_ROOT_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../..") + # parse the version number from gdal_version.h and include in GDAL_MAJOR_VERSION and GDAL_MINOR_VERSION -file(READ ${PROJECT_SOURCE_DIR}/gcore/gdal_version.h.in GDAL_VERSION_H_CONTENTS) +file(READ ${GDAL_ROOT_SOURCE_DIR}/gcore/gdal_version.h.in GDAL_VERSION_H_CONTENTS) string(REGEX MATCH "GDAL_VERSION_MAJOR[ \t]+([0-9]+)" GDAL_VERSION_MAJOR ${GDAL_VERSION_H_CONTENTS}) string(REGEX MATCH "([0-9]+)" @@ -35,12 +37,16 @@ string(REGEX MATCH "GDAL_VERSION_BUILD[ \t]+([0-9]+)" string(REGEX MATCH "([0-9]+)" GDAL_VERSION_BUILD ${GDAL_VERSION_BUILD}) -if ((EXISTS "${PROJECT_SOURCE_DIR}/gcore/gdal_version.h") AND NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")) +if (STANDALONE) + return() +endif() + +if ((EXISTS "${GDAL_ROOT_SOURCE_DIR}/gcore/gdal_version.h") AND NOT ("${GDAL_ROOT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")) # Try to detect issues when building with cmake out of source tree, but against a previous build done in source tree - message(FATAL_ERROR "${PROJECT_SOURCE_DIR}/gcore/gdal_version.h was found, and likely conflicts with ${PROJECT_BINARY_DIR}/gcore/gdal_version.h") + message(FATAL_ERROR "${GDAL_ROOT_SOURCE_DIR}/gcore/gdal_version.h was found, and likely conflicts with ${PROJECT_BINARY_DIR}/gcore/gdal_version.h") endif () -if (EXISTS ${PROJECT_SOURCE_DIR}/.git) +if (EXISTS ${GDAL_ROOT_SOURCE_DIR}/.git) set(GDAL_DEV_SUFFIX "dev") else() set(GDAL_DEV_SUFFIX "") @@ -52,11 +58,11 @@ set(GDAL_RELEASE_DATE "$ENV{GDAL_RELEASE_DATE}") add_custom_target(generate_gdal_version_h COMMAND ${CMAKE_COMMAND} - "-DSOURCE_DIR=${PROJECT_SOURCE_DIR}" + "-DSOURCE_DIR=${GDAL_ROOT_SOURCE_DIR}" "-DBINARY_DIR=${PROJECT_BINARY_DIR}" "-DGDAL_SHA1SUM=${GDAL_SHA1SUM}" "-DGDAL_RELEASE_DATE=${GDAL_RELEASE_DATE}" - -P "${PROJECT_SOURCE_DIR}/cmake/helpers/generate_gdal_version_h.cmake" + -P "${GDAL_ROOT_SOURCE_DIR}/cmake/helpers/generate_gdal_version_h.cmake" VERBATIM) if (WIN32 AND NOT MINGW) From a3db07e69f0155bed5e3e57ba1b3c568f69299f9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:22:32 +0200 Subject: [PATCH 23/72] SystemSummary.cmake: report C++17 related flags rather than C++11 ones --- cmake/modules/thirdparty/SystemSummary.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/modules/thirdparty/SystemSummary.cmake b/cmake/modules/thirdparty/SystemSummary.cmake index 5251c4840a26..54164304fd09 100644 --- a/cmake/modules/thirdparty/SystemSummary.cmake +++ b/cmake/modules/thirdparty/SystemSummary.cmake @@ -23,8 +23,8 @@ macro(gather_flags with_linker result) # add the main flags without a config list(APPEND ${result} CMAKE_C_FLAGS) list(APPEND ${result} CMAKE_CXX_FLAGS) - list(APPEND ${result} CMAKE_CXX11_STANDARD_COMPILE_OPTION) - list(APPEND ${result} CMAKE_CXX11_EXTENSION_COMPILE_OPTION) + list(APPEND ${result} CMAKE_CXX17_STANDARD_COMPILE_OPTION) + list(APPEND ${result} CMAKE_CXX17_EXTENSION_COMPILE_OPTION) if(${with_linker}) list(APPEND ${result} CMAKE_EXE_LINKER_FLAGS) From 76ec763fc832eff0b8e03371707a78d9452e4adc Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:24:18 +0200 Subject: [PATCH 24/72] CMake: move code from CheckDependentLibraries.cmake to CheckDependentLibrariesCommon.cmake --- cmake/helpers/CheckDependentLibraries.cmake | 271 +---------------- .../CheckDependentLibrariesCommon.cmake | 281 ++++++++++++++++++ 2 files changed, 283 insertions(+), 269 deletions(-) create mode 100644 cmake/helpers/CheckDependentLibrariesCommon.cmake diff --git a/cmake/helpers/CheckDependentLibraries.cmake b/cmake/helpers/CheckDependentLibraries.cmake index 7335af0e9751..0dddba9da97c 100644 --- a/cmake/helpers/CheckDependentLibraries.cmake +++ b/cmake/helpers/CheckDependentLibraries.cmake @@ -8,275 +8,7 @@ Detect GDAL dependencies and set variable HAVE_* #]=======================================================================] -include(CheckFunctionExists) -include(CMakeDependentOption) -include(FeatureSummary) -include(DefineFindPackage2) -include(CheckSymbolExists) - -option( - GDAL_USE_EXTERNAL_LIBS - "Whether detected external libraries should be used by default. This should be set before CMakeCache.txt is created." - ON) - -set(GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES ON OFF WHEN_NO_EXTERNAL) -set( - GDAL_USE_INTERNAL_LIBS WHEN_NO_EXTERNAL - CACHE STRING "Control how internal libraries should be used by default. This should be set before CMakeCache.txt is created.") -set_property(CACHE GDAL_USE_INTERNAL_LIBS PROPERTY STRINGS ${GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES}) -if(NOT GDAL_USE_INTERNAL_LIBS IN_LIST GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES) - message(FATAL_ERROR "GDAL_USE_INTERNAL_LIBS must be one of ${GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES}") -endif() - -set(GDAL_IMPORT_DEPENDENCIES [[ -include(CMakeFindDependencyMacro) -include("${CMAKE_CURRENT_LIST_DIR}/DefineFindPackage2.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/GdalFindModulePath.cmake") -]]) -if(TARGET Threads::Threads) - string(APPEND GDAL_IMPORT_DEPENDENCIES "find_dependency(Threads)\n") -endif() - -# Check that the configuration has a valid value for INTERFACE_INCLUDE_DIRECTORIES. This aimed at avoiding issues like -# https://github.com/OSGeo/gdal/issues/5324 -function (gdal_check_target_is_valid target res_var) - get_target_property(_interface_include_directories ${target} "INTERFACE_INCLUDE_DIRECTORIES") - if(_interface_include_directories) - foreach(_dir IN LISTS _interface_include_directories) - if(NOT EXISTS "${_dir}") - message(WARNING "Target ${target} references ${_dir} as a INTERFACE_INCLUDE_DIRECTORIES, but it does not exist. Ignoring that target.") - set(${res_var} FALSE PARENT_SCOPE) - return() - endif() - endforeach() - elseif("${target}" STREQUAL "geotiff_library" AND DEFINED GeoTIFF_INCLUDE_DIRS) - # geotiff-config.cmake of GeoTIFF 1.7.0 doesn't define a INTERFACE_INCLUDE_DIRECTORIES - # property, but a GeoTIFF_INCLUDE_DIRS variable. - set_target_properties(${target} PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${GeoTIFF_INCLUDE_DIRS}") - else() - message(WARNING "Target ${target} has no INTERFACE_INCLUDE_DIRECTORIES property. Ignoring that target.") - set(${res_var} FALSE PARENT_SCOPE) - return() - endif() - set(${res_var} TRUE PARENT_SCOPE) -endfunction() - -# Package acceptance based on a candidate target list. -# If a matching target is found, sets ${name}_FOUND to TRUE, -# ${name}_INCLUDE_DIRS to "" and ${name}_LIBRARIES to the target name. -# If `REQUIRED` is used, ${name}_FOUND is set to FALSE if no target matches. -function(gdal_check_package_target name) - if("REQUIRED" IN_LIST ARGN) - list(REMOVE_ITEM ARGN "REQUIRED") - set(${name}_FOUND FALSE PARENT_SCOPE) - endif() - foreach(target IN LISTS ARGN) - if(TARGET ${target}) - gdal_check_target_is_valid(${target} _is_valid) - if (_is_valid) - set(${name}_TARGET "${target}" PARENT_SCOPE) - set(${name}_FOUND TRUE PARENT_SCOPE) - return() - endif() - endif() - endforeach() -endfunction() - -# Macro to declare a dependency on an external package. -# If not marked with the ALWAYS_ON_WHEN_FOUND option, dependencies can be -# marked for user control with either the CAN_DISABLE or DISABLED_BY_DEFAULT -# option. User control is done via a cache variable GDAL_USE_{name in upper case} -# with the default value ON for CAN_DISABLE or OFF for DISABLED_BY_DEFAULT. -# The RECOMMENDED option is used for the feature summary. -# The VERSION, CONFIG, MODULE, COMPONENTS and NAMES parameters are passed to find_package(). -# Using NAMES with find_package() implies config mode. However, gdal_check_package() -# attempts another find_package() without NAMES if the config mode attempt was not -# successful, allowing a fallback to Find modules. -# The TARGETS parameter can define a list of candidate targets. If given, a -# package will only be accepted if it defines one of the given targets. The matching -# target name will be saved in ${name}_TARGET. -# The NAMES and TARGETS map to GDAL_CHECK_PACKAGE_${name}_NAMES and -# GDAL_CHECK_PACKAGE_${name}_TARGETS cache variables which can be used to -# overwrite the default config and targets names. -# The required find_dependency() commands for exported config are appended to -# the GDAL_IMPORT_DEPENDENCIES string (when BUILD_SHARED_LIBS=OFF). -macro (gdal_check_package name purpose) - set(_options CONFIG MODULE CAN_DISABLE RECOMMENDED DISABLED_BY_DEFAULT ALWAYS_ON_WHEN_FOUND) - set(_oneValueArgs VERSION NAMES) - set(_multiValueArgs COMPONENTS TARGETS PATHS) - cmake_parse_arguments(_GCP "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" ${ARGN}) - string(TOUPPER ${name} key) - set(_find_dependency "") - set(_find_dependency_args "") - if(FIND_PACKAGE2_${name}_ENABLED) - find_package2(${name} QUIET OUT_DEPENDENCY _find_dependency) - else() - set(_find_package_args) - # For some reason passing the HDF5 version requirement cause a linking error of the libkea driver on Conda Windows builds... - if (_GCP_VERSION AND NOT ("${name}" STREQUAL "TileDB") AND NOT ("${name}" STREQUAL "HDF5")) - list(APPEND _find_package_args ${_GCP_VERSION}) - endif () - if (_GCP_CONFIG) - list(APPEND _find_package_args CONFIG) - endif () - if (_GCP_MODULE) - list(APPEND _find_package_args MODULE) - endif () - if (_GCP_COMPONENTS) - list(APPEND _find_package_args COMPONENTS ${_GCP_COMPONENTS}) - endif () - if (_GCP_PATHS) - list(APPEND _find_package_args PATHS ${_GCP_PATHS}) - endif () - if (_GCP_NAMES) - set(GDAL_CHECK_PACKAGE_${name}_NAMES "${_GCP_NAMES}" CACHE STRING "Config file name for ${name}") - mark_as_advanced(GDAL_CHECK_PACKAGE_${name}_NAMES) - endif () - if (_GCP_TARGETS) - set(GDAL_CHECK_PACKAGE_${name}_TARGETS "${_GCP_TARGETS}" CACHE STRING "Target name candidates for ${name}") - mark_as_advanced(GDAL_CHECK_PACKAGE_${name}_TARGETS) - endif () - if (GDAL_CHECK_PACKAGE_${name}_NAMES) - find_package(${name} NAMES ${GDAL_CHECK_PACKAGE_${name}_NAMES} ${_find_package_args}) - gdal_check_package_target(${name} ${GDAL_CHECK_PACKAGE_${name}_TARGETS} REQUIRED) - if (${name}_FOUND) - get_filename_component(_find_dependency_args "${${name}_CONFIG}" NAME) - string(REPLACE ";" " " _find_dependency_args "${name} ${_find_package_args} NAMES ${GDAL_CHECK_PACKAGE_${name}_NAMES} CONFIGS ${_find_dependency_args}") - endif () - endif () - if (NOT ${name}_FOUND) - find_package(${name} ${_find_package_args}) - if (${name}_FOUND) - gdal_check_package_target(${name} ${GDAL_CHECK_PACKAGE_${name}_TARGETS}) - elseif (${key}_FOUND) # Some find modules do not set _FOUND - gdal_check_package_target(${key} ${GDAL_CHECK_PACKAGE_${name}_TARGETS}) - set(${name}_FOUND "${key}_FOUND") - endif () - if (${name}_FOUND) - string(REPLACE ";" " " _find_dependency_args "${name} ${_find_package_args}") - endif() - endif () - endif () - if (${key}_FOUND OR ${name}_FOUND) - if(_GCP_VERSION) - - if( "${name}" STREQUAL "TileDB" AND NOT DEFINED TileDB_VERSION) - get_property(_dirs TARGET TileDB::tiledb_shared PROPERTY INTERFACE_INCLUDE_DIRECTORIES) - foreach(_dir IN LISTS _dirs) - set(TILEDB_VERSION_FILENAME "${_dir}/tiledb/tiledb_version.h") - if(EXISTS ${TILEDB_VERSION_FILENAME}) - file(READ ${TILEDB_VERSION_FILENAME} _tiledb_version_contents) - string(REGEX REPLACE "^.*TILEDB_VERSION_MAJOR +([0-9]+).*$" "\\1" TILEDB_VERSION_MAJOR "${_tiledb_version_contents}") - string(REGEX REPLACE "^.*TILEDB_VERSION_MINOR +([0-9]+).*$" "\\1" TILEDB_VERSION_MINOR "${_tiledb_version_contents}") - set(TileDB_VERSION "${TILEDB_VERSION_MAJOR}.${TILEDB_VERSION_MINOR}") - endif() - endforeach() - endif() - - if (DEFINED ${name}_VERSION_STRING AND NOT DEFINED ${name}_VERSION) - set(${name}_VERSION "${${name}_VERSION_STRING}") - endif() - - if( "${${name}_VERSION}" STREQUAL "") - message(WARNING "${name} has unknown version. Assuming it is at least matching the minimum version required of ${_GCP_VERSION}") - set(HAVE_${key} ON) - elseif( ${name}_VERSION VERSION_LESS ${_GCP_VERSION}) - message(WARNING "Ignoring ${name} because it is at version ${${name}_VERSION}, whereas the minimum version required is ${_GCP_VERSION}") - set(HAVE_${key} OFF) - else() - set(HAVE_${key} ON) - endif() - else() - set(HAVE_${key} ON) - endif() - else () - set(HAVE_${key} OFF) - endif () - if (purpose STREQUAL "") - - else () - if (_GCP_RECOMMENDED) - set_package_properties( - ${name} PROPERTIES - PURPOSE ${purpose} - TYPE RECOMMENDED) - else () - set_package_properties(${name} PROPERTIES PURPOSE ${purpose}) - endif () - endif () - - if (_GCP_CAN_DISABLE OR _GCP_DISABLED_BY_DEFAULT) - set(_gcpp_status ON) - if (GDAL_USE_${key}) - if (NOT HAVE_${key}) - message(FATAL_ERROR "Configured to use ${key}, but not found") - endif () - elseif (NOT GDAL_USE_EXTERNAL_LIBS) - set(_gcpp_status OFF) - if (HAVE_${key} AND NOT GDAL_USE_${key}) - message(STATUS - "${key} has been found, but is disabled due to GDAL_USE_EXTERNAL_LIBS=OFF. Enable it by setting GDAL_USE_${key}=ON" - ) - set(_find_dependency_args "") - endif () - endif () - if (_gcpp_status AND _GCP_DISABLED_BY_DEFAULT) - set(_gcpp_status OFF) - if (HAVE_${key} AND NOT GDAL_USE_${key}) - message(STATUS "${key} has been found, but is disabled by default. Enable it by setting GDAL_USE_${key}=ON") - set(_find_dependency_args "") - endif () - endif () - cmake_dependent_option(GDAL_USE_${key} "Set ON to use ${key}" ${_gcpp_status} "HAVE_${key}" OFF) - elseif (NOT _GCP_ALWAYS_ON_WHEN_FOUND) - message(FATAL_ERROR "Programming error: missing CAN_DISABLE or DISABLED_BY_DEFAULT option for component ${name}") - endif () - - if(_find_dependency_args) - string(REPLACE "\"" "\\\"" _find_dependency_args "${_find_dependency_args}") - set(_find_dependency "find_dependency(${_find_dependency_args})\n") - endif() - if(NOT BUILD_SHARED_LIBS AND GDAL_USE_${key} AND _find_dependency) - string(APPEND GDAL_IMPORT_DEPENDENCIES "${_find_dependency}") - endif() - unset(_find_dependency_args) - unset(_find_dependency) -endmacro () - -function (split_libpath _lib) - if (_lib) - # split lib_line into -L and -l linker options - get_filename_component(_path ${${_lib}} PATH) - get_filename_component(_name ${${_lib}} NAME_WE) - string(REGEX REPLACE "^lib" "" _name ${_name}) - set(${_lib} -L${_path} -l${_name}) - endif () -endfunction () - -function (gdal_internal_library libname) - set(_options REQUIRED) - set(_oneValueArgs) - set(_multiValueArgs) - cmake_parse_arguments(_GIL "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" ${ARGN}) - if ("${GDAL_USE_INTERNAL_LIBS}" STREQUAL "ON") - set(_default_value ON) - elseif ("${GDAL_USE_INTERNAL_LIBS}" STREQUAL "OFF") - set(_default_value OFF) - elseif( GDAL_USE_${libname} ) - set(_default_value OFF) - else() - set(_default_value ON) - endif() - set(GDAL_USE_${libname}_INTERNAL - ${_default_value} - CACHE BOOL "Use internal ${libname} copy (if set to ON, has precedence over GDAL_USE_${libname})") - if (_GIL_REQUIRED - AND (NOT GDAL_USE_${libname}) - AND (NOT GDAL_USE_${libname}_INTERNAL)) - message(FATAL_ERROR "GDAL_USE_${libname} or GDAL_USE_${libname}_INTERNAL must be set to ON") - endif () -endfunction () +include(CheckDependentLibrariesCommon) # Custom find_package definitions @@ -313,6 +45,7 @@ if (Iconv_FOUND) size_t ret = iconv(conv, &in, &ilen, &out, &olen); return (size_t)ret; }") + include(CheckCXXSourceCompiles) check_cxx_source_compiles("${ICONV_CONST_TEST_CODE}" _ICONV_SECOND_ARGUMENT_IS_NOT_CONST) if (_ICONV_SECOND_ARGUMENT_IS_NOT_CONST) set(ICONV_CPP_CONST "") diff --git a/cmake/helpers/CheckDependentLibrariesCommon.cmake b/cmake/helpers/CheckDependentLibrariesCommon.cmake new file mode 100644 index 000000000000..ebb353829fcb --- /dev/null +++ b/cmake/helpers/CheckDependentLibrariesCommon.cmake @@ -0,0 +1,281 @@ +# Distributed under the GDAL/OGR MIT style License. See accompanying file LICENSE.TXT. + +#[=======================================================================[.rst: +CheckDependentLibraries.cmake +----------------------------- + +Detect GDAL dependencies and set variable HAVE_* + +#]=======================================================================] + +include(CheckFunctionExists) +include(CMakeDependentOption) +include(FeatureSummary) +include(DefineFindPackage2) +include(CheckSymbolExists) + +option( + GDAL_USE_EXTERNAL_LIBS + "Whether detected external libraries should be used by default. This should be set before CMakeCache.txt is created." + ON) + +set(GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES ON OFF WHEN_NO_EXTERNAL) +set( + GDAL_USE_INTERNAL_LIBS WHEN_NO_EXTERNAL + CACHE STRING "Control how internal libraries should be used by default. This should be set before CMakeCache.txt is created.") +set_property(CACHE GDAL_USE_INTERNAL_LIBS PROPERTY STRINGS ${GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES}) +if(NOT GDAL_USE_INTERNAL_LIBS IN_LIST GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES) + message(FATAL_ERROR "GDAL_USE_INTERNAL_LIBS must be one of ${GDAL_USE_INTERNAL_LIBS_ALLOWED_VALUES}") +endif() + +set(GDAL_IMPORT_DEPENDENCIES [[ +include(CMakeFindDependencyMacro) +include("${CMAKE_CURRENT_LIST_DIR}/DefineFindPackage2.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/GdalFindModulePath.cmake") +]]) +if(TARGET Threads::Threads) + string(APPEND GDAL_IMPORT_DEPENDENCIES "find_dependency(Threads)\n") +endif() + +# Check that the configuration has a valid value for INTERFACE_INCLUDE_DIRECTORIES. This aimed at avoiding issues like +# https://github.com/OSGeo/gdal/issues/5324 +function (gdal_check_target_is_valid target res_var) + get_target_property(_interface_include_directories ${target} "INTERFACE_INCLUDE_DIRECTORIES") + if(_interface_include_directories) + foreach(_dir IN LISTS _interface_include_directories) + if(NOT EXISTS "${_dir}") + message(WARNING "Target ${target} references ${_dir} as a INTERFACE_INCLUDE_DIRECTORIES, but it does not exist. Ignoring that target.") + set(${res_var} FALSE PARENT_SCOPE) + return() + endif() + endforeach() + elseif("${target}" STREQUAL "geotiff_library" AND DEFINED GeoTIFF_INCLUDE_DIRS) + # geotiff-config.cmake of GeoTIFF 1.7.0 doesn't define a INTERFACE_INCLUDE_DIRECTORIES + # property, but a GeoTIFF_INCLUDE_DIRS variable. + set_target_properties(${target} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${GeoTIFF_INCLUDE_DIRS}") + else() + message(WARNING "Target ${target} has no INTERFACE_INCLUDE_DIRECTORIES property. Ignoring that target.") + set(${res_var} FALSE PARENT_SCOPE) + return() + endif() + set(${res_var} TRUE PARENT_SCOPE) +endfunction() + +# Package acceptance based on a candidate target list. +# If a matching target is found, sets ${name}_FOUND to TRUE, +# ${name}_INCLUDE_DIRS to "" and ${name}_LIBRARIES to the target name. +# If `REQUIRED` is used, ${name}_FOUND is set to FALSE if no target matches. +function(gdal_check_package_target name) + if("REQUIRED" IN_LIST ARGN) + list(REMOVE_ITEM ARGN "REQUIRED") + set(${name}_FOUND FALSE PARENT_SCOPE) + endif() + foreach(target IN LISTS ARGN) + if(TARGET ${target}) + gdal_check_target_is_valid(${target} _is_valid) + if (_is_valid) + set(${name}_TARGET "${target}" PARENT_SCOPE) + set(${name}_FOUND TRUE PARENT_SCOPE) + return() + endif() + endif() + endforeach() +endfunction() + +# Macro to declare a dependency on an external package. +# If not marked with the ALWAYS_ON_WHEN_FOUND option, dependencies can be +# marked for user control with either the CAN_DISABLE or DISABLED_BY_DEFAULT +# option. User control is done via a cache variable GDAL_USE_{name in upper case} +# with the default value ON for CAN_DISABLE or OFF for DISABLED_BY_DEFAULT. +# The RECOMMENDED option is used for the feature summary. +# The VERSION, CONFIG, MODULE, COMPONENTS and NAMES parameters are passed to find_package(). +# Using NAMES with find_package() implies config mode. However, gdal_check_package() +# attempts another find_package() without NAMES if the config mode attempt was not +# successful, allowing a fallback to Find modules. +# The TARGETS parameter can define a list of candidate targets. If given, a +# package will only be accepted if it defines one of the given targets. The matching +# target name will be saved in ${name}_TARGET. +# The NAMES and TARGETS map to GDAL_CHECK_PACKAGE_${name}_NAMES and +# GDAL_CHECK_PACKAGE_${name}_TARGETS cache variables which can be used to +# overwrite the default config and targets names. +# The required find_dependency() commands for exported config are appended to +# the GDAL_IMPORT_DEPENDENCIES string (when BUILD_SHARED_LIBS=OFF). +macro (gdal_check_package name purpose) + set(_options CONFIG MODULE CAN_DISABLE RECOMMENDED DISABLED_BY_DEFAULT ALWAYS_ON_WHEN_FOUND) + set(_oneValueArgs VERSION NAMES) + set(_multiValueArgs COMPONENTS TARGETS PATHS) + cmake_parse_arguments(_GCP "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" ${ARGN}) + string(TOUPPER ${name} key) + set(_find_dependency "") + set(_find_dependency_args "") + if(FIND_PACKAGE2_${name}_ENABLED) + find_package2(${name} QUIET OUT_DEPENDENCY _find_dependency) + else() + set(_find_package_args) + # For some reason passing the HDF5 version requirement cause a linking error of the libkea driver on Conda Windows builds... + if (_GCP_VERSION AND NOT ("${name}" STREQUAL "TileDB") AND NOT ("${name}" STREQUAL "HDF5")) + list(APPEND _find_package_args ${_GCP_VERSION}) + endif () + if (_GCP_CONFIG) + list(APPEND _find_package_args CONFIG) + endif () + if (_GCP_MODULE) + list(APPEND _find_package_args MODULE) + endif () + if (_GCP_COMPONENTS) + list(APPEND _find_package_args COMPONENTS ${_GCP_COMPONENTS}) + endif () + if (_GCP_PATHS) + list(APPEND _find_package_args PATHS ${_GCP_PATHS}) + endif () + if (_GCP_NAMES) + set(GDAL_CHECK_PACKAGE_${name}_NAMES "${_GCP_NAMES}" CACHE STRING "Config file name for ${name}") + mark_as_advanced(GDAL_CHECK_PACKAGE_${name}_NAMES) + endif () + if (_GCP_TARGETS) + set(GDAL_CHECK_PACKAGE_${name}_TARGETS "${_GCP_TARGETS}" CACHE STRING "Target name candidates for ${name}") + mark_as_advanced(GDAL_CHECK_PACKAGE_${name}_TARGETS) + endif () + if (GDAL_CHECK_PACKAGE_${name}_NAMES) + find_package(${name} NAMES ${GDAL_CHECK_PACKAGE_${name}_NAMES} ${_find_package_args}) + gdal_check_package_target(${name} ${GDAL_CHECK_PACKAGE_${name}_TARGETS} REQUIRED) + if (${name}_FOUND) + get_filename_component(_find_dependency_args "${${name}_CONFIG}" NAME) + string(REPLACE ";" " " _find_dependency_args "${name} ${_find_package_args} NAMES ${GDAL_CHECK_PACKAGE_${name}_NAMES} CONFIGS ${_find_dependency_args}") + endif () + endif () + if (NOT ${name}_FOUND) + find_package(${name} ${_find_package_args}) + if (${name}_FOUND) + gdal_check_package_target(${name} ${GDAL_CHECK_PACKAGE_${name}_TARGETS}) + elseif (${key}_FOUND) # Some find modules do not set _FOUND + gdal_check_package_target(${key} ${GDAL_CHECK_PACKAGE_${name}_TARGETS}) + set(${name}_FOUND "${key}_FOUND") + endif () + if (${name}_FOUND) + string(REPLACE ";" " " _find_dependency_args "${name} ${_find_package_args}") + endif() + endif () + endif () + if (${key}_FOUND OR ${name}_FOUND) + if(_GCP_VERSION) + + if( "${name}" STREQUAL "TileDB" AND NOT DEFINED TileDB_VERSION) + get_property(_dirs TARGET TileDB::tiledb_shared PROPERTY INTERFACE_INCLUDE_DIRECTORIES) + foreach(_dir IN LISTS _dirs) + set(TILEDB_VERSION_FILENAME "${_dir}/tiledb/tiledb_version.h") + if(EXISTS ${TILEDB_VERSION_FILENAME}) + file(READ ${TILEDB_VERSION_FILENAME} _tiledb_version_contents) + string(REGEX REPLACE "^.*TILEDB_VERSION_MAJOR +([0-9]+).*$" "\\1" TILEDB_VERSION_MAJOR "${_tiledb_version_contents}") + string(REGEX REPLACE "^.*TILEDB_VERSION_MINOR +([0-9]+).*$" "\\1" TILEDB_VERSION_MINOR "${_tiledb_version_contents}") + set(TileDB_VERSION "${TILEDB_VERSION_MAJOR}.${TILEDB_VERSION_MINOR}") + endif() + endforeach() + endif() + + if (DEFINED ${name}_VERSION_STRING AND NOT DEFINED ${name}_VERSION) + set(${name}_VERSION "${${name}_VERSION_STRING}") + endif() + + if( "${${name}_VERSION}" STREQUAL "") + message(WARNING "${name} has unknown version. Assuming it is at least matching the minimum version required of ${_GCP_VERSION}") + set(HAVE_${key} ON) + elseif( ${name}_VERSION VERSION_LESS ${_GCP_VERSION}) + message(WARNING "Ignoring ${name} because it is at version ${${name}_VERSION}, whereas the minimum version required is ${_GCP_VERSION}") + set(HAVE_${key} OFF) + else() + set(HAVE_${key} ON) + endif() + else() + set(HAVE_${key} ON) + endif() + else () + set(HAVE_${key} OFF) + endif () + if (purpose STREQUAL "") + + else () + if (_GCP_RECOMMENDED) + set_package_properties( + ${name} PROPERTIES + PURPOSE ${purpose} + TYPE RECOMMENDED) + else () + set_package_properties(${name} PROPERTIES PURPOSE ${purpose}) + endif () + endif () + + if (_GCP_CAN_DISABLE OR _GCP_DISABLED_BY_DEFAULT) + set(_gcpp_status ON) + if (GDAL_USE_${key}) + if (NOT HAVE_${key}) + message(FATAL_ERROR "Configured to use ${key}, but not found") + endif () + elseif (NOT GDAL_USE_EXTERNAL_LIBS) + set(_gcpp_status OFF) + if (HAVE_${key} AND NOT GDAL_USE_${key}) + message(STATUS + "${key} has been found, but is disabled due to GDAL_USE_EXTERNAL_LIBS=OFF. Enable it by setting GDAL_USE_${key}=ON" + ) + set(_find_dependency_args "") + endif () + endif () + if (_gcpp_status AND _GCP_DISABLED_BY_DEFAULT) + set(_gcpp_status OFF) + if (HAVE_${key} AND NOT GDAL_USE_${key}) + message(STATUS "${key} has been found, but is disabled by default. Enable it by setting GDAL_USE_${key}=ON") + set(_find_dependency_args "") + endif () + endif () + cmake_dependent_option(GDAL_USE_${key} "Set ON to use ${key}" ${_gcpp_status} "HAVE_${key}" OFF) + elseif (NOT _GCP_ALWAYS_ON_WHEN_FOUND) + message(FATAL_ERROR "Programming error: missing CAN_DISABLE or DISABLED_BY_DEFAULT option for component ${name}") + endif () + + if(_find_dependency_args) + string(REPLACE "\"" "\\\"" _find_dependency_args "${_find_dependency_args}") + set(_find_dependency "find_dependency(${_find_dependency_args})\n") + endif() + if(NOT BUILD_SHARED_LIBS AND GDAL_USE_${key} AND _find_dependency) + string(APPEND GDAL_IMPORT_DEPENDENCIES "${_find_dependency}") + endif() + unset(_find_dependency_args) + unset(_find_dependency) +endmacro () + +function (split_libpath _lib) + if (_lib) + # split lib_line into -L and -l linker options + get_filename_component(_path ${${_lib}} PATH) + get_filename_component(_name ${${_lib}} NAME_WE) + string(REGEX REPLACE "^lib" "" _name ${_name}) + set(${_lib} -L${_path} -l${_name}) + endif () +endfunction () + +function (gdal_internal_library libname) + set(_options REQUIRED) + set(_oneValueArgs) + set(_multiValueArgs) + cmake_parse_arguments(_GIL "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" ${ARGN}) + if ("${GDAL_USE_INTERNAL_LIBS}" STREQUAL "ON") + set(_default_value ON) + elseif ("${GDAL_USE_INTERNAL_LIBS}" STREQUAL "OFF") + set(_default_value OFF) + elseif( GDAL_USE_${libname} ) + set(_default_value OFF) + else() + set(_default_value ON) + endif() + set(GDAL_USE_${libname}_INTERNAL + ${_default_value} + CACHE BOOL "Use internal ${libname} copy (if set to ON, has precedence over GDAL_USE_${libname})") + if (_GIL_REQUIRED + AND (NOT GDAL_USE_${libname}) + AND (NOT GDAL_USE_${libname}_INTERNAL)) + message(FATAL_ERROR "GDAL_USE_${libname} or GDAL_USE_${libname}_INTERNAL must be set to ON") + endif () +endfunction () + +# vim: ts=4 sw=4 sts=4 et From e755508bea4708667588fde365a5e90c3461ad37 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:25:06 +0200 Subject: [PATCH 25/72] GdalDriverHelper.cmake: add infrastructure for STANDALONE mode --- cmake/helpers/GdalDriverHelper.cmake | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cmake/helpers/GdalDriverHelper.cmake b/cmake/helpers/GdalDriverHelper.cmake index b352ae1ca08b..ad07cb3d407c 100644 --- a/cmake/helpers/GdalDriverHelper.cmake +++ b/cmake/helpers/GdalDriverHelper.cmake @@ -155,7 +155,11 @@ function(add_gdal_driver) set(_COND ${_DRIVER_PLUGIN_CAPABLE_IF}) endif() - get_target_property(PLUGIN_OUTPUT_DIR ${GDAL_LIB_TARGET_NAME} PLUGIN_OUTPUT_DIR) + if(STANDALONE) + set(PLUGIN_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}") + else() + get_target_property(PLUGIN_OUTPUT_DIR ${GDAL_LIB_TARGET_NAME} PLUGIN_OUTPUT_DIR) + endif() if (_DRIVER_PLUGIN_CAPABLE OR _DRIVER_PLUGIN_CAPABLE_IF) set(_INITIAL_VALUE OFF) @@ -293,7 +297,9 @@ function(add_gdal_driver) else() message(FATAL_ERROR "Driver ${_DRIVER_TARGET} should declare DRIVER_NO_SHARED_SYMBOL_WITH_CORE") endif() - _set_driver_core_sources(${_KEY} ${_DRIVER_TARGET} ${_DRIVER_CORE_SOURCES}) + if(NOT STANDALONE) + _set_driver_core_sources(${_KEY} ${_DRIVER_TARGET} ${_DRIVER_CORE_SOURCES}) + endif() endif () else () @@ -325,7 +331,9 @@ function(add_gdal_driver) target_compile_options(${_DRIVER_TARGET} PRIVATE $<$:${GDAL_CXX_WARNING_FLAGS}>) endif() target_compile_options(${_DRIVER_TARGET} PRIVATE $<$:${GDAL_C_WARNING_FLAGS}>) - add_dependencies(${_DRIVER_TARGET} generate_gdal_version_h) + if (NOT STANDALONE) + add_dependencies(${_DRIVER_TARGET} generate_gdal_version_h) + endif() endfunction() # Detect whether driver is built as PLUGIN or not. @@ -483,7 +491,7 @@ macro(gdal_dependent_format format desc depends) cmake_dependent_option(GDAL_ENABLE_DRIVER_${key} "Set ON to build ${desc} format" ${GDAL_BUILD_OPTIONAL_DRIVERS} "${depends}" OFF) add_feature_info(gdal_${key} GDAL_ENABLE_DRIVER_${key} "${desc}") - if ((GDAL_ENABLE_DRIVER_${key} AND NOT _GDF_SKIP_ADD_SUBDIRECTORY) OR GDAL_REGISTER_DRIVER_${key}_FOR_LATER_PLUGIN) + if (NOT STANDALONE AND (GDAL_ENABLE_DRIVER_${key} AND NOT _GDF_SKIP_ADD_SUBDIRECTORY) OR GDAL_REGISTER_DRIVER_${key}_FOR_LATER_PLUGIN) add_subdirectory(${format}) endif () endmacro() @@ -525,7 +533,7 @@ macro(ogr_dependent_driver name desc depend) "${depend}" OFF) endif() add_feature_info(ogr_${key} OGR_ENABLE_DRIVER_${key} "${desc}") - if (OGR_ENABLE_DRIVER_${key} OR OGR_REGISTER_DRIVER_${key}_FOR_LATER_PLUGIN) + if (NOT STANDALONE AND OGR_ENABLE_DRIVER_${key} OR OGR_REGISTER_DRIVER_${key}_FOR_LATER_PLUGIN) add_subdirectory(${name}) endif () endmacro() From 8f46cf0fceca26b4a02243742ef1f2213984e307 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:25:35 +0200 Subject: [PATCH 26/72] CMake: add SetupStandalonePlugin.cmake --- cmake/helpers/SetupStandalonePlugin.cmake | 57 +++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 cmake/helpers/SetupStandalonePlugin.cmake diff --git a/cmake/helpers/SetupStandalonePlugin.cmake b/cmake/helpers/SetupStandalonePlugin.cmake new file mode 100644 index 000000000000..53fee7fdcd72 --- /dev/null +++ b/cmake/helpers/SetupStandalonePlugin.cmake @@ -0,0 +1,57 @@ +# Distributed under the GDAL/OGR MIT License. See accompanying file LICENSE.TXT. +# This file is included by drivers that want to be built as plugin against an +# installed GDAL library (and thus not requiring to build libgdal itself) + +include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/modules/init.cmake") + +# Hint used to alter the behavior of a number of .cmake files +set(STANDALONE ON) + +# Detect installed GDAL +find_package(GDAL REQUIRED) +set(GDAL_VERSION_IMPORTED ${GDAL_VERSION}) +set(GDAL_LIB_TARGET_NAME GDAL::GDAL) + +# Check that we build the plugin against a GDAL version that matches the one +# of the sources +include(GdalVersion) +set(GDAL_VERSION_MAJOR_SOURCE ${GDAL_VERSION_MAJOR}) +set(GDAL_VERSION_MINOR_SOURCE ${GDAL_VERSION_MINOR}) +set(GDAL_VERSION_REV_SOURCE ${GDAL_VERSION_REV}) +if(NOT "${GDAL_VERSION_IMPORTED}" MATCHES "${GDAL_VERSION_MAJOR_SOURCE}.${GDAL_VERSION_MINOR_SOURCE}.${GDAL_VERSION_REV_SOURCE}") + if (NOT IGNORE_GDAL_VERSION_MISMATCH) + message(FATAL_ERROR "Building plugin against GDAL sources ${GDAL_VERSION_MAJOR_SOURCE}.${GDAL_VERSION_MINOR_SOURCE}.${GDAL_VERSION_REV_SOURCE} whereas linked GDAL library is at version ${GDAL_VERSION_IMPORTED}. This is not a nominally supported configuration. You can bypass this check by setting the IGNORE_GDAL_VERSION_MISMATCH variable.") + endif() +endif() + +include(GdalCAndCXXStandards) +include(GdalStandardIncludes) + +include(CheckDependentLibrariesCommon) + +include(GdalCompilationFlags) + +set(GDAL_ENABLE_PLUGINS ON) +set(GDAL_BUILD_OPTIONAL_DRIVERS ON) +set(OGR_ENABLE_PLUGINS ON) +set(OGR_BUILD_OPTIONAL_DRIVERS ON) +include(GdalDriverHelper) + +include(GNUInstallDirs) +# Used by GdalDriverHelper's add_gdal_driver() +set(INSTALL_PLUGIN_DIR + "${CMAKE_INSTALL_LIBDIR}/gdalplugins" + CACHE PATH "Installation sub-directory for plugins") + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") + +macro(standalone_driver_finalize VAR) + include(SystemSummary) + include(driver_declaration.cmake) + if (NOT ${VAR}) + message(FATAL_ERROR "${VAR} is not set, due to missing build requirements") + endif() + system_summary(DESCRIPTION "${PROJECT_NAME} is now configured on") + feature_summary(DESCRIPTION "Enabled drivers and features and found dependency packages" WHAT ALL) +endmacro() From 9e85ace225b241aaa6cc2493557600bb770c61fe Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:27:06 +0200 Subject: [PATCH 27/72] CMake: make MrSID buildable in standalone mode --- cmake/helpers/CheckDependentLibraries.cmake | 8 ++---- .../CheckDependentLibrariesGeoTIFF.cmake | 5 ++++ .../CheckDependentLibrariesMrSID.cmake | 1 + doc/source/drivers/raster/mrsid.rst | 25 +++++++++++++++++++ frmts/CMakeLists.txt | 2 +- frmts/mrsid/CMakeLists.txt | 11 ++++++++ frmts/mrsid/driver_declaration.cmake | 1 + 7 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 cmake/helpers/CheckDependentLibrariesGeoTIFF.cmake create mode 100644 cmake/helpers/CheckDependentLibrariesMrSID.cmake create mode 100644 frmts/mrsid/driver_declaration.cmake diff --git a/cmake/helpers/CheckDependentLibraries.cmake b/cmake/helpers/CheckDependentLibraries.cmake index 0dddba9da97c..448cebba887b 100644 --- a/cmake/helpers/CheckDependentLibraries.cmake +++ b/cmake/helpers/CheckDependentLibraries.cmake @@ -163,11 +163,7 @@ gdal_check_package(ZSTD "ZSTD compression library" CAN_DISABLE ${ZSTD_NAMES_AND_ gdal_check_package(SFCGAL "gdal core supports ISO 19107:2013 and OGC Simple Features Access 1.2 for 3D operations" CAN_DISABLE) -gdal_check_package(GeoTIFF "libgeotiff library (external)" CAN_DISABLE RECOMMENDED - NAMES GeoTIFF - TARGETS geotiff_library GEOTIFF::GEOTIFF -) -gdal_internal_library(GEOTIFF REQUIRED) +include(CheckDependentLibrariesGeoTIFF) gdal_check_package(PNG "PNG compression library (external)" CAN_DISABLE RECOMMENDED VERSION "1.6") gdal_internal_library(PNG) @@ -348,7 +344,7 @@ gdal_check_package(FreeXL "Enable XLS driver" CAN_DISABLE) define_find_package2(GTA gta/gta.h gta PKGCONFIG_NAME gta) gdal_check_package(GTA "Enable GTA driver" CAN_DISABLE) -gdal_check_package(MRSID "MrSID raster SDK" CAN_DISABLE) +include(CheckDependentLibrariesMrSID) set(GDAL_USE_ARMADILLO_OLD ${GDAL_USE_ARMADILLO}) gdal_check_package(Armadillo "C++ library for linear algebra (used for TPS transformation)" CAN_DISABLE) diff --git a/cmake/helpers/CheckDependentLibrariesGeoTIFF.cmake b/cmake/helpers/CheckDependentLibrariesGeoTIFF.cmake new file mode 100644 index 000000000000..82ea7cbc70fe --- /dev/null +++ b/cmake/helpers/CheckDependentLibrariesGeoTIFF.cmake @@ -0,0 +1,5 @@ +gdal_check_package(GeoTIFF "libgeotiff library (external)" CAN_DISABLE RECOMMENDED + NAMES GeoTIFF + TARGETS geotiff_library GEOTIFF::GEOTIFF +) +gdal_internal_library(GEOTIFF REQUIRED) diff --git a/cmake/helpers/CheckDependentLibrariesMrSID.cmake b/cmake/helpers/CheckDependentLibrariesMrSID.cmake new file mode 100644 index 000000000000..e2f37b39feae --- /dev/null +++ b/cmake/helpers/CheckDependentLibrariesMrSID.cmake @@ -0,0 +1 @@ +gdal_check_package(MRSID "MrSID raster SDK" CAN_DISABLE) diff --git a/doc/source/drivers/raster/mrsid.rst b/doc/source/drivers/raster/mrsid.rst index fbe96a84c8a6..ec77d3e4ceaa 100644 --- a/doc/source/drivers/raster/mrsid.rst +++ b/doc/source/drivers/raster/mrsid.rst @@ -63,6 +63,31 @@ GeoKeys, stored in MrSID files. This bug was fixed in MrSID software version 1.5, but if you have older encoders or files, created with older encoders, you cannot use georeference information from them. +Standalone plugin compilation +----------------------------- + +.. versionadded:: 3.10 + +While this driver may be built as part of a whole GDAL build, either in libgdal +itself, or as a plugin, it is also possible to only build this driver as a plugin, +against an already built libgdal. + +The version of the GDAL sources used to build the driver must match the version +of the libgdal it is built against. + +For example, from a "build_mrsid" directory under the root of the GDAL source tree: + +:: + + cmake -S ../frmts/mrsid -DCMAKE_PREFIX_PATH=/path/to/GDAL_installation_prefix -DMRSID_ROOT=/path/to/mrsid_sdk_root + cmake --build . + + +Note that such a plugin, when used against a libgdal not aware of it, will be +systematically loaded at GDAL driver initialization time, and will not benefit from +`deferred plugin loading capabilities `. For that, libgdal itself must be built with the +CMake variable GDAL_REGISTER_DRIVER_MRSID_FOR_LATER_PLUGIN=ON set. + See Also: --------- diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index fc9787f9580c..fe27c646f5f5 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -165,7 +165,7 @@ gdal_dependent_format(jp2lura "JPEG-2000 (based on Luratech)" "GDAL_USE_LURATECH # ESRI ArcSDE C API SDK gdal_dependent_format(sde "ESRI ArcSDE Raster" "HAVE_SDE") # LizardTech's decoding software development kit (DSDK) -gdal_dependent_format(mrsid "Multi-resolution Seamless Image Database" "GDAL_USE_MRSID") +include(mrsid/driver_declaration.cmake) gdal_dependent_format(georaster "Oracle Spatial GeoRaster" "GDAL_USE_ORACLE" DRIVER_NAME_OPTION GEOR) gdal_dependent_format(ecw "ERDAS JPEG2000 (.jp2)" "GDAL_USE_ECW") diff --git a/frmts/mrsid/CMakeLists.txt b/frmts/mrsid/CMakeLists.txt index c91ab9d269ed..bc5ea5fe6ab8 100644 --- a/frmts/mrsid/CMakeLists.txt +++ b/frmts/mrsid/CMakeLists.txt @@ -1,3 +1,14 @@ +cmake_minimum_required(VERSION 3.16...3.28) + +if(NOT DEFINED PROJECT_SOURCE_DIR) + # Standalone plugin building + project(gdal_MrSID) + include("${PROJECT_SOURCE_DIR}/../../cmake/helpers/SetupStandalonePlugin.cmake" ) + include(CheckDependentLibrariesGeoTIFF) + include(CheckDependentLibrariesMrSID) + standalone_driver_finalize(GDAL_ENABLE_DRIVER_MRSID) +endif() + option(GDAL_ENABLE_DRIVER_JP2MRSID "Whether to enable JPEG2000 support with MrSID SDK" OFF) add_gdal_driver(TARGET gdal_MrSID diff --git a/frmts/mrsid/driver_declaration.cmake b/frmts/mrsid/driver_declaration.cmake new file mode 100644 index 000000000000..007b65737707 --- /dev/null +++ b/frmts/mrsid/driver_declaration.cmake @@ -0,0 +1 @@ +gdal_dependent_format(mrsid "Multi-resolution Seamless Image Database" "GDAL_USE_MRSID") From 36055b6de63d95b7c6e831be80d9b3dcfbf5ee54 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 21:13:11 +0200 Subject: [PATCH 28/72] tiff_srs.py: skip 2 tests if libgeotiff < 1.6.0 --- autotest/gcore/tiff_srs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/autotest/gcore/tiff_srs.py b/autotest/gcore/tiff_srs.py index 71a922ad89f6..a4d9d6c9c9a1 100755 --- a/autotest/gcore/tiff_srs.py +++ b/autotest/gcore/tiff_srs.py @@ -621,7 +621,7 @@ def test_tiff_srs_proj4(proj4): def _create_geotiff1_1_from_copy_and_compare(srcfilename, options=[]): if int(gdal.GetDriverByName("GTiff").GetMetadataItem("LIBGEOTIFF")) < 1600: - pytest.skip() + pytest.skip("libgeotiff >= 1.6.0 required") src_ds = gdal.Open(srcfilename) tmpfile = "/vsimem/tmp.tif" @@ -1250,9 +1250,11 @@ def test_tiff_srs_projected_GTCitationGeoKey_with_underscore_and_GeogTOWGS84GeoK def test_tiff_srs_write_compound_with_non_epsg_vert_crs(): - """Test bugfix for https://github.com/OSGeo/gdal/issues/7833""" + if int(gdal.GetDriverByName("GTiff").GetMetadataItem("LIBGEOTIFF")) < 1600: + pytest.skip("libgeotiff >= 1.6.0 required") + filename = "/vsimem/test_tiff_srs_write_compound_with_non_epsg_vert_crs.tif" srs = osr.SpatialReference() srs.SetFromUserInput( @@ -1349,6 +1351,9 @@ def test_tiff_srs_read_compound_without_EPSG_code(): """Test case where identification of code for CompoundCRS (added for bugfix of https://github.com/OSGeo/gdal/issues/7982) doesn't trigger""" + if int(gdal.GetDriverByName("GTiff").GetMetadataItem("LIBGEOTIFF")) < 1600: + pytest.skip("libgeotiff >= 1.6.0 required") + filename = "/vsimem/test_tiff_srs_read_compound_without_EPSG_code.tif" srs = osr.SpatialReference() # WGS 84 + NAP height, unlikely to have a EPSG code ever From db2e14d0d97c1b131253ea01971b72475d273a5f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 19:03:44 +0200 Subject: [PATCH 29/72] CI: test building MrSID driver in standalone mode --- .github/workflows/ubuntu_20.04/Dockerfile.ci | 1 + .github/workflows/ubuntu_20.04/build.sh | 20 ++++++++++++++------ frmts/mrsid/CMakeLists.txt | 3 +++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ubuntu_20.04/Dockerfile.ci b/.github/workflows/ubuntu_20.04/Dockerfile.ci index 50aa2c17d8a5..531c52ba75b1 100644 --- a/.github/workflows/ubuntu_20.04/Dockerfile.ci +++ b/.github/workflows/ubuntu_20.04/Dockerfile.ci @@ -37,6 +37,7 @@ RUN apt-get update -y \ libfreexl-dev \ libfyba-dev \ libgeos-dev \ + libgeotiff-dev \ libgif-dev \ libhdf4-alt-dev \ libhdf5-serial-dev \ diff --git a/.github/workflows/ubuntu_20.04/build.sh b/.github/workflows/ubuntu_20.04/build.sh index 510a463b79cf..1b7b17ff3512 100755 --- a/.github/workflows/ubuntu_20.04/build.sh +++ b/.github/workflows/ubuntu_20.04/build.sh @@ -5,12 +5,12 @@ set -eu export CXXFLAGS="-march=native -O2 -Wodr -flto-odr-type-merging -Werror" export CFLAGS="-O2 -march=native -Werror" -cmake ${GDAL_SOURCE_DIR:=..} \ +cmake "${GDAL_SOURCE_DIR:=..}" \ -DUSE_CCACHE=ON \ -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DGDAL_USE_TIFF_INTERNAL=ON \ - -DGDAL_USE_GEOTIFF_INTERNAL=ON \ + -DCMAKE_INSTALL_PREFIX=/tmp/install-gdal \ + -DGDAL_USE_TIFF_INTERNAL=OFF \ + -DGDAL_USE_GEOTIFF_INTERNAL=OFF \ -DECW_ROOT=/opt/libecwj2-3.3 \ -DMRSID_ROOT=/usr/local \ -DFileGDB_ROOT=/usr/local/FileGDB_API \ @@ -20,5 +20,13 @@ cmake ${GDAL_SOURCE_DIR:=..} \ unset CXXFLAGS unset CFLAGS -make -j$(nproc) -make -j$(nproc) install DESTDIR=/tmp/install-gdal +make "-j$(nproc)" +make "-j$(nproc)" install + +# Test building MrSID driver in standalone mode +mkdir build_mrsid +cd build_mrsid +cmake -S ${GDAL_SOURCE_DIR:=..}/frmts/mrsid -DMRSID_ROOT=/usr/local -DCMAKE_PREFIX_PATH=/tmp/install-gdal +cmake --build . "-j$(nproc)" +test -f gdal_MrSID.so +cd .. diff --git a/frmts/mrsid/CMakeLists.txt b/frmts/mrsid/CMakeLists.txt index bc5ea5fe6ab8..5a7021927847 100644 --- a/frmts/mrsid/CMakeLists.txt +++ b/frmts/mrsid/CMakeLists.txt @@ -6,6 +6,9 @@ if(NOT DEFINED PROJECT_SOURCE_DIR) include("${PROJECT_SOURCE_DIR}/../../cmake/helpers/SetupStandalonePlugin.cmake" ) include(CheckDependentLibrariesGeoTIFF) include(CheckDependentLibrariesMrSID) + if (GDAL_USE_GEOTIFF_INTERNAL) + message(FATAL_ERROR "Internal libgeotiff not supported for MrSID standalone plugin build") + endif() standalone_driver_finalize(GDAL_ENABLE_DRIVER_MRSID) endif() From 2cea7b80e3b076a2074035d670e61f5dd833dd1a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 18:28:02 +0200 Subject: [PATCH 30/72] CMake: make Kakadu driver buildable in standalone mode --- cmake/helpers/CheckDependentLibraries.cmake | 2 +- .../CheckDependentLibrariesKakadu.cmake | 1 + doc/source/drivers/raster/jp2kak.rst | 25 +++++++++++++++++++ frmts/CMakeLists.txt | 2 +- frmts/jp2kak/CMakeLists.txt | 10 ++++++++ frmts/jp2kak/driver_declaration.cmake | 1 + 6 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 cmake/helpers/CheckDependentLibrariesKakadu.cmake create mode 100644 frmts/jp2kak/driver_declaration.cmake diff --git a/cmake/helpers/CheckDependentLibraries.cmake b/cmake/helpers/CheckDependentLibraries.cmake index 448cebba887b..97ede1ee7d16 100644 --- a/cmake/helpers/CheckDependentLibraries.cmake +++ b/cmake/helpers/CheckDependentLibraries.cmake @@ -519,7 +519,7 @@ option(GDAL_USE_PUBLICDECOMPWT "Set ON to build MSG driver and download external https://gitlab.eumetsat.int/open-source/PublicDecompWT" OFF) # proprietary libraries KAKADU -gdal_check_package(KDU "Enable KAKADU" CAN_DISABLE) +include(CheckDependentLibrariesKakadu) gdal_check_package(LURATECH "Enable JP2Lura driver" CAN_DISABLE) gdal_check_package(Arrow "Apache Arrow C++ library" CONFIG CAN_DISABLE) diff --git a/cmake/helpers/CheckDependentLibrariesKakadu.cmake b/cmake/helpers/CheckDependentLibrariesKakadu.cmake new file mode 100644 index 000000000000..5ef1b0f4383d --- /dev/null +++ b/cmake/helpers/CheckDependentLibrariesKakadu.cmake @@ -0,0 +1 @@ +gdal_check_package(KDU "Enable KAKADU" CAN_DISABLE) diff --git a/doc/source/drivers/raster/jp2kak.rst b/doc/source/drivers/raster/jp2kak.rst index da787ca31420..cf0a2ecbc741 100644 --- a/doc/source/drivers/raster/jp2kak.rst +++ b/doc/source/drivers/raster/jp2kak.rst @@ -284,6 +284,31 @@ unistd.h in kdu_arch.cpp. This means that \_SC_NPROCESSORS_ONLN and always return 0. Therefore the jp2kak driver might not default to creating worker threads. +Standalone plugin compilation +----------------------------- + +.. versionadded:: 3.10 + +While this driver may be built as part of a whole GDAL build, either in libgdal +itself, or as a plugin, it is also possible to only build this driver as a plugin, +against an already built libgdal. + +The version of the GDAL sources used to build the driver must match the version +of the libgdal it is built against. + +For example, from a "build_jp2kak" directory under the root of the GDAL source tree: + +:: + + cmake -S ../frmts/jp2kak -DCMAKE_PREFIX_PATH=/path/to/GDAL_installation_prefix -DKDU_ROOT=/path/to/kakadu_root + cmake --build . + + +Note that such a plugin, when used against a libgdal not aware of it, will be +systematically loaded at GDAL driver initialization time, and will not benefit from +`deferred plugin loading capabilities `. For that, libgdal itself must be built with the +CMake variable GDAL_REGISTER_DRIVER_JP2KAK_FOR_LATER_PLUGIN=ON set. + See Also -------- diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index fe27c646f5f5..2ee5e9e0a921 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -158,7 +158,7 @@ gdal_dependent_format(basisu_ktx2 "Basis Universal and KTX2 texture formats" "GD # ###################################################################################################################### # driver with proprietary libraries Kakadu software SDK -gdal_dependent_format(jp2kak "JPEG-2000 (based on Kakadu)" "GDAL_USE_KDU") +include(jp2kak/driver_declaration.cmake) gdal_dependent_format(jpipkak "JPIP Streaming" "GDAL_USE_KDU") # Luratech SDK gdal_dependent_format(jp2lura "JPEG-2000 (based on Luratech)" "GDAL_USE_LURATECH") diff --git a/frmts/jp2kak/CMakeLists.txt b/frmts/jp2kak/CMakeLists.txt index fa95a832ab21..194e5c01cf56 100644 --- a/frmts/jp2kak/CMakeLists.txt +++ b/frmts/jp2kak/CMakeLists.txt @@ -1,3 +1,13 @@ +cmake_minimum_required(VERSION 3.16...3.28) + +if(NOT DEFINED PROJECT_SOURCE_DIR) + # Standalone plugin building + project(gdal_JP2KAK) + include("${PROJECT_SOURCE_DIR}/../../cmake/helpers/SetupStandalonePlugin.cmake" ) + include(CheckDependentLibrariesKakadu) + standalone_driver_finalize(GDAL_ENABLE_DRIVER_JP2KAK) +endif() + add_gdal_driver(TARGET gdal_JP2KAK SOURCES jp2kakdataset.cpp jp2kak_headers.h diff --git a/frmts/jp2kak/driver_declaration.cmake b/frmts/jp2kak/driver_declaration.cmake new file mode 100644 index 000000000000..118d43fd80b5 --- /dev/null +++ b/frmts/jp2kak/driver_declaration.cmake @@ -0,0 +1 @@ +gdal_dependent_format(jp2kak "JPEG-2000 (based on Kakadu)" "GDAL_USE_KDU") From 38fbe8255f7602aa4029c17850a0c74cc1487d90 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 20:07:45 +0200 Subject: [PATCH 31/72] CMake: make OCI driver buildable in standalone mode --- .../helpers/CheckDependentLibrariesOCI.cmake | 2 ++ doc/source/drivers/vector/oci.rst | 30 +++++++++++++++++-- ogr/ogrsf_frmts/CMakeLists.txt | 2 +- ogr/ogrsf_frmts/oci/CMakeLists.txt | 10 +++++++ ogr/ogrsf_frmts/oci/driver_declaration.cmake | 1 + 5 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 cmake/helpers/CheckDependentLibrariesOCI.cmake create mode 100644 ogr/ogrsf_frmts/oci/driver_declaration.cmake diff --git a/cmake/helpers/CheckDependentLibrariesOCI.cmake b/cmake/helpers/CheckDependentLibrariesOCI.cmake new file mode 100644 index 000000000000..76d2644b1965 --- /dev/null +++ b/cmake/helpers/CheckDependentLibrariesOCI.cmake @@ -0,0 +1,2 @@ +set(Oracle_CAN_USE_CLNTSH_AS_MAIN_LIBRARY ON) +gdal_check_package(Oracle "Enable Oracle OCI driver" CAN_DISABLE) diff --git a/doc/source/drivers/vector/oci.rst b/doc/source/drivers/vector/oci.rst index 9f8b1fa18381..764767c2c9d5 100644 --- a/doc/source/drivers/vector/oci.rst +++ b/doc/source/drivers/vector/oci.rst @@ -353,8 +353,34 @@ coordinate reference system, and converting timestamps to UTC. ogr2ogr -f GPKG output.gpkg -nln new_layer_name -nlt POLYGON -s_srs EPSG:25832 -t_srs EPSG:25832 -dsco DATETIME_FORMAT=UTC OCI:username/password@host_name:port_number/service_name:MY_SCHEMA.MY_VIEW -sql "SELECT COLUMN_A, COLUMN_B, GEOMETRY FROM MY_SCHEMA.MY_VIEW" +Standalone plugin compilation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 3.10 + +While this driver may be built as part of a whole GDAL build, either in libgdal +itself, or as a plugin, it is also possible to only build this driver as a plugin, +against an already built libgdal. + +The version of the GDAL sources used to build the driver must match the version +of the libgdal it is built against. + +For example, from a "build_oci" directory under the root of the GDAL source tree: + +:: + + cmake -S ../ogr/ogrsf_frmts/oci -DCMAKE_PREFIX_PATH=/path/to/GDAL_installation_prefix -DOracle_ROOT=/path/to/instantclient_sdk_root + cmake --build . + + +Note that such a plugin, when used against a libgdal not aware of it, will be +systematically loaded at GDAL driver initialization time, and will not benefit from +`deferred plugin loading capabilities `. For that, libgdal itself must be built with the +CMake variable OGR_REGISTER_DRIVER_OCI_FOR_LATER_PLUGIN=ON set. + Credits ~~~~~~~ -I would like to thank `SRC, LLC `__ for -its financial support of the development of this driver. +`SRC, LLC `__ for its financial support of +the initial development of this driver. + diff --git a/ogr/ogrsf_frmts/CMakeLists.txt b/ogr/ogrsf_frmts/CMakeLists.txt index 690e4a0ef339..96bd9c670383 100644 --- a/ogr/ogrsf_frmts/CMakeLists.txt +++ b/ogr/ogrsf_frmts/CMakeLists.txt @@ -107,7 +107,7 @@ ogr_dependent_driver(gtfs "GTFS" "OGR_ENABLE_DRIVER_CSV") # ###################################################################################################################### # proprietary libraries -ogr_dependent_driver(oci "Oracle OCI" "GDAL_USE_ORACLE") +include(oci/driver_declaration.cmake) ogr_dependent_driver(idb "IDB" "GDAL_USE_IDB") ogr_dependent_driver(ods ODS "GDAL_USE_EXPAT") ogr_dependent_driver(ogdi "OGDI" "GDAL_USE_OGDI") diff --git a/ogr/ogrsf_frmts/oci/CMakeLists.txt b/ogr/ogrsf_frmts/oci/CMakeLists.txt index 577e1b1c2809..e3cc52932dc4 100644 --- a/ogr/ogrsf_frmts/oci/CMakeLists.txt +++ b/ogr/ogrsf_frmts/oci/CMakeLists.txt @@ -1,3 +1,13 @@ +cmake_minimum_required(VERSION 3.16...3.28) + +if(NOT DEFINED PROJECT_SOURCE_DIR) + # Standalone plugin building + project(ogr_OCI) + include("${PROJECT_SOURCE_DIR}/../../../cmake/helpers/SetupStandalonePlugin.cmake" ) + include(CheckDependentLibrariesOCI) + standalone_driver_finalize(OGR_ENABLE_DRIVER_OCI) +endif() + set(SOURCE oci_utils.cpp ocitest.cpp diff --git a/ogr/ogrsf_frmts/oci/driver_declaration.cmake b/ogr/ogrsf_frmts/oci/driver_declaration.cmake new file mode 100644 index 000000000000..cb0e78641e79 --- /dev/null +++ b/ogr/ogrsf_frmts/oci/driver_declaration.cmake @@ -0,0 +1 @@ +ogr_dependent_driver(oci "Oracle OCI" "GDAL_USE_ORACLE") From d147a5109e116af7172ad15149fe1cf71cf1e561 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 20:08:09 +0200 Subject: [PATCH 32/72] CI: test building OCI driver in standalone mode --- .github/workflows/ubuntu_20.04/build.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ubuntu_20.04/build.sh b/.github/workflows/ubuntu_20.04/build.sh index 1b7b17ff3512..e2a8cdde3517 100755 --- a/.github/workflows/ubuntu_20.04/build.sh +++ b/.github/workflows/ubuntu_20.04/build.sh @@ -30,3 +30,15 @@ cmake -S ${GDAL_SOURCE_DIR:=..}/frmts/mrsid -DMRSID_ROOT=/usr/local -DCMAKE_PREF cmake --build . "-j$(nproc)" test -f gdal_MrSID.so cd .. + +# Test building OCI driver in standalone mode +mkdir build_oci +cd build_oci +wget https://download.oracle.com/otn_software/linux/instantclient/1923000/instantclient-basic-linux.x64-19.23.0.0.0dbru.zip +wget https://download.oracle.com/otn_software/linux/instantclient/1923000/instantclient-sdk-linux.x64-19.23.0.0.0dbru.zip +unzip -o instantclient-basic-linux.x64-19.23.0.0.0dbru.zip +unzip -o instantclient-sdk-linux.x64-19.23.0.0.0dbru.zip +cmake -S "${GDAL_SOURCE_DIR:=..}/ogr/ogrsf_frmts/oci" "-DOracle_ROOT=$PWD/instantclient_19_23" -DCMAKE_PREFIX_PATH=/tmp/install-gdal +cmake --build . "-j$(nproc)" +test -f ogr_OCI.so +cd .. From 367c0bb3b188ae9da1fa83d78cd497033d040dcd Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 27 Jun 2024 22:45:11 +0200 Subject: [PATCH 33/72] GML writer: fix missing SRS in Featurecollection's boundedBy element (3.9.1 regression) Fixes #10332 Was caused by the fix 3674ec7570f324ea43eef08b6f5ce3a29de2741d for #10071 The regression mostly occurs when writing a GML file using ogr2ogr when the source layer has a named geometry column, e.g if the source is a GeoPackage or if using -sql The problem was acutally latent and also could have occured before 3.9.1 if using CreateLayer() without a SRS + CreateGeomField() --- autotest/ogr/ogr_gml.py | 92 ++++++++++++++++++++++++ ogr/ogrsf_frmts/gml/ogr_gml.h | 18 ++++- ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp | 88 +++++++++++++---------- ogr/ogrsf_frmts/gml/ogrgmllayer.cpp | 34 ++------- 4 files changed, 161 insertions(+), 71 deletions(-) diff --git a/autotest/ogr/ogr_gml.py b/autotest/ogr/ogr_gml.py index eadda1826340..fc70d74f1766 100755 --- a/autotest/ogr/ogr_gml.py +++ b/autotest/ogr/ogr_gml.py @@ -4356,3 +4356,95 @@ def test_ogr_gml_geom_link_to_immediate_child(): "data/gml/link_to_immediate_child.gml", open_options=["WRITE_GFS=NO"] ) assert ds + + +############################################################################### +# Test scenario of https://github.com/OSGeo/gdal/issues/10332 + + +@pytest.mark.parametrize("use_create_geom_field", [False, True]) +@pytest.mark.parametrize("has_srs", [False, True]) +def test_ogr_gml_ogr2ogr_from_layer_with_name_geom_field( + tmp_vsimem, use_create_geom_field, has_srs +): + + ds = gdal.GetDriverByName("Memory").Create("", 0, 0, 0, gdal.GDT_Unknown) + if has_srs: + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + else: + srs = None + if use_create_geom_field: + lyr = ds.CreateLayer("test", geom_type=ogr.wkbNone) + my_geom_field = ogr.GeomFieldDefn("my_geom", ogr.wkbUnknown) + my_geom_field.SetSpatialRef(srs) + lyr.CreateGeomField(my_geom_field) + else: + lyr = ds.CreateLayer("test", geom_type=ogr.wkbUnknown, srs=srs) + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometryDirectly(ogr.CreateGeometryFromWkt("POINT(2 49)")) + lyr.CreateFeature(f) + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometryDirectly(ogr.CreateGeometryFromWkt("POINT(3 50)")) + lyr.CreateFeature(f) + + out_filename = str(tmp_vsimem / "out.gml") + gdal.VectorTranslate(out_filename, ds, format="GML") + + f = gdal.VSIFOpenL(out_filename, "rb") + assert f + try: + data = gdal.VSIFReadL(1, 10000, f) + finally: + gdal.VSIFCloseL(f) + + if has_srs: + assert ( + b'49 250 3' + in data + ) + else: + assert ( + b"2 493 50" + in data + ) + + +############################################################################### + + +@pytest.mark.parametrize("first_layer_has_srs", [False, True]) +def test_ogr_gml_ogr2ogr_from_layers_with_inconsistent_srs( + tmp_vsimem, first_layer_has_srs +): + + ds = gdal.GetDriverByName("Memory").Create("", 0, 0, 0, gdal.GDT_Unknown) + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + lyr = ds.CreateLayer( + "test", geom_type=ogr.wkbUnknown, srs=(srs if first_layer_has_srs else None) + ) + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometryDirectly(ogr.CreateGeometryFromWkt("POINT(2 49)")) + lyr.CreateFeature(f) + + lyr = ds.CreateLayer( + "test2", geom_type=ogr.wkbUnknown, srs=(None if first_layer_has_srs else srs) + ) + f = ogr.Feature(lyr.GetLayerDefn()) + f.SetGeometryDirectly(ogr.CreateGeometryFromWkt("POINT(3 50)")) + lyr.CreateFeature(f) + + out_filename = str(tmp_vsimem / "out.gml") + gdal.VectorTranslate(out_filename, ds, format="GML") + + f = gdal.VSIFOpenL(out_filename, "rb") + assert f + try: + data = gdal.VSIFReadL(1, 10000, f) + finally: + gdal.VSIFCloseL(f) + + assert b"" in data diff --git a/ogr/ogrsf_frmts/gml/ogr_gml.h b/ogr/ogrsf_frmts/gml/ogr_gml.h index b5e0cda9b9e3..4ccedcbbfb61 100644 --- a/ogr/ogrsf_frmts/gml/ogr_gml.h +++ b/ogr/ogrsf_frmts/gml/ogr_gml.h @@ -61,7 +61,6 @@ class OGRGMLLayer final : public OGRLayer char *pszFIDPrefix; bool bWriter; - bool bSameSRS; OGRGMLDataSource *poDS; @@ -137,8 +136,14 @@ class OGRGMLDataSource final : public OGRDataSource OGRGMLSRSNameFormat eSRSNameFormat; bool bWriteSpaceIndentation; - OGRSpatialReference *poWriteGlobalSRS; - bool bWriteGlobalSRS; + //! Whether all geometry fields of all layers have the same SRS (or no SRS at all) + bool m_bWriteGlobalSRS = true; + + //! The global SRS (may be null), that is valid only if m_bWriteGlobalSRS == true + std::unique_ptr m_poWriteGlobalSRS{}; + + //! Whether at least one geometry field has been created + bool m_bWriteGlobalSRSInit = false; // input related parameters. CPLString osFilename; @@ -300,6 +305,13 @@ class OGRGMLDataSource final : public OGRDataSource const char *GetSRSDimensionLoc() const; bool GMLFeatureCollection() const; + void DeclareNewWriteSRS(const OGRSpatialReference *poSRS); + + bool HasWriteGlobalSRS() const + { + return m_bWriteGlobalSRS; + } + virtual OGRLayer *ExecuteSQL(const char *pszSQLCommand, OGRGeometry *poSpatialFilter, const char *pszDialect) override; diff --git a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp index 4ef9d4f6b47a..d9463991ef84 100644 --- a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp +++ b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp @@ -86,9 +86,8 @@ OGRGMLDataSource::OGRGMLDataSource() nBoundedByLocation(-1), nSchemaInsertLocation(-1), bIsOutputGML3(false), bIsOutputGML3Deegree(false), bIsOutputGML32(false), eSRSNameFormat(SRSNAME_SHORT), bWriteSpaceIndentation(true), - poWriteGlobalSRS(nullptr), bWriteGlobalSRS(false), poReader(nullptr), - bOutIsTempFile(false), bExposeGMLId(false), bExposeFid(false), - bIsWFS(false), bUseGlobalSRSName(false), + poReader(nullptr), bOutIsTempFile(false), bExposeGMLId(false), + bExposeFid(false), bIsWFS(false), bUseGlobalSRSName(false), m_bInvertAxisOrderIfLatLong(false), m_bConsiderEPSGAsURN(false), m_eSwapCoordinates(GML_SWAP_AUTO), m_bGetSecondaryGeometryOption(false), eReadMode(STANDARD), poStoredGMLFeature(nullptr), @@ -126,13 +125,13 @@ OGRGMLDataSource::~OGRGMLDataSource() if (!bFpOutputIsNonSeekable && nBoundedByLocation != -1 && VSIFSeekL(fpOutput, nBoundedByLocation, SEEK_SET) == 0) { - if (bWriteGlobalSRS && sBoundingRect.IsInit() && IsGML3Output()) + if (m_bWriteGlobalSRS && sBoundingRect.IsInit() && IsGML3Output()) { bool bCoordSwap = false; char *pszSRSName = - poWriteGlobalSRS - ? GML_GetSRSName(poWriteGlobalSRS, eSRSNameFormat, - &bCoordSwap) + m_poWriteGlobalSRS + ? GML_GetSRSName(m_poWriteGlobalSRS.get(), + eSRSNameFormat, &bCoordSwap) : CPLStrdup(""); char szLowerCorner[75] = {}; char szUpperCorner[75] = {}; @@ -201,7 +200,7 @@ OGRGMLDataSource::~OGRGMLDataSource() szLowerCorner, szUpperCorner); CPLFree(pszSRSName); } - else if (bWriteGlobalSRS && sBoundingRect.IsInit()) + else if (m_bWriteGlobalSRS && sBoundingRect.IsInit()) { if (bWriteSpaceIndentation) VSIFPrintfL(fpOutput, " "); @@ -268,8 +267,6 @@ OGRGMLDataSource::~OGRGMLDataSource() delete poReader; } - delete poWriteGlobalSRS; - delete poStoredGMLFeature; if (osXSDFilename.compare(CPLSPrintf("/vsimem/tmp_gml_xsd_%p.xsd", this)) == @@ -2001,6 +1998,48 @@ void OGRGMLDataSource::WriteTopElements() } } +/************************************************************************/ +/* DeclareNewWriteSRS() */ +/************************************************************************/ + +// Check that all SRS passed to ICreateLayer() and CreateGeomField() +// are the same (or all null) + +void OGRGMLDataSource::DeclareNewWriteSRS(const OGRSpatialReference *poSRS) +{ + if (m_bWriteGlobalSRS) + { + if (!m_bWriteGlobalSRSInit) + { + m_bWriteGlobalSRSInit = true; + if (poSRS) + { + m_poWriteGlobalSRS.reset(poSRS->Clone()); + m_poWriteGlobalSRS->SetAxisMappingStrategy( + OAMS_TRADITIONAL_GIS_ORDER); + } + } + else + { + if (m_poWriteGlobalSRS) + { + const char *const apszOptions[] = { + "IGNORE_DATA_AXIS_TO_SRS_AXIS_MAPPING=YES", nullptr}; + if (!poSRS || + !poSRS->IsSame(m_poWriteGlobalSRS.get(), apszOptions)) + { + m_bWriteGlobalSRS = false; + } + } + else + { + if (poSRS) + m_bWriteGlobalSRS = false; + } + } + } +} + /************************************************************************/ /* ICreateLayer() */ /************************************************************************/ @@ -2037,37 +2076,9 @@ OGRGMLDataSource::ICreateLayer(const char *pszLayerName, pszLayerName, pszCleanLayerName); } - // Set or check validity of global SRS. if (nLayers == 0) { WriteTopElements(); - if (poSRS) - { - poWriteGlobalSRS = poSRS->Clone(); - poWriteGlobalSRS->SetAxisMappingStrategy( - OAMS_TRADITIONAL_GIS_ORDER); - } - bWriteGlobalSRS = true; - } - else if (bWriteGlobalSRS) - { - if (poWriteGlobalSRS != nullptr) - { - const char *const apszOptions[] = { - "IGNORE_DATA_AXIS_TO_SRS_AXIS_MAPPING=YES", nullptr}; - if (poSRS == nullptr || - !poSRS->IsSame(poWriteGlobalSRS, apszOptions)) - { - delete poWriteGlobalSRS; - poWriteGlobalSRS = nullptr; - bWriteGlobalSRS = false; - } - } - else - { - if (poSRS != nullptr) - bWriteGlobalSRS = false; - } } // Create the layer object. @@ -2081,6 +2092,7 @@ OGRGMLDataSource::ICreateLayer(const char *pszLayerName, pszGeomFieldName = "geometryProperty"; poGeomFieldDefn->SetName(pszGeomFieldName); poGeomFieldDefn->SetNullable(poSrcGeomFieldDefn->IsNullable()); + DeclareNewWriteSRS(poSRS); if (poSRS != nullptr) { auto poSRSClone = poSRS->Clone(); diff --git a/ogr/ogrsf_frmts/gml/ogrgmllayer.cpp b/ogr/ogrsf_frmts/gml/ogrgmllayer.cpp index d5a9a3009d33..bb47fa9e9ab7 100644 --- a/ogr/ogrsf_frmts/gml/ogrgmllayer.cpp +++ b/ogr/ogrsf_frmts/gml/ogrgmllayer.cpp @@ -44,7 +44,7 @@ OGRGMLLayer::OGRGMLLayer(const char *pszName, bool bWriterIn, : poFeatureDefn(new OGRFeatureDefn( pszName + (STARTS_WITH_CI(pszName, "ogr:") ? 4 : 0))), iNextGMLId(0), bInvalidFIDFound(false), pszFIDPrefix(nullptr), - bWriter(bWriterIn), bSameSRS(false), poDS(poDSIn), + bWriter(bWriterIn), poDS(poDSIn), poFClass(!bWriter ? poDS->GetReader()->GetClass(pszName) : nullptr), // Reader's should get the corresponding GMLFeatureClass and cache it. hCacheSRS(GML_BuildOGRGeometryFromList_CreateCache()), @@ -708,33 +708,6 @@ OGRErr OGRGMLLayer::ICreateFeature(OGRFeature *poFeature) poDS->PrintLine(fp, ""); } - if (iNextGMLId == 0) - { - bSameSRS = true; - for (int iGeomField = 1; - iGeomField < poFeatureDefn->GetGeomFieldCount(); iGeomField++) - { - OGRGeomFieldDefn *poFieldDefn0 = poFeatureDefn->GetGeomFieldDefn(0); - OGRGeomFieldDefn *poFieldDefn = - poFeatureDefn->GetGeomFieldDefn(iGeomField); - const OGRSpatialReference *poSRS0 = poFieldDefn0->GetSpatialRef(); - const OGRSpatialReference *poSRS = poFieldDefn->GetSpatialRef(); - if (poSRS0 != nullptr && poSRS == nullptr) - { - bSameSRS = false; - } - else if (poSRS0 == nullptr && poSRS != nullptr) - { - bSameSRS = false; - } - else if (poSRS0 != nullptr && poSRS != nullptr && poSRS0 != poSRS && - !poSRS0->IsSame(poSRS)) - { - bSameSRS = false; - } - } - } - if (poFeature->GetFID() == OGRNullFID) poFeature->SetFID(iNextGMLId++); @@ -794,7 +767,7 @@ OGRErr OGRGMLLayer::ICreateFeature(OGRFeature *poFeature) const int nCoordDimension = poGeom->getCoordinateDimension(); poGeom->getEnvelope(&sGeomBounds); - if (bSameSRS) + if (poDS->HasWriteGlobalSRS()) poDS->GrowExtents(&sGeomBounds, nCoordDimension); if (poGeom->getSpatialReference() == nullptr && @@ -1246,7 +1219,8 @@ OGRErr OGRGMLLayer::CreateGeomField(const OGRGeomFieldDefn *poField, /* Enforce XML naming semantics on element name. */ /* -------------------------------------------------------------------- */ OGRGeomFieldDefn oCleanCopy(poField); - auto poSRSOri = poField->GetSpatialRef(); + const auto poSRSOri = poField->GetSpatialRef(); + poDS->DeclareNewWriteSRS(poSRSOri); if (poSRSOri) { auto poSRS = poSRSOri->Clone(); From 3643e2af50acaa9aab138edc7216b6ad8c55b1b2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 28 Jun 2024 16:47:53 +0200 Subject: [PATCH 34/72] GTiff: emit clearer error when attempting to create external JPEG compressed overviews on dataset with color table --- autotest/gcore/tiff_ovr.py | 28 ++++++++++++++++++++++++++++ frmts/gtiff/gt_overview.cpp | 13 ++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/autotest/gcore/tiff_ovr.py b/autotest/gcore/tiff_ovr.py index 43577d218158..ba49b4b3fb45 100755 --- a/autotest/gcore/tiff_ovr.py +++ b/autotest/gcore/tiff_ovr.py @@ -1602,6 +1602,34 @@ def test_tiff_ovr_42(tmp_path, both_endian): ds = None +############################################################################### +# Test (failed) attempt at creating JPEG external overviews on dataset with color table + + +@pytest.mark.require_creation_option("GTiff", "JPEG") +@gdaltest.enable_exceptions() +def test_tiff_ovr_jpeg_on_color_table(tmp_path): + + tif_fname = str(tmp_path / "test_tiff_ovr_jpeg_on_color_table.tif") + + ct_data = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 255)] + + ct = gdal.ColorTable() + for i, data in enumerate(ct_data): + ct.SetColorEntry(i, data) + + ds = gdal.GetDriverByName("GTiff").Create(tif_fname, 1, 1) + ds.GetRasterBand(1).SetRasterColorTable(ct) + ds = None + + with gdal.Open(tif_fname) as ds: + with pytest.raises( + Exception, + match="Cannot create JPEG compressed overviews on a raster with a color table", + ): + ds.BuildOverviews("NEAREST", overviewlist=[2], options=["COMPRESS=JPEG"]) + + ############################################################################### # Make sure that 16bit overviews with JPEG compression are handled using 12-bit # jpeg-in-tiff (#3539) diff --git a/frmts/gtiff/gt_overview.cpp b/frmts/gtiff/gt_overview.cpp index 38ca89e434c2..980dd544076c 100644 --- a/frmts/gtiff/gt_overview.cpp +++ b/frmts/gtiff/gt_overview.cpp @@ -561,8 +561,19 @@ CPLErr GTIFFBuildOverviewsEx(const char *pszFilename, int nBands, papoBandList[0]->GetRasterDataType() == GDT_UInt16) && !STARTS_WITH_CI(pszResampling, "AVERAGE_BIT2")) { + // Would also apply to other lossy compression scheme, but for JPEG, + // this at least avoids a later cryptic error message from libtiff: + // "JPEGSetupEncode:PhotometricInterpretation 3 not allowed for JPEG" + if (nCompression == COMPRESSION_JPEG) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot create JPEG compressed overviews on a raster " + "with a color table"); + return CE_Failure; + } + nPhotometric = PHOTOMETRIC_PALETTE; - // Should set the colormap up at this point too! + // Color map is set up after } else if (nBands >= 3 && papoBandList[0]->GetColorInterpretation() == GCI_RedBand && From c308cf3d9a59c9cdf2916585de273cdf54ab9bb4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 28 Jun 2024 20:28:02 +0200 Subject: [PATCH 35/72] gdallocationinfo: avoid extra newline character in -valonly mode if coordinate is outside raster extent (3.9.0 regression) Fixes #10336 --- apps/gdallocationinfo.cpp | 7 +++- autotest/utilities/test_gdallocationinfo.py | 45 +++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/apps/gdallocationinfo.cpp b/apps/gdallocationinfo.cpp index d317e2599b5c..72599f0ef1d2 100644 --- a/apps/gdallocationinfo.cpp +++ b/apps/gdallocationinfo.cpp @@ -432,7 +432,12 @@ MAIN_START(argc, argv) osXML += "Location is off this file! No further details " "to report."; else if (bValOnly) - printf("\n"); + { + for (int i = 1; i < static_cast(anBandList.size()); i++) + { + printf("%s", osFieldSep.c_str()); + } + } else if (!bQuiet) printf("\nLocation is off this file! No further details to " "report.\n"); diff --git a/autotest/utilities/test_gdallocationinfo.py b/autotest/utilities/test_gdallocationinfo.py index 2b8c80cd7a2e..0b64f4c19333 100755 --- a/autotest/utilities/test_gdallocationinfo.py +++ b/autotest/utilities/test_gdallocationinfo.py @@ -254,3 +254,48 @@ def test_gdallocationinfo_echo(gdallocationinfo_path): strin="1 2", ) assert "1,2,132" in ret + + +############################################################################### +# Test out of raster coordinates + + +def test_gdallocationinfo_out_of_raster_coordinates_valonly(gdallocationinfo_path): + + ret = gdaltest.runexternal( + gdallocationinfo_path + " -valonly ../gcore/data/byte.tif", + strin="1 2\n-1 -1\n1 2", + ) + + ret = ret.replace("\r\n", "\n") + assert "132\n\n132\n" in ret + + ret = gdaltest.runexternal( + gdallocationinfo_path + ' -E -valonly -field_sep "," ../gcore/data/byte.tif', + strin="1 2\n-1 -1\n1 2", + ) + + ret = ret.replace("\r\n", "\n") + assert "1,2,132\n-1,-1,\n1,2,132\n" in ret + + +def test_gdallocationinfo_out_of_raster_coordinates_valonly_multiband( + gdallocationinfo_path, +): + + ret = gdaltest.runexternal( + gdallocationinfo_path + " -valonly ../gcore/data/rgbsmall.tif", + strin="1 2\n-1 -1\n1 2", + ) + + ret = ret.replace("\r\n", "\n") + assert "0\n0\n0\n\n\n\n0\n0\n0\n" in ret + + ret = gdaltest.runexternal( + gdallocationinfo_path + + ' -E -valonly -field_sep "," ../gcore/data/rgbsmall.tif', + strin="1 2\n-1 -1\n1 2", + ) + + ret = ret.replace("\r\n", "\n") + assert "1,2,0,0,0\n-1,-1,,,\n1,2,0,0,0\n" in ret From aede51d223f8e5ddad4dd874c5ce46e10d6e573f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 30 Jun 2024 22:13:39 +0200 Subject: [PATCH 36/72] ISGDataset: fix performance inefficiencies (CID 1549336, 1549337, 1549338, 1549339) --- frmts/aaigrid/aaigriddataset.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frmts/aaigrid/aaigriddataset.cpp b/frmts/aaigrid/aaigriddataset.cpp index 96d8325a8be1..910b77a3c8f0 100644 --- a/frmts/aaigrid/aaigriddataset.cpp +++ b/frmts/aaigrid/aaigriddataset.cpp @@ -864,9 +864,9 @@ int ISGDataset::ParseHeader(const char *pszHeader, const char *) return FALSE; } - auto parseDMS = [](auto str) + const auto parseDMS = [](CPLString &str) { - std::string degreeSymbol{"\xc2\xb0"}; + const std::string degreeSymbol{"\xc2\xb0"}; str.replaceAll(degreeSymbol, "D"); return CPLDMSToDec(str); }; From 7886dba34a17f076d6860c9e619106a4ca42e060 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 02:44:54 +0000 Subject: [PATCH 37/72] Bump github/codeql-action from 3.25.7 to 3.25.11 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.25.7 to 3.25.11. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/f079b8493333aace61c81488f8bd40919487bd9f...b611370bb5703a7efb587f9d136a52ea24c5c38c) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bdac6681fb81..1fbb297467a7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -107,7 +107,7 @@ jobs: # We do that after running CMake to avoid CodeQL to trigger during CMake time, # in particular during HDF5 detection which is terribly slow (https://github.com/OSGeo/gdal/issues/9549) - name: Initialize CodeQL - uses: github/codeql-action/init@f079b8493333aace61c81488f8bd40919487bd9f # v3.25.7 + uses: github/codeql-action/init@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -127,6 +127,6 @@ jobs: cmake --build build -j$(nproc) - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@f079b8493333aace61c81488f8bd40919487bd9f # v3.25.7 + uses: github/codeql-action/analyze@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index d5727d248b87..c4205a9037f5 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@f079b8493333aace61c81488f8bd40919487bd9f # v3.25.7 + uses: github/codeql-action/upload-sarif@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11 with: sarif_file: results.sarif From a8f16a01a1f188afb7df9330ad4934d29818d65b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 02:45:03 +0000 Subject: [PATCH 38/72] Bump actions/checkout from 4.1.6 to 4.1.7 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.6 to 4.1.7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/a5ac7e51b41094c92402da3b24376905380afc29...692973e3d937129bcbf40652eb9f2f61becf3332) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/android_cmake.yml | 2 +- .github/workflows/clang_static_analyzer.yml | 2 +- .github/workflows/cmake_builds.yml | 12 ++++++------ .github/workflows/code_checks.yml | 16 ++++++++-------- .github/workflows/codeql.yml | 2 +- .github/workflows/conda.yml | 2 +- .github/workflows/coverity_scan.yml | 2 +- .github/workflows/doc_build.yml | 2 +- .github/workflows/linux_build.yml | 2 +- .github/workflows/macos.yml | 2 +- .github/workflows/scorecard.yml | 2 +- .github/workflows/slow_tests.yml | 2 +- .github/workflows/windows_build.yml | 2 +- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/android_cmake.yml b/.github/workflows/android_cmake.yml index d81b50ff90dd..0200112eeaf1 100644 --- a/.github/workflows/android_cmake.yml +++ b/.github/workflows/android_cmake.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Cache uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 diff --git a/.github/workflows/clang_static_analyzer.yml b/.github/workflows/clang_static_analyzer.yml index f1cddfbfb00f..452a8cd3c392 100644 --- a/.github/workflows/clang_static_analyzer.yml +++ b/.github/workflows/clang_static_analyzer.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Run run: docker run --rm -v $PWD:$PWD ubuntu:22.04 sh -c "cd $PWD && apt update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends sudo software-properties-common && DEBIAN_FRONTEND=noninteractive sh ./ci/travis/csa_common/before_install.sh && sh ./ci/travis/csa_common/install.sh && sh ./ci/travis/csa_common/script.sh" diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index 825ec91eaa3a..80b0fe371307 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -32,7 +32,7 @@ jobs: cache-name: cmake-ubuntu-focal steps: - name: Checkout GDAL - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Setup cache uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 id: cache @@ -312,7 +312,7 @@ jobs: run: | git config --global core.autocrlf false - name: Checkout GDAL - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install development packages uses: msys2/setup-msys2@d0e80f58dffbc64f6a3a1f43527d469b4fc7b6c8 # v2.23.0 with: @@ -405,7 +405,7 @@ jobs: run: | git config --global core.autocrlf false - name: Checkout GDAL - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: populate JAVA_HOME shell: pwsh @@ -512,7 +512,7 @@ jobs: run: | git config --global core.autocrlf false - name: Checkout GDAL - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: activate-environment: gdalenv @@ -589,7 +589,7 @@ jobs: with: xcode-version: 14.3 - name: Checkout GDAL - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Setup cache uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 id: cache @@ -669,7 +669,7 @@ jobs: run: | git config --global core.autocrlf false - name: Checkout GDAL - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: activate-environment: gdalenv diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 321cd50a897e..18a29bdba220 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install Requirements run: | @@ -47,7 +47,7 @@ jobs: container: ubuntu:24.04 steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install Requirements run: | @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Detect tabulations run: ./scripts/detect_tabulations.sh @@ -104,7 +104,7 @@ jobs: linting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 @@ -113,7 +113,7 @@ jobs: container: ghcr.io/osgeo/proj-docs steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Run doxygen run: | @@ -124,7 +124,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install Requirements run: | @@ -143,7 +143,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: @@ -159,7 +159,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install requirements run: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bdac6681fb81..ca7f92f22319 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -44,7 +44,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install dependencies run: | diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index eaba06ddd159..fdb2624f3949 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -37,7 +37,7 @@ jobs: CACHE_NUMBER: 0 steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Support longpaths run: git config --system core.longpaths true diff --git a/.github/workflows/coverity_scan.yml b/.github/workflows/coverity_scan.yml index 7b560539f657..746b7864a198 100644 --- a/.github/workflows/coverity_scan.yml +++ b/.github/workflows/coverity_scan.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Login to GHCR if: env.CONTAINER_REGISTRY == 'ghcr.io' diff --git a/.github/workflows/doc_build.yml b/.github/workflows/doc_build.yml index ee75f953b0c0..91f6db8b444b 100644 --- a/.github/workflows/doc_build.yml +++ b/.github/workflows/doc_build.yml @@ -24,7 +24,7 @@ jobs: container: ghcr.io/osgeo/proj-docs steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Setup environment shell: bash -l {0} run: | diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 1ec856e4e5c5..ef1eb6501999 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -174,7 +174,7 @@ jobs: sudo sysctl vm.mmap_rnd_bits=28 - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Login to Docker Hub if: env.CONTAINER_REGISTRY == 'docker.io' diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 715058dfc41a..2221d7e29664 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -25,7 +25,7 @@ jobs: runs-on: macos-14 steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index d5727d248b87..aef0af85a012 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -36,7 +36,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: persist-credentials: false diff --git a/.github/workflows/slow_tests.yml b/.github/workflows/slow_tests.yml index c7fc3b084495..f7cec9357de0 100644 --- a/.github/workflows/slow_tests.yml +++ b/.github/workflows/slow_tests.yml @@ -47,7 +47,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Login to GHCR if: env.CONTAINER_REGISTRY == 'ghcr.io' diff --git a/.github/workflows/windows_build.yml b/.github/workflows/windows_build.yml index a75a70f5878c..475510c9fedf 100644 --- a/.github/workflows/windows_build.yml +++ b/.github/workflows/windows_build.yml @@ -56,7 +56,7 @@ jobs: git config --global core.autocrlf false - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set environment shell: pwsh From f9ec1514bd3d9abb6a7ac5032e4303cf78ec9d4a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 1 Jul 2024 22:23:54 +0200 Subject: [PATCH 39/72] WMTS: add utility methods to get tile matrix / tile matrix limits extent --- frmts/wmts/wmtsdataset.cpp | 72 ++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/frmts/wmts/wmtsdataset.cpp b/frmts/wmts/wmtsdataset.cpp index 01f8aad9cc5a..3da867eb240b 100644 --- a/frmts/wmts/wmtsdataset.cpp +++ b/frmts/wmts/wmtsdataset.cpp @@ -78,6 +78,16 @@ class WMTSTileMatrix int nTileHeight; int nMatrixWidth; int nMatrixHeight; + + OGREnvelope GetExtent() const + { + OGREnvelope sExtent; + sExtent.MinX = dfTLX; + sExtent.MaxX = dfTLX + nMatrixWidth * dfPixelSize * nTileWidth; + sExtent.MaxY = dfTLY; + sExtent.MinY = dfTLY - nMatrixHeight * dfPixelSize * nTileHeight; + return sExtent; + } }; /************************************************************************/ @@ -94,6 +104,18 @@ class WMTSTileMatrixLimits int nMaxTileRow; int nMinTileCol; int nMaxTileCol; + + OGREnvelope GetExtent(const WMTSTileMatrix &oTM) const + { + OGREnvelope sExtent; + const double dfTileWidthUnits = oTM.dfPixelSize * oTM.nTileWidth; + const double dfTileHeightUnits = oTM.dfPixelSize * oTM.nTileHeight; + sExtent.MinX = oTM.dfTLX + nMinTileCol * dfTileWidthUnits; + sExtent.MaxY = oTM.dfTLY - nMinTileRow * dfTileHeightUnits; + sExtent.MaxX = oTM.dfTLX + (nMaxTileCol + 1) * dfTileWidthUnits; + sExtent.MinY = oTM.dfTLY - (nMaxTileRow + 1) * dfTileHeightUnits; + return sExtent; + } }; /************************************************************************/ @@ -1763,15 +1785,11 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) } if (poRevCT != nullptr) { - const double dfX0 = oTM.dfTLX; - const double dfY1 = oTM.dfTLY; - const double dfX1 = - oTM.dfTLX + oTM.nMatrixWidth * oTM.dfPixelSize * - oTM.nTileWidth; - const double dfY0 = - oTM.dfTLY - oTM.nMatrixHeight * - oTM.dfPixelSize * - oTM.nTileHeight; + const auto sTMExtent = oTM.GetExtent(); + const double dfX0 = sTMExtent.MinX; + const double dfY1 = sTMExtent.MaxY; + const double dfX1 = sTMExtent.MaxX; + const double dfY0 = sTMExtent.MinY; double dfXMin = std::numeric_limits::infinity(); double dfYMin = @@ -1927,12 +1945,7 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) CPLDebug("WMTS", "Using TM level %s bounding box as layer extent", oTM.osIdentifier.c_str()); - sAOI.MinX = oTM.dfTLX; - sAOI.MaxY = oTM.dfTLY; - sAOI.MaxX = - oTM.dfTLX + oTM.nMatrixWidth * oTM.dfPixelSize * oTM.nTileWidth; - sAOI.MinY = oTM.dfTLY - - oTM.nMatrixHeight * oTM.dfPixelSize * oTM.nTileHeight; + sAOI = oTM.GetExtent(); bHasAOI = TRUE; } @@ -1953,6 +1966,7 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) // Clip with implied BoundingBox of the most precise TM // Useful for http://tileserver.maptiler.com/wmts const WMTSTileMatrix &oTM = oTMS.aoTM.back(); + const OGREnvelope sTMExtent = oTM.GetExtent(); OGREnvelope sAOINew(sAOI); // For @@ -1962,15 +1976,11 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) // initial coding. So do X clipping in default mode. if (!bExtendBeyondDateLine) { - sAOINew.MinX = std::max(sAOI.MinX, oTM.dfTLX); - sAOINew.MaxX = std::min( - sAOI.MaxX, oTM.dfTLX + oTM.nMatrixWidth * oTM.dfPixelSize * - oTM.nTileWidth); + sAOINew.MinX = std::max(sAOI.MinX, sTMExtent.MinX); + sAOINew.MaxX = std::min(sAOI.MaxX, sTMExtent.MaxX); } - sAOINew.MaxY = std::min(sAOI.MaxY, oTM.dfTLY); - sAOINew.MinY = std::max(sAOI.MinY, oTM.dfTLY - oTM.nMatrixHeight * - oTM.dfPixelSize * - oTM.nTileHeight); + sAOINew.MaxY = std::min(sAOI.MaxY, sTMExtent.MaxY); + sAOINew.MinY = std::max(sAOI.MinY, sTMExtent.MinY); if (sAOI != sAOINew) { CPLDebug( @@ -2001,20 +2011,8 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) const WMTSTileMatrixLimits &oTMLimits = aoMapTileMatrixLimits[oTM.osIdentifier]; - double dfTileWidthUnits = oTM.dfPixelSize * oTM.nTileWidth; - double dfTileHeightUnits = oTM.dfPixelSize * oTM.nTileHeight; - sAOINew.MinX = - std::max(sAOI.MinX, oTM.dfTLX + oTMLimits.nMinTileCol * - dfTileWidthUnits); - sAOINew.MaxY = - std::min(sAOI.MaxY, oTM.dfTLY - oTMLimits.nMinTileRow * - dfTileHeightUnits); - sAOINew.MaxX = std::min( - sAOI.MaxX, - oTM.dfTLX + (oTMLimits.nMaxTileCol + 1) * dfTileWidthUnits); - sAOINew.MinY = std::max( - sAOI.MinY, oTM.dfTLY - (oTMLimits.nMaxTileRow + 1) * - dfTileHeightUnits); + const OGREnvelope sTMLimitsExtent = oTMLimits.GetExtent(oTM); + sAOINew.Intersect(sTMLimitsExtent); if (sAOI != sAOINew) { From 7f95866615e63f4819d1b248a797484c13975078 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 1 Jul 2024 22:25:04 +0200 Subject: [PATCH 40/72] WMTS: make sure not to request tiles outside of tile matrix / tile matrix limits, if the dataset extent goes beyond them --- frmts/wmts/wmtsdataset.cpp | 55 ++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/frmts/wmts/wmtsdataset.cpp b/frmts/wmts/wmtsdataset.cpp index 3da867eb240b..3462a186c0b9 100644 --- a/frmts/wmts/wmtsdataset.cpp +++ b/frmts/wmts/wmtsdataset.cpp @@ -2193,10 +2193,10 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) int(0.5 + (poDS->adfGT[3] - sAOI.MinY) / oTM.dfPixelSize); } - int nRasterXSize = int(0.5 + poDS->nRasterXSize / oTM.dfPixelSize * - poDS->adfGT[1]); - int nRasterYSize = int(0.5 + poDS->nRasterYSize / oTM.dfPixelSize * - poDS->adfGT[1]); + const int nRasterXSize = int( + 0.5 + poDS->nRasterXSize / oTM.dfPixelSize * poDS->adfGT[1]); + const int nRasterYSize = int( + 0.5 + poDS->nRasterYSize / oTM.dfPixelSize * poDS->adfGT[1]); if (!poDS->apoDatasets.empty() && (nRasterXSize < 128 || nRasterYSize < 128)) { @@ -2205,14 +2205,28 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) CPLString osURL( Replace(osURLTileTemplate, "{TileMatrix}", oTM.osIdentifier)); - double dfTileWidthUnits = oTM.dfPixelSize * oTM.nTileWidth; - double dfTileHeightUnits = oTM.dfPixelSize * oTM.nTileHeight; + const double dfTileWidthUnits = oTM.dfPixelSize * oTM.nTileWidth; + const double dfTileHeightUnits = oTM.dfPixelSize * oTM.nTileHeight; + + // Get bounds of this tile matrix / tile matrix limits + auto sTMExtent = oTM.GetExtent(); + if (aoMapTileMatrixLimits.find(oTM.osIdentifier) != + aoMapTileMatrixLimits.end()) + { + const WMTSTileMatrixLimits &oTMLimits = + aoMapTileMatrixLimits[oTM.osIdentifier]; + sTMExtent.Intersect(oTMLimits.GetExtent(oTM)); + } // Compute the shift in terms of tiles between AOI and TM origin - int nTileX = (int)(floor(poDS->adfGT[0] - oTM.dfTLX + 1e-10) / - dfTileWidthUnits); - int nTileY = (int)(floor(oTM.dfTLY - poDS->adfGT[3] + 1e-10) / - dfTileHeightUnits); + const int nTileX = static_cast( + floor(std::max(sTMExtent.MinX, poDS->adfGT[0]) - oTM.dfTLX + + 1e-10) / + dfTileWidthUnits); + const int nTileY = static_cast( + floor(oTM.dfTLY - std::min(poDS->adfGT[3], sTMExtent.MaxY) + + 1e-10) / + dfTileHeightUnits); // Compute extent of this zoom level slightly larger than the AOI // and aligned on tile boundaries at this TM @@ -2225,8 +2239,13 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) dfLRY = dfULY + floor((dfLRY - dfULY) / dfTileHeightUnits + 1e-10) * dfTileHeightUnits; - double dfSizeX = 0.5 + (dfLRX - dfULX) / oTM.dfPixelSize; - double dfSizeY = 0.5 + (dfULY - dfLRY) / oTM.dfPixelSize; + // Clip TMS extent to the one of this TM + if (!bExtendBeyondDateLine) + dfLRX = std::min(dfLRX, sTMExtent.MaxX); + dfLRY = std::max(dfLRY, sTMExtent.MinY); + + const double dfSizeX = 0.5 + (dfLRX - dfULX) / oTM.dfPixelSize; + const double dfSizeY = 0.5 + (dfULY - dfLRY) / oTM.dfPixelSize; if (dfSizeX > INT_MAX || dfSizeY > INT_MAX) { continue; @@ -2239,13 +2258,15 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) poDS->oTMS = oTMS; } - int nSizeX = static_cast(dfSizeX); - int nSizeY = static_cast(dfSizeY); + const int nSizeX = static_cast(dfSizeX); + const int nSizeY = static_cast(dfSizeY); - double dfDateLineX = + const double dfDateLineX = oTM.dfTLX + oTM.nMatrixWidth * dfTileWidthUnits; - int nSizeX1 = int(0.5 + (dfDateLineX - dfULX) / oTM.dfPixelSize); - int nSizeX2 = int(0.5 + (dfLRX - dfDateLineX) / oTM.dfPixelSize); + const int nSizeX1 = + int(0.5 + (dfDateLineX - dfULX) / oTM.dfPixelSize); + const int nSizeX2 = + int(0.5 + (dfLRX - dfDateLineX) / oTM.dfPixelSize); if (bExtendBeyondDateLine && dfDateLineX > dfLRX) { CPLDebug("WMTS", "ExtendBeyondDateLine ignored in that case"); From d0410b80a7ca206fecf4f968af048e10f5ea432b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 1 Jul 2024 22:25:19 +0200 Subject: [PATCH 41/72] WMTS: clip layer extent with union of extent of tile matrices Fixes #10348 --- .../clip_WGS84BoundingBox_with_tilematrix.xml | 152 ++++++++++++++++++ autotest/gdrivers/wmts.py | 12 ++ frmts/wmts/wmtsdataset.cpp | 15 ++ 3 files changed, 179 insertions(+) create mode 100644 autotest/gdrivers/data/wmts/clip_WGS84BoundingBox_with_tilematrix.xml diff --git a/autotest/gdrivers/data/wmts/clip_WGS84BoundingBox_with_tilematrix.xml b/autotest/gdrivers/data/wmts/clip_WGS84BoundingBox_with_tilematrix.xml new file mode 100644 index 000000000000..bdf9dc4d910d --- /dev/null +++ b/autotest/gdrivers/data/wmts/clip_WGS84BoundingBox_with_tilematrix.xml @@ -0,0 +1,152 @@ + + + + + title + + 0.10594674240568917 45.2375427360256 + 20.448891294525627 56.84787345153812 + + de_basemapde_web_raster_grau + + image/png + + DE_EPSG_25832_ADV + + + + + DE_EPSG_25832_ADV + EPSG:25832 + + 00 + 17471320.750897426 + -46133.17 6301219.54 + 256 + 256 + 1 + 1 + + + 01 + 8735660.375448713 + -46133.17 6301219.54 + 256 + 256 + 2 + 2 + + + 02 + 4367830.187724357 + -46133.17 6301219.54 + 256 + 256 + 4 + 4 + + + 03 + 2183915.0938621783 + -46133.17 6301219.54 + 256 + 256 + 8 + 8 + + + 04 + 1091957.5469310891 + -46133.17 6301219.54 + 256 + 256 + 16 + 16 + + + 05 + 545978.7734655463 + -46133.17 6301219.54 + 256 + 256 + 32 + 32 + + + 06 + 272989.38673277246 + -46133.17 6301219.54 + 256 + 256 + 64 + 64 + + + 07 + 136494.69336638605 + -46133.17 6301219.54 + 256 + 256 + 128 + 128 + + + 08 + 68247.3466831932 + -46133.17 6301219.54 + 256 + 256 + 256 + 256 + + + 09 + 34123.673341596535 + -46133.17 6301219.54 + 256 + 256 + 512 + 512 + + + 10 + 17061.836670798286 + -46133.17 6301219.54 + 256 + 256 + 1024 + 1024 + + + 11 + 8530.918335399143 + -46133.17 6301219.54 + 256 + 256 + 2048 + 2048 + + + 12 + 4265.4591676995715 + -46133.17 6301219.54 + 256 + 256 + 4096 + 4096 + + + 13 + 2132.729583849782 + -46133.17 6301219.54 + 256 + 256 + 8192 + 8192 + + + + diff --git a/autotest/gdrivers/wmts.py b/autotest/gdrivers/wmts.py index e941167ad9ee..93e117939475 100755 --- a/autotest/gdrivers/wmts.py +++ b/autotest/gdrivers/wmts.py @@ -1871,6 +1871,18 @@ def test_wmts_check_no_overflow_zoom_level(): gdal.Unlink(inputXml) +############################################################################### +# Test fix for https://github.com/OSGeo/gdal/issues/10348 + + +def test_wmts_clip_extent_with_union_of_tile_matrix_extent(): + + ds = gdal.Open("data/wmts/clip_WGS84BoundingBox_with_tilematrix.xml") + assert ds.GetGeoTransform() == pytest.approx( + (-46133.17, 0.5971642834779389, 0.0, 6301219.54, 0.0, -0.5971642834779389) + ) + + ############################################################################### # Test when local wmts tiles are missing diff --git a/frmts/wmts/wmtsdataset.cpp b/frmts/wmts/wmtsdataset.cpp index 3462a186c0b9..d2245c9ca301 100644 --- a/frmts/wmts/wmtsdataset.cpp +++ b/frmts/wmts/wmtsdataset.cpp @@ -1928,6 +1928,21 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) } } + // Clip the computed AOI with the union of the extent of the tile + // matrices + if (bHasAOI && !bExtendBeyondDateLine) + { + OGREnvelope sUnionTM; + for (const WMTSTileMatrix &oTM : oTMS.aoTM) + { + if (!sUnionTM.IsInit()) + sUnionTM = oTM.GetExtent(); + else + sUnionTM.Merge(oTM.GetExtent()); + } + sAOI.Intersect(sUnionTM); + } + // Otherwise default to BoundingBox of the TMS if (!bHasAOI && oTMS.bBoundingBoxValid && (eExtentMethod == AUTO || eExtentMethod == TILE_MATRIX_SET)) From ba3e4f68209e0b36ff295256e11bd3101a2b1ad5 Mon Sep 17 00:00:00 2001 From: Andrea Giudiceandrea Date: Mon, 1 Jul 2024 23:07:45 +0200 Subject: [PATCH 42/72] docs ogrinfo: add a note to -so option with -json --- doc/source/programs/ogrinfo.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/programs/ogrinfo.rst b/doc/source/programs/ogrinfo.rst index 66be7c7106bb..3bcd4773924d 100644 --- a/doc/source/programs/ogrinfo.rst +++ b/doc/source/programs/ogrinfo.rst @@ -76,6 +76,8 @@ edit data. Summary Only: suppress listing of individual features and show only summary information like projection, schema, feature count and extents. + In JSON output, -so is implicit and listing of features can be enabled + with :option:`-features`. .. option:: -features From c30f96e611f2c46a776ddecd06285f0fbe0d676c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 2 Jul 2024 22:02:04 +0200 Subject: [PATCH 43/72] gdalwarp: fix -srcnodata/-dstnodata to accept negative value in first position (3.9.0 regression) --- apps/gdalwarp_lib.cpp | 18 ++++++++++++++-- autotest/utilities/test_gdalwarp_lib.py | 28 +++++++++++++++++++++++++ doc/source/programs/gdalwarp.rst | 4 ++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 6fb54762fc6a..b79dbf15b75f 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -5747,12 +5747,12 @@ GDALWarpAppOptionsGetParser(GDALWarpAppOptions *psOptions, .help(_("Set max warp memory.")); argParser->add_argument("-srcnodata") - .metavar("[ ...]") + .metavar("\"[ ]...\"") .store_into(psOptions->osSrcNodata) .help(_("Nodata masking values for input bands.")); argParser->add_argument("-dstnodata") - .metavar("[ ...]") + .metavar("\"[ ]...\"") .store_into(psOptions->osDstNodata) .help(_("Nodata masking values for output bands.")); @@ -6126,6 +6126,20 @@ GDALWarpAppOptionsNew(char **papszArgv, } psOptions->bCreateOutput = true; } + // argparser will be confused if the value of a string argument + // starts with a negative sign. + else if (EQUAL(papszArgv[i], "-srcnodata") && i + 1 < nArgc) + { + ++i; + psOptions->osSrcNodata = papszArgv[i]; + } + // argparser will be confused if the value of a string argument + // starts with a negative sign. + else if (EQUAL(papszArgv[i], "-dstnodata") && i + 1 < nArgc) + { + ++i; + psOptions->osDstNodata = papszArgv[i]; + } else { aosArgv.AddString(papszArgv[i]); diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index a3f7b300a64d..7f641242e18b 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -1473,6 +1473,34 @@ def test_gdalwarp_lib_127(): assert ds.GetRasterBand(1).Checksum() == 4672, "bad checksum" +@pytest.mark.parametrize("srcNodata", [float("-inf"), -1]) +def test_gdalwarp_lib_srcnodata(srcNodata): + + ds = gdal.Warp( + "", + "../gcore/data/byte.tif", + format="MEM", + srcNodata=srcNodata, + outputType=gdal.GDT_Float32, + ) + assert ds.GetRasterBand(1).GetNoDataValue() == srcNodata, "bad nodata value" + assert ds.GetRasterBand(1).Checksum() == 4672, "bad checksum" + + +@pytest.mark.parametrize("dstNodata", [float("-inf"), -1]) +def test_gdalwarp_lib_dstnodata(dstNodata): + + ds = gdal.Warp( + "", + "../gcore/data/byte.tif", + format="MEM", + dstNodata=dstNodata, + outputType=gdal.GDT_Float32, + ) + assert ds.GetRasterBand(1).GetNoDataValue() == dstNodata, "bad nodata value" + assert ds.GetRasterBand(1).Checksum() == 4672, "bad checksum" + + ############################################################################### # Test automatic densification of cutline (#6375) diff --git a/doc/source/programs/gdalwarp.rst b/doc/source/programs/gdalwarp.rst index 68d85cb1c8d2..ee878a3c30a4 100644 --- a/doc/source/programs/gdalwarp.rst +++ b/doc/source/programs/gdalwarp.rst @@ -33,8 +33,8 @@ Synopsis [-order <1|2|3>] [-refine_gcps []] [-to =]... [-et ] [-wm ] - [-srcnodata [ ...]] - [-dstnodata [ ...]] [-tap] + [-srcnodata "[ ]..."] + [-dstnodata "[ ]..."] [-tap] [-wt Byte|Int8|[U]Int{16|32|64}|CInt{16|32}|[C]Float{32|64}] [-cutline |] [-cutline_srs ] [-cwhere ] From 1d2ca0724ec0b5edca00bc9d02d1de5a74f7c1a3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 2 Jul 2024 22:02:38 +0200 Subject: [PATCH 44/72] gdalbuildvrt: -fix -srcnodata/-vrtnodata to accept negative value in first position (3.9.0 regression) --- apps/gdalbuildvrt_lib.cpp | 15 +++++++++++++++ autotest/utilities/test_gdalbuildvrt_lib.py | 10 +++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/gdalbuildvrt_lib.cpp b/apps/gdalbuildvrt_lib.cpp index ae642786f383..31e720fcc866 100644 --- a/apps/gdalbuildvrt_lib.cpp +++ b/apps/gdalbuildvrt_lib.cpp @@ -2268,6 +2268,21 @@ GDALBuildVRTOptionsNew(char **papszArgv, psOptionsForBinary->osDstFilename = papszArgv[i + 1]; ++i; } + // argparser will be confused if the value of a string argument + // starts with a negative sign. + else if (EQUAL(papszArgv[i], "-srcnodata") && i + 1 < nArgc) + { + ++i; + psOptions->osSrcNoData = papszArgv[i]; + } + // argparser will be confused if the value of a string argument + // starts with a negative sign. + else if (EQUAL(papszArgv[i], "-vrtnodata") && i + 1 < nArgc) + { + ++i; + psOptions->osVRTNoData = papszArgv[i]; + } + else { aosArgv.AddString(papszArgv[i]); diff --git a/autotest/utilities/test_gdalbuildvrt_lib.py b/autotest/utilities/test_gdalbuildvrt_lib.py index 94f449d1e282..74e41222ed6e 100755 --- a/autotest/utilities/test_gdalbuildvrt_lib.py +++ b/autotest/utilities/test_gdalbuildvrt_lib.py @@ -280,15 +280,15 @@ def test_gdalbuildvrt_lib_separate_nodata_2(tmp_vsimem): src2_ds.GetRasterBand(1).SetNoDataValue(2) gdal.BuildVRT( - tmp_vsimem / "out.vrt", [src1_ds, src2_ds], separate=True, srcNodata="3 4" + tmp_vsimem / "out.vrt", [src1_ds, src2_ds], separate=True, srcNodata="-3 4" ) f = gdal.VSIFOpenL(tmp_vsimem / "out.vrt", "rb") data = gdal.VSIFReadL(1, 10000, f) gdal.VSIFCloseL(f) - assert b"3" in data - assert b"3" in data + assert b"-3" in data + assert b"-3" in data assert b"4" in data assert b"4" in data @@ -309,14 +309,14 @@ def test_gdalbuildvrt_lib_separate_nodata_3(tmp_vsimem): [src1_ds, src2_ds], separate=True, srcNodata="3 4", - VRTNodata="5 6", + VRTNodata="-5 6", ) f = gdal.VSIFOpenL(tmp_vsimem / "out.vrt", "rb") data = gdal.VSIFReadL(1, 10000, f) gdal.VSIFCloseL(f) - assert b"5" in data + assert b"-5" in data assert b"3" in data assert b"6" in data assert b"4" in data From 79ae4414601ae9d6e7137853e537e011e758f4e4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 2 Jul 2024 22:03:19 +0200 Subject: [PATCH 45/72] gdal_translate: fix -a_nodata to accept '-inf' as input (3.9.0 regression) --- apps/gdal_translate_lib.cpp | 30 +++++++++++-------- autotest/utilities/test_gdal_translate_lib.py | 12 ++++++++ 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/apps/gdal_translate_lib.cpp b/apps/gdal_translate_lib.cpp index 0dbbd9193253..79f1e4895551 100644 --- a/apps/gdal_translate_lib.cpp +++ b/apps/gdal_translate_lib.cpp @@ -3032,19 +3032,6 @@ GDALTranslateOptionsGetParser(GDALTranslateOptions *psOptions, argParser->add_argument("-a_nodata") .metavar("|none") - .action( - [psOptions](const std::string &s) - { - if (EQUAL(s.c_str(), "none")) - { - psOptions->bUnsetNoData = true; - } - else - { - psOptions->bSetNoData = true; - psOptions->osNoData = s; - } - }) .help(_("Assign a specified nodata value to output bands.")); argParser->add_argument("-a_gt") @@ -3383,6 +3370,23 @@ GDALTranslateOptionsNew(char **papszArgv, psOptions->anColorInterp[nIndex] = GetColorInterp(papszArgv[i]); } + // argparser will be confused if the value of a string argument + // starts with a negative sign. + else if (EQUAL(papszArgv[i], "-a_nodata") && papszArgv[i + 1]) + { + ++i; + const std::string s = papszArgv[i]; + if (EQUAL(s.c_str(), "none")) + { + psOptions->bUnsetNoData = true; + } + else + { + psOptions->bSetNoData = true; + psOptions->osNoData = s; + } + } + else { aosArgv.AddString(papszArgv[i]); diff --git a/autotest/utilities/test_gdal_translate_lib.py b/autotest/utilities/test_gdal_translate_lib.py index 8948856bd329..07375a0f0801 100755 --- a/autotest/utilities/test_gdal_translate_lib.py +++ b/autotest/utilities/test_gdal_translate_lib.py @@ -292,6 +292,18 @@ def test_gdal_translate_lib_nodata_int64(): ds = None +############################################################################### +# Test nodata=-inf + + +def test_gdal_translate_lib_nodata_minus_inf(): + + ds = gdal.Translate( + "", "../gcore/data/float32.tif", format="MEM", noData=float("-inf") + ) + assert ds.GetRasterBand(1).GetNoDataValue() == float("-inf"), "Bad nodata value" + + ############################################################################### # Test srcWin option From ca6fd7fcea9186437aa1cbf845013583f1c03112 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 2 Jul 2024 22:06:22 +0200 Subject: [PATCH 46/72] argparse: accept '-inf' as value for numeric arguments --- apps/argparse/argparse.hpp | 4 ++++ autotest/utilities/test_gdal_translate_lib.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/apps/argparse/argparse.hpp b/apps/argparse/argparse.hpp index 7b589e7c79c8..a52142848192 100644 --- a/apps/argparse/argparse.hpp +++ b/apps/argparse/argparse.hpp @@ -1411,6 +1411,10 @@ class Argument { * '+' '-' */ static bool is_decimal_literal(std::string_view s) { + if (s == "inf") { + return true; + } + auto is_digit = [](auto c) constexpr { switch (c) { case '0': diff --git a/autotest/utilities/test_gdal_translate_lib.py b/autotest/utilities/test_gdal_translate_lib.py index 07375a0f0801..180b3c4e9d41 100755 --- a/autotest/utilities/test_gdal_translate_lib.py +++ b/autotest/utilities/test_gdal_translate_lib.py @@ -1154,6 +1154,20 @@ def test_gdal_translate_lib_scale_and_unscale_incompatible(): ) +############################################################################### +# Test -a_offset -inf (dummy example, but to proove -inf works as a value +# numeric value) + + +@gdaltest.enable_exceptions() +def test_gdal_translate_lib_assign_offset(): + + out_ds = gdal.Translate( + "", gdal.Open("../gcore/data/byte.tif"), options="-f MEM -a_offset -inf" + ) + assert out_ds.GetRasterBand(1).GetOffset() == float("-inf") + + ############################################################################### # Test option argument handling From 868dac16f28a15c93f15c90194b780f062122452 Mon Sep 17 00:00:00 2001 From: Sam Gillingham Date: Wed, 3 Jul 2024 06:10:14 +1000 Subject: [PATCH 47/72] KEA: don't derive from PAM classes (#10351) --- frmts/kea/keaband.cpp | 12 ++++++------ frmts/kea/keaband.h | 4 ++-- frmts/kea/keadataset.h | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frmts/kea/keaband.cpp b/frmts/kea/keaband.cpp index 905ec37d03c3..5b452c74a401 100644 --- a/frmts/kea/keaband.cpp +++ b/frmts/kea/keaband.cpp @@ -399,7 +399,7 @@ void KEARasterBand::SetDescription(const char *pszDescription) try { this->m_pImageIO->setImageBandDescription(this->nBand, pszDescription); - GDALPamRasterBand::SetDescription(pszDescription); + GDALRasterBand::SetDescription(pszDescription); } catch (const kealib::KEAIOException &) { @@ -721,9 +721,9 @@ CPLErr KEARasterBand::GetDefaultHistogram(double *pdfMin, double *pdfMax, { if (bForce) { - return GDALPamRasterBand::GetDefaultHistogram(pdfMin, pdfMax, pnBuckets, - ppanHistogram, bForce, fn, - pProgressData); + return GDALRasterBand::GetDefaultHistogram(pdfMin, pdfMax, pnBuckets, + ppanHistogram, bForce, fn, + pProgressData); } else { @@ -1379,7 +1379,7 @@ GDALRasterBand *KEARasterBand::GetMaskBand() { // use the base class implementation - GDAL will delete // fprintf( stderr, "returning base GetMaskBand()\n" ); - m_pMaskBand = GDALPamRasterBand::GetMaskBand(); + m_pMaskBand = GDALRasterBand::GetMaskBand(); } } catch (const kealib::KEAException &) @@ -1399,7 +1399,7 @@ int KEARasterBand::GetMaskFlags() // need to return the base class one since we are using // the base class implementation of GetMaskBand() // fprintf( stderr, "returning base GetMaskFlags()\n" ); - return GDALPamRasterBand::GetMaskFlags(); + return GDALRasterBand::GetMaskFlags(); } } catch (const kealib::KEAException &) diff --git a/frmts/kea/keaband.h b/frmts/kea/keaband.h index 9ae18b5b30d7..882b5e528a71 100644 --- a/frmts/kea/keaband.h +++ b/frmts/kea/keaband.h @@ -31,14 +31,14 @@ #ifndef KEABAND_H #define KEABAND_H -#include "gdal_pam.h" +#include "gdal_priv.h" #include "keadataset.h" class KEAOverview; class KEAMaskBand; // Provides the implementation of a GDAL raster band -class KEARasterBand CPL_NON_FINAL : public GDALPamRasterBand +class KEARasterBand CPL_NON_FINAL : public GDALRasterBand { private: LockedRefCount *m_pRefCount = nullptr; // reference count of m_pImageIO diff --git a/frmts/kea/keadataset.h b/frmts/kea/keadataset.h index 63505b1826e5..5f56ba5a4691 100644 --- a/frmts/kea/keadataset.h +++ b/frmts/kea/keadataset.h @@ -31,14 +31,14 @@ #ifndef KEADATASET_H #define KEADATASET_H -#include "gdal_pam.h" +#include "gdal_priv.h" #include "cpl_multiproc.h" #include "libkea_headers.h" class LockedRefCount; // class that implements a GDAL dataset -class KEADataset final : public GDALPamDataset +class KEADataset final : public GDALDataset { static H5::H5File *CreateLL(const char *pszFilename, int nXSize, int nYSize, int nBands, GDALDataType eType, From 1f937502c230e5224ca7254da21fa1e0b642d538 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 1 Jul 2024 17:19:35 +0200 Subject: [PATCH 48/72] GDALMDArray::GetResampled(): take into account good_wavelengths array for EMIT dataset orthorectification When the /sensor_band_parameters/good_wavelengths variable is present, use it to decide of the current array along the band dimension are valid. --- .../fake_EMIT_L2A_with_good_wavelengths.nc | Bin 0 -> 16803 bytes autotest/gdrivers/netcdf_multidim.py | 82 +++++++++++ gcore/gdal_priv.h | 12 +- gcore/gdalmultidim.cpp | 17 ++- gcore/gdalmultidim_gltorthorectification.cpp | 134 ++++++++++++++---- 5 files changed, 206 insertions(+), 39 deletions(-) create mode 100644 autotest/gdrivers/data/netcdf/fake_EMIT_L2A_with_good_wavelengths.nc diff --git a/autotest/gdrivers/data/netcdf/fake_EMIT_L2A_with_good_wavelengths.nc b/autotest/gdrivers/data/netcdf/fake_EMIT_L2A_with_good_wavelengths.nc new file mode 100644 index 0000000000000000000000000000000000000000..bddcb696313b22c5e643288f76df47761d018903 GIT binary patch literal 16803 zcmeHOdu&tJ8UJqLfRh*!$}8njE`e^)g($I|S6Nz2;>4jQiEKk*lo=D>Bp${$*f)es z+seAF6`K0T`WS1cRcQNT6;pyw zj^o$?LSPWx1Mzpy`+NM(x!-x*6M@dIMHQrW-MBEPg;73;Y}KKx!$ZK;e`lz>K0)q;!lr84zhkp{4Q4e|$2>HuaVpStR( zh6a`3#D9rY#!8PKHd2;shdQAKRCD0p*%N_nGVl6+fBm(KPvwcqo#eB~aTayK2I^WP zzC@@VGR=5&EOyjL8mS>`ICJD4mGebK|G#8!*XPgViORgQX!$fN>I!0gQq1D8NmW+Y}MV##RQ7!++1s_Kz#=DgeA*WD4`SNmXhSa0>#)^++KyLO6bw6Tp6 z!u!IxZR$nW>1}OS0vCE>WS>#SL^Q&ch}a{wlsJe zYWMFikkjmOH&af3sAs1id+SacMrtfEFkD;LP@s5gTSFri-__d_*;gAkhOju6tZ!&+ z@iuwfZSJPVwl=S)5D#sQZnxlJPY((iL|s^;aje|i7m5UsHJNOthE%%W7mgI-*ef}9 z?E#=czX-Kk14D*srDLhgpqV~Q%B^(_}Xgcw@sgq06tOJz>VDJdxt95}_4R;uwJ zKnXInM6VXR5;qS;V1jy3sxd_ljOD)%W(JdY(P zNWt5^GWMP%fEu15JmY?(dxq?5%@QN$05PsXnGmVhx6@CpkVxC`k41_MCo($zkLfAH z>gep!;|ZL!GYK=L9~`yxlxa<79gU?Ev4csYUgzKj7m7(3S}aCrwAZi(;)CAK#Nk97 zdzpqIOyls_$)}#17v@(en`-_IO;XOQGynPxUwxeQG8gn#w?Cop^`%dY&j1j@nZ2!cmE^ShKbdFC&>Av1r!>Sn6x zSGUT}f2(+ydFKD&*$*nY;Pug)WqHpmCw;Q1`Fh`>`C$1Si>0lmTE1L4J6C7<-*|TY z?)ogwwQqIHlUw-Sg^PD*EiXohRw|tk_VTWlDsNCP;0?OSuOV+xIHI3i^Gk>D4>=}1 zv+jlKZ7~p+X&$S8H!`Pwp8AnqID}y=YU^(%R9V|*u8ktPiU&g%9 z+PBvsnWG@AAECxHSSDO~cEdVMGaXN)VwORi6>!Itznz_JMJf41{F6moLFq#Tue|aOe29-5prh&=ZTMh^TT27N z{%X;VoJkBYuoMom5SLis8B$qZyiiCeY4YrzhJ;AAX^9OnjH$5163-|KBX527$@7$H zhnuDWdSUWBKe0`AtE=8!Bed&vt+P18gDHWDXK}(xZk)i98z;29d-=Z4b$9dyDV^tV z@r+T%i$kG|k(r;)eRF!L!X0Ck)43f6EvIvNMsGGI$_EYnKH+rkS1p&P z$;GQ6F3eTKW=MJe&S+#faErEYJ?W zUh~^uR-_%34AsmqC)m{1fUN;r1GWZi4cHp6HDGJN)_|=6TLZQR=CuZLcO~}uq^KK} z#I6Ig*_Gg(0;ajU4z%jcjf>Up?!P8fhjT_qsOkTax zWh;4RqPU{{lCpBPAMrVX8!$-4Bk!W!>62VB()hGwhTEc|2Hh5=yJz{wN{mk4ttob# zG~vRlYIMn|_*s%aD+s`YlQ%o{v0)=^=y;KkPnGBv)M&<_3r_dyje5IImqGP-%!<|N z4H!KblQBB>qZ?0GckJitmxqLNcPd0<=pRGl9}z}fZ{ig=-J>wt=@2yqA#?d6fH2bvpEA)ssVDp5>(3 ziIKfpAzN1^8QC*jC_#%8C}(#Rn=grM3y)#3RRVgkW7z4Eikd}E87WW9!S_lm7au+} z4bf1i5agSs`X;~0-H2trmf+zBPZXMHhnQ~obt#oO0RG}Kk??} z%aiBx6qDn(O4<5R;swLa9+e~yovXMyc~lmBi{XSqr2usC%kv`Zo+-)Yx+bf22P1ko zc6cOd=t0vwG&QqKuFq^+#_##c?fdqZ007{xPdLtBoN#>gxhM78S4uWGOUZGTlH(s9 zIW#7uJj*M*ThWut_e%KtAC`i@_~H0nn<{=Gq}m~;rN!unIU(BH(VsR)jI@;y7t-j& zv=;xk)=?u(A0D*3>+o9y{EVR8Ti;UesT+ee=ZAlb@um z$l!-3W;%K>mWoG5Vrje+XdxmVQ1H+kj1won!I~8=H3tKYrqilSaKe0M?P+OUuu95Xu3w*@o@-;Yo^$M{}==HMF=l{u?Q zUh`<^#>h}D7O=8n88BYrxjPe9?f89+m!6fKy0MA-#n3 y5>kCT{=KT9nDiniO5xJ|lEOAg^W#pH&a_&^naW!9%Pr?WPnM89vl~I>>F - CreateGLTOrthorectified(const std::shared_ptr &poParent, - const std::shared_ptr &poGLTX, - const std::shared_ptr &poGLTY, - int nGLTIndexOffset, - const std::vector &adfGeoTransform); + static std::shared_ptr CreateGLTOrthorectified( + const std::shared_ptr &poParent, + const std::shared_ptr &poRootGroup, + const std::shared_ptr &poGLTX, + const std::shared_ptr &poGLTY, int nGLTIndexOffset, + const std::vector &adfGeoTransform, CSLConstList papszOptions); //! @endcond diff --git a/gcore/gdalmultidim.cpp b/gcore/gdalmultidim.cpp index ab7ece1b95f1..9fd53204c1d8 100644 --- a/gcore/gdalmultidim.cpp +++ b/gcore/gdalmultidim.cpp @@ -8441,7 +8441,18 @@ bool GDALMDArrayResampled::IRead(const GUInt64 *arrayStartIdx, * orthorectifying a NASA EMIT dataset. * * For NASA EMIT datasets, if apoNewDims[] and poTargetSRS is NULL, the - * geometry lookup table (GLT) is used for fast orthorectification. + * geometry lookup table (GLT) is used by default for fast orthorectification. + * + * Options available are: + *
    + *
  • EMIT_ORTHORECTIFICATION=YES/NO: defaults to YES for a NASA EMIT dataset. + * Can be set to NO to use generic reprojection method. + *
    • + *
  • USE_GOOD_WAVELENGTHS=YES/NO: defaults to YES. Only used for EMIT + * orthorectification to take into account the value of the + * /sensor_band_parameters/good_wavelengths array to decide if slices of the + * current array along the band dimension are valid.
    • + *
* * @param apoNewDims New dimensions. Its size should be GetDimensionCount(). * apoNewDims[i] can be NULL to let the method automatically @@ -8511,9 +8522,9 @@ std::shared_ptr GDALMDArray::GetResampled( poGLT_Y->GetDimensions()[1]->GetName() == "ortho_x") { return CreateGLTOrthorectified( - self, poGLT_X, poGLT_Y, + self, poRootGroup, poGLT_X, poGLT_Y, /* nGLTIndexOffset = */ -1, - poAttrGeotransform->ReadAsDoubleArray()); + poAttrGeotransform->ReadAsDoubleArray(), papszOptions); } } } diff --git a/gcore/gdalmultidim_gltorthorectification.cpp b/gcore/gdalmultidim_gltorthorectification.cpp index 62e6446528aa..4baf11e31863 100644 --- a/gcore/gdalmultidim_gltorthorectification.cpp +++ b/gcore/gdalmultidim_gltorthorectification.cpp @@ -49,6 +49,7 @@ class GLTOrthoRectifiedArray final : public GDALPamMDArray std::shared_ptr m_poGLTX{}; std::shared_ptr m_poGLTY{}; int m_nGLTIndexOffset = 0; + std::vector m_abyBandValidity{}; protected: GLTOrthoRectifiedArray( @@ -76,9 +77,11 @@ class GLTOrthoRectifiedArray final : public GDALPamMDArray public: static std::shared_ptr Create(const std::shared_ptr &poParent, + const std::shared_ptr &poRootGroup, const std::shared_ptr &poGLTX, const std::shared_ptr &poGLTY, int nGLTIndexOffset, - const std::vector &adfGeoTransform) + const std::vector &adfGeoTransform, + CSLConstList papszOptions) { std::vector> apoNewDims; @@ -124,6 +127,32 @@ class GLTOrthoRectifiedArray final : public GDALPamMDArray newAr->m_poSRS.reset(oSRS.Clone()); newAr->m_poSRS->SetDataAxisToSRSAxisMapping( {/*latIdx = */ 1, /* lonIdx = */ 2}); + if (CPLTestBool(CSLFetchNameValueDef(papszOptions, + "USE_GOOD_WAVELENGTHS", "YES")) && + poParent->GetDimensionCount() == 3) + { + const auto poGoodWaveLengths = poRootGroup->OpenMDArrayFromFullname( + "/sensor_band_parameters/good_wavelengths"); + if (poGoodWaveLengths && + poGoodWaveLengths->GetDimensionCount() == 1 && + poGoodWaveLengths->GetDimensions()[0]->GetSize() == + poParent->GetDimensions()[2]->GetSize() && + poGoodWaveLengths->GetDimensions()[0]->GetSize() < + 1000 * 1000 && + poGoodWaveLengths->GetDataType().GetClass() == GEDTC_NUMERIC) + { + const GUInt64 arrayStartIdx[] = {0}; + const size_t count[] = {static_cast( + poGoodWaveLengths->GetDimensions()[0]->GetSize())}; + const GInt64 arrayStep[] = {1}; + const GPtrDiff_t bufferStride[] = {1}; + newAr->m_abyBandValidity.resize(count[0], 1); + CPL_IGNORE_RET_VAL(poGoodWaveLengths->Read( + arrayStartIdx, count, arrayStep, bufferStride, + GDALExtendedDataType::Create(GDT_Byte), + newAr->m_abyBandValidity.data())); + } + } return newAr; } @@ -206,6 +235,59 @@ bool GLTOrthoRectifiedArray::IRead(const GUInt64 *arrayStartIdx, if (bufferDataType.GetClass() != GEDTC_NUMERIC) return false; + const auto eBufferDT = bufferDataType.GetNumericDataType(); + auto pRawNoDataValue = GetRawNoDataValue(); + std::vector abyNoData(16); + if (pRawNoDataValue) + GDALCopyWords(pRawNoDataValue, GetDataType().GetNumericDataType(), 0, + abyNoData.data(), eBufferDT, 0, 1); + + const auto nBufferDTSize = bufferDataType.GetSize(); + const int nCopyWordsDstStride = + m_apoDims.size() == 3 + ? static_cast(bufferStride[2] * nBufferDTSize) + : 0; + const int nCopyWordsCount = + m_apoDims.size() == 3 ? static_cast(count[2]) : 1; + + const auto FillBufferWithNodata = + [count, bufferStride, pDstBuffer, nBufferDTSize, &eBufferDT, + nCopyWordsDstStride, nCopyWordsCount, &abyNoData]() + { + for (size_t iY = 0; iY < count[0]; ++iY) + { + for (size_t iX = 0; iX < count[1]; ++iX) + { + GByte *pabyDstBuffer = + static_cast(pDstBuffer) + + (iY * bufferStride[0] + iX * bufferStride[1]) * + static_cast(nBufferDTSize); + GDALCopyWords(abyNoData.data(), eBufferDT, 0, pabyDstBuffer, + eBufferDT, nCopyWordsDstStride, nCopyWordsCount); + } + } + }; + + bool bSomeBandInvalids = false; + if (!m_abyBandValidity.empty()) + { + size_t nCountInvalid = 0; + for (size_t i = 0; i < count[2]; ++i) + { + const size_t iBand = + static_cast(arrayStartIdx[2] + i * arrayStep[2]); + CPLAssert(iBand < m_abyBandValidity.size()); + if (!m_abyBandValidity[iBand]) + nCountInvalid++; + } + if (nCountInvalid == count[2]) + { + FillBufferWithNodata(); + return true; + } + bSomeBandInvalids = true; + } + const size_t nXYValsCount = count[0] * count[1]; const auto eInt32DT = GDALExtendedDataType::Create(GDT_Int32); std::vector anGLTX; @@ -263,34 +345,9 @@ bool GLTOrthoRectifiedArray::IRead(const GUInt64 *arrayStartIdx, } } - const auto eBufferDT = bufferDataType.GetNumericDataType(); - auto pRawNoDataValue = GetRawNoDataValue(); - std::vector abyNoData(16); - if (pRawNoDataValue) - GDALCopyWords(pRawNoDataValue, GetDataType().GetNumericDataType(), 0, - abyNoData.data(), eBufferDT, 0, 1); - - const auto nBufferDTSize = bufferDataType.GetSize(); - const int nCopyWordsDstStride = - m_apoDims.size() == 3 - ? static_cast(bufferStride[2] * nBufferDTSize) - : 0; - const int nCopyWordsCount = - m_apoDims.size() == 3 ? static_cast(count[2]) : 1; if (nMinX > nMaxX || nMinY > nMaxY) { - for (size_t iY = 0; iY < count[0]; ++iY) - { - for (size_t iX = 0; iX < count[1]; ++iX) - { - GByte *pabyDstBuffer = - static_cast(pDstBuffer) + - (iY * bufferStride[0] + iX * bufferStride[1]) * - static_cast(nBufferDTSize); - GDALCopyWords(abyNoData.data(), eBufferDT, 0, pabyDstBuffer, - eBufferDT, nCopyWordsDstStride, nCopyWordsCount); - } - } + FillBufferWithNodata(); return true; } @@ -363,6 +420,21 @@ bool GLTOrthoRectifiedArray::IRead(const GUInt64 *arrayStartIdx, GDALCopyWords(pabySrcBuffer, eBufferDT, static_cast(nBufferDTSize), pabyDstBuffer, eBufferDT, nCopyWordsDstStride, nCopyWordsCount); + if (bSomeBandInvalids) + { + for (size_t i = 0; i < count[2]; ++i) + { + const size_t iBand = static_cast( + arrayStartIdx[2] + i * arrayStep[2]); + if (!m_abyBandValidity[iBand]) + { + GDALCopyWords(abyNoData.data(), eBufferDT, 0, + pabyDstBuffer + + i * nCopyWordsDstStride, + eBufferDT, 0, 1); + } + } + } } else { @@ -383,12 +455,14 @@ bool GLTOrthoRectifiedArray::IRead(const GUInt64 *arrayStartIdx, /* static */ std::shared_ptr GDALMDArray::CreateGLTOrthorectified( const std::shared_ptr &poParent, + const std::shared_ptr &poRootGroup, const std::shared_ptr &poGLTX, const std::shared_ptr &poGLTY, int nGLTIndexOffset, - const std::vector &adfGeoTransform) + const std::vector &adfGeoTransform, CSLConstList papszOptions) { - return GLTOrthoRectifiedArray::Create(poParent, poGLTX, poGLTY, - nGLTIndexOffset, adfGeoTransform); + return GLTOrthoRectifiedArray::Create(poParent, poRootGroup, poGLTX, poGLTY, + nGLTIndexOffset, adfGeoTransform, + papszOptions); } //! @endcond From b63b5dd7934a1ce391da17aedd2e4b6f3ad37025 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 2 Jul 2024 22:21:16 +0200 Subject: [PATCH 49/72] Fix compilation against openssl-libs-3.2.2-3 of fedora:rawhide --- port/cpl_http.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/port/cpl_http.cpp b/port/cpl_http.cpp index ac35c47f3fe3..4f3991b1ef6f 100644 --- a/port/cpl_http.cpp +++ b/port/cpl_http.cpp @@ -59,7 +59,6 @@ #ifdef HAVE_OPENSSL_CRYPTO #include #include -#include #include #if defined(_WIN32) From 2cb3e6632281f32b4170dad2f4f5d49f6ba20657 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Thu, 4 Jul 2024 10:02:34 +1000 Subject: [PATCH 50/72] [pdf] Fix link to pdfcomposition.xsd in docs [skip ci] --- doc/source/drivers/raster/pdf.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/drivers/raster/pdf.rst b/doc/source/drivers/raster/pdf.rst index e1642071be51..01267dfc0514 100644 --- a/doc/source/drivers/raster/pdf.rst +++ b/doc/source/drivers/raster/pdf.rst @@ -558,7 +558,7 @@ datatype = GDT_Unknown and :co:`COMPOSITION_FILE` must be the single creation option. The XML schema against which the composition file must validate is -`pdfcomposition.xsd `__ +`pdfcomposition.xsd `__ Example on how to use the API: From e5536ffe8ce02b47fa516d5604ba1247b3df40c2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 6 Jul 2024 10:05:22 +0200 Subject: [PATCH 51/72] Doc: fix link --- doc/source/drivers/raster/vrt_multidimensional.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/drivers/raster/vrt_multidimensional.rst b/doc/source/drivers/raster/vrt_multidimensional.rst index 1e4fa912a744..0392b73c35c7 100644 --- a/doc/source/drivers/raster/vrt_multidimensional.rst +++ b/doc/source/drivers/raster/vrt_multidimensional.rst @@ -36,7 +36,7 @@ Here's an example of such a file: .vrt Format ----------- -A `XML schema of the GDAL VRT format `_ +A `XML schema of the GDAL VRT format `_ is available. Virtual files stored on disk are kept in an XML format with the following From 151e9119aa8ac96f0af2100e4116ef4134fa4120 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Sat, 6 Jul 2024 04:21:46 -0400 Subject: [PATCH 52/72] Viewshed: support observer above and below the raster. (#10352) --- alg/viewshed.cpp | 96 ++++++++++------- alg/viewshed.h | 14 ++- autotest/cpp/test_viewshed.cpp | 126 ++++++++++++++++++++++- autotest/utilities/test_gdal_viewshed.py | 11 -- doc/source/programs/gdal_viewshed.rst | 8 +- 5 files changed, 204 insertions(+), 51 deletions(-) diff --git a/alg/viewshed.cpp b/alg/viewshed.cpp index 18af2999a9eb..6fb84ad6f736 100644 --- a/alg/viewshed.cpp +++ b/alg/viewshed.cpp @@ -282,19 +282,9 @@ bool Viewshed::calcOutputExtent(int nX, int nY) oOutExtent.xStop = GDALGetRasterBandXSize(pSrcBand); oOutExtent.yStop = GDALGetRasterBandYSize(pSrcBand); - if (!oOutExtent.containsY(nY)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Observer position above or below the raster " - "not currently supported"); - return false; - } if (!oOutExtent.contains(nX, nY)) - { CPLError(CE_Warning, CPLE_AppDefined, "NOTE: The observer location falls outside of the DEM area"); - //ABELL - Make sure observer Z is specified. - } constexpr double EPSILON = 1e-8; if (oOpts.maxDistance > 0) @@ -419,7 +409,8 @@ bool Viewshed::emitProgress(double fraction) /// @param nX X location of the observer. /// @param vThisLineVal Line height data. /// @return [left, right) Leftmost and one past the rightmost cell in the line within -/// the max distance +/// the max distance. Indices are limited to the raster extent (right may be just +/// outside the raster). std::pair Viewshed::adjustHeight(int nYOffset, int nX, std::vector &vThisLineVal) { @@ -799,15 +790,15 @@ void Viewshed::setOutput(double &dfResult, double &dfCellVal, double dfZ) /// /// @param nX X location of the observer /// @param nY Y location of the observer -/// @param nLine Line number being processed (should always be the same as nY) /// @param vLastLineVal Vector in which to store the read line. Becomes the last line /// in further processing. /// @return True on success, false otherwise. -bool Viewshed::processFirstLine(int nX, int nY, int nLine, +bool Viewshed::processFirstLine(int nX, int nY, std::vector &vLastLineVal) { + int nLine = oOutExtent.clampY(nY); int nYOffset = nLine - nY; - assert(nYOffset == 0); + std::vector vResult(oOutExtent.xSize()); std::vector vThisLineVal(oOutExtent.xSize()); @@ -832,21 +823,30 @@ bool Viewshed::processFirstLine(int nX, int nY, int nLine, // iLeft and iRight are the processing limits for the line. const auto [iLeft, iRight] = adjustHeight(nYOffset, nX, vThisLineVal); - auto t1 = std::async( - std::launch::async, [&, left = iLeft]() - { processFirstLineLeft(nX, nX - 1, left - 1, vResult, vThisLineVal); }); - - auto t2 = std::async( - std::launch::async, [&, right = iRight]() - { processFirstLineRight(nX, nX + 1, right, vResult, vThisLineVal); }); - t1.wait(); - t2.wait(); + if (!oCurExtent.containsY(nY)) + processFirstLineTopOrBottom(iLeft, iRight, vResult, vThisLineVal); + else + { + auto t1 = std::async(std::launch::async, + [&, left = iLeft]() { + processFirstLineLeft(nX, nX - 1, left - 1, + vResult, vThisLineVal); + }); + + auto t2 = std::async(std::launch::async, + [&, right = iRight]() { + processFirstLineRight(nX, nX + 1, right, + vResult, vThisLineVal); + }); + t1.wait(); + t2.wait(); + } // Make the current line the last line. vLastLineVal = std::move(vThisLineVal); // Create the output writer. - if (!writeLine(nY, vResult)) + if (!writeLine(nLine, vResult)) return false; if (!lineProgress()) @@ -854,6 +854,30 @@ bool Viewshed::processFirstLine(int nX, int nY, int nLine, return true; } +// If the observer is above or below the raster, set all cells in the first line near the +// observer as observable provided they're in range. Mark cells out of range as such. +/// @param iLeft Leftmost observable raster position in range of the target line. +/// @param iRight One past the rightmost observable raster position of the target line. +/// @param vResult Result line. +/// @param vThisLineVal Heights of the cells in the target line +void Viewshed::processFirstLineTopOrBottom(int iLeft, int iRight, + std::vector &vResult, + std::vector &vThisLineVal) +{ + double *pResult = vResult.data() + iLeft; + double *pThis = vThisLineVal.data() + iLeft; + for (int iPixel = iLeft; iPixel < iRight; ++iPixel, ++pResult, pThis++) + { + if (oOpts.outputMode == OutputMode::Normal) + *pResult = oOpts.visibleVal; + else + setOutput(*pResult, *pThis, *pThis); + } + std::fill(vResult.begin(), vResult.begin() + iLeft, oOpts.outOfRangeVal); + std::fill(vResult.begin() + iRight, vResult.begin() + oCurExtent.xStop, + oOpts.outOfRangeVal); +} + /// Process a line above or below the observer. /// /// @param nX X location of the observer @@ -989,7 +1013,7 @@ bool Viewshed::run(GDALRasterBandH band, GDALProgressFunc pfnProgress, std::vector vFirstLineVal(oCurExtent.xSize()); - if (!processFirstLine(nX, nY, nY, vFirstLineVal)) + if (!processFirstLine(nX, nY, vFirstLineVal)) return false; if (oOpts.cellMode == CellMode::Edge) @@ -1002,29 +1026,31 @@ bool Viewshed::run(GDALRasterBandH band, GDALProgressFunc pfnProgress, oZcalc = doMax; // scan upwards + int yStart = oCurExtent.clampY(nY); std::atomic err(false); auto tUp = std::async(std::launch::async, [&]() { std::vector vLastLineVal = vFirstLineVal; - for (int nLine = nY - 1; + for (int nLine = yStart - 1; nLine >= oCurExtent.yStart && !err; nLine--) if (!processLine(nX, nY, nLine, vLastLineVal)) err = true; }); // scan downwards - auto tDown = std::async( - std::launch::async, - [&]() - { - std::vector vLastLineVal = vFirstLineVal; + auto tDown = + std::async(std::launch::async, + [&]() + { + std::vector vLastLineVal = vFirstLineVal; - for (int nLine = nY + 1; nLine < oCurExtent.yStop && !err; nLine++) - if (!processLine(nX, nY, nLine, vLastLineVal)) - err = true; - }); + for (int nLine = yStart + 1; + nLine < oCurExtent.yStop && !err; nLine++) + if (!processLine(nX, nY, nLine, vLastLineVal)) + err = true; + }); tUp.wait(); tDown.wait(); diff --git a/alg/viewshed.h b/alg/viewshed.h index 349ccc8e6ba3..13a1ab558091 100644 --- a/alg/viewshed.h +++ b/alg/viewshed.h @@ -132,6 +132,14 @@ class Viewshed return xSize() ? std::clamp(nX, xStart, xStop - 1) : xStart; } + /// \brief Clamp the argument to be in the window in the Y dimension. + /// \param nY Value to clamp. + /// \return Clamped value. + int clampY(int nY) const + { + return ySize() ? std::clamp(nY, yStart, yStop - 1) : yStart; + } + /// \brief Shift the X dimension by nShift. /// \param nShift Amount to shift void shiftX(int nShift) @@ -225,14 +233,16 @@ class Viewshed bool writeLine(int nLine, std::vector &vResult); bool processLine(int nX, int nY, int nLine, std::vector &vLastLineVal); - bool processFirstLine(int nX, int nY, int nLine, - std::vector &vLastLineVal); + bool processFirstLine(int nX, int nY, std::vector &vLastLineVal); void processFirstLineLeft(int nX, int iStart, int iEnd, std::vector &vResult, std::vector &vThisLineVal); void processFirstLineRight(int nX, int iStart, int iEnd, std::vector &vResult, std::vector &vThisLineVal); + void processFirstLineTopOrBottom(int iLeft, int iRight, + std::vector &vResult, + std::vector &vThisLineVal); void processLineLeft(int nX, int nYOffset, int iStart, int iEnd, std::vector &vResult, std::vector &vThisLineVal, diff --git a/autotest/cpp/test_viewshed.cpp b/autotest/cpp/test_viewshed.cpp index 7311265cb834..95382ba0a814 100644 --- a/autotest/cpp/test_viewshed.cpp +++ b/autotest/cpp/test_viewshed.cpp @@ -308,7 +308,7 @@ TEST(Viewshed, oor_right) } } -// Test an observer to the right of the raster. +// Test an observer to the left of the raster. TEST(Viewshed, oor_left) { // clang-format off @@ -369,4 +369,128 @@ TEST(Viewshed, oor_left) } } +// Test an observer above the raster +TEST(Viewshed, oor_above) +{ + // clang-format off + const int xlen = 5; + const int ylen = 3; + std::array in + { + 1, 2, 0, 4, 1, + 0, 0, 2, 1, 0, + 1, 0, 0, 3, 3 + }; + // clang-format on + + { + Viewshed::Options opts = stdOptions(2, -2); + opts.outputMode = Viewshed::OutputMode::DEM; + DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); + GDALRasterBand *band = ds->GetRasterBand(1); + std::array out; + CPLErr err = band->RasterIO(GF_Read, 0, 0, xlen, ylen, out.data(), xlen, + ylen, GDT_Float64, 0, 0, nullptr); + + EXPECT_EQ(err, CE_None); + + // clang-format off + std::array expected + { + 1, 2, 0, 4, 1, + 2.5, 2, 0, 4, 4.5, + 3, 8 / 3.0, 8 / 3.0, 14 / 3.0, 17 / 3.0 + }; + // clang-format on + + for (size_t i = 0; i < out.size(); ++i) + EXPECT_DOUBLE_EQ(out[i], expected[i]); + } + + { + Viewshed::Options opts = stdOptions(-2, -2); + opts.outputMode = Viewshed::OutputMode::DEM; + DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); + GDALRasterBand *band = ds->GetRasterBand(1); + std::array out; + CPLErr err = band->RasterIO(GF_Read, 0, 0, xlen, ylen, out.data(), xlen, + ylen, GDT_Float64, 0, 0, nullptr); + EXPECT_EQ(err, CE_None); + + // clang-format off + std::array expected + { + 1, 2, 0, 4, 1, + 0, 1.5, 2.5, 1.25, 3.15, + 1, 0.5, 2, 3, 2.2 + }; + // clang-format on + + for (size_t i = 0; i < out.size(); ++i) + EXPECT_DOUBLE_EQ(out[i], expected[i]); + } +} + +// Test an observer below the raster +TEST(Viewshed, oor_below) +{ + // clang-format off + const int xlen = 5; + const int ylen = 3; + std::array in + { + 1, 2, 0, 4, 1, + 0, 0, 2, 1, 0, + 1, 0, 0, 3, 3 + }; + // clang-format on + + { + Viewshed::Options opts = stdOptions(2, 4); + opts.outputMode = Viewshed::OutputMode::DEM; + DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); + GDALRasterBand *band = ds->GetRasterBand(1); + std::array out; + CPLErr err = band->RasterIO(GF_Read, 0, 0, xlen, ylen, out.data(), xlen, + ylen, GDT_Float64, 0, 0, nullptr); + + EXPECT_EQ(err, CE_None); + + // clang-format off + std::array expected + { + 1 / 3.0, 2 / 3.0, 8 / 3.0, 11 / 3.0, 5, + 0.5, 0, 0, 3, 4.5, + 1, 0, 0, 3, 3 + }; + // clang-format on + + for (size_t i = 0; i < out.size(); ++i) + EXPECT_DOUBLE_EQ(out[i], expected[i]); + } + + { + Viewshed::Options opts = stdOptions(6, 4); + opts.outputMode = Viewshed::OutputMode::DEM; + DatasetPtr ds = runViewshed(in.data(), xlen, ylen, opts); + GDALRasterBand *band = ds->GetRasterBand(1); + std::array out; + CPLErr err = band->RasterIO(GF_Read, 0, 0, xlen, ylen, out.data(), xlen, + ylen, GDT_Float64, 0, 0, nullptr); + EXPECT_EQ(err, CE_None); + + // clang-format off + std::array expected + { + 4.2, 6, 6, 1.5, 1, + 1.35, 2.25, 4.5, 4.5, 0, + 1, 0, 0, 3, 3 + }; + // clang-format on + + for (size_t i = 0; i < out.size(); ++i) + EXPECT_DOUBLE_EQ(out[i], expected[i]); + } +} + } // namespace gdal diff --git a/autotest/utilities/test_gdal_viewshed.py b/autotest/utilities/test_gdal_viewshed.py index 42dadb733cf8..ca572b803e66 100755 --- a/autotest/utilities/test_gdal_viewshed.py +++ b/autotest/utilities/test_gdal_viewshed.py @@ -369,17 +369,6 @@ def test_gdal_viewshed_invalid_band(gdal_viewshed_path, tmp_path): ############################################################################### -def test_gdal_viewshed_invalid_observer_point(gdal_viewshed_path, tmp_path): - - _, err = gdaltest.runexternal_out_and_err( - f"{gdal_viewshed_path} -ox 0 -oy 0 ../gdrivers/data/n43.tif {tmp_path}/tmp.tif" - ) - assert "Observer position above or below" in err - - -############################################################################### - - def test_gdal_viewshed_invalid_output_driver(gdal_viewshed_path, tmp_path): _, err = gdaltest.runexternal_out_and_err( diff --git a/doc/source/programs/gdal_viewshed.rst b/doc/source/programs/gdal_viewshed.rst index 12d121f84a57..5ff9316dc74e 100644 --- a/doc/source/programs/gdal_viewshed.rst +++ b/doc/source/programs/gdal_viewshed.rst @@ -60,11 +60,15 @@ Byte. With the -mode flag can also return a minimum visible height raster of typ .. option:: -ox - The X position of the observer (in SRS units). + The X position of the observer (in SRS units). If the coordinate is outside of the + raster, all space between the observer and the raster is assumed not to occlude + visibility of the raster. .. option:: -oy - The Y position of the observer (in SRS units). + The Y position of the observer (in SRS units). If the coordinate is outside of the + raster, all space between the observer and the raster is assumed not to occlude + visibility of the raster. .. option:: -oz From b73e49e423954a248305c3e0a5921d94694bfc6b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 6 Jul 2024 10:23:37 +0200 Subject: [PATCH 53/72] =?UTF-8?q?OpenFileGDB:=20fix=20attribute=20filter?= =?UTF-8?q?=20when=20an=20equality=20comparison=20is=20involved=E2=80=A6?= =?UTF-8?q?=20(#10346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit with a field with an index created with a LOWER(field_name) expression Fixes #10345 --- autotest/ogr/ogr_openfilegdb.py | 21 ++++++ ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp | 75 +++++++++++++++---- ogr/ogrsf_frmts/openfilegdb/filegdbtable.h | 3 +- .../openfilegdb/ogropenfilegdblayer.cpp | 36 ++++++++- 4 files changed, 118 insertions(+), 17 deletions(-) diff --git a/autotest/ogr/ogr_openfilegdb.py b/autotest/ogr/ogr_openfilegdb.py index 3ea017d1f425..7d3f41f63591 100755 --- a/autotest/ogr/ogr_openfilegdb.py +++ b/autotest/ogr/ogr_openfilegdb.py @@ -810,6 +810,9 @@ def test_ogr_openfilegdb_4(): ("str IN ('aaa ')", [], IDX_USED), ("str IN ('aaaX')", [], IDX_USED), ("str IN ('aaaXX')", [], IDX_USED), + ("str ILIKE 'a'", [1], IDX_NOT_USED), + ("str ILIKE 'a%'", [1, 2, 3], IDX_NOT_USED), + ("str ILIKE 'aaa '", [], IDX_NOT_USED), ], ) def test_ogr_openfilegdb_str_indexed_truncated( @@ -832,6 +835,24 @@ def test_ogr_openfilegdb_str_indexed_truncated( assert [f.GetFID() for f in lyr] == fids, (where_clause, fids) +def test_ogr_openfilegdb_ilike(): + + ds = ogr.Open("data/filegdb/Domains.gdb/a00000001.gdbtable") + lyr = ds.GetLayer(0) + + lyr.SetAttributeFilter("Name = 'Roads'") + assert lyr.GetFeatureCount() == 1 + + lyr.SetAttributeFilter("Name ILIKE 'Roads'") + assert lyr.GetFeatureCount() == 1 + + lyr.SetAttributeFilter("Name = 'Roadsx'") + assert lyr.GetFeatureCount() == 0 + + lyr.SetAttributeFilter("Name ILIKE 'Roadsx'") + assert lyr.GetFeatureCount() == 0 + + ############################################################################### # Test opening an unzipped dataset diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp index d6fee21adf7c..1c3ddf445609 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp @@ -993,6 +993,8 @@ static const char *FileGDBSQLOpToStr(FileGDBSQLOp op) return ">="; case FGSO_GT: return ">"; + case FGSO_ILIKE: + return "ILIKE"; } return "unknown_op"; } @@ -1117,10 +1119,36 @@ int FileGDBIndexIterator::SetConstraint(int nFieldIdx, FileGDBSQLOp op, eFieldType != FGFT_TIME && eFieldType != FGFT_DATETIME_WITH_OFFSET); - const char *pszAtxName = CPLFormFilename( - CPLGetPath(poParent->GetFilename().c_str()), - CPLGetBasename(poParent->GetFilename().c_str()), - CPLSPrintf("%s.atx", poField->GetIndex()->GetIndexName().c_str())); + const auto poIndex = poField->GetIndex(); + + // Only supports ILIKE on a field string if the index expression starts + // with LOWER() and the string to compare with is only ASCII without + // wildcards + if (eOGRFieldType == OFTString && + STARTS_WITH_CI(poIndex->GetExpression().c_str(), "LOWER(")) + { + if (eOp == FGSO_ILIKE) + { + if (!CPLIsASCII(psValue->String, strlen(psValue->String)) || + strchr(psValue->String, '%') || strchr(psValue->String, '_')) + { + return FALSE; + } + } + else if (eOp != FGSO_ISNOTNULL) + { + return FALSE; + } + } + else if (eOp == FGSO_ILIKE) + { + return FALSE; + } + + const char *pszAtxName = + CPLFormFilename(CPLGetPath(poParent->GetFilename().c_str()), + CPLGetBasename(poParent->GetFilename().c_str()), + CPLSPrintf("%s.atx", poIndex->GetIndexName().c_str())); if (!ReadTrailer(pszAtxName)) return FALSE; @@ -1288,13 +1316,23 @@ int FileGDBIndexIterator::SetConstraint(int nFieldIdx, FileGDBSQLOp op, /************************************************************************/ static int FileGDBUTF16StrCompare(const GUInt16 *pasFirst, - const GUInt16 *pasSecond, int nStrLen) + const GUInt16 *pasSecond, int nStrLen, + bool bCaseInsensitive) { for (int i = 0; i < nStrLen; i++) { - if (pasFirst[i] < pasSecond[i]) + GUInt16 chA = pasFirst[i]; + GUInt16 chB = pasSecond[i]; + if (bCaseInsensitive) + { + if (chA >= 'a' && chA <= 'z') + chA -= 'a' - 'A'; + if (chB >= 'a' && chB <= 'z') + chB -= 'a' - 'A'; + } + if (chA < chB) return -1; - if (pasFirst[i] > pasSecond[i]) + if (chA > chB) return 1; } return 0; @@ -1439,12 +1477,17 @@ bool FileGDBIndexIterator::FindPages(int iLevel, uint64_t nPage) nStrLen * sizeof(GUInt16)); for (int j = 0; j < nStrLen; j++) CPL_LSBPTR16(&asMax[j]); + // Note: we have an inconsistency. OGR SQL equality operator + // is advertized to be case insensitive, but we have always + // implemented FGSO_EQ as case sensitive. #ifdef DEBUG_INDEX_CONSISTENCY - returnErrorIf(i > 0 && FileGDBUTF16StrCompare(pasMax, asLastMax, - nStrLen) < 0); + returnErrorIf(i > 0 && + FileGDBUTF16StrCompare(pasMax, asLastMax, nStrLen, + eOp == FGSO_ILIKE) < 0); memcpy(asLastMax, pasMax, nStrLen * 2); #endif - nComp = FileGDBUTF16StrCompare(asUTF16Str, pasMax, nStrLen); + nComp = FileGDBUTF16StrCompare(asUTF16Str, pasMax, nStrLen, + eOp == FGSO_ILIKE); break; } @@ -1494,6 +1537,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, uint64_t nPage) break; case FGSO_EQ: + case FGSO_ILIKE: if (iFirstPageIdx[iLevel] < 0) { if (nComp <= 0) @@ -1533,7 +1577,7 @@ bool FileGDBIndexIterator::FindPages(int iLevel, uint64_t nPage) } break; - default: + case FGSO_ISNOTNULL: CPLAssert(false); break; } @@ -1806,7 +1850,11 @@ int64_t FileGDBIndexIterator::GetNextRow() nStrLen * 2); for (int j = 0; j < nStrLen; j++) CPL_LSBPTR16(&asVal[j]); - nComp = FileGDBUTF16StrCompare(asUTF16Str, asVal, nStrLen); + // Note: we have an inconsistency. OGR SQL equality operator + // is advertized to be case insensitive, but we have always + // implemented FGSO_EQ as case sensitive. + nComp = FileGDBUTF16StrCompare(asUTF16Str, asVal, nStrLen, + eOp == FGSO_ILIKE); break; } @@ -1858,6 +1906,7 @@ int64_t FileGDBIndexIterator::GetNextRow() break; case FGSO_EQ: + case FGSO_ILIKE: if (nComp < 0 && bAscending) { bEOF = true; @@ -1874,7 +1923,7 @@ int64_t FileGDBIndexIterator::GetNextRow() bMatch = nComp < 0; break; - default: + case FGSO_ISNOTNULL: CPLAssert(false); break; } diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h index d6a184011ac7..6fa44cac9892 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.h @@ -794,7 +794,8 @@ typedef enum FGSO_LE, FGSO_EQ, FGSO_GE, - FGSO_GT + FGSO_GT, + FGSO_ILIKE } FileGDBSQLOp; /************************************************************************/ diff --git a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp index 1c33b099cc7e..28d2686d1c28 100644 --- a/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/ogropenfilegdblayer.cpp @@ -1332,7 +1332,8 @@ OGROpenFileGDBLayer::BuildIteratorFromExprNode(swq_expr_node *poNode) } else if (poNode->eNodeType == SNT_OPERATION && - OGROpenFileGDBIsComparisonOp(poNode->nOperation) && + (OGROpenFileGDBIsComparisonOp(poNode->nOperation) || + poNode->nOperation == SWQ_ILIKE) && poNode->nSubExprCount == 2) { swq_expr_node *poColumn = GetColumnSubNode(poNode); @@ -1376,6 +1377,9 @@ OGROpenFileGDBLayer::BuildIteratorFromExprNode(swq_expr_node *poNode) case SWQ_GT: eOp = FGSO_GT; break; + case SWQ_ILIKE: + eOp = FGSO_ILIKE; + break; default: CPLAssert(false); break; @@ -1405,6 +1409,9 @@ OGROpenFileGDBLayer::BuildIteratorFromExprNode(swq_expr_node *poNode) case SWQ_GT: eOp = FGSO_LT; break; + case SWQ_ILIKE: + eOp = FGSO_ILIKE; + break; default: CPLAssert(false); break; @@ -1417,10 +1424,31 @@ OGROpenFileGDBLayer::BuildIteratorFromExprNode(swq_expr_node *poNode) if (poField->GetType() == FGFT_STRING && poFieldDefn->GetType() == OFTString) { + // If we have an equality comparison, but the index + // uses LOWER(), transform it to a ILIKE comparison + if (eOp == FGSO_EQ && poField->HasIndex() && + STARTS_WITH_CI( + poField->GetIndex()->GetExpression().c_str(), + "LOWER(")) + { + // Note: FileGDBIndexIterator::SetConstraint() + // checks that the string to compare with has no + // wildcard + eOp = FGSO_ILIKE; + + // In theory, a ILIKE is not sufficient as it is + // case insensitive, whereas one could expect + // equality testing to be case sensitive... but + // it is not in OGR SQL... + // So we can comment the below line + // bIteratorSufficient = false; + } + // As the index use ' ' as padding value, we cannot // fully trust the index. - if ((eOp == FGSO_EQ && poNode->nOperation != SWQ_NE) || - eOp == FGSO_GE) + else if ((eOp == FGSO_EQ && + poNode->nOperation != SWQ_NE) || + eOp == FGSO_GE) bIteratorSufficient = false; else return nullptr; @@ -1453,6 +1481,8 @@ OGROpenFileGDBLayer::BuildIteratorFromExprNode(swq_expr_node *poNode) } } } + else if (eOp == FGSO_ILIKE) + return nullptr; FileGDBIterator *poIter = FileGDBIterator::Build( m_poLyrTable, nTableColIdx, TRUE, eOp, From 388d3ae7e47cba64e1050023bed43c4b04fe4277 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 29 Jun 2024 18:51:08 +0200 Subject: [PATCH 54/72] ogr2ogr: speed-up -clipsrc/-clipdst by avoiding GEOS when possible If the envelope of the feature's geometry is contained in the envelope of the -clipsrc/-clipdst geometry, we can just avoid any GEOS operation. --- apps/ogr2ogr_lib.cpp | 203 ++++++++++++++++++++++++++----------------- 1 file changed, 125 insertions(+), 78 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 0d24991f801d..e154fcf189ac 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -578,14 +578,19 @@ class LayerTranslator int m_nCoordDim = 0; GeomOperation m_eGeomOp = GEOMOP_NONE; double m_dfGeomOpParam = 0; + OGRGeometry *m_poClipSrcOri = nullptr; bool m_bWarnedClipSrcSRS = false; std::unique_ptr m_poClipSrcReprojectedToSrcSRS; const OGRSpatialReference *m_poClipSrcReprojectedToSrcSRS_SRS = nullptr; + OGREnvelope m_oClipSrcEnv{}; + OGRGeometry *m_poClipDstOri = nullptr; bool m_bWarnedClipDstSRS = false; std::unique_ptr m_poClipDstReprojectedToDstSRS; const OGRSpatialReference *m_poClipDstReprojectedToDstSRS_SRS = nullptr; + OGREnvelope m_oClipDstEnv{}; + bool m_bExplodeCollections = false; bool m_bNativeData = false; GIntBig m_nLimit = -1; @@ -598,8 +603,10 @@ class LayerTranslator const GDALVectorTranslateOptions *psOptions); private: - const OGRGeometry *GetDstClipGeom(const OGRSpatialReference *poGeomSRS); - const OGRGeometry *GetSrcClipGeom(const OGRSpatialReference *poGeomSRS); + std::pair + GetDstClipGeom(const OGRSpatialReference *poGeomSRS); + std::pair + GetSrcClipGeom(const OGRSpatialReference *poGeomSRS); }; static OGRLayer *GetLayerAndOverwriteIfNecessary(GDALDataset *poDstDS, @@ -6002,13 +6009,22 @@ bool LayerTranslator::Translate( if (nDstGeomFieldCount == 0 && poStolenGeometry && m_poClipSrcOri) { - const OGRGeometry *poClipGeom = + if (poStolenGeometry->IsEmpty()) + goto end_loop; + + const auto [poClipGeom, poClipGeomEnvelope] = GetSrcClipGeom(poStolenGeometry->getSpatialReference()); - if (poClipGeom != nullptr && - !poClipGeom->Intersects(poStolenGeometry.get())) + if (poClipGeom && poClipGeomEnvelope) { - goto end_loop; + OGREnvelope oEnv; + poStolenGeometry->getEnvelope(&oEnv); + if (!poClipGeomEnvelope->Contains(oEnv) && + !(poClipGeomEnvelope->Intersects(oEnv) && + poClipGeom->Intersects(poStolenGeometry.get()))) + { + goto end_loop; + } } } @@ -6207,49 +6223,51 @@ bool LayerTranslator::Translate( if (m_poClipSrcOri) { + if (poDstGeometry->IsEmpty()) + goto end_loop; - const OGRGeometry *poClipGeom = + const auto [poClipGeom, poClipGeomEnvelope] = GetSrcClipGeom(poDstGeometry->getSpatialReference()); - std::unique_ptr poClipped; - if (poClipGeom != nullptr) - { - OGREnvelope oClipEnv; - OGREnvelope oDstEnv; + if (!(poClipGeom && poClipGeomEnvelope)) + goto end_loop; - poClipGeom->getEnvelope(&oClipEnv); - poDstGeometry->getEnvelope(&oDstEnv); + OGREnvelope oDstEnv; + poDstGeometry->getEnvelope(&oDstEnv); - if (oClipEnv.Intersects(oDstEnv)) + if (!poClipGeomEnvelope->Contains(oDstEnv)) + { + std::unique_ptr poClipped; + if (poClipGeomEnvelope->Intersects(oDstEnv)) { poClipped.reset( poClipGeom->Intersection(poDstGeometry.get())); } - } + if (poClipped == nullptr || poClipped->IsEmpty()) + { + goto end_loop; + } - if (poClipped == nullptr || poClipped->IsEmpty()) - { - goto end_loop; - } + const int nDim = poDstGeometry->getDimension(); + if (poClipped->getDimension() < nDim && + wkbFlatten(poDstFDefn->GetGeomFieldDefn(iGeom) + ->GetType()) != wkbUnknown) + { + CPLDebug( + "OGR2OGR", + "Discarding feature " CPL_FRMT_GIB + " of layer %s, " + "as its intersection with -clipsrc is a %s " + "whereas the input is a %s", + nSrcFID, poSrcLayer->GetName(), + OGRToOGCGeomType(poClipped->getGeometryType()), + OGRToOGCGeomType( + poDstGeometry->getGeometryType())); + goto end_loop; + } - const int nDim = poDstGeometry->getDimension(); - if (poClipped->getDimension() < nDim && - wkbFlatten( - poDstFDefn->GetGeomFieldDefn(iGeom)->GetType()) != - wkbUnknown) - { - CPLDebug( - "OGR2OGR", - "Discarding feature " CPL_FRMT_GIB " of layer %s, " - "as its intersection with -clipsrc is a %s " - "whereas the input is a %s", - nSrcFID, poSrcLayer->GetName(), - OGRToOGCGeomType(poClipped->getGeometryType()), - OGRToOGCGeomType(poDstGeometry->getGeometryType())); - goto end_loop; + poDstGeometry = std::move(poClipped); } - - poDstGeometry = std::move(poClipped); } OGRCoordinateTransformation *const poCT = @@ -6387,51 +6405,54 @@ bool LayerTranslator::Translate( { if (m_poClipDstOri) { - const OGRGeometry *poClipGeom = GetDstClipGeom( + if (poDstGeometry->IsEmpty()) + goto end_loop; + + auto [poClipGeom, poClipGeomEnvelope] = GetDstClipGeom( poDstGeometry->getSpatialReference()); - if (poClipGeom == nullptr) + if (!poClipGeom || !poClipGeomEnvelope) { goto end_loop; } - std::unique_ptr poClipped; - - OGREnvelope oClipEnv; OGREnvelope oDstEnv; - - poClipGeom->getEnvelope(&oClipEnv); poDstGeometry->getEnvelope(&oDstEnv); - if (oClipEnv.Intersects(oDstEnv)) + if (!poClipGeomEnvelope->Contains(oDstEnv)) { - poClipped.reset( - poClipGeom->Intersection(poDstGeometry.get())); - } + std::unique_ptr poClipped; + if (poClipGeomEnvelope->Intersects(oDstEnv)) + { + poClipped.reset(poClipGeom->Intersection( + poDstGeometry.get())); + } - if (poClipped == nullptr || poClipped->IsEmpty()) - { - goto end_loop; - } + if (poClipped == nullptr || poClipped->IsEmpty()) + { + goto end_loop; + } - const int nDim = poDstGeometry->getDimension(); - if (poClipped->getDimension() < nDim && - wkbFlatten(poDstFDefn->GetGeomFieldDefn(iGeom) - ->GetType()) != wkbUnknown) - { - CPLDebug( - "OGR2OGR", - "Discarding feature " CPL_FRMT_GIB - " of layer %s, " - "as its intersection with -clipdst is a %s " - "whereas the input is a %s", - nSrcFID, poSrcLayer->GetName(), - OGRToOGCGeomType(poClipped->getGeometryType()), - OGRToOGCGeomType( - poDstGeometry->getGeometryType())); - goto end_loop; - } + const int nDim = poDstGeometry->getDimension(); + if (poClipped->getDimension() < nDim && + wkbFlatten(poDstFDefn->GetGeomFieldDefn(iGeom) + ->GetType()) != wkbUnknown) + { + CPLDebug( + "OGR2OGR", + "Discarding feature " CPL_FRMT_GIB + " of layer %s, " + "as its intersection with -clipdst is a %s " + "whereas the input is a %s", + nSrcFID, poSrcLayer->GetName(), + OGRToOGCGeomType( + poClipped->getGeometryType()), + OGRToOGCGeomType( + poDstGeometry->getGeometryType())); + goto end_loop; + } - poDstGeometry = std::move(poClipped); + poDstGeometry = std::move(poClipped); + } } if (psOptions->dfXYRes != @@ -6603,7 +6624,13 @@ bool LayerTranslator::Translate( /* LayerTranslator::GetDstClipGeom() */ /************************************************************************/ -const OGRGeometry * +/** Returns the destination clip geometry and its envelope + * + * @param poGeomSRS The SRS into which the destination clip geometry should be + * expressed. + * @return the destination clip geometry and its envelope, or (nullptr, nullptr) + */ +std::pair LayerTranslator::GetDstClipGeom(const OGRSpatialReference *poGeomSRS) { if (m_poClipDstReprojectedToDstSRS_SRS != poGeomSRS) @@ -6616,7 +6643,7 @@ LayerTranslator::GetDstClipGeom(const OGRSpatialReference *poGeomSRS) if (m_poClipDstReprojectedToDstSRS->transformTo(poGeomSRS) != OGRERR_NONE) { - return nullptr; + return std::make_pair(nullptr, nullptr); } m_poClipDstReprojectedToDstSRS_SRS = poGeomSRS; } @@ -6633,17 +6660,30 @@ LayerTranslator::GetDstClipGeom(const OGRSpatialReference *poGeomSRS) "same as the feature's geometry"); } } + m_oClipDstEnv = OGREnvelope(); } - return m_poClipDstReprojectedToDstSRS ? m_poClipDstReprojectedToDstSRS.get() - : m_poClipDstOri; + const auto poGeom = m_poClipDstReprojectedToDstSRS + ? m_poClipDstReprojectedToDstSRS.get() + : m_poClipDstOri; + if (poGeom && !m_oClipDstEnv.IsInit()) + { + poGeom->getEnvelope(&m_oClipDstEnv); + } + return std::make_pair(poGeom, poGeom ? &m_oClipDstEnv : nullptr); } /************************************************************************/ /* LayerTranslator::GetSrcClipGeom() */ /************************************************************************/ -const OGRGeometry * +/** Returns the source clip geometry and its envelope + * + * @param poGeomSRS The SRS into which the source clip geometry should be + * expressed. + * @return the source clip geometry and its envelope, or (nullptr, nullptr) + */ +std::pair LayerTranslator::GetSrcClipGeom(const OGRSpatialReference *poGeomSRS) { if (m_poClipSrcReprojectedToSrcSRS_SRS != poGeomSRS) @@ -6656,7 +6696,7 @@ LayerTranslator::GetSrcClipGeom(const OGRSpatialReference *poGeomSRS) if (m_poClipSrcReprojectedToSrcSRS->transformTo(poGeomSRS) != OGRERR_NONE) { - return nullptr; + return std::make_pair(nullptr, nullptr); } m_poClipSrcReprojectedToSrcSRS_SRS = poGeomSRS; } @@ -6672,10 +6712,17 @@ LayerTranslator::GetSrcClipGeom(const OGRSpatialReference *poGeomSRS) "same as the feature's geometry"); } } + m_oClipSrcEnv = OGREnvelope(); } - return m_poClipSrcReprojectedToSrcSRS ? m_poClipSrcReprojectedToSrcSRS.get() - : m_poClipSrcOri; + const auto poGeom = m_poClipSrcReprojectedToSrcSRS + ? m_poClipSrcReprojectedToSrcSRS.get() + : m_poClipSrcOri; + if (poGeom && !m_oClipSrcEnv.IsInit()) + { + poGeom->getEnvelope(&m_oClipSrcEnv); + } + return std::make_pair(poGeom, poGeom ? &m_oClipSrcEnv : nullptr); } /************************************************************************/ From a3dffa272026778061a7369d1b9dd3792254e457 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 29 Jun 2024 20:06:45 +0200 Subject: [PATCH 55/72] ogr2ogr: add -skipinvalid to skip features whose geometry is not valid w.r.t Simple Features --- apps/ogr2ogr_lib.cpp | 25 ++++++++++++++++++++++++- autotest/utilities/test_ogr2ogr_lib.py | 24 ++++++++++++++++++++++++ doc/source/programs/ogr2ogr.rst | 14 ++++++++++++-- swig/include/python/gdal_python.i | 6 ++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index e154fcf189ac..601f243c7352 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -270,6 +270,9 @@ struct GDALVectorTranslateOptions /*! Whether to run MakeValid */ bool bMakeValid = false; + /*! Whether to run OGRGeometry::IsValid */ + bool bSkipInvalidGeom = false; + /*! list of field types to convert to a field of type string in the destination layer. Valid types are: Integer, Integer64, Real, String, Date, Time, DateTime, Binary, IntegerList, Integer64List, RealList, @@ -575,6 +578,7 @@ class LayerTranslator int m_eGType = -1; GeomTypeConversion m_eGeomTypeConversion = GTC_DEFAULT; bool m_bMakeValid = false; + bool m_bSkipInvalidGeom = false; int m_nCoordDim = 0; GeomOperation m_eGeomOp = GEOMOP_NONE; double m_dfGeomOpParam = 0; @@ -2887,6 +2891,7 @@ GDALDatasetH GDALVectorTranslate(const char *pszDest, GDALDatasetH hDstDS, oTranslator.m_eGType = psOptions->eGType; oTranslator.m_eGeomTypeConversion = psOptions->eGeomTypeConversion; oTranslator.m_bMakeValid = psOptions->bMakeValid; + oTranslator.m_bSkipInvalidGeom = psOptions->bSkipInvalidGeom; oTranslator.m_nCoordDim = psOptions->nCoordDim; oTranslator.m_eGeomOp = psOptions->eGeomOp; oTranslator.m_dfGeomOpParam = psOptions->dfGeomOpParam; @@ -3923,7 +3928,7 @@ bool SetupTargetLayer::CanUseWriteArrowBatch( m_bExactFieldNameMatch && !m_bForceNullable && !m_bResolveDomains && !m_bUnsetDefault && psOptions->nFIDToFetch == OGRNullFID && psOptions->dfXYRes == OGRGeomCoordinatePrecision::UNKNOWN && - !psOptions->bMakeValid) + !psOptions->bMakeValid && !psOptions->bSkipInvalidGeom) { struct ArrowArrayStream streamSrc; const char *const apszOptions[] = {"SILENCE_GET_SCHEMA_ERROR=YES", @@ -6501,6 +6506,9 @@ bool LayerTranslator::Translate( } } + if (m_bSkipInvalidGeom && !poDstGeometry->IsValid()) + goto end_loop; + if (m_eGeomTypeConversion != GTC_DEFAULT) { OGRwkbGeometryType eTargetType = @@ -7293,6 +7301,21 @@ static std::unique_ptr GDALVectorTranslateOptionsGetParser( .help(_("Fix geometries to be valid regarding the rules of the Simple " "Features specification.")); + argParser->add_argument("-skipinvalid") + .flag() + .action( + [psOptions](const std::string &) + { + if (!OGRGeometryFactory::haveGEOS()) + { + throw std::invalid_argument( + "-skipinvalid only supported for builds against GEOS"); + } + psOptions->bSkipInvalidGeom = true; + }) + .help(_("Whether to skip features with invalid geometries regarding the" + "rules of the Simple Features specification.")); + argParser->add_argument("-wrapdateline") .store_into(psOptions->bWrapDateline) .help(_("Split geometries crossing the dateline meridian.")); diff --git a/autotest/utilities/test_ogr2ogr_lib.py b/autotest/utilities/test_ogr2ogr_lib.py index b94847000ba0..d704aeb1c87d 100755 --- a/autotest/utilities/test_ogr2ogr_lib.py +++ b/autotest/utilities/test_ogr2ogr_lib.py @@ -2806,3 +2806,27 @@ def test_ogr2ogr_lib_two_gcps(): f = out_lyr.GetNextFeature() assert f.GetGeometryRef().GetX(0) == pytest.approx(250) assert f.GetGeometryRef().GetY(0) == pytest.approx(350) + + +############################################################################### +# Test -skipInvalid + + +@pytest.mark.require_geos +@gdaltest.enable_exceptions() +def test_ogr2ogr_lib_skip_invalid(tmp_vsimem): + + srcDS = gdal.GetDriverByName("Memory").Create("", 0, 0, 0, gdal.GDT_Unknown) + srcLayer = srcDS.CreateLayer("test") + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT(1 2)")) + srcLayer.CreateFeature(f) + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POLYGON((0 0,1 1,0 1,1 0,0 0))")) + srcLayer.CreateFeature(f) + + with gdal.quiet_errors(): + ds = gdal.VectorTranslate("", srcDS, format="Memory", skipInvalid=True) + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + ds = None diff --git a/doc/source/programs/ogr2ogr.rst b/doc/source/programs/ogr2ogr.rst index 25599491d77c..357811628ca1 100644 --- a/doc/source/programs/ogr2ogr.rst +++ b/doc/source/programs/ogr2ogr.rst @@ -43,7 +43,8 @@ Synopsis [-t_coord_epoch ] [-ct ] [-spat_srs ] [-geomfield ] [-segmentize ] [-simplify ] - [-makevalid] [-wrapdateline] [-datelineoffset ] + [-makevalid] [-skipinvalid] + [-wrapdateline] [-datelineoffset ] [-clipsrc [ ]|||spat_extent] [-clipsrcsql ] [-clipsrclayer ] [-clipsrcwhere ] @@ -474,7 +475,16 @@ output coordinate system or even reprojecting the features during translation. :cpp:func:`OGRGeometryFactory::removeLowerDimensionSubGeoms`, on geometries to ensure they are valid regarding the rules of the Simple Features specification. - .. versionadded: 3.1 (requires GEOS 3.8 or later) + .. versionadded: 3.1 (requires GEOS) + +.. option:: -skipinvalid + + Run the :cpp:func:`OGRGeometry::IsValid` operation on geometries to check if + they are valid regarding the rules of the Simple Features specification. + If they are not, the feature is skipped. This check is done after all other + geometry operations. + + .. versionadded: 3.10 (requires GEOS) .. option:: -fieldTypeToString All|[,]... diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index 017499fd9917..44561749de7a 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -3051,6 +3051,7 @@ def VectorTranslateOptions(options=None, format=None, simplifyTolerance=None, segmentizeMaxDist=None, makeValid=False, + skipInvalid=False, mapFieldType=None, explodeCollections=False, zField=None, @@ -3151,6 +3152,9 @@ def VectorTranslateOptions(options=None, format=None, maximum distance between consecutive nodes of a line geometry makeValid: run MakeValid() on geometries + skipInvalid: + whether to skip features with invalid geometries regarding the rules of + the Simple Features specification. mapFieldType: converts any field of the specified type to another type. Valid types are: Integer, Integer64, Real, String, Date, Time, DateTime, Binary, IntegerList, @@ -3334,6 +3338,8 @@ def VectorTranslateOptions(options=None, format=None, new_options += ['-segmentize', str(segmentizeMaxDist)] if makeValid: new_options += ['-makevalid'] + if skipInvalid: + new_options += ['-skipinvalid'] if mapFieldType is not None: new_options += ['-mapFieldType'] if isinstance(mapFieldType, str): From b15e2567f33c0b1d0f5e292b8db71ef4cc674ce7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 25 Jun 2024 02:17:36 +0200 Subject: [PATCH 56/72] OSM: add a [general] section at top of osmconf.ini to make it INI compliant (and Python's configparser friendly) Refs https://github.com/geopandas/pyogrio/issues/419#issuecomment-2187640864 --- autotest/ogr/ogr_osm.py | 18 ++++++++++++++++++ ogr/ogrsf_frmts/osm/data/osmconf.ini | 2 ++ ogr/ogrsf_frmts/osm/ogr_osm.h | 2 ++ ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp | 17 +++++++++++++++++ 4 files changed, 39 insertions(+) diff --git a/autotest/ogr/ogr_osm.py b/autotest/ogr/ogr_osm.py index f50d66b04a68..09666b00edbd 100755 --- a/autotest/ogr/ogr_osm.py +++ b/autotest/ogr/ogr_osm.py @@ -929,3 +929,21 @@ def test_ogr_osm_tags_json_special_characters(): assert lyr_defn.GetFieldDefn(other_tags_idx).GetSubType() == ogr.OFSTJSON f = lyr.GetNextFeature() assert f["other_tags"] == """{"foo":"x'\\\\\\"\\t\\n\\ry"}""" + + +############################################################################### +# Test that osmconf.ini can be parsed with Python's configparser + + +def test_ogr_osmconf_ini(): + + import configparser + + with ogr.Open("data/osm/test_json.pbf") as ds: + with ds.ExecuteSQL("SHOW config_file_path") as sql_lyr: + f = sql_lyr.GetNextFeature() + osmconf_ini_filename = f.GetField(0) + config = configparser.ConfigParser() + config.read_file(open(osmconf_ini_filename)) + assert "general" in config + assert "closed_ways_are_polygons" in config["general"] diff --git a/ogr/ogrsf_frmts/osm/data/osmconf.ini b/ogr/ogrsf_frmts/osm/data/osmconf.ini index b9b85fd80236..d44029f30def 100644 --- a/ogr/ogrsf_frmts/osm/data/osmconf.ini +++ b/ogr/ogrsf_frmts/osm/data/osmconf.ini @@ -1,6 +1,8 @@ # # Configuration file for importing OSM data into OGR # +# NOTE: remove the below "[general]" line for GDAL < 3.10 +[general] # put here the name of keys, or key=value, for ways that are assumed to be polygons if they are closed # see http://wiki.openstreetmap.org/wiki/Map_Features diff --git a/ogr/ogrsf_frmts/osm/ogr_osm.h b/ogr/ogrsf_frmts/osm/ogr_osm.h index c4e29cafda93..5407175d58a5 100644 --- a/ogr/ogrsf_frmts/osm/ogr_osm.h +++ b/ogr/ogrsf_frmts/osm/ogr_osm.h @@ -394,6 +394,8 @@ class OGROSMDataSource final : public OGRDataSource std::vector> m_apoLayers{}; char *m_pszName = nullptr; + std::string m_osConfigFile{}; + OGREnvelope m_sExtent{}; bool m_bExtentValid = false; diff --git a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp index 0ec6621f1a13..a00d31dab42d 100644 --- a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp +++ b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp @@ -3400,6 +3400,8 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) return false; } + m_osConfigFile = pszFilename; + VSILFILE *fpConf = VSIFOpenL(pszFilename, "rb"); if (fpConf == nullptr) return false; @@ -3422,6 +3424,12 @@ bool OGROSMDataSource::ParseConf(char **papszOpenOptionsIn) pszLine++; const_cast(pszLine)[strlen(pszLine) - 1] = '\0'; /* Evil but OK */ + + if (strcmp(pszLine, "general") == 0) + { + continue; + } + int i = 0; for (auto &&poLayer : m_apoLayers) { @@ -4385,6 +4393,15 @@ OGRLayer *OGROSMDataSource::ExecuteSQL(const char *pszSQLCommand, return new OGROSMSingleFeatureLayer("GetBytesRead", szVal); } + /* -------------------------------------------------------------------- */ + /* Special SHOW config_file_path command */ + /* -------------------------------------------------------------------- */ + if (strcmp(pszSQLCommand, "SHOW config_file_path") == 0) + { + return new OGROSMSingleFeatureLayer("config_file_path", + m_osConfigFile.c_str()); + } + if (m_poResultSetLayer != nullptr) { CPLError(CE_Failure, CPLE_NotSupported, From ab6f3d3d8ab57194fd26042b4f47fbb4cbb0d82b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 6 Jul 2024 11:33:11 +0200 Subject: [PATCH 57/72] gdal2tiles: fix exception with --nodata-values-pct-threshold but not --excluded-values on a multi-band raster Co-authored-by: James Scott-Brown / @jamesscottbrown --- autotest/pyscripts/test_gdal2tiles.py | 8 ++++---- .../python/gdal-utils/osgeo_utils/gdal2tiles.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/autotest/pyscripts/test_gdal2tiles.py b/autotest/pyscripts/test_gdal2tiles.py index a89e2ef4a952..b3da97a4de72 100755 --- a/autotest/pyscripts/test_gdal2tiles.py +++ b/autotest/pyscripts/test_gdal2tiles.py @@ -643,7 +643,7 @@ def test_gdal2tiles_nodata_values_pct_threshold(script_path, tmp_path): input_tif = str(tmp_path / "test_gdal2tiles_nodata_values_pct_threshold.tif") output_folder = str(tmp_path / "test_gdal2tiles_nodata_values_pct_threshold") - src_ds = gdal.GetDriverByName("GTiff").Create(input_tif, 256, 256, 1, gdal.GDT_Byte) + src_ds = gdal.GetDriverByName("GTiff").Create(input_tif, 256, 256, 3, gdal.GDT_Byte) src_ds.GetRasterBand(1).SetNoDataValue(20) src_ds.GetRasterBand(1).WriteRaster( 0, 0, 2, 2, struct.pack("B" * 4, 10, 20, 30, 40) @@ -665,7 +665,7 @@ def test_gdal2tiles_nodata_values_pct_threshold(script_path, tmp_path): ) ds = gdal.Open(f"{output_folder}/0/0/0.png") - assert struct.unpack("B" * 2, ds.ReadRaster(0, 0, 1, 1)) == ( + assert struct.unpack("B" * 2, ds.ReadRaster(0, 0, 1, 1, band_list=[1, 4])) == ( round((10 + 30 + 40) / 3), 255, ) @@ -677,7 +677,7 @@ def test_gdal2tiles_nodata_values_pct_threshold(script_path, tmp_path): ) ds = gdal.Open(f"{output_folder}/0/0/0.png") - assert struct.unpack("B" * 2, ds.ReadRaster(0, 0, 1, 1)) == ( + assert struct.unpack("B" * 2, ds.ReadRaster(0, 0, 1, 1, band_list=[1, 4])) == ( round((10 + 30 + 40) / 3), 255, ) @@ -689,7 +689,7 @@ def test_gdal2tiles_nodata_values_pct_threshold(script_path, tmp_path): ) ds = gdal.Open(f"{output_folder}/0/0/0.png") - assert struct.unpack("B" * 2, ds.ReadRaster(0, 0, 1, 1)) == (0, 0) + assert struct.unpack("B" * 2, ds.ReadRaster(0, 0, 1, 1, band_list=[1, 4])) == (0, 0) @pytest.mark.require_driver("JPEG") diff --git a/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py b/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py index 2b3cb201733c..1ed7c69a0469 100644 --- a/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py +++ b/swig/python/gdal-utils/osgeo_utils/gdal2tiles.py @@ -895,13 +895,22 @@ def scale_query_to_tile(dsquery, dstile, options, tilefilename=""): options.excluded_values or options.nodata_values_pct_threshold < 100 ): + warp_options = "-r average" + + assert options.nodata_values_pct_threshold is not None + warp_options += ( + f" -wo NODATA_VALUES_PCT_THRESHOLD={options.nodata_values_pct_threshold}" + ) + + if options.excluded_values: + assert options.excluded_values_pct_threshold is not None + warp_options += f" -wo EXCLUDED_VALUES={options.excluded_values}" + warp_options += f" -wo EXCLUDED_VALUES_PCT_THRESHOLD={options.excluded_values_pct_threshold}" + gdal.Warp( dstile, dsquery, - options="-r average " - + f"-wo EXCLUDED_VALUES={options.excluded_values} " - + f"-wo EXCLUDED_VALUES_PCT_THRESHOLD={options.excluded_values_pct_threshold} " - + f"-wo NODATA_VALUES_PCT_THRESHOLD={options.nodata_values_pct_threshold}", + options=warp_options, ) elif options.resampling == "average": From 5a8484677c66339dc440a12a6ab78b26f54c07f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauren=C8=9Biu=20Nicola?= Date: Thu, 4 Jul 2024 17:35:51 +0300 Subject: [PATCH 58/72] cpl_http: Add support for GDAL_HTTP_VERSION=2PRIOR_KNOWLEDGE --- doc/source/user/configoptions.rst | 5 +++-- port/cpl_http.cpp | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst index 69a0f90b6a39..aa2bcaa0209c 100644 --- a/doc/source/user/configoptions.rst +++ b/doc/source/user/configoptions.rst @@ -815,14 +815,15 @@ Networking options - .. config:: GDAL_HTTP_VERSION :since: 2.3 - :choices: 1.0, 1.1, 2, 2TLS + :choices: 1.0, 1.1, 2, 2TLS, 2PRIOR_KNOWLEDGE Specifies which HTTP version to use. Will default to 1.1 generally (except on some controlled environments, like Google Compute Engine VMs, where 2TLS will be the default). Support for HTTP/2 requires curl 7.33 or later, built against nghttp2. "2TLS" means that HTTP/2 will be attempted for HTTPS connections only. Whereas "2" means that HTTP/2 will be attempted for HTTP or - HTTPS. The interest of enabling HTTP/2 is the use of HTTP/2 multiplexing when + HTTPS. "2PRIOR_KNOWLEDGE" means that the server will be assumed to support + HTTP/2. The interest of enabling HTTP/2 is the use of HTTP/2 multiplexing when reading GeoTIFFs stored on /vsicurl/ and related virtual file systems. - .. config:: GDAL_HTTP_MULTIPLEX diff --git a/port/cpl_http.cpp b/port/cpl_http.cpp index 4f3991b1ef6f..e64adea82bfe 100644 --- a/port/cpl_http.cpp +++ b/port/cpl_http.cpp @@ -1150,12 +1150,15 @@ int CPLHTTPPopFetchCallback(void) * and use the first one found as the CAINFO value (GDAL >= 2.1.3). The * GDAL_CURL_CA_BUNDLE environment variable may also be used to set the * CAINFO value in GDAL >= 3.2. - *
  • HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3). Specify HTTP version to use. + *
  • HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3)/2PRIOR_KNOWLEDGE (GDAL >= 3.10). + * Specify HTTP version to use. * Will default to 1.1 generally (except on some controlled environments, * like Google Compute Engine VMs, where 2TLS will be the default). * Support for HTTP/2 requires curl 7.33 or later, built against nghttp2. * "2TLS" means that HTTP/2 will be attempted for HTTPS connections only. * Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS. + * "2PRIOR_KNOWLEDGE" means that the server will be assumed to support + * HTTP/2. * Corresponding configuration option: GDAL_HTTP_VERSION. *
    • *
  • SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines @@ -2136,6 +2139,17 @@ void *CPLHTTPSetOptions(void *pcurl, const char *pszURL, CURL_HTTP_VERSION_2_0); } } + else if (pszHttpVersion && strcmp(pszHttpVersion, "2PRIOR_KNOWLEDGE") == 0) + { + if (bSupportHTTP2) + { + // Assume HTTP/2 is supported by the server. The cURL docs indicate + // that it makes no difference for HTTPS, but it does seem to work + // in practice. + unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION, + CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE); + } + } else if (pszHttpVersion == nullptr || strcmp(pszHttpVersion, "2TLS") == 0) { if (bSupportHTTP2) From 954c21f5b750267ac7c0b597b74d977783e15057 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 6 Jul 2024 12:56:41 +0200 Subject: [PATCH 59/72] typo fix [ci skip] --- autotest/utilities/test_gdal_translate_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autotest/utilities/test_gdal_translate_lib.py b/autotest/utilities/test_gdal_translate_lib.py index 180b3c4e9d41..d3e32650feb0 100755 --- a/autotest/utilities/test_gdal_translate_lib.py +++ b/autotest/utilities/test_gdal_translate_lib.py @@ -1155,7 +1155,7 @@ def test_gdal_translate_lib_scale_and_unscale_incompatible(): ############################################################################### -# Test -a_offset -inf (dummy example, but to proove -inf works as a value +# Test -a_offset -inf (dummy example, but to prove -inf works as a value # numeric value) From 912ee6c9beabf7090355fec5245aad050fce3c36 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 6 Jul 2024 12:55:17 +0200 Subject: [PATCH 60/72] WKT geometry importer: accept (non conformant) PointZ/PointZM without space as generated by current QGIS versions Cf thread at https://lists.osgeo.org/pipermail/qgis-developer/2024-July/066906.html --- autotest/ogr/ogr_wkbwkt_geom.py | 33 +++++++++++ ogr/ogrgeometry.cpp | 99 +++++++++++++++++---------------- 2 files changed, 84 insertions(+), 48 deletions(-) diff --git a/autotest/ogr/ogr_wkbwkt_geom.py b/autotest/ogr/ogr_wkbwkt_geom.py index 6930da09fcfc..26fab54f5f97 100755 --- a/autotest/ogr/ogr_wkbwkt_geom.py +++ b/autotest/ogr/ogr_wkbwkt_geom.py @@ -166,6 +166,7 @@ def test_ogr_wkbwkt_test_broken_geom(): "POINT Z(EMPTY)", "POINT Z(A)", "POINT Z(0 1", + "POINTZ M EMPTY", "LINESTRING", "LINESTRING UNKNOWN", "LINESTRING(", @@ -568,6 +569,38 @@ def test_ogr_wkbwkt_test_import_wkt_sf12(): ) +############################################################################### +# Test importing non-conformant WKT with Z/M modifier directly appended to +# geometry type name + + +@pytest.mark.parametrize( + "input_wkt,expected_output_wkt", + [ + ("POINTZ EMPTY", "POINT Z EMPTY"), + ("POINTM EMPTY", "POINT M EMPTY"), + ("POINTZM EMPTY", "POINT ZM EMPTY"), + ("POINTZ (0 1 2)", "POINT Z (0 1 2)"), + ("POINTM (0 1 2)", "POINT M (0 1 2)"), + ("POINTZM (0 1 2 3)", "POINT ZM (0 1 2 3)"), + ], +) +def test_ogr_wkbwkt_test_import_wkt_z_m_modifier_without_space( + input_wkt, expected_output_wkt +): + + geom = ogr.CreateGeometryFromWkt(input_wkt) + assert geom is not None + out_wkt = geom.ExportToIsoWkt() + assert out_wkt == expected_output_wkt + + # Test with input in lower case + geom = ogr.CreateGeometryFromWkt(input_wkt.lower()) + assert geom is not None + out_wkt = geom.ExportToIsoWkt() + assert out_wkt == expected_output_wkt + + ############################################################################### # Test that importing the wkb that would be equivalent to MULTIPOINT(POLYGON((0 0)) # doesn't work diff --git a/ogr/ogrgeometry.cpp b/ogr/ogrgeometry.cpp index 770a64b4e2f2..07d2275b4cba 100644 --- a/ogr/ogrgeometry.cpp +++ b/ogr/ogrgeometry.cpp @@ -1793,19 +1793,33 @@ OGRErr OGRGeometry::importPreambleFromWkt(const char **ppszInput, int *pbHasZ, /* -------------------------------------------------------------------- */ bool bHasM = false; bool bHasZ = false; - bool bIsoWKT = true; + bool bAlreadyGotDimension = false; char szToken[OGR_WKT_TOKEN_MAX] = {}; pszInput = OGRWktReadToken(pszInput, szToken); if (szToken[0] != '\0') { // Postgis EWKT: POINTM instead of POINT M. + // Current QGIS versions (at least <= 3.38) also export POINTZ. const size_t nTokenLen = strlen(szToken); - if (szToken[nTokenLen - 1] == 'M') + if (szToken[nTokenLen - 1] == 'M' || szToken[nTokenLen - 1] == 'm') { szToken[nTokenLen - 1] = '\0'; bHasM = true; - bIsoWKT = false; + bAlreadyGotDimension = true; + + if (nTokenLen > 2 && (szToken[nTokenLen - 2] == 'Z' || + szToken[nTokenLen - 2] == 'z')) + { + bHasZ = true; + szToken[nTokenLen - 2] = '\0'; + } + } + else if (szToken[nTokenLen - 1] == 'Z' || szToken[nTokenLen - 1] == 'z') + { + szToken[nTokenLen - 1] = '\0'; + bHasZ = true; + bAlreadyGotDimension = true; } } @@ -1813,55 +1827,44 @@ OGRErr OGRGeometry::importPreambleFromWkt(const char **ppszInput, int *pbHasZ, return OGRERR_CORRUPT_DATA; /* -------------------------------------------------------------------- */ - /* Check for EMPTY ... */ + /* Check for Z, M or ZM */ /* -------------------------------------------------------------------- */ - const char *pszPreScan = OGRWktReadToken(pszInput, szToken); - if (!bIsoWKT) - { - // Go on. - } - else if (EQUAL(szToken, "EMPTY")) - { - *ppszInput = const_cast(pszPreScan); - *pbIsEmpty = true; - *pbHasM = bHasM; - empty(); - return OGRERR_NONE; - } - /* -------------------------------------------------------------------- */ - /* Check for Z, M or ZM. Will ignore the Measure */ - /* -------------------------------------------------------------------- */ - else if (EQUAL(szToken, "Z")) - { - bHasZ = true; - } - else if (EQUAL(szToken, "M")) - { - bHasM = true; - } - else if (EQUAL(szToken, "ZM")) + if (!bAlreadyGotDimension) { - bHasZ = true; - bHasM = true; + const char *pszNewInput = OGRWktReadToken(pszInput, szToken); + if (EQUAL(szToken, "Z")) + { + pszInput = pszNewInput; + bHasZ = true; + } + else if (EQUAL(szToken, "M")) + { + pszInput = pszNewInput; + bHasM = true; + } + else if (EQUAL(szToken, "ZM")) + { + pszInput = pszNewInput; + bHasZ = true; + bHasM = true; + } } *pbHasZ = bHasZ; *pbHasM = bHasM; - if (bIsoWKT && (bHasZ || bHasM)) + /* -------------------------------------------------------------------- */ + /* Check for EMPTY ... */ + /* -------------------------------------------------------------------- */ + const char *pszNewInput = OGRWktReadToken(pszInput, szToken); + if (EQUAL(szToken, "EMPTY")) { - pszInput = pszPreScan; - pszPreScan = OGRWktReadToken(pszInput, szToken); - if (EQUAL(szToken, "EMPTY")) - { - *ppszInput = pszPreScan; - empty(); - if (bHasZ) - set3D(TRUE); - if (bHasM) - setMeasured(TRUE); - *pbIsEmpty = true; - return OGRERR_NONE; - } + *ppszInput = pszNewInput; + *pbIsEmpty = true; + if (bHasZ) + set3D(TRUE); + if (bHasM) + setMeasured(TRUE); + return OGRERR_NONE; } if (!EQUAL(szToken, "(")) @@ -1870,10 +1873,10 @@ OGRErr OGRGeometry::importPreambleFromWkt(const char **ppszInput, int *pbHasZ, if (!bHasZ && !bHasM) { // Test for old-style XXXXXXXXX(EMPTY). - pszPreScan = OGRWktReadToken(pszPreScan, szToken); + pszNewInput = OGRWktReadToken(pszNewInput, szToken); if (EQUAL(szToken, "EMPTY")) { - pszPreScan = OGRWktReadToken(pszPreScan, szToken); + pszNewInput = OGRWktReadToken(pszNewInput, szToken); if (EQUAL(szToken, ",")) { @@ -1885,7 +1888,7 @@ OGRErr OGRGeometry::importPreambleFromWkt(const char **ppszInput, int *pbHasZ, } else { - *ppszInput = pszPreScan; + *ppszInput = pszNewInput; empty(); *pbIsEmpty = true; return OGRERR_NONE; From 4cc46a6659d0777329586ba4d2842d7b2e6fa9cf Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 6 Jul 2024 13:06:22 +0200 Subject: [PATCH 61/72] Modernize test_ogr_wkbwkt_test_import_wkt_sf12() --- autotest/ogr/ogr_wkbwkt_geom.py | 136 +++++++++++++++++--------------- 1 file changed, 71 insertions(+), 65 deletions(-) diff --git a/autotest/ogr/ogr_wkbwkt_geom.py b/autotest/ogr/ogr_wkbwkt_geom.py index 26fab54f5f97..aec4dcc5001a 100755 --- a/autotest/ogr/ogr_wkbwkt_geom.py +++ b/autotest/ogr/ogr_wkbwkt_geom.py @@ -373,76 +373,79 @@ def test_ogr_wkbwkt_test_broken_geom(): # Test importing WKT SF1.2 -def test_ogr_wkbwkt_test_import_wkt_sf12(): - - list_wkt_tuples = [ +@pytest.mark.parametrize( + "input_wkt,expected_output_wkt", + [ ("POINT EMPTY", "POINT EMPTY"), - ("POINT Z EMPTY", "POINT EMPTY"), - ("POINT M EMPTY", "POINT EMPTY"), - ("POINT ZM EMPTY", "POINT EMPTY"), + ("POINT Z EMPTY", "POINT Z EMPTY"), + ("POINT M EMPTY", "POINT M EMPTY"), + ("POINT ZM EMPTY", "POINT ZM EMPTY"), ("POINT (0 1)", "POINT (0 1)"), - ("POINT Z (0 1 2)", "POINT (0 1 2)"), - ("POINT M (0 1 2)", "POINT (0 1)"), - ("POINT ZM (0 1 2 3)", "POINT (0 1 2)"), + ("POINT Z (0 1 2)", "POINT Z (0 1 2)"), + ("POINT M (0 1 2)", "POINT M (0 1 2)"), + ("POINT ZM (0 1 2 3)", "POINT ZM (0 1 2 3)"), ("LINESTRING EMPTY", "LINESTRING EMPTY"), - ("LINESTRING Z EMPTY", "LINESTRING EMPTY"), - ("LINESTRING M EMPTY", "LINESTRING EMPTY"), - ("LINESTRING ZM EMPTY", "LINESTRING EMPTY"), + ("LINESTRING Z EMPTY", "LINESTRING Z EMPTY"), + ("LINESTRING M EMPTY", "LINESTRING M EMPTY"), + ("LINESTRING ZM EMPTY", "LINESTRING ZM EMPTY"), ("LINESTRING (0 1,2 3)", "LINESTRING (0 1,2 3)"), - ("LINESTRING Z (0 1 2,3 4 5)", "LINESTRING (0 1 2,3 4 5)"), - ("LINESTRING M (0 1 2,3 4 5)", "LINESTRING (0 1,3 4)"), - ("LINESTRING ZM (0 1 2 3,4 5 6 7)", "LINESTRING (0 1 2,4 5 6)"), + ("LINESTRING Z (0 1 2,3 4 5)", "LINESTRING Z (0 1 2,3 4 5)"), + ("LINESTRING M (0 1 2,3 4 5)", "LINESTRING M (0 1 2,3 4 5)"), + ("LINESTRING ZM (0 1 2 3,4 5 6 7)", "LINESTRING ZM (0 1 2 3,4 5 6 7)"), ("POLYGON EMPTY", "POLYGON EMPTY"), ("POLYGON (EMPTY)", "POLYGON EMPTY"), - ("POLYGON Z EMPTY", "POLYGON EMPTY"), - ("POLYGON Z (EMPTY)", "POLYGON EMPTY"), - ("POLYGON M EMPTY", "POLYGON EMPTY"), - ("POLYGON ZM EMPTY", "POLYGON EMPTY"), + ("POLYGON Z EMPTY", "POLYGON Z EMPTY"), + ("POLYGON Z (EMPTY)", "POLYGON Z EMPTY"), + ("POLYGON M EMPTY", "POLYGON M EMPTY"), + ("POLYGON ZM EMPTY", "POLYGON ZM EMPTY"), ("POLYGON ((0 1,2 3,4 5,0 1))", "POLYGON ((0 1,2 3,4 5,0 1))"), ("POLYGON ((0 1,2 3,4 5,0 1),EMPTY)", "POLYGON ((0 1,2 3,4 5,0 1))"), ("POLYGON (EMPTY,(0 1,2 3,4 5,0 1))", "POLYGON EMPTY"), ("POLYGON (EMPTY,(0 1,2 3,4 5,0 1),EMPTY)", "POLYGON EMPTY"), ( "POLYGON Z ((0 1 10,2 3 20,4 5 30,0 1 10),(0 1 10,2 3 20,4 5 30,0 1 10))", - "POLYGON ((0 1 10,2 3 20,4 5 30,0 1 10),(0 1 10,2 3 20,4 5 30,0 1 10))", + "POLYGON Z ((0 1 10,2 3 20,4 5 30,0 1 10),(0 1 10,2 3 20,4 5 30,0 1 10))", + ), + ( + "POLYGON M ((0 1 10,2 3 20,4 5 30,0 1 10))", + "POLYGON M ((0 1 10,2 3 20,4 5 30,0 1 10))", ), - ("POLYGON M ((0 1 10,2 3 20,4 5 30,0 1 10))", "POLYGON ((0 1,2 3,4 5,0 1))"), ( "POLYGON ZM ((0 1 10 100,2 3 20 200,4 5 30 300,0 1 10 10))", - "POLYGON ((0 1 10,2 3 20,4 5 30,0 1 10))", + "POLYGON ZM ((0 1 10 100,2 3 20 200,4 5 30 300,0 1 10 10))", ), ("MULTIPOINT EMPTY", "MULTIPOINT EMPTY"), ("MULTIPOINT (EMPTY)", "MULTIPOINT EMPTY"), - ("MULTIPOINT Z EMPTY", "MULTIPOINT EMPTY"), - ("MULTIPOINT Z (EMPTY)", "MULTIPOINT EMPTY"), - ("MULTIPOINT M EMPTY", "MULTIPOINT EMPTY"), - ("MULTIPOINT ZM EMPTY", "MULTIPOINT EMPTY"), + ("MULTIPOINT Z EMPTY", "MULTIPOINT Z EMPTY"), + ("MULTIPOINT Z (EMPTY)", "MULTIPOINT Z EMPTY"), + ("MULTIPOINT M EMPTY", "MULTIPOINT M EMPTY"), + ("MULTIPOINT ZM EMPTY", "MULTIPOINT ZM EMPTY"), ( "MULTIPOINT (0 1,2 3)", - "MULTIPOINT (0 1,2 3)", + "MULTIPOINT ((0 1),(2 3))", ), # Not SF1.2 compliant but recognized - ("MULTIPOINT ((0 1),(2 3))", "MULTIPOINT (0 1,2 3)"), + ("MULTIPOINT ((0 1),(2 3))", "MULTIPOINT ((0 1),(2 3))"), ( "MULTIPOINT ((0 1),EMPTY)", - "MULTIPOINT (0 1)", + "MULTIPOINT ((0 1))", ), # We don't output empty points in multipoint ( "MULTIPOINT (EMPTY,(0 1))", - "MULTIPOINT (0 1)", + "MULTIPOINT ((0 1))", ), # We don't output empty points in multipoint ( "MULTIPOINT (EMPTY,(0 1),EMPTY)", - "MULTIPOINT (0 1)", + "MULTIPOINT ((0 1))", ), # We don't output empty points in multipoint - ("MULTIPOINT Z ((0 1 2),(3 4 5))", "MULTIPOINT (0 1 2,3 4 5)"), - ("MULTIPOINT M ((0 1 2),(3 4 5))", "MULTIPOINT (0 1,3 4)"), - ("MULTIPOINT ZM ((0 1 2 3),(4 5 6 7))", "MULTIPOINT (0 1 2,4 5 6)"), + ("MULTIPOINT Z ((0 1 2),(3 4 5))", "MULTIPOINT Z ((0 1 2),(3 4 5))"), + ("MULTIPOINT M ((0 1 2),(3 4 5))", "MULTIPOINT M ((0 1 2),(3 4 5))"), + ("MULTIPOINT ZM ((0 1 2 3),(4 5 6 7))", "MULTIPOINT ZM ((0 1 2 3),(4 5 6 7))"), ("MULTILINESTRING EMPTY", "MULTILINESTRING EMPTY"), ("MULTILINESTRING (EMPTY)", "MULTILINESTRING EMPTY"), - ("MULTILINESTRING Z EMPTY", "MULTILINESTRING EMPTY"), - ("MULTILINESTRING Z (EMPTY)", "MULTILINESTRING EMPTY"), - ("MULTILINESTRING M EMPTY", "MULTILINESTRING EMPTY"), - ("MULTILINESTRING ZM EMPTY", "MULTILINESTRING EMPTY"), + ("MULTILINESTRING Z EMPTY", "MULTILINESTRING Z EMPTY"), + ("MULTILINESTRING Z (EMPTY)", "MULTILINESTRING Z EMPTY"), + ("MULTILINESTRING M EMPTY", "MULTILINESTRING M EMPTY"), + ("MULTILINESTRING ZM EMPTY", "MULTILINESTRING ZM EMPTY"), ("MULTILINESTRING ((0 1,2 3,4 5,0 1))", "MULTILINESTRING ((0 1,2 3,4 5,0 1))"), ( "MULTILINESTRING ((0 1,2 3,4 5,0 1),EMPTY)", @@ -458,22 +461,22 @@ def test_ogr_wkbwkt_test_import_wkt_sf12(): ), ( "MULTILINESTRING Z ((0 1 10,2 3 20,4 5 30,0 1 10),(0 1 10,2 3 20,4 5 30,0 1 10))", - "MULTILINESTRING ((0 1 10,2 3 20,4 5 30,0 1 10),(0 1 10,2 3 20,4 5 30,0 1 10))", + "MULTILINESTRING Z ((0 1 10,2 3 20,4 5 30,0 1 10),(0 1 10,2 3 20,4 5 30,0 1 10))", ), ( "MULTILINESTRING M ((0 1 10,2 3 20,4 5 30,0 1 10))", - "MULTILINESTRING ((0 1,2 3,4 5,0 1))", + "MULTILINESTRING M ((0 1 10,2 3 20,4 5 30,0 1 10))", ), ( "MULTILINESTRING ZM ((0 1 10 100,2 3 20 200,4 5 30 300,0 1 10 10))", - "MULTILINESTRING ((0 1 10,2 3 20,4 5 30,0 1 10))", + "MULTILINESTRING ZM ((0 1 10 100,2 3 20 200,4 5 30 300,0 1 10 10))", ), ("MULTIPOLYGON EMPTY", "MULTIPOLYGON EMPTY"), ("MULTIPOLYGON (EMPTY)", "MULTIPOLYGON EMPTY"), - ("MULTIPOLYGON Z EMPTY", "MULTIPOLYGON EMPTY"), - ("MULTIPOLYGON Z (EMPTY)", "MULTIPOLYGON EMPTY"), - ("MULTIPOLYGON M EMPTY", "MULTIPOLYGON EMPTY"), - ("MULTIPOLYGON ZM EMPTY", "MULTIPOLYGON EMPTY"), + ("MULTIPOLYGON Z EMPTY", "MULTIPOLYGON Z EMPTY"), + ("MULTIPOLYGON Z (EMPTY)", "MULTIPOLYGON Z EMPTY"), + ("MULTIPOLYGON M EMPTY", "MULTIPOLYGON M EMPTY"), + ("MULTIPOLYGON ZM EMPTY", "MULTIPOLYGON ZM EMPTY"), ("MULTIPOLYGON ((EMPTY))", "MULTIPOLYGON EMPTY"), ("MULTIPOLYGON (((0 1,2 3,4 5,0 1)))", "MULTIPOLYGON (((0 1,2 3,4 5,0 1)))"), ( @@ -511,31 +514,31 @@ def test_ogr_wkbwkt_test_import_wkt_sf12(): ), ( "MULTIPOLYGON Z (((0 1 10,2 3 20,4 5 30,0 1 10)),((0 1 10,2 3 20,4 5 30,0 1 10)))", - "MULTIPOLYGON (((0 1 10,2 3 20,4 5 30,0 1 10)),((0 1 10,2 3 20,4 5 30,0 1 10)))", + "MULTIPOLYGON Z (((0 1 10,2 3 20,4 5 30,0 1 10)),((0 1 10,2 3 20,4 5 30,0 1 10)))", ), ( "MULTIPOLYGON M (((0 1 10,2 3 20,4 5 30,0 1 10)))", - "MULTIPOLYGON (((0 1,2 3,4 5,0 1)))", + "MULTIPOLYGON M (((0 1 10,2 3 20,4 5 30,0 1 10)))", ), ( "MULTIPOLYGON ZM (((0 1 10 100,2 3 20 200,4 5 30 300,0 1 10 10)))", - "MULTIPOLYGON (((0 1 10,2 3 20,4 5 30,0 1 10)))", + "MULTIPOLYGON ZM (((0 1 10 100,2 3 20 200,4 5 30 300,0 1 10 10)))", ), ("GEOMETRYCOLLECTION EMPTY", "GEOMETRYCOLLECTION EMPTY"), - ("GEOMETRYCOLLECTION Z EMPTY", "GEOMETRYCOLLECTION EMPTY"), - ("GEOMETRYCOLLECTION M EMPTY", "GEOMETRYCOLLECTION EMPTY"), - ("GEOMETRYCOLLECTION ZM EMPTY", "GEOMETRYCOLLECTION EMPTY"), + ("GEOMETRYCOLLECTION Z EMPTY", "GEOMETRYCOLLECTION Z EMPTY"), + ("GEOMETRYCOLLECTION M EMPTY", "GEOMETRYCOLLECTION M EMPTY"), + ("GEOMETRYCOLLECTION ZM EMPTY", "GEOMETRYCOLLECTION ZM EMPTY"), ( "GEOMETRYCOLLECTION Z (POINT Z (0 1 2),LINESTRING Z (0 1 2,3 4 5))", - "GEOMETRYCOLLECTION (POINT (0 1 2),LINESTRING (0 1 2,3 4 5))", + "GEOMETRYCOLLECTION Z (POINT Z (0 1 2),LINESTRING Z (0 1 2,3 4 5))", ), ( "GEOMETRYCOLLECTION M (POINT M (0 1 2),LINESTRING M (0 1 2,3 4 5))", - "GEOMETRYCOLLECTION (POINT (0 1),LINESTRING (0 1,3 4))", + "GEOMETRYCOLLECTION M (POINT M (0 1 2),LINESTRING M (0 1 2,3 4 5))", ), ( "GEOMETRYCOLLECTION ZM (POINT ZM (0 1 2 10),LINESTRING ZM (0 1 2 10,3 4 5 20))", - "GEOMETRYCOLLECTION (POINT (0 1 2),LINESTRING (0 1 2,3 4 5))", + "GEOMETRYCOLLECTION ZM (POINT ZM (0 1 2 10),LINESTRING ZM (0 1 2 10,3 4 5 20))", ), ( "GEOMETRYCOLLECTION (POINT EMPTY,LINESTRING EMPTY,POLYGON EMPTY,MULTIPOINT EMPTY,MULTILINESTRING EMPTY,MULTIPOLYGON EMPTY,GEOMETRYCOLLECTION EMPTY)", @@ -543,7 +546,7 @@ def test_ogr_wkbwkt_test_import_wkt_sf12(): ), ( "GEOMETRYCOLLECTION (POINT Z EMPTY,LINESTRING Z EMPTY,POLYGON Z EMPTY,MULTIPOINT Z EMPTY,MULTILINESTRING Z EMPTY,MULTIPOLYGON Z EMPTY,GEOMETRYCOLLECTION Z EMPTY)", - "GEOMETRYCOLLECTION (POINT EMPTY,LINESTRING EMPTY,POLYGON EMPTY,MULTIPOINT EMPTY,MULTILINESTRING EMPTY,MULTIPOLYGON EMPTY,GEOMETRYCOLLECTION EMPTY)", + "GEOMETRYCOLLECTION Z (POINT Z EMPTY,LINESTRING Z EMPTY,POLYGON Z EMPTY,MULTIPOINT Z EMPTY,MULTILINESTRING Z EMPTY,MULTIPOLYGON Z EMPTY,GEOMETRYCOLLECTION Z EMPTY)", ), # Not SF1.2 compliant but recognized ( @@ -556,17 +559,20 @@ def test_ogr_wkbwkt_test_import_wkt_sf12(): ("MULTICURVE (EMPTY)", "MULTICURVE EMPTY"), ("MULTISURFACE EMPTY", "MULTISURFACE EMPTY"), ("MULTISURFACE (EMPTY)", "MULTISURFACE EMPTY"), - ] + ], +) +def test_ogr_wkbwkt_test_import_wkt_sf12(input_wkt, expected_output_wkt): - for wkt_tuple in list_wkt_tuples: - geom = ogr.CreateGeometryFromWkt(wkt_tuple[0]) - assert geom is not None, "could not instantiate geometry %s" % wkt_tuple[0] - out_wkt = geom.ExportToWkt() - assert out_wkt == wkt_tuple[1], "in=%s, out=%s, expected=%s." % ( - wkt_tuple[0], - out_wkt, - wkt_tuple[1], - ) + geom = ogr.CreateGeometryFromWkt(input_wkt) + assert geom is not None + out_wkt = geom.ExportToIsoWkt() + assert out_wkt == expected_output_wkt + + # Test with input in lower case + geom = ogr.CreateGeometryFromWkt(input_wkt.lower()) + assert geom is not None + out_wkt = geom.ExportToIsoWkt() + assert out_wkt == expected_output_wkt ############################################################################### From 1112d481d33932c9438a2f1f86295c4f82e368d3 Mon Sep 17 00:00:00 2001 From: Riivo Kolka Date: Sat, 6 Jul 2024 21:50:15 +0300 Subject: [PATCH 62/72] Doc: revise build steps on Windows with conda and Visual Studio (#10372) * Revise Visual Studio build steps * Mention that Visual Studio must be installed * clcache comes from conda-forge * newer (default) python, otherwise could not find numpy --- doc/source/development/dev_environment.rst | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/doc/source/development/dev_environment.rst b/doc/source/development/dev_environment.rst index 810d05e7bc19..e8dff89f9fb4 100644 --- a/doc/source/development/dev_environment.rst +++ b/doc/source/development/dev_environment.rst @@ -80,6 +80,13 @@ Install miniconda Install `miniconda `_ +Install Visual Studio ++++++++++++++++++++++ + +Install `Visual Studio `_ +In Visual Studio Installer Workloads tab check Desktop development with C++. +Only the latest Community Edition (2022) is available. + Install GDAL dependencies +++++++++++++++++++++++++ @@ -90,8 +97,8 @@ Start a Conda enabled console and assuming there is a c:\\dev directory cd c:\dev conda create --name gdal conda activate gdal - conda install --yes --quiet curl libiconv icu git python=3.7 swig numpy pytest zlib clcache - conda install --yes --quiet -c conda-forge compilers + conda install --yes --quiet curl libiconv icu git python swig numpy pytest zlib + conda install --yes --quiet -c conda-forge compilers clcache conda install --yes --quiet -c conda-forge \ cmake proj geos hdf4 hdf5 \ libnetcdf openjpeg poppler libtiff libpng xerces-c expat libxml2 kealib json-c \ @@ -100,9 +107,9 @@ Start a Conda enabled console and assuming there is a c:\\dev directory .. note:: - The ``compilers`` package will install ``vs2017_win-64`` (at time of writing) - to set the appropriate environment for cmake to pick up. It is also possible - to use the ``vs2019_win-64`` package if Visual Studio 2019 is to be used. + The ``compilers`` package will install ``vs2019_win-64`` (at time of writing) + to set the appropriate environment for cmake to pick up. It also finds and works + with Visual Studio 2022 if that is installed. Checkout GDAL sources +++++++++++++++++++++ @@ -156,6 +163,11 @@ This can be done by sourcing the following from the build directory: For Windows, a similar ``scripts/setdevenv.bat`` script exists (it currently assumes a Release build). +.. code-block:: console + + cd c:\dev\gdal\build + ..\scripts\setdevenv.bat + To verify that environment variables have been set correctly, you can check the version of a GDAL binary: .. code-block:: bash From be77d2a9e6fd98a12c84f4a52fceaf9d90a859b6 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 10:47:35 +0200 Subject: [PATCH 63/72] OGR SQL lexer: fix memory leak on invalid expression (master only) Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=69985 --- autotest/ogr/ogr_sql_test.py | 2 ++ ogr/CMakeLists.txt | 2 +- ogr/swq_parser.cpp | 12 ++++++++++++ ogr/swq_parser.y | 6 +++--- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index cba0f8a0ad33..a3041c326e81 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -717,6 +717,8 @@ def ds_for_invalid_statements(): "SELECT 1 - FROM my_layer", "SELECT 1 * FROM my_layer", "SELECT 1 % FROM my_layer", + "SELECT x.", + "SELECT x AS", "SELECT *", "SELECT * FROM", "SELECT * FROM foo", diff --git a/ogr/CMakeLists.txt b/ogr/CMakeLists.txt index 8cb85f7513ee..cc59f2a665a8 100644 --- a/ogr/CMakeLists.txt +++ b/ogr/CMakeLists.txt @@ -180,7 +180,7 @@ add_custom_target(check_swq_parser_md5 ALL COMMAND ${CMAKE_COMMAND} "-DIN_FILE=swq_parser.y" "-DTARGET=generate_swq_parser" - "-DEXPECTED_MD5SUM=6c82680a0abb18a011e872929a4a4ece" + "-DEXPECTED_MD5SUM=9aa3dfdf31343882217863b82a009d64 " "-DFILENAME_CMAKE=${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt" -P "${PROJECT_SOURCE_DIR}/cmake/helpers/check_md5sum.cmake" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" diff --git a/ogr/swq_parser.cpp b/ogr/swq_parser.cpp index f12ed64fa289..ced67fba01e9 100644 --- a/ogr/swq_parser.cpp +++ b/ogr/swq_parser.cpp @@ -1421,6 +1421,10 @@ yydestruct (const char *yymsg, { delete (*yyvaluep); } break; + case YYSYMBOL_identifier: /* identifier */ + { delete (*yyvaluep); } + break; + case YYSYMBOL_field_value: /* field_value */ { delete (*yyvaluep); } break; @@ -1433,6 +1437,14 @@ yydestruct (const char *yymsg, { delete (*yyvaluep); } break; + case YYSYMBOL_as_clause: /* as_clause */ + { delete (*yyvaluep); } + break; + + case YYSYMBOL_as_clause_with_hidden: /* as_clause_with_hidden */ + { delete (*yyvaluep); } + break; + case YYSYMBOL_table_def: /* table_def */ { delete (*yyvaluep); } break; diff --git a/ogr/swq_parser.y b/ogr/swq_parser.y index 6383b25827d8..0adb7651a5d4 100644 --- a/ogr/swq_parser.y +++ b/ogr/swq_parser.y @@ -116,9 +116,9 @@ %token SWQT_RESERVED_KEYWORD "reserved keyword" /* Any grammar rule that does $$ = must be listed afterwards */ -/* as well as SWQT_INTEGER_NUMBER SWQT_FLOAT_NUMBER SWQT_STRING SWQT_IDENTIFIER SWQT_HIDDEN that are allocated by swqlex() */ -%destructor { delete $$; } SWQT_INTEGER_NUMBER SWQT_FLOAT_NUMBER SWQT_STRING SWQT_IDENTIFIER SWQT_HIDDEN -%destructor { delete $$; } value_expr_list field_value value_expr value_expr_non_logical type_def table_def +/* as well as SWQT_INTEGER_NUMBER SWQT_FLOAT_NUMBER SWQT_STRING SWQT_IDENTIFIER SWQT_HIDDEN that are allocated by swqlex(), and identifier */ +%destructor { delete $$; } SWQT_INTEGER_NUMBER SWQT_FLOAT_NUMBER SWQT_STRING SWQT_IDENTIFIER SWQT_HIDDEN identifier +%destructor { delete $$; } value_expr_list field_value value_expr value_expr_non_logical type_def table_def as_clause as_clause_with_hidden %% From f0393ae302fa91d64633989e73f838f8cc20c14e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 10:53:07 +0200 Subject: [PATCH 64/72] SRP: avoid potential integer overflow when ZNA == INT_MIN Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=70095 --- frmts/adrg/srpdataset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frmts/adrg/srpdataset.cpp b/frmts/adrg/srpdataset.cpp index e507adf25abf..018da7fb2320 100644 --- a/frmts/adrg/srpdataset.cpp +++ b/frmts/adrg/srpdataset.cpp @@ -824,7 +824,7 @@ bool SRPDataset::GetFromRecord(const char *pszFileName, DDFRecord *record) } else { - if (std::abs(ZNA) >= 1 && std::abs(ZNA) <= 60) + if (ZNA >= -60 && ZNA <= 60 && ZNA != 0) { m_oSRS.SetUTM(std::abs(ZNA), ZNA > 0); m_oSRS.SetWellKnownGeogCS("WGS84"); From d91a902c7224bdc2e210b05fbefbdfa3e17060bb Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 10:57:42 +0200 Subject: [PATCH 65/72] OGRAVCE00Layer::FormPolygonGeometry(): avoid integer overflow when nArcId == INT_MIN Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=70096 --- ogr/ogrsf_frmts/avc/ogravce00layer.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ogr/ogrsf_frmts/avc/ogravce00layer.cpp b/ogr/ogrsf_frmts/avc/ogravce00layer.cpp index e4f87f5a98d7..6d364338242b 100644 --- a/ogr/ogrsf_frmts/avc/ogravce00layer.cpp +++ b/ogr/ogrsf_frmts/avc/ogravce00layer.cpp @@ -314,8 +314,11 @@ bool OGRAVCE00Layer::FormPolygonGeometry(OGRFeature *poFeature, AVCPal *psPAL) for (int iArc = 0; iArc < psPAL->numArcs; iArc++) { - if (psPAL->pasArcs[iArc].nArcId == 0) + if (psPAL->pasArcs[iArc].nArcId == 0 || + psPAL->pasArcs[iArc].nArcId == INT_MIN) + { continue; + } // If the other side of the line is the same polygon then this // arc is a "bridge" arc and can be discarded. If we don't discard @@ -325,8 +328,8 @@ bool OGRAVCE00Layer::FormPolygonGeometry(OGRFeature *poFeature, AVCPal *psPAL) if (psPAL->pasArcs[iArc].nAdjPoly == psPAL->nPolyId) continue; - OGRFeature *poArc = - poArcLayer->GetFeature(std::abs(psPAL->pasArcs[iArc].nArcId)); + auto poArc = std::unique_ptr( + poArcLayer->GetFeature(std::abs(psPAL->pasArcs[iArc].nArcId))); if (poArc == nullptr) return false; @@ -334,8 +337,7 @@ bool OGRAVCE00Layer::FormPolygonGeometry(OGRFeature *poFeature, AVCPal *psPAL) if (poArc->GetGeometryRef() == nullptr) return false; - oArcs.addGeometry(poArc->GetGeometryRef()); - OGRFeature::DestroyFeature(poArc); + oArcs.addGeometryDirectly(poArc->StealGeometry()); } OGRErr eErr; From bde84002410eac1ee1efaf18152261d35a5d8b41 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 11:42:24 +0200 Subject: [PATCH 66/72] ECW: emit error message before emitting progression, and one cleanup --- frmts/ecw/ecwcreatecopy.cpp | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/frmts/ecw/ecwcreatecopy.cpp b/frmts/ecw/ecwcreatecopy.cpp index 39d91ddaf133..5654e382cd57 100644 --- a/frmts/ecw/ecwcreatecopy.cpp +++ b/frmts/ecw/ecwcreatecopy.cpp @@ -1415,16 +1415,13 @@ static GDALDataset *ECWCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, oCompressor.pProgressData = pProgressData; oCompressor.m_poSrcDS = poSrcDS; - if (!pfnProgress(0.0, nullptr, pProgressData)) - return nullptr; - - char **papszBandDescriptions = (char **)CPLMalloc(nBands * sizeof(char *)); + CPLStringList aosBandDescriptions; for (int i = 0; i < nBands; i++) { /* Make a copy since ECWGetColorInterpretationName() can return a string * generated */ /* by CPLSPrintf(), which has just a few rotating entries. */ - papszBandDescriptions[i] = CPLStrdup(ECWGetColorInterpretationName( + aosBandDescriptions.AddString(ECWGetColorInterpretationName( poSrcDS->GetRasterBand(i + 1)->GetColorInterpretation(), i)); } @@ -1433,29 +1430,27 @@ static GDALDataset *ECWCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, pszAreaOrPoint != nullptr && EQUAL(pszAreaOrPoint, GDALMD_AOP_POINT); if (oCompressor.Initialize(pszFilename, papszOptions, nXSize, nYSize, - nBands, papszBandDescriptions, bRGBColorSpace, - eType, poSRS, adfGeoTransform, + nBands, aosBandDescriptions.List(), + bRGBColorSpace, eType, poSRS, adfGeoTransform, poSrcDS->GetGCPCount(), poSrcDS->GetGCPs(), bIsJPEG2000, bPixelIsPoint, poSrcDS->GetMetadata("RPC"), poSrcDS) != CE_None) { - for (int i = 0; i < nBands; i++) - CPLFree(papszBandDescriptions[i]); - CPLFree(papszBandDescriptions); return nullptr; } /* -------------------------------------------------------------------- */ /* Start the compression. */ /* -------------------------------------------------------------------- */ + + if (!pfnProgress(0.0, nullptr, pProgressData)) + return nullptr; + CNCSError oErr = oCompressor.Write(); if (oErr.GetErrorNumber() != NCS_SUCCESS) { ECWReportError(oErr); - for (int i = 0; i < nBands; i++) - CPLFree(papszBandDescriptions[i]); - CPLFree(papszBandDescriptions); return nullptr; } @@ -1463,9 +1458,6 @@ static GDALDataset *ECWCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, /* Cleanup, and return read-only handle. */ /* -------------------------------------------------------------------- */ oCompressor.CloseDown(); - for (int i = 0; i < nBands; i++) - CPLFree(papszBandDescriptions[i]); - CPLFree(papszBandDescriptions); pfnProgress(1.001, nullptr, pProgressData); /* -------------------------------------------------------------------- */ From 1899e2f37978cb4d660614d008b5219d7ab21f3b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 22:57:57 +0200 Subject: [PATCH 67/72] cpl_vsi_mem.cpp: avoid accessing bEOF under lock (CID 1549765, 1549764) --- port/cpl_vsi_mem.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/port/cpl_vsi_mem.cpp b/port/cpl_vsi_mem.cpp index 17f57ef9fa88..a4ccaedd9266 100644 --- a/port/cpl_vsi_mem.cpp +++ b/port/cpl_vsi_mem.cpp @@ -421,25 +421,36 @@ size_t VSIMemHandle::Read(void *pBuffer, size_t nSize, size_t nCount) return 0; } + bool bEOFTmp = bEOF; + // Do not access/modify bEOF under the lock to avoid confusing Coverity + // Scan since we access it in other methods outside of the lock. + const auto DoUnderLock = + [this, nOffset, pBuffer, nSize, &nBytesToRead, &nCount, &bEOFTmp] { CPL_SHARED_LOCK oLock(poFile->m_oMutex); if (poFile->nLength <= nOffset || nBytesToRead + nOffset < nBytesToRead) { - bEOF = true; - return 0; + bEOFTmp = true; + return false; } if (nBytesToRead + nOffset > poFile->nLength) { nBytesToRead = static_cast(poFile->nLength - nOffset); nCount = nBytesToRead / nSize; - bEOF = true; + bEOFTmp = true; } if (nBytesToRead) memcpy(pBuffer, poFile->pabyData + nOffset, static_cast(nBytesToRead)); - } + return true; + }; + + bool bRet = DoUnderLock(); + bEOF = bEOFTmp; + if (!bRet) + return 0; m_nOffset += nBytesToRead; From 93903529698d566b2f0837dcd64966e72ec2b4b9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 23:04:32 +0200 Subject: [PATCH 68/72] gdalmultidim_gltorthorectification.cpp: avoid warning about using a moved pointer (CID 1549763) --- gcore/gdalmultidim_gltorthorectification.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcore/gdalmultidim_gltorthorectification.cpp b/gcore/gdalmultidim_gltorthorectification.cpp index 4baf11e31863..dfa7c38826d2 100644 --- a/gcore/gdalmultidim_gltorthorectification.cpp +++ b/gcore/gdalmultidim_gltorthorectification.cpp @@ -129,14 +129,14 @@ class GLTOrthoRectifiedArray final : public GDALPamMDArray {/*latIdx = */ 1, /* lonIdx = */ 2}); if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "USE_GOOD_WAVELENGTHS", "YES")) && - poParent->GetDimensionCount() == 3) + newAr->m_poParent->GetDimensionCount() == 3) { const auto poGoodWaveLengths = poRootGroup->OpenMDArrayFromFullname( "/sensor_band_parameters/good_wavelengths"); if (poGoodWaveLengths && poGoodWaveLengths->GetDimensionCount() == 1 && poGoodWaveLengths->GetDimensions()[0]->GetSize() == - poParent->GetDimensions()[2]->GetSize() && + newAr->m_poParent->GetDimensions()[2]->GetSize() && poGoodWaveLengths->GetDimensions()[0]->GetSize() < 1000 * 1000 && poGoodWaveLengths->GetDataType().GetClass() == GEDTC_NUMERIC) From 7866ee852c6b4ec25e5fd12e0401fc960cfa4b68 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 23:11:06 +0200 Subject: [PATCH 69/72] ogrshapedatasource.cpp: consistently access variable under lock (CID 1549762) --- ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp b/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp index 69e45b7d4b50..eb102e8c3b8a 100644 --- a/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp +++ b/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp @@ -1500,8 +1500,10 @@ bool OGRShapeDataSource::UncompressIfNeeded() return false; } m_psLockFile = f; + CPLAcquireMutex(m_poRefreshLockFileMutex, 1000); m_bExitRefreshLockFileThread = false; m_bRefreshLockFileThreadStarted = false; + CPLReleaseMutex(m_poRefreshLockFileMutex); // Config option mostly for testing purposes // coverity[tainted_data] m_dfRefreshLockDelay = CPLAtof(CPLGetConfigOption( From a6b3453b0c9cab84baa650c73c51152469788635 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 23:20:10 +0200 Subject: [PATCH 70/72] OGR_GPKG_FillArrowArray_Step(): make sure to access nCountRows only under mutex (CID 1549761) --- .../gpkg/ogrgeopackagetablelayer.cpp | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp index be706dbebb4a..074dcad5d3cd 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp @@ -7771,33 +7771,36 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, auto psFillArrowArray = static_cast( sqlite3_user_data(pContext)); - if (psFillArrowArray->nCountRows >= - psFillArrowArray->psHelper->m_nMaxBatchSize) { - if (psFillArrowArray->bAsynchronousMode) + std::unique_lock oLock(psFillArrowArray->oMutex); + if (psFillArrowArray->nCountRows >= + psFillArrowArray->psHelper->m_nMaxBatchSize) { - std::unique_lock oLock(psFillArrowArray->oMutex); - psFillArrowArray->psHelper->Shrink(psFillArrowArray->nCountRows); - psFillArrowArray->oCV.notify_one(); - while (psFillArrowArray->nCountRows > 0) + if (psFillArrowArray->bAsynchronousMode) { - psFillArrowArray->oCV.wait(oLock); + psFillArrowArray->psHelper->Shrink( + psFillArrowArray->nCountRows); + psFillArrowArray->oCV.notify_one(); + while (psFillArrowArray->nCountRows > 0) + { + psFillArrowArray->oCV.wait(oLock); + } + // Note that psFillArrowArray->psHelper.get() will generally now be + // different from before the wait() + } + else + { + // should not happen ! + psFillArrowArray->osErrorMsg = "OGR_GPKG_FillArrowArray_Step() " + "got more rows than expected!"; + sqlite3_interrupt(psFillArrowArray->hDB); + psFillArrowArray->bErrorOccurred = true; + return; } - // Note that psFillArrowArray->psHelper.get() will generally now be - // different from before the wait() } - else - { - // should not happen ! - psFillArrowArray->osErrorMsg = - "OGR_GPKG_FillArrowArray_Step() got more rows than expected!"; - sqlite3_interrupt(psFillArrowArray->hDB); - psFillArrowArray->bErrorOccurred = true; + if (psFillArrowArray->nCountRows < 0) return; - } } - if (psFillArrowArray->nCountRows < 0) - return; if (psFillArrowArray->nMemLimit == 0) psFillArrowArray->nMemLimit = OGRArrowArrayHelper::GetMemLimit(); @@ -7943,7 +7946,7 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, } } - if (psFillArrowArray->nCountRows > 0) + if (iFeat > 0) { auto panOffsets = static_cast( const_cast(psArray->buffers[1])); @@ -7956,7 +7959,7 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, "OGR_GPKG_FillArrowArray_Step(): premature " "notification of %d features to consumer due " "to too big array", - psFillArrowArray->nCountRows); + iFeat); psFillArrowArray->bMemoryLimitReached = true; if (psFillArrowArray->bAsynchronousMode) { @@ -8081,7 +8084,7 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, const void *pabyData = sqlite3_value_blob(argv[iCol]); if (pabyData != nullptr || nBytes == 0) { - if (psFillArrowArray->nCountRows > 0) + if (iFeat > 0) { auto panOffsets = static_cast( const_cast(psArray->buffers[1])); @@ -8094,7 +8097,7 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, "OGR_GPKG_FillArrowArray_Step(): " "premature notification of %d features to " "consumer due to too big array", - psFillArrowArray->nCountRows); + iFeat); psFillArrowArray->bMemoryLimitReached = true; if (psFillArrowArray->bAsynchronousMode) { @@ -8171,7 +8174,7 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, if (pszTxt != nullptr) { const size_t nBytes = strlen(pszTxt); - if (psFillArrowArray->nCountRows > 0) + if (iFeat > 0) { auto panOffsets = static_cast( const_cast(psArray->buffers[1])); @@ -8184,7 +8187,7 @@ void OGR_GPKG_FillArrowArray_Step(sqlite3_context *pContext, int /*argc*/, "OGR_GPKG_FillArrowArray_Step(): " "premature notification of %d features to " "consumer due to too big array", - psFillArrowArray->nCountRows); + iFeat); psFillArrowArray->bMemoryLimitReached = true; if (psFillArrowArray->bAsynchronousMode) { From f3acb03b0f3be6d2f8ca6e1add8d09195772420a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 23:27:56 +0200 Subject: [PATCH 71/72] ograrrowlayer.hpp: silence false positive (CID 1548670) --- ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp index b364925ea9a9..51b71cf4b332 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp @@ -2385,8 +2385,9 @@ inline OGRFeature *OGRArrowLayer::ReadFeature( arrow::LargeBinaryArray::offset_type out_length = 0; const uint8_t *data = castArray->GetValue(nIdxInBatch, &out_length); - if (out_length <= INT_MAX - 1) + if (out_length >= 0 && out_length <= INT_MAX - 1) { + // coverity[overflow_sink] poFeature->SetField(i, static_cast(out_length), data); } else From f1fd72ca85c4abfabc6d383305157fd5d0feb69d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 7 Jul 2024 23:32:26 +0200 Subject: [PATCH 72/72] gdalchecksum.cpp: silence false positive (CID 1548660) --- alg/gdalchecksum.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/alg/gdalchecksum.cpp b/alg/gdalchecksum.cpp index 64f6158433b8..e1f5e38f1b1d 100644 --- a/alg/gdalchecksum.cpp +++ b/alg/gdalchecksum.cpp @@ -368,5 +368,6 @@ int CPL_STDCALL GDALChecksumImage(GDALRasterBandH hBand, int nXOff, int nYOff, CPLFree(panLineData); } + // coverity[return_overflow] return nChecksum; }