Merge remote-tracking branch 'upstream/master'

autotest/ogr/ogr_gpkg.py

@@ -10540,3 +10540,49 @@ def test_gpkg_create_more_than_2000_fields(tmp_vsimem):
         with pytest.raises(Exception, match="Limit of 2000 columns reached"):
             lyr.CreateField(ogr.FieldDefn("foo"))
         assert lyr.GetLayerDefn().GetFieldCount() == 2000 - 2
+
+
+###############################################################################
+# Test that secure_delete is turned on
+
+
+@gdaltest.enable_exceptions()
+def test_gpkg_secure_delete(tmp_vsimem):
+
+    filename = str(tmp_vsimem / "secure_delete.gpkg")
+    with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds:
+
+        with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr:
+            f = sql_lyr.GetNextFeature()
+            assert f.GetField(0) == 1
+
+        lyr = ds.CreateLayer("test")
+        lyr.CreateField(ogr.FieldDefn("foo"))
+        f = ogr.Feature(lyr.GetLayerDefn())
+        f["foo"] = "very_secret"
+        lyr.CreateFeature(f)
+
+    f = gdal.VSIFOpenL(filename, "rb")
+    data = gdal.VSIFReadL(1, 100000, f)
+    gdal.VSIFCloseL(f)
+    assert b"very_secret" in data
+
+    with ogr.Open(filename, update=1) as ds:
+
+        with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr:
+            f = sql_lyr.GetNextFeature()
+            assert f.GetField(0) == 1
+
+        lyr = ds.GetLayer(0)
+        lyr.DeleteFeature(1)
+
+    f = gdal.VSIFOpenL(filename, "rb")
+    data = gdal.VSIFReadL(1, 100000, f)
+    gdal.VSIFCloseL(f)
+    assert b"very_secret" not in data
+
+    with gdaltest.config_option("OGR_SQLITE_PRAGMA", "secure_delete=0"):
+        with ogr.Open(filename, update=1) as ds:
+            with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr:
+                f = sql_lyr.GetNextFeature()
+                assert f.GetField(0) == 0

autotest/osr/osr_basic.py

@@ -2478,3 +2478,13 @@ def test_osr_basic_export_wkt_utm_south():
 
     j = json.loads(srs.ExportToPROJJSON())
     assert j["conversion"]["id"]["code"] == 16101
+
+
+###############################################################################
+
+
+def test_osr_basic_GetAuthorityListFromDatabase():
+
+    ret = osr.GetAuthorityListFromDatabase()
+    assert "EPSG" in ret
+    assert "PROJ" in ret

doc/source/drivers/vector/gpkg.rst

@@ -809,6 +809,18 @@ Examples
         -sql "SELECT poly.id, other.foo FROM poly JOIN other_schema.other USING (id)" \
         -oo PRELUDE_STATEMENTS="ATTACH DATABASE 'other.gpkg' AS other_schema"
 
+Secure deletion
+---------------
+
+Depending on how SQLite3 is built, `secure deletion <https://www.sqlite.org/pragma.html#pragma_secure_delete>`__
+might or might not be enabled.
+Starting with GDAL 3.10, secure deletion is always enabled, unless
+``SECURE_DELETE`` is specified through the :config:`OGR_SQLITE_PRAGMA`
+configuration option.
+Note that secure deletion does not recover potential lost space, so running
+a `VACUUM <https://sqlite.org/lang_vacuum.html>`__ query is recommended to fully
+optimized a database that has been subject to updates or deletions.
+
 See Also
 --------
 

doc/source/drivers/vector/sqlite.rst

@@ -556,6 +556,17 @@ and optimize it.
 
    ogrinfo db.sqlite -sql "VACUUM"
 
+Secure deletion
+---------------
+
+Depending on how SQLite3 is built, `secure deletion <https://www.sqlite.org/pragma.html#pragma_secure_delete>`__
+might or might not be enabled.
+Starting with GDAL 3.10, secure deletion is always enabled, unless
+``SECURE_DELETE`` is specified through the :config:`OGR_SQLITE_PRAGMA`
+configuration option.
+Note that secure deletion does not recover potential lost space, so running
+a `VACUUM <https://sqlite.org/lang_vacuum.html>`__ query is recommended to fully
+optimized a database that has been subject to updates or deletions.
 
 Example
 -------

ogr/ogr_srs_api.h

@@ -1005,6 +1005,8 @@ OSRGetCRSInfoListFromDatabase(const char *pszAuthName,
 
 void CPL_DLL OSRDestroyCRSInfoList(OSRCRSInfo **list);
 
+char CPL_DLL **OSRGetAuthorityListFromDatabase(void);
+
 /* -------------------------------------------------------------------- */
 /*      OGRCoordinateTransform C API.                                   */
 /* -------------------------------------------------------------------- */

ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp

@@ -228,6 +228,10 @@ inline bool OGRArrowLayer::IsHandledListOrMapType(
            itemTypeId == arrow::Type::HALF_FLOAT ||
            itemTypeId == arrow::Type::FLOAT ||
            itemTypeId == arrow::Type::DOUBLE ||
+#if ARROW_VERSION_MAJOR >= 18
+           itemTypeId == arrow::Type::DECIMAL32 ||
+           itemTypeId == arrow::Type::DECIMAL64 ||
+#endif
            itemTypeId == arrow::Type::DECIMAL128 ||
            itemTypeId == arrow::Type::DECIMAL256 ||
            itemTypeId == arrow::Type::STRING ||
@@ -422,6 +426,10 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR(
                                    // nanosecond accuracy
             break;
 
+#if ARROW_VERSION_MAJOR >= 18
+        case arrow::Type::DECIMAL32:
+        case arrow::Type::DECIMAL64:
+#endif
         case arrow::Type::DECIMAL128:
         case arrow::Type::DECIMAL256:
         {
@@ -468,6 +476,10 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR(
                     eSubType = OFSTFloat32;
                     break;
                 case arrow::Type::DOUBLE:
+#if ARROW_VERSION_MAJOR >= 18
+                case arrow::Type::DECIMAL32:
+                case arrow::Type::DECIMAL64:
+#endif
                 case arrow::Type::DECIMAL128:
                 case arrow::Type::DECIMAL256:
                     eType = OFTRealList;
@@ -1290,6 +1302,23 @@ static void AddToArray(CPLJSONArray &oArray, const arrow::Array *array,
                 static_cast<const arrow::DoubleArray *>(array)->Value(nIdx));
             break;
         }
+
+#if ARROW_VERSION_MAJOR >= 18
+        case arrow::Type::DECIMAL32:
+        {
+            oArray.Add(CPLAtof(static_cast<const arrow::Decimal32Array *>(array)
+                                   ->FormatValue(nIdx)
+                                   .c_str()));
+            break;
+        }
+        case arrow::Type::DECIMAL64:
+        {
+            oArray.Add(CPLAtof(static_cast<const arrow::Decimal64Array *>(array)
+                                   ->FormatValue(nIdx)
+                                   .c_str()));
+            break;
+        }
+#endif
         case arrow::Type::DECIMAL128:
         {
             oArray.Add(
@@ -1470,6 +1499,25 @@ static void AddToDict(CPLJSONObject &oDict, const std::string &osKey,
                 static_cast<const arrow::DoubleArray *>(array)->Value(nIdx));
             break;
         }
+
+#if ARROW_VERSION_MAJOR >= 18
+        case arrow::Type::DECIMAL32:
+        {
+            oDict.Add(osKey,
+                      CPLAtof(static_cast<const arrow::Decimal32Array *>(array)
+                                  ->FormatValue(nIdx)
+                                  .c_str()));
+            break;
+        }
+        case arrow::Type::DECIMAL64:
+        {
+            oDict.Add(osKey,
+                      CPLAtof(static_cast<const arrow::Decimal64Array *>(array)
+                                  ->FormatValue(nIdx)
+                                  .c_str()));
+            break;
+        }
+#endif
         case arrow::Type::DECIMAL128:
         {
             oDict.Add(osKey,
@@ -1757,6 +1805,48 @@ static void ReadList(OGRFeature *poFeature, int i, int64_t nIdxInArray,
             break;
         }
 
+#if ARROW_VERSION_MAJOR >= 18
+        case arrow::Type::DECIMAL32:
+        {
+            const auto values = std::static_pointer_cast<arrow::Decimal32Array>(
+                array->values());
+            const auto nIdxStart = array->value_offset(nIdxInArray);
+            const int nCount = array->value_length(nIdxInArray);
+            std::vector<double> aValues;
+            aValues.reserve(nCount);
+            for (int k = 0; k < nCount; k++)
+            {
+                if (values->IsNull(nIdxStart + k))
+                    aValues.push_back(std::numeric_limits<double>::quiet_NaN());
+                else
+                    aValues.push_back(
+                        CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
+            }
+            poFeature->SetField(i, nCount, aValues.data());
+            break;
+        }
+
+        case arrow::Type::DECIMAL64:
+        {
+            const auto values = std::static_pointer_cast<arrow::Decimal64Array>(
+                array->values());
+            const auto nIdxStart = array->value_offset(nIdxInArray);
+            const int nCount = array->value_length(nIdxInArray);
+            std::vector<double> aValues;
+            aValues.reserve(nCount);
+            for (int k = 0; k < nCount; k++)
+            {
+                if (values->IsNull(nIdxStart + k))
+                    aValues.push_back(std::numeric_limits<double>::quiet_NaN());
+                else
+                    aValues.push_back(
+                        CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
+            }
+            poFeature->SetField(i, nCount, aValues.data());
+            break;
+        }
+#endif
+
         case arrow::Type::DECIMAL128:
         {
             const auto values =
@@ -2406,6 +2496,26 @@ inline OGRFeature *OGRArrowLayer::ReadFeature(
                 break;
             }
 
+#if ARROW_VERSION_MAJOR >= 18
+            case arrow::Type::DECIMAL32:
+            {
+                const auto castArray =
+                    static_cast<const arrow::Decimal32Array *>(array);
+                poFeature->SetField(
+                    i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
+                break;
+            }
+
+            case arrow::Type::DECIMAL64:
+            {
+                const auto castArray =
+                    static_cast<const arrow::Decimal64Array *>(array);
+                poFeature->SetField(
+                    i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
+                break;
+            }
+#endif
+
             case arrow::Type::DECIMAL128:
             {
                 const auto castArray =
@@ -3900,6 +4010,34 @@ inline bool OGRArrowLayer::SkipToNextFeatureDueToAttributeFilter() const
                 break;
             }
 
+#if ARROW_VERSION_MAJOR >= 18
+            case arrow::Type::DECIMAL32:
+            {
+                const auto castArray =
+                    static_cast<const arrow::Decimal32Array *>(array);
+                if (!ConstraintEvaluator(
+                        constraint,
+                        CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
+                {
+                    return true;
+                }
+                break;
+            }
+
+            case arrow::Type::DECIMAL64:
+            {
+                const auto castArray =
+                    static_cast<const arrow::Decimal64Array *>(array);
+                if (!ConstraintEvaluator(
+                        constraint,
+                        CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
+                {
+                    return true;
+                }
+                break;
+            }
+#endif
+
             case arrow::Type::DECIMAL128:
             {
                 const auto castArray =

ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp

@@ -184,7 +184,19 @@ inline void OGRArrowWriterLayer::CreateSchemaCommon()
             {
                 const int nPrecision = poFieldDefn->GetPrecision();
                 if (nWidth != 0 && nPrecision != 0)
-                    dt = arrow::decimal(nWidth, nPrecision);
+                {
+                    // Since arrow 18.0, we could use arrow::smallest_decimal()
+                    // to return the smallest representation (i.e. possibly
+                    // decimal32 and decimal64). But for now keep decimal128
+                    // as the minimum for backwards compatibility.
+                    // GetValueDecimal() and other functions in
+                    // ogrlayerarrow.cpp would have to be adapted for decimal32
+                    // and decimal64 compatibility.
+                    if (nWidth > 38)
+                        dt = arrow::decimal256(nWidth, nPrecision);
+                    else
+                        dt = arrow::decimal128(nWidth, nPrecision);
+                }
                 else if (eSubDT == OFSTFloat32)
                     dt = arrow::float32();
                 else

ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp

@@ -1427,6 +1427,10 @@ bool OGRParquetLayer::ReadNextBatch()
 {
     m_nIdxInBatch = 0;
 
+    const int nNumGroups = m_poArrowReader->num_row_groups();
+    if (nNumGroups == 0)
+        return false;
+
     if (m_bSingleBatch)
     {
         CPLAssert(m_iRecordBatch == 0);
@@ -1468,7 +1472,6 @@ bool OGRParquetLayer::ReadNextBatch()
         }
         else
         {
-            const int nNumGroups = m_poArrowReader->num_row_groups();
             OGRField sMin;
             OGRField sMax;
             OGR_RawField_SetNull(&sMin);

ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp

@@ -1372,6 +1372,7 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
 #endif
 
     bool bPageSizeFound = false;
+    bool bSecureDeleteFound = false;
 
     const char *pszSqlitePragma =
         CPLGetConfigOption("OGR_SQLITE_PRAGMA", nullptr);
@@ -1456,7 +1457,7 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
             {
                 if (STARTS_WITH_CI(papszTokens[i], "PAGE_SIZE"))
                     bPageSizeFound = true;
-                if (STARTS_WITH_CI(papszTokens[i], "JOURNAL_MODE"))
+                else if (STARTS_WITH_CI(papszTokens[i], "JOURNAL_MODE"))
                 {
                     const char *pszEqual = strchr(papszTokens[i], '=');
                     if (pszEqual)
@@ -1467,6 +1468,8 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
                         continue;
                     }
                 }
+                else if (STARTS_WITH_CI(papszTokens[i], "SECURE_DELETE"))
+                    bSecureDeleteFound = true;
 
                 const char *pszSQL = CPLSPrintf("PRAGMA %s", papszTokens[i]);
 
@@ -1646,6 +1649,16 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
             sqlite3_exec(hDB, pszSQL, nullptr, nullptr, nullptr));
     }
 
+    if (!bSecureDeleteFound)
+    {
+        // Turn on secure_delete by default (unless the user specifies a
+        // value of this pragma through OGR_SQLITE_PRAGMA)
+        // For example, Debian and Conda-Forge SQLite3 builds already turn on
+        // secure_delete.
+        CPL_IGNORE_RET_VAL(sqlite3_exec(hDB, "PRAGMA secure_delete = 1",
+                                        nullptr, nullptr, nullptr));
+    }
+
     SetCacheSize();
     SetSynchronous();
     if (bLoadExtensions)