Updated API for compatibility with latest rocksdb

.github/workflows/debian.yml

@@ -26,7 +26,7 @@ jobs:
         run: |
           apt-get update
           apt-get install --no-install-recommends -y \
-              build-essential librocksdb-dev cython3 \
+              build-essential librocksdb-dev cython3 python3-pkgconfig \
               python3-dev python3-pip python3-pytest \
               libsnappy-dev libbz2-dev liblz4-dev libz-dev
 

.github/workflows/dist.yml

@@ -49,7 +49,7 @@ jobs:
           # Avoid re-building the C library in every iteration by testing for
           # the build directory.
           CIBW_BEFORE_BUILD: >
-            yum install -y bzip2-devel lz4-devel snappy-devel zlib-devel
+            yum install -y python3-pkgconfig bzip2-devel lz4-devel snappy-devel zlib-devel
             python3-Cython &&
             test -d ${{ env.LIBROCKSDB_PATH }} || (
             git clone https://github.com/facebook/rocksdb --depth 1

docs/api/options.rst

@@ -504,57 +504,6 @@ Options objects
         | *Type:* ``bool``
         | *Default:* ``False``
 
-    .. py:attribute:: soft_rate_limit
-
-        .. IMPORTANT::
-
-            NOT SUPPORTED ANYMORE -- this option is no longer used
-
-        Puts are delayed 0-1 ms when any level has a compaction score that exceeds
-        soft_rate_limit. This is ignored when == 0.0.
-        CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
-        hold, RocksDB will set soft_rate_limit = hard_rate_limit.
-        A value of ``0`` means disabled.
-
-        | *Type:* ``float``
-        | *Default:* ``0``
-
-    .. py:attribute:: hard_rate_limit
-
-        .. IMPORTANT::
-
-            NOT SUPPORTED ANYMORE -- this option is no longer used
-
-        Puts are delayed 1ms at a time when any level has a compaction score that
-        exceeds hard_rate_limit. This is ignored when <= 1.0.
-        A value fo ``0`` means disabled.
-
-        | *Type:* ``float``
-        | *Default:* ``0``
-
-    .. py:attribute:: rate_limit_delay_max_milliseconds
-
-        .. IMPORTANT::
-
-            NOT SUPPORTED ANYMORE -- this option is no longer used
-
-        Max time a put will be stalled when hard_rate_limit is enforced. If 0, then
-        there is no limit.
-
-        | *Type:* ``int``
-        | *Default:* ``1000``
-
-    .. py:attribute:: purge_redundant_kvs_while_flush
-
-        .. IMPORTANT::
-
-            NOT SUPPORTED ANYMORE -- this option is no longer used
-
-        Purge duplicate/deleted keys when a memtable is flushed to storage.
-
-        | *Type:* ``bool``
-        | *Default:* ``True``
-
 
 
 .. py:class:: rocksdb.Options
@@ -819,14 +768,6 @@ Options objects
         | *Type:* ``bool``
         | *Default:* ``True``
 
-    .. py:attribute:: skip_log_error_on_recovery
-
-        Skip log corruption error on recovery
-        (If client is ok with losing most recent changes)
-
-        | *Type:* ``bool``
-        | *Default:* ``False``
-
     .. py:attribute:: stats_dump_period_sec
 
         If not zero, dump rocksdb.stats to LOG every stats_dump_period_sec

pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools", "wheel", "cython"]
+requires = ["setuptools", "wheel", "cython", "pkgconfig"]
 build-backend = "setuptools.build_meta"

rocksdb/_rocksdb.pyx

@@ -1112,24 +1112,6 @@ cdef class ColumnFamilyOptions(object):
         def __set__(self, value):
             self.copts.max_bytes_for_level_multiplier_additional = value
 
-    property soft_rate_limit:
-        def __get__(self):
-            return self.copts.soft_rate_limit
-        def __set__(self, value):
-            self.copts.soft_rate_limit = value
-
-    property hard_rate_limit:
-        def __get__(self):
-            return self.copts.hard_rate_limit
-        def __set__(self, value):
-            self.copts.hard_rate_limit = value
-
-    property rate_limit_delay_max_milliseconds:
-        def __get__(self):
-            return self.copts.rate_limit_delay_max_milliseconds
-        def __set__(self, value):
-            self.copts.rate_limit_delay_max_milliseconds = value
-
     property arena_block_size:
         def __get__(self):
             return self.copts.arena_block_size
@@ -1142,12 +1124,6 @@ cdef class ColumnFamilyOptions(object):
         def __set__(self, value):
             self.copts.disable_auto_compactions = value
 
-    property purge_redundant_kvs_while_flush:
-        def __get__(self):
-            return self.copts.purge_redundant_kvs_while_flush
-        def __set__(self, value):
-            self.copts.purge_redundant_kvs_while_flush = value
-
     # FIXME: remove to util/options_helper.h
     #  property allow_os_buffer:
         #  def __get__(self):
@@ -1531,12 +1507,6 @@ cdef class Options(ColumnFamilyOptions):
         def __set__(self, value):
             self.opts.is_fd_close_on_exec = value
 
-    property skip_log_error_on_recovery:
-        def __get__(self):
-            return self.opts.skip_log_error_on_recovery
-        def __set__(self, value):
-            self.opts.skip_log_error_on_recovery = value
-
     property stats_dump_period_sec:
         def __get__(self):
             return self.opts.stats_dump_period_sec

rocksdb/backup.pxd

@@ -9,9 +9,11 @@ from .status cimport Status
 from .db cimport DB
 from .env cimport Env
 
+# TODO: For rocksdb >= 6.21.0, change to `rocksdb/utilities/backup_engine.h`.
 cdef extern from "rocksdb/utilities/backupable_db.h" namespace "rocksdb":
     ctypedef uint32_t BackupID
 
+    # TODO: For rocksdb >= 6.21.0, rename to `BackupEngineOptions`.
     cdef cppclass BackupableDBOptions:
         BackupableDBOptions(const string& backup_dir)
 
@@ -29,6 +31,7 @@ cdef extern from "rocksdb/utilities/backupable_db.h" namespace "rocksdb":
         Status RestoreDBFromBackup(BackupID, string&, string&) nogil except+
         Status RestoreDBFromLatestBackup(string&, string&) nogil except+
 
+    # TODO: For rocksdb >= 6.21.0, swap order of first two parameters.
     cdef Status BackupEngine_Open "rocksdb::BackupEngine::Open"(
             Env*,
             BackupableDBOptions&,

rocksdb/options.pxd

@@ -134,7 +134,6 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb":
         cpp_bool use_direct_io_for_flush_and_compaction
         cpp_bool allow_fallocate
         cpp_bool is_fd_close_on_exec
-        cpp_bool skip_log_error_on_recovery
         unsigned int stats_dump_period_sec
         unsigned int stats_persist_period_sec
         cpp_bool persist_stats_to_disk

rocksdb/tests/test_options.py

@@ -1,7 +1,9 @@
-import unittest
 import sys
+import unittest
+
 import rocksdb
 
+
 class TestFilterPolicy(rocksdb.interfaces.FilterPolicy):
     def create_filter(self, keys):
         return b'nix'
@@ -230,7 +232,6 @@ def test_rocksdb_options(self):
                 ('use_direct_io_for_flush_and_compaction', False, True),
                 ('allow_fallocate', True, False),
                 ('is_fd_close_on_exec', True, False),
-                ('skip_log_error_on_recovery', False, True),
                 ('stats_dump_period_sec', 600, 3600),
                 ('stats_persist_period_sec', 600, 3600),
                 ('persist_stats_to_disk', False, True),

setup.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
 
+import os
 import platform
-import setuptools
 import sys
 
+import pkgconfig
+from Cython.Build import cythonize
+from setuptools import Extension, setup
 
 extra_compile_args = [
     '-std=c++11',
@@ -19,24 +22,30 @@
     extra_compile_args += ['-mmacosx-version-min=10.7', '-stdlib=libc++']
 
 if sys.version_info < (3 , 0):
-    raise Exception("python-rocksdb requires Python 3.x")
-
-setuptools.setup(
-    ext_modules=[
-        setuptools.Extension(
-            'rocksdb._rocksdb',
-            [
-                'rocksdb/_rocksdb.pyx',
-            ],
-            extra_compile_args=extra_compile_args,
-            language='c++',
-            libraries=[
-                'rocksdb',
-                'snappy',
-                'bz2',
-                'z',
-                'lz4',
-            ],
-        ),
+    raise Exception('python-rocksdb requires Python 3.x')
+
+try:
+    ext_args = pkgconfig.parse('rocksdb')
+except pkgconfig.PackageNotFoundError:
+    include_path = os.environ.get('INCLUDE_PATH')
+    library_path = os.environ.get('LIBRARY_PATH')
+
+    ext_args = {
+        'include_dirs': include_path.split(os.pathsep) if include_path else [],
+        'library_dirs': library_path.split(os.pathsep) if library_path else [],
+        'libraries': ['rocksdb', 'snappy', 'bz2', 'z', 'lz4'],
+    }
+
+rocksdb_extension = Extension(
+    'rocksdb._rocksdb',
+    [
+        'rocksdb/_rocksdb.pyx',
     ],
+    extra_compile_args=extra_compile_args,
+    language='c++',
+    **ext_args,
+)
+
+setup(
+    ext_modules=cythonize([rocksdb_extension]),
 )