fix: full_scan can't scan data completely in some occassions (#825)

apache · Oct 25, 2021 · 90fea54 · 90fea54
1 parent 8232fac
commit 90fea54
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 15 deletions.
diff --git a/src/base/rrdb_types.cpp b/src/base/rrdb_types.cpp
diff --git a/src/client_lib/pegasus_client_impl.cpp b/src/client_lib/pegasus_client_impl.cpp
@@ -1179,7 +1179,7 @@ int pegasus_client_impl::get_scanner(const std::string &hash_key,
     if (c < 0 || (c == 0 && o.start_inclusive && o.stop_inclusive)) {
         v.push_back(pegasus_key_hash(start));
     }
-    scanner = new pegasus_scanner_impl(_client, std::move(v), o, start, stop, false);
+    scanner = new pegasus_scanner_impl(_client, std::move(v), o, start, stop, false, false);
 
     return PERR_OK;
 }
@@ -1223,7 +1223,8 @@ void pegasus_client_impl::async_get_unordered_scanners(
                     std::vector<uint64_t> hash(s);
                     for (int j = 0; j < s; j++)
                         hash[j] = --count;
-                    scanners[i] = new pegasus_scanner_impl(_client, std::move(hash), options, true);
+                    scanners[i] =
+                        new pegasus_scanner_impl(_client, std::move(hash), options, true, true);
                 }
             }
         }

diff --git a/src/client_lib/pegasus_client_impl.h b/src/client_lib/pegasus_client_impl.h
@@ -266,13 +266,15 @@ class pegasus_client_impl : public pegasus_client
         pegasus_scanner_impl(::dsn::apps::rrdb_client *client,
                              std::vector<uint64_t> &&hash,
                              const scan_options &options,
-                             bool validate_partition_hash);
+                             bool validate_partition_hash,
+                             bool full_scan);
         pegasus_scanner_impl(::dsn::apps::rrdb_client *client,
                              std::vector<uint64_t> &&hash,
                              const scan_options &options,
                              const ::dsn::blob &start_key,
                              const ::dsn::blob &stop_key,
-                             bool validate_partition_hash);
+                             bool validate_partition_hash,
+                             bool full_scan);
 
     private:
         ::dsn::apps::rrdb_client *_client;
@@ -291,6 +293,7 @@ class pegasus_client_impl : public pegasus_client
         std::list<async_scan_next_callback_t> _queue;
         volatile bool _rpc_started;
         bool _validate_partition_hash;
+        bool _full_scan;
 
         void _async_next_internal();
         void _start_scan();

diff --git a/src/client_lib/pegasus_scanner_impl.cpp b/src/client_lib/pegasus_scanner_impl.cpp
@@ -29,8 +29,10 @@ namespace client {
 pegasus_client_impl::pegasus_scanner_impl::pegasus_scanner_impl(::dsn::apps::rrdb_client *client,
                                                                 std::vector<uint64_t> &&hash,
                                                                 const scan_options &options,
-                                                                bool validate_partition_hash)
-    : pegasus_scanner_impl(client, std::move(hash), options, _min, _max, validate_partition_hash)
+                                                                bool validate_partition_hash,
+                                                                bool full_scan)
+    : pegasus_scanner_impl(
+          client, std::move(hash), options, _min, _max, validate_partition_hash, full_scan)
 {
     _options.start_inclusive = true;
     _options.stop_inclusive = false;
@@ -41,7 +43,8 @@ pegasus_client_impl::pegasus_scanner_impl::pegasus_scanner_impl(::dsn::apps::rrd
                                                                 const scan_options &options,
                                                                 const ::dsn::blob &start_key,
                                                                 const ::dsn::blob &stop_key,
-                                                                bool validate_partition_hash)
+                                                                bool validate_partition_hash,
+                                                                bool full_scan)
     : _client(client),
       _start_key(start_key),
       _stop_key(stop_key),
@@ -50,7 +53,8 @@ pegasus_client_impl::pegasus_scanner_impl::pegasus_scanner_impl(::dsn::apps::rrd
       _p(-1),
       _context(SCAN_CONTEXT_ID_COMPLETED),
       _rpc_started(false),
-      _validate_partition_hash(validate_partition_hash)
+      _validate_partition_hash(validate_partition_hash),
+      _full_scan(full_scan)
 {
 }
 
@@ -225,6 +229,7 @@ void pegasus_client_impl::pegasus_scanner_impl::_start_scan()
     req.no_value = _options.no_value;
     req.__set_validate_partition_hash(_validate_partition_hash);
     req.__set_return_expire_ts(_options.return_expire_ts);
+    req.__set_full_scan(_full_scan);
 
     dassert(!_rpc_started, "");
     _rpc_started = true;

diff --git a/src/idl/rrdb.thrift b/src/idl/rrdb.thrift
@@ -255,6 +255,7 @@ struct get_scanner_request
     10:dsn.blob    sort_key_filter_pattern;
     11:optional bool    validate_partition_hash;
     12:optional bool    return_expire_ts;
+    13:optional bool full_scan; // true means client want to build 'full scan' context with the server side, false otherwise
 }
 
 struct scan_request

diff --git a/src/include/rrdb/rrdb_types.h b/src/include/rrdb/rrdb_types.h
diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp
@@ -945,7 +945,7 @@ void pegasus_server_impl::on_get_scanner(get_scanner_rpc rpc)
     if (_data_cf_opts.prefix_extractor) {
         ::dsn::blob start_hash_key, tmp;
         pegasus_restore_key(request.start_key, start_hash_key, tmp);
-        if (start_hash_key.size() == 0) {
+        if (start_hash_key.size() == 0 || request.full_scan) {
             // hash_key is not passed, only happened when do full scan (scanners got by
             // get_unordered_scanners) on a partition, we have to do total order seek on rocksDB.
             rd_opts.total_order_seek = true;