PARQUET-1463: [C++] Utilize common hashing machinery for dictionary e…

…ncoding
apache · Nov 26, 2018 · f909f0a · f909f0a
1 parent 54b0af8
commit f909f0a
Show file tree

Hide file tree

Showing 4 changed files with 98 additions and 223 deletions.
diff --git a/cpp/src/arrow/util/hashing-test.cc b/cpp/src/arrow/util/hashing-test.cc
@@ -370,6 +370,14 @@ TEST(BinaryMemoTable, Basics) {
     table.CopyValues(4 /* start offset */, reinterpret_cast<uint8_t*>(&values[0]));
     ASSERT_EQ(values, expected_values);
   }
+  {
+    std::vector<std::string> expected({B, C, D, E, F});
+    std::vector<std::string> actual;
+    table.VisitValues(1 /* start offset */, [&](const util::string_view& v) {
+      actual.emplace_back(v.data(), v.length());
+    });
+    ASSERT_EQ(actual, expected);
+  }
 }
 
 TEST(BinaryMemoTable, Stress) {

diff --git a/cpp/src/arrow/util/hashing.h b/cpp/src/arrow/util/hashing.h
@@ -34,6 +34,7 @@
 
 #include "arrow/array.h"
 #include "arrow/buffer.h"
+#include "arrow/builder.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
@@ -605,6 +606,17 @@ class BinaryMemoTable {
     CopyValues(0, out_size, out_data);
   }
 
+  // Visit the stored values in insertion order.
+  // The visitor function should have the signature `void(util::string_view)`
+  // or `void(const util::string_view&)`.
+  template <typename VisitFunc>
+  void VisitValues(int32_t start, VisitFunc&& visit) const {
+    for (uint32_t i = start; i < offsets_.size() - 1; ++i) {
+      visit(
+          util::string_view(values_.data() + offsets_[i], offsets_[i + 1] - offsets_[i]));
+    }
+  }
+
  protected:
   struct Payload {
     int32_t memo_index;

diff --git a/cpp/src/parquet/encoding-benchmark.cc b/cpp/src/parquet/encoding-benchmark.cc
@@ -110,8 +110,7 @@ static void DecodeDict(std::vector<typename Type::c_type>& values,
 
   DictEncoder<Type> encoder(descr.get(), &pool, allocator);
   for (int i = 0; i < num_values; ++i) {
-    // No SSE
-    encoder.template Put<false>(values[i]);
+    encoder.Put(values[i]);
   }
 
   std::shared_ptr<ResizableBuffer> dict_buffer =