reduce memory allocation for filter when result_size_hint equals -1

Signed-off-by: gengliqi <[email protected]>
pingcap · Feb 7, 2023 · 6c8d54a · 6c8d54a
1 parent f939582
commit 6c8d54a
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 14 deletions.
diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp
@@ -305,8 +305,10 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
     auto res = this->create(0, scale);
     Container & res_data = res->getData();
 
-    if (result_size_hint)
-        res_data.reserve(result_size_hint > 0 ? result_size_hint : size);
+    if (result_size_hint < 0)
+        res_data.reserve(countBytesInFilter(filt));
+    else if (result_size_hint > 0)
+        res_data.reserve(result_size_hint);
 
     const UInt8 * filt_pos = filt.data();
     const UInt8 * filt_end = filt_pos + size;

diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include <Columns/ColumnFixedString.h>
+#include <Columns/ColumnsCommon.h>
 #include <Common/Arena.h>
 #include <Common/HashTable/Hash.h>
 #include <Common/SipHash.h>
@@ -204,8 +205,10 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
 
     auto res = ColumnFixedString::create(n);
 
-    if (result_size_hint)
-        res->chars.reserve(result_size_hint > 0 ? result_size_hint * n : chars.size());
+    if (result_size_hint < 0)
+        res->chars.reserve(countBytesInFilter(filt) * n);
+    else if (result_size_hint > 0)
+        res->chars.reserve(result_size_hint * n);
 
     const UInt8 * filt_pos = &filt[0];
     const UInt8 * filt_end = filt_pos + col_size;

diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include <Columns/ColumnVector.h>
+#include <Columns/ColumnsCommon.h>
 #include <Common/Arena.h>
 #include <Common/Exception.h>
 #include <Common/HashTable/Hash.h>
@@ -212,8 +213,10 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
     auto res = this->create();
     Container & res_data = res->getData();
 
-    if (result_size_hint)
-        res_data.reserve(result_size_hint > 0 ? result_size_hint : size);
+    if (result_size_hint < 0)
+        res_data.reserve(countBytesInFilter(filt));
+    else if (result_size_hint > 0)
+        res_data.reserve(result_size_hint);
 
     const UInt8 * filt_pos = &filt[0];
     const UInt8 * filt_end = filt_pos + size;

diff --git a/dbms/src/Columns/ColumnsCommon.cpp b/dbms/src/Columns/ColumnsCommon.cpp
@@ -122,9 +122,9 @@ struct ResultOffsetsBuilder
         : res_offsets(*res_offsets_)
     {}
 
-    void reserve(ssize_t result_size_hint, size_t src_size)
+    void reserve(size_t result_size_hint)
     {
-        res_offsets.reserve(result_size_hint > 0 ? result_size_hint : src_size);
+        res_offsets.reserve(result_size_hint);
     }
 
     void insertOne(size_t array_size)
@@ -165,7 +165,7 @@ struct ResultOffsetsBuilder
 struct NoResultOffsetsBuilder
 {
     explicit NoResultOffsetsBuilder(IColumn::Offsets *) {}
-    void reserve(ssize_t, size_t) {}
+    void reserve(size_t) {}
     void insertOne(size_t) {}
 
     template <size_t SIMD_BYTES>
@@ -196,11 +196,12 @@ void filterArraysImplGeneric(
 
     if (result_size_hint)
     {
-        result_offsets_builder.reserve(result_size_hint, size);
-
         if (result_size_hint < 0)
-            res_elems.reserve(src_elems.size());
-        else if (result_size_hint < 1000000000 && src_elems.size() < 1000000000) /// Avoid overflow.
+            result_size_hint = countBytesInFilter(filt);
+
+        result_offsets_builder.reserve(result_size_hint);
+
+        if (result_size_hint < 1000000000 && src_elems.size() < 1000000000) /// Avoid overflow.
             res_elems.reserve((result_size_hint * src_elems.size() + size - 1) / size);
     }
 

diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h
@@ -216,7 +216,7 @@ class IColumn : public COWPtr<IColumn>
       * Is used in WHERE and HAVING operations.
       * If result_size_hint > 0, then makes advance reserve(result_size_hint) for the result column;
       *  if 0, then don't makes reserve(),
-      *  otherwise (i.e. < 0), makes reserve() using size of source column.
+      *  otherwise (i.e. < 0), makes reserve() using size of column after being filtered.
       */
     using Filter = PaddedPODArray<UInt8>;
     virtual Ptr filter(const Filter & filt, ssize_t result_size_hint) const = 0;