Support hll function: hll_empty(), hll_hash(), hll_cardinality() (apa…

…che#36)
HappenLee · Jul 13, 2021 · c3d41d7 · c3d41d7
1 parent b876a55
commit c3d41d7
Show file tree

Hide file tree

Showing 16 changed files with 463 additions and 16 deletions.
diff --git a/be/src/exprs/hll_function.cpp b/be/src/exprs/hll_function.cpp
@@ -29,6 +29,10 @@ using doris_udf::StringVal;
 void HllFunctions::init() {}
 
 StringVal HllFunctions::hll_hash(FunctionContext* ctx, const StringVal& input) {
+    return AnyValUtil::from_string_temp(ctx, hll_hash(input));
+}
+
+std::string HllFunctions::hll_hash(const StringVal& input) {
     HyperLogLog hll;
     if (!input.is_null) {
         uint64_t hash_value = HashUtil::murmur_hash64A(input.ptr, input.len, HashUtil::MURMUR_SEED);
@@ -37,14 +41,16 @@ StringVal HllFunctions::hll_hash(FunctionContext* ctx, const StringVal& input) {
     std::string buf;
     buf.resize(hll.max_serialized_size());
     buf.resize(hll.serialize((uint8_t*)buf.c_str()));
-    return AnyValUtil::from_string_temp(ctx, buf);
+
+    return buf;
 }
 
 void HllFunctions::hll_init(FunctionContext*, StringVal* dst) {
     dst->is_null = false;
     dst->len = sizeof(HyperLogLog);
     dst->ptr = (uint8_t*)new HyperLogLog();
 }
+
 StringVal HllFunctions::hll_empty(FunctionContext* ctx) {
     return AnyValUtil::from_string_temp(ctx, HyperLogLog::empty());
 }

diff --git a/be/src/exprs/hll_function.h b/be/src/exprs/hll_function.h
@@ -18,6 +18,7 @@
 #ifndef DORIS_BE_SRC_QUERY_EXPRS_HLL_FUNCTION_H
 #define DORIS_BE_SRC_QUERY_EXPRS_HLL_FUNCTION_H
 
+#include <string>
 #include "udf/udf.h"
 
 namespace doris {
@@ -26,6 +27,8 @@ class HllFunctions {
 public:
     static void init();
     static StringVal hll_hash(FunctionContext* ctx, const StringVal& dest_base);
+    static std::string hll_hash(const StringVal& dest_base);
+
     static StringVal hll_empty(FunctionContext* ctx);
     static void hll_init(FunctionContext*, StringVal* dst);
 

diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
@@ -80,6 +80,9 @@ set(VEC_FILES
   functions/function_helpers.cpp
   functions/functions_logical.cpp
   functions/function_cast.cpp
+  functions/hll_cardinality.cpp
+  functions/hll_empty.cpp
+  functions/hll_hash.cpp
   functions/plus.cpp
   functions/modulo.cpp
   functions/multiply.cpp

diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp
@@ -46,6 +46,7 @@ class DataTypeFactory {
             instance.regist_data_type("Int64", DataTypePtr(std::make_shared<DataTypeInt64>()));
             instance.regist_data_type("Float32", DataTypePtr(std::make_shared<DataTypeFloat32>()));
             instance.regist_data_type("Float64", DataTypePtr(std::make_shared<DataTypeFloat64>()));
+            instance.regist_data_type("String", DataTypePtr(std::make_shared<DataTypeString>()));
         });
         return instance;
     }

diff --git a/be/src/vec/functions/abs.cpp b/be/src/vec/functions/abs.cpp
@@ -47,6 +47,7 @@ struct AbsImpl {
 struct NameAbs {
     static constexpr auto name = "abs";
 };
+
 using FunctionAbs = FunctionUnaryArithmetic<AbsImpl, NameAbs, false>;
 
 //template <> struct FunctionUnaryArithmeticMonotonicity<NameAbs>

diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
@@ -953,8 +953,8 @@ class FunctionCast final : public IFunctionBase {
         if (callOnIndexAndDataType<void>(to_type->getTypeId(), make_default_wrapper)) return ret;
 
         switch (to_type->getTypeId()) {
-        // case TypeIndex::String:
-        //     return createStringWrapper(from_type);
+//         case TypeIndex::String:
+//             return createStringWrapper(from_type);
         // case TypeIndex::FixedString:
         //     return createFixedStringWrapper(from_type, checkAndGetDataType<DataTypeFixedString>(to_type.get())->getN());
 

diff --git a/be/src/vec/functions/function_const.h b/be/src/vec/functions/function_const.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include "vec/data_types/data_types_number.h"
+#include "vec/columns/columns_number.h"
+#include "vec/functions/function.h"
+
+
+namespace doris::vectorized {
+
+template<typename Impl>
+class FunctionConst : public IFunction {
+public:
+    static constexpr auto name = Impl::name;
+
+    static FunctionPtr create() { return std::make_shared<FunctionConst>(); }
+
+private:
+    String getName() const override { return name; }
+
+    size_t getNumberOfArguments() const override { return 0; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override {
+        return Impl::get_return_type();
+    }
+
+    void executeImpl(Block &block, const ColumnNumbers &, size_t result, size_t input_rows_count) override {
+        block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(input_rows_count,
+                                                                                                 Impl::init_value());
+    }
+};
+
+}
diff --git a/be/src/vec/functions/function_string_or_array_to_t.h b/be/src/vec/functions/function_string_or_array_to_t.h
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_types_number.h"
+#include "vec/functions/function.h"
+#include "vec/functions/function_helpers.h"
+#include "vec/columns/column_vector.h"
+#include "vec/columns/column_string.h"
+//#include<vec/columns/column_fixed_string.h>
+//#include<vec/columns/column_array.h>
+
+namespace doris::vectorized
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_COLUMN;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+
+template <typename Impl, typename Name, typename ResultType>
+class FunctionStringOrArrayToT : public IFunction
+{
+public:
+    static constexpr auto name = Name::name;
+//    static FunctionPtr create(const Context &)
+    static FunctionPtr create() {
+        return std::make_shared<FunctionStringOrArrayToT>();
+    }
+
+    String getName() const override
+    {
+        return name;
+    }
+
+    size_t getNumberOfArguments() const override
+    {
+        return 1;
+    }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (!isStringOrFixedString(arguments[0])
+            && !isArray(arguments[0]))
+            throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return std::make_shared<DataTypeNumber<ResultType>>();
+    }
+
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
+    {
+        const ColumnPtr column = block.getByPosition(arguments[0]).column;
+        if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
+        {
+            auto col_res = ColumnVector<ResultType>::create();
+
+            typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
+            vec_res.resize(col->size());
+            Impl::vector(col->getChars(), col->getOffsets(), vec_res);
+
+            block.getByPosition(result).column = std::move(col_res);
+        }
+//        else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
+//        {
+//            if (Impl::is_fixed_to_constant)
+//            {
+//                ResultType res = 0;
+//                Impl::vector_fixed_to_constant(col_fixed->getChars(), col_fixed->getN(), res);
+//
+//                block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(col_fixed->size(), toField(res));
+//            }
+//            else
+//            {
+//                auto col_res = ColumnVector<ResultType>::create();
+//
+//                typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
+//                vec_res.resize(col_fixed->size());
+//                Impl::vector_fixed_to_vector(col_fixed->getChars(), col_fixed->getN(), vec_res);
+//
+//                block.getByPosition(result).column = std::move(col_res);
+//            }
+//        }
+//        else if (const ColumnArray * col_arr = checkAndGetColumn<ColumnArray>(column.get()))
+//        {
+//            auto col_res = ColumnVector<ResultType>::create();
+//
+//            typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
+//            vec_res.resize(col_arr->size());
+//            Impl::array(col_arr->getOffsets(), vec_res);
+//
+//            block.getByPosition(result).column = std::move(col_res);
+//        }
+        else
+            throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of argument of function " + getName(),
+                ErrorCodes::ILLEGAL_COLUMN);
+    }
+};
+
+}
diff --git a/be/src/vec/functions/function_string_to_string.h b/be/src/vec/functions/function_string_to_string.h
@@ -0,0 +1,97 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_types_number.h"
+#include "vec/functions/function.h"
+#include "vec/functions/function_helpers.h"
+#include "vec/columns/column_vector.h"
+#include "vec/columns/column_string.h"
+//#include<vec/columns/column_fixed_string.h>
+//#include<vec/columns/column_array.h>
+
+namespace doris::vectorized
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_COLUMN;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+
+template <typename Impl, typename Name, bool is_injective = false>
+class FunctionStringToString : public IFunction
+{
+public:
+    static constexpr auto name = Name::name;
+//    static FunctionPtr create(const Context &)
+    static FunctionPtr create()
+    {
+        return std::make_shared<FunctionStringToString>();
+    }
+
+    String getName() const override
+    {
+        return name;
+    }
+
+    size_t getNumberOfArguments() const override
+    {
+        return 1;
+    }
+
+    bool isInjective(const Block &) override
+    {
+        return is_injective;
+    }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (!isStringOrFixedString(arguments[0]))
+            throw Exception(
+                "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        return arguments[0];
+    }
+
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
+    {
+        const ColumnPtr column = block.getByPosition(arguments[0]).column;
+        if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
+        {
+            auto col_res = ColumnString::create();
+            Impl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
+            block.getByPosition(result).column = std::move(col_res);
+        }
+//        else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
+//        {
+//            auto col_res = ColumnFixedString::create(col_fixed->getN());
+//            Impl::vector_fixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars());
+//            block.getByPosition(result).column = std::move(col_res);
+//        }
+        else
+            throw Exception(
+                "Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of argument of function " + getName(),
+                ErrorCodes::ILLEGAL_COLUMN);
+    }
+};
+
+}
diff --git a/be/src/vec/functions/hll_cardinality.cpp b/be/src/vec/functions/hll_cardinality.cpp
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//#include <Functions/FunctionFactory.h>
+#include "exprs/hll_function.h"
+#include "udf/udf.h"
+
+#include "vec/data_types/number_traits.h"
+#include "vec/functions/function_string_or_array_to_t.h"
+#include "vec/functions/simple_function_factory.h"
+
+namespace doris::vectorized {
+
+struct HLLCardinalityImpl {
+    static void vector(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, PaddedPODArray<Int64>& res) {
+        auto size = res.size();
+        for (int i = 0; i < size; ++i) {
+            auto val = HllFunctions::hll_cardinality(nullptr,
+                    StringVal((uint8_t*)&data[offsets[i - 1]], offsets[i] - offsets[i - 1] - 1));
+            res[i] = val.val;
+        }
+    }
+};
+
+struct NameHLLCardinality {
+    static constexpr auto name = "hll_cardinality";
+};
+
+using FunctionHLLCardinality = FunctionStringOrArrayToT<HLLCardinalityImpl, NameHLLCardinality, Int64>;
+
+void registerFunctionHLLCardinality(SimpleFunctionFactory& factory) {
+    factory.registerFunction<FunctionHLLCardinality>();
+}
+
+}