support date and datetime. (apache#35)

* 1. support date and datetime. 2. remove some not used file. * remove time zone
stdpain · Jul 1, 2021 · 6aa454c · 6aa454c
1 parent b64fe1d
commit 6aa454c
Show file tree

Hide file tree

Showing 70 changed files with 549 additions and 6,141 deletions.
diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h
@@ -272,13 +272,15 @@ class DateTimeValue {
     int hour() const { return _hour; }
     int minute() const { return _minute; }
     int second() const { return _second; }
+    int microsecond() const { return _microsecond; }
+    int neg() const { return _neg; }
 
     bool check_loss_accuracy_cast_to_date() {
         auto loss_accuracy = _hour != 0 || _minute != 0 || _second != 0 || _microsecond != 0;
         cast_to_date();
         return loss_accuracy;
     }
-    
+
     void cast_to_date() {
         _hour = 0;
         _minute = 0;

diff --git a/be/src/runtime/row_batch.cpp b/be/src/runtime/row_batch.cpp
@@ -26,11 +26,10 @@
 #include "runtime/string_value.h"
 #include "runtime/tuple_row.h"
 //#include "runtime/mem_tracker.h"
-#include "vec/columns/column_vector.h"
-#include "vec/core/block.h"
-
 #include "gen_cpp/Data_types.h"
 #include "gen_cpp/data.pb.h"
+#include "vec/columns/column_vector.h"
+#include "vec/core/block.h"
 
 using std::vector;
 
@@ -553,11 +552,12 @@ vectorized::Block RowBatch::convert_to_vec_block() const {
             if (slot_desc->is_nullable() && tuple->is_null(slot_desc->null_indicator_offset())) {
                 columns[i]->insertData(nullptr, 0);
             } else if (slot_desc->type().is_string_type()) {
-                auto string_value = static_cast<const StringValue*>(tuple->get_slot(slot_desc->tuple_offset()));
+                auto string_value =
+                        static_cast<const StringValue*>(tuple->get_slot(slot_desc->tuple_offset()));
                 columns[i]->insertData(string_value->ptr, string_value->len);
             } else {
                 columns[i]->insertData(
-                        static_cast<const char *>(tuple->get_slot(slot_desc->tuple_offset())),
+                        static_cast<const char*>(tuple->get_slot(slot_desc->tuple_offset())),
                         slot_desc->slot_size());
             }
         }

diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
@@ -27,6 +27,8 @@
 #include "olap/hll.h"
 #include "runtime/primitive_type.h"
 #include "thrift/protocol/TDebugProtocol.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
 #include "vec/data_types/data_type_string.h"
 #include "vec/data_types/data_types_decimal.h"
 #include "vec/data_types/data_types_number.h"
@@ -176,6 +178,8 @@ struct TypeDescriptor {
 
     inline bool is_date_type() const { return type == TYPE_DATE || type == TYPE_DATETIME; }
 
+    inline bool is_datetime_type() const { return type == TYPE_DATETIME; }
+
     inline bool is_decimal_type() const { return (type == TYPE_DECIMAL || type == TYPE_DECIMALV2); }
 
     inline bool is_var_len_string_type() const {
@@ -297,10 +301,11 @@ struct TypeDescriptor {
             return std::make_shared<vectorized::DataTypeInt64>();
 
         case TYPE_LARGEINT:
+            return std::make_shared<vectorized::DataTypeInt128>();
         case TYPE_DATE:
+            return std::make_shared<vectorized::DataTypeDate>();
         case TYPE_DATETIME:
-            return std::make_shared<vectorized::DataTypeInt128>();
-
+            return std::make_shared<vectorized::DataTypeDateTime>();
         case TYPE_DOUBLE:
             return std::make_shared<vectorized::DataTypeFloat64>();
 

diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
@@ -56,6 +56,8 @@ set(VEC_FILES
   data_types/data_types_number.cpp
   data_types/get_least_supertype.cpp
   data_types/nested_utils.cpp
+  data_types/data_type_date.cpp
+  data_types/data_type_date_time.cpp
   exec/vaggregation_node.cpp
   exec/volap_scan_node.cpp
   exec/vsort_node.cpp

diff --git a/be/src/vec/columns/column_decimal.cpp b/be/src/vec/columns/column_decimal.cpp
@@ -15,20 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "vec/columns/column_decimal.h"
+
+#include "vec/columns/columns_common.h"
 #include "vec/common/arena.h"
 #include "vec/common/assert_cast.h"
 #include "vec/common/exception.h"
 #include "vec/common/sip_hash.h"
 #include "vec/common/unaligned.h"
 
-//#include <IO/WriteHelpers.h>
-
-#include "vec/columns/column_decimal.h"
-#include "vec/columns/columns_common.h"
-//#include <DataStreams/ColumnGathererStream.h>
-
 template <typename T>
-bool decimalLess(T x, T y, UInt32 x_scale, UInt32 y_scale);
+bool decimalLess(T x, T y, doris::vectorized::UInt32 x_scale, doris::vectorized::UInt32 y_scale);
 
 namespace doris::vectorized {
 
@@ -78,17 +75,15 @@ void ColumnDecimal<T>::updateHashWithValue(size_t n, SipHash& hash) const {
 }
 
 template <typename T>
-void ColumnDecimal<T>::getPermutation(bool reverse, size_t limit, int , IColumn::Permutation & res) const
-{
+void ColumnDecimal<T>::getPermutation(bool reverse, size_t limit, int,
+                                      IColumn::Permutation& res) const {
 #if 1 /// TODO: perf test
-    if (data.size() <= std::numeric_limits<UInt32>::max())
-    {
+    if (data.size() <= std::numeric_limits<UInt32>::max()) {
         PaddedPODArray<UInt32> tmp_res;
         permutation(reverse, limit, tmp_res);
 
         res.resize(tmp_res.size());
-        for (size_t i = 0; i < tmp_res.size(); ++i)
-            res[i] = tmp_res[i];
+        for (size_t i = 0; i < tmp_res.size(); ++i) res[i] = tmp_res[i];
         return;
     }
 #endif

diff --git a/be/src/vec/columns/column_string.cpp b/be/src/vec/columns/column_string.cpp
@@ -22,9 +22,6 @@
 #include "vec/common/arena.h"
 #include "vec/common/assert_cast.h"
 #include "vec/common/memcmp_small.h"
-#include "vec/core/defines.h"
-//#include <DataStreams/ColumnGathererStream.h>
-
 #include "vec/common/unaligned.h"
 
 namespace doris::vectorized {
@@ -233,25 +230,20 @@ struct ColumnString::less {
     }
 };
 
-void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res) const
-{
+void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_direction_hint*/,
+                                  Permutation& res) const {
     size_t s = offsets.size();
     res.resize(s);
-    for (size_t i = 0; i < s; ++i)
-        res[i] = i;
+    for (size_t i = 0; i < s; ++i) res[i] = i;
 
-    if (limit >= s)
-        limit = 0;
+    if (limit >= s) limit = 0;
 
-    if (limit)
-    {
+    if (limit) {
         if (reverse)
             std::partial_sort(res.begin(), res.begin() + limit, res.end(), less<false>(*this));
         else
             std::partial_sort(res.begin(), res.begin() + limit, res.end(), less<true>(*this));
-    }
-    else
-    {
+    } else {
         if (reverse)
             std::sort(res.begin(), res.end(), less<false>(*this));
         else

diff --git a/be/src/vec/columns/columns_number.h b/be/src/vec/columns/columns_number.h
@@ -34,6 +34,7 @@ using ColumnInt8 = ColumnVector<Int8>;
 using ColumnInt16 = ColumnVector<Int16>;
 using ColumnInt32 = ColumnVector<Int32>;
 using ColumnInt64 = ColumnVector<Int64>;
+using ColumnInt128 = ColumnVector<Int128>;
 
 using ColumnFloat32 = ColumnVector<Float32>;
 using ColumnFloat64 = ColumnVector<Float64>;

diff --git a/be/src/vec/common/aggregation_common.h b/be/src/vec/common/aggregation_common.h
@@ -1,19 +1,16 @@
 #pragma once
 
-#include <vec/columns/column.h>
-#include <vec/columns/columns_number.h>
-#include <vec/common/arena.h>
-#include <vec/common/assert_cast.h>
-#include <vec/common/hash_table/hash.h>
-#include <vec/common/memcpy_small.h>
-#include <vec/common/sip_hash.h>
-#include <vec/common/string_ref.h>
-#include <vec/common/uint128.h>
-#include <vec/core/defines.h>
-
 #include <array>
-// #include <vec/columns/columnFixedString.h>
-// #include <vec/columns/columnLowCardinality.h>
+
+#include "vec/columns/column.h"
+#include "vec/columns/columns_number.h"
+#include "vec/common/arena.h"
+#include "vec/common/assert_cast.h"
+#include "vec/common/hash_table/hash.h"
+#include "vec/common/memcpy_small.h"
+#include "vec/common/sip_hash.h"
+#include "vec/common/string_ref.h"
+#include "vec/common/uint128.h"
 
 template <>
 struct DefaultHash<StringRef> : public StringRefHash {};

diff --git a/be/src/vec/common/allocator.h b/be/src/vec/common/allocator.h
@@ -31,9 +31,6 @@
 #define ALLOCATOR_ASLR 1
 #endif
 
-//#include <pcg_random.hpp>
-// #include <vec/Common/thread_local_rng.h>
-
 #if !defined(__APPLE__) && !defined(__FreeBSD__)
 #include <malloc.h>
 #endif
@@ -48,13 +45,9 @@
 /// Thread sanitizer does not intercept mremap. The usage of mremap will lead to false positives.
 #define DISABLE_MREMAP 1
 #endif
-#include "vec/common/mremap.h"
-
-// #include <vec/Common/MemoryTracker.h>
-#include "vec/common/exception.h"
-// #include <vec/Common/formatReadable.h>
-
 #include "vec/common/allocator_fwd.h"
+#include "vec/common/exception.h"
+#include "vec/common/mremap.h"
 
 /// Required for older Darwin builds, that lack definition of MAP_ANONYMOUS
 #ifndef MAP_ANONYMOUS

diff --git a/be/src/vec/common/arena.h b/be/src/vec/common/arena.h
@@ -26,10 +26,9 @@
 #if __has_include(<sanitizer/asan_interface.h>)
 #include <sanitizer/asan_interface.h>
 #endif
+#include "vec/common/allocator.h"
 #include "vec/common/memcpy_small.h"
 #include "vec/core/defines.h"
-//#include <vec/Common/ProfileEvents.h>
-#include "vec/common/allocator.h"
 
 //namespace ProfileEvents
 //{

diff --git a/be/src/vec/common/args_to_config.h b/be/src/vec/common/args_to_config.h