From 30158ad5a068208237f3c30e6c9eb60454bae402 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 30 Oct 2017 08:50:28 +0100 Subject: [PATCH 001/177] ARROW-1718: [C++/Python] Implement casts from timestamp to date32/64, properly handle NumPy datetime64[D] -> date32 This was sort of a can of worms. cc @xhochy @cpcloud @BryanCutler Author: Wes McKinney Closes #1258 from wesm/ARROW-1718 and squashes the following commits: c693889 [Wes McKinney] Use explicit static_cast to convert int64->int32 586face [Wes McKinney] Use correct syntax for int64 literals in MSVC. remove incorrect comments d61b7a4 [Wes McKinney] Downcast datetime64[D] as int64_t to int32_t e6f8b62 [Wes McKinney] Build with clang in Travis CI b187a09 [Wes McKinney] Remove now unneeded template specialization for Date32Type, but another test fails 284b9ba [Wes McKinney] Complete unit tests for timestamp->date32/64 6ca361f [Wes McKinney] tweak test case 68f3a32 [Wes McKinney] Test case, c++ unittest placeholder 383f730 [Wes McKinney] Implement builtin converter for date32, test datetime.date, ints, overflows d52bd77 [Wes McKinney] One failing test case --- .travis.yml | 4 +- cpp/src/arrow/compute/cast.cc | 99 +++++++++--- cpp/src/arrow/compute/compute-test.cc | 64 ++++++++ cpp/src/arrow/python/builtin_convert.cc | 27 +++- cpp/src/arrow/python/numpy_to_arrow.cc | 153 +++++++++---------- cpp/src/arrow/python/util/datetime.h | 5 + python/pyarrow/tests/test_convert_builtin.py | 19 +++ python/pyarrow/tests/test_convert_pandas.py | 11 ++ 8 files changed, 278 insertions(+), 104 deletions(-) diff --git a/.travis.yml b/.travis.yml index 039ae95208b74..6419548a622f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,12 +51,12 @@ matrix: os: linux group: deprecated before_script: - - export CC="gcc-4.9" - - export CXX="g++-4.9" - export ARROW_TRAVIS_USE_TOOLCHAIN=1 - export ARROW_TRAVIS_VALGRIND=1 - export ARROW_TRAVIS_PLASMA=1 - export ARROW_TRAVIS_CLANG_FORMAT=1 + - export CC="clang-4.0" + - export CXX="clang++-4.0" - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc index 68a2b12379e34..114ab9af0d0d0 100644 --- a/cpp/src/arrow/compute/cast.cc +++ b/cpp/src/arrow/compute/cast.cc @@ -69,11 +69,18 @@ namespace arrow { namespace compute { +constexpr int64_t kMillisecondsInDay = 86400000; + template -inline const T* GetValuesAs(const ArrayData& data, int i) { +inline const T* GetValues(const ArrayData& data, int i) { return reinterpret_cast(data.buffers[i]->data()) + data.offset; } +template +inline T* GetMutableValues(const ArrayData* data, int i) { + return reinterpret_cast(data->buffers[i]->mutable_data()) + data->offset; +} + namespace { void CopyData(const Array& input, ArrayData* output) { @@ -164,7 +171,7 @@ struct CastFunctorbuffers[1]->data(), in_data->offset, in_data->length); - auto out = reinterpret_cast(output->buffers[1]->mutable_data()); + auto out = GetMutableValues(output, 1); for (int64_t i = 0; i < input.length(); ++i) { *out++ = bit_reader.IsSet() ? kOne : kZero; bit_reader.Next(); @@ -214,8 +221,8 @@ struct CastFunctor::v using in_type = typename I::c_type; DCHECK_EQ(output->offset, 0); - const in_type* in_data = GetValuesAs(*input.data(), 1); - uint8_t* out_data = reinterpret_cast(output->buffers[1]->mutable_data()); + const in_type* in_data = GetValues(*input.data(), 1); + uint8_t* out_data = GetMutableValues(output, 1); for (int64_t i = 0; i < input.length(); ++i) { BitUtil::SetBitTo(out_data, i, (*in_data++) != 0); } @@ -233,8 +240,8 @@ struct CastFunctor(*input.data(), 1); - auto out_data = reinterpret_cast(output->buffers[1]->mutable_data()); + const in_type* in_data = GetValues(*input.data(), 1); + auto out_data = GetMutableValues(output, 1); if (!options.allow_int_overflow) { constexpr in_type kMax = static_cast(std::numeric_limits::max()); @@ -276,8 +283,8 @@ struct CastFunctor(*input.data(), 1); - auto out_data = reinterpret_cast(output->buffers[1]->mutable_data()); + const in_type* in_data = GetValues(*input.data(), 1); + auto out_data = GetMutableValues(output, 1); for (int64_t i = 0; i < input.length(); ++i) { *out_data++ = static_cast(*in_data++); } @@ -288,13 +295,16 @@ struct CastFunctor -inline void ShiftTime(FunctionContext* ctx, const CastOptions& options, - const bool is_multiply, const int64_t factor, const Array& input, - ArrayData* output) { - const in_type* in_data = GetValuesAs(*input.data(), 1); - auto out_data = reinterpret_cast(output->buffers[1]->mutable_data()); +void ShiftTime(FunctionContext* ctx, const CastOptions& options, const bool is_multiply, + const int64_t factor, const Array& input, ArrayData* output) { + const in_type* in_data = GetValues(*input.data(), 1); + auto out_data = GetMutableValues(output, 1); - if (is_multiply) { + if (factor == 1) { + for (int64_t i = 0; i < input.length(); i++) { + out_data[i] = static_cast(in_data[i]); + } + } else if (is_multiply) { for (int64_t i = 0; i < input.length(); i++) { out_data[i] = static_cast(in_data[i] * factor); } @@ -352,6 +362,52 @@ struct CastFunctor { } }; +template <> +struct CastFunctor { + void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, + ArrayData* output) { + const auto& in_type = static_cast(*input.type()); + + static const int64_t kTimestampToDateFactors[4] = { + 86400LL, // SECOND + 86400LL * 1000LL, // MILLI + 86400LL * 1000LL * 1000LL, // MICRO + 86400LL * 1000LL * 1000LL * 1000LL, // NANO + }; + + const int64_t factor = kTimestampToDateFactors[static_cast(in_type.unit())]; + ShiftTime(ctx, options, false, factor, input, output); + } +}; + +template <> +struct CastFunctor { + void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, + ArrayData* output) { + const auto& in_type = static_cast(*input.type()); + + std::pair conversion = + kTimeConversionTable[static_cast(in_type.unit())] + [static_cast(TimeUnit::MILLI)]; + + ShiftTime(ctx, options, conversion.first, conversion.second, input, + output); + + // Ensure that intraday milliseconds have been zeroed out + auto out_data = GetMutableValues(output, 1); + for (int64_t i = 0; i < input.length(); ++i) { + const int64_t remainder = out_data[i] % kMillisecondsInDay; + if (ARROW_PREDICT_FALSE(!options.allow_time_truncate && input.IsValid(i) && + remainder > 0)) { + ctx->SetStatus( + Status::Invalid("Timestamp value had non-zero intraday milliseconds")); + break; + } + out_data[i] -= remainder; + } + } +}; + // ---------------------------------------------------------------------- // From one time32 or time64 to another @@ -385,8 +441,6 @@ struct CastFunctor struct CastFunctor { void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, @@ -415,7 +469,7 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices, internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(), indices.length()); - const index_c_type* in = GetValuesAs(*indices.data(), 1); + const index_c_type* in = GetValues(*indices.data(), 1); uint8_t* out = output->buffers[1]->mutable_data(); int32_t byte_width = @@ -479,7 +533,7 @@ Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices, internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(), indices.length()); - const index_c_type* in = GetValuesAs(*indices.data(), 1); + const index_c_type* in = GetValues(*indices.data(), 1); for (int64_t i = 0; i < indices.length(); ++i) { if (valid_bits_reader.IsSet()) { int32_t length; @@ -550,7 +604,7 @@ void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary, internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(), indices.length()); - const index_c_type* in = GetValuesAs(*indices.data(), 1); + const index_c_type* in = GetValues(*indices.data(), 1); for (int64_t i = 0; i < indices.length(); ++i) { if (valid_bits_reader.IsSet()) { out[i] = dictionary[in[i]]; @@ -575,7 +629,7 @@ struct CastFunctortype)) << "Dictionary type: " << values_type << " target type: " << (*output->type); - const c_type* dictionary = GetValuesAs(*type.dictionary()->data(), 1); + const c_type* dictionary = GetValues(*type.dictionary()->data(), 1); auto out = reinterpret_cast(output->buffers[1]->mutable_data()); const Array& indices = *dict_array.indices(); @@ -755,7 +809,10 @@ class CastKernel : public UnaryKernel { FN(Time64Type, Time32Type); \ FN(Time64Type, Time64Type); -#define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType); +#define TIMESTAMP_CASES(FN, IN_TYPE) \ + FN(TimestampType, TimestampType); \ + FN(TimestampType, Date32Type); \ + FN(TimestampType, Date64Type); #define DICTIONARY_CASES(FN, IN_TYPE) \ FN(IN_TYPE, NullType); \ diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index 8a7ef923b4719..61d53c4d50b33 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -355,6 +355,70 @@ TEST_F(TestCast, TimestampToTimestamp) { timestamp(TimeUnit::SECOND), options); } +TEST_F(TestCast, TimestampToDate32_Date64) { + CastOptions options; + + vector is_valid = {true, true, false}; + + // 2000-01-01, 2000-01-02, null + vector v_nano = {946684800000000000, 946771200000000000, 0}; + vector v_micro = {946684800000000, 946771200000000, 0}; + vector v_milli = {946684800000, 946771200000, 0}; + vector v_second = {946684800, 946771200, 0}; + vector v_day = {10957, 10958, 0}; + + // Simple conversions + CheckCase( + timestamp(TimeUnit::NANO), v_nano, is_valid, date64(), v_milli, options); + CheckCase( + timestamp(TimeUnit::MICRO), v_micro, is_valid, date64(), v_milli, options); + CheckCase( + timestamp(TimeUnit::MILLI), v_milli, is_valid, date64(), v_milli, options); + CheckCase( + timestamp(TimeUnit::SECOND), v_second, is_valid, date64(), v_milli, options); + + CheckCase( + timestamp(TimeUnit::NANO), v_nano, is_valid, date32(), v_day, options); + CheckCase( + timestamp(TimeUnit::MICRO), v_micro, is_valid, date32(), v_day, options); + CheckCase( + timestamp(TimeUnit::MILLI), v_milli, is_valid, date32(), v_day, options); + CheckCase( + timestamp(TimeUnit::SECOND), v_second, is_valid, date32(), v_day, options); + + // Disallow truncate, failures + vector v_nano_fail = {946684800000000001, 946771200000000001, 0}; + vector v_micro_fail = {946684800000001, 946771200000001, 0}; + vector v_milli_fail = {946684800001, 946771200001, 0}; + vector v_second_fail = {946684801, 946771201, 0}; + + options.allow_time_truncate = false; + CheckFails(timestamp(TimeUnit::NANO), v_nano_fail, is_valid, date64(), + options); + CheckFails(timestamp(TimeUnit::MICRO), v_micro_fail, is_valid, date64(), + options); + CheckFails(timestamp(TimeUnit::MILLI), v_milli_fail, is_valid, date64(), + options); + CheckFails(timestamp(TimeUnit::SECOND), v_second_fail, is_valid, + date64(), options); + + CheckFails(timestamp(TimeUnit::NANO), v_nano_fail, is_valid, date32(), + options); + CheckFails(timestamp(TimeUnit::MICRO), v_micro_fail, is_valid, date32(), + options); + CheckFails(timestamp(TimeUnit::MILLI), v_milli_fail, is_valid, date32(), + options); + CheckFails(timestamp(TimeUnit::SECOND), v_second_fail, is_valid, + date32(), options); + + // Make sure that nulls are excluded from the truncation checks + vector v_second_nofail = {946684800, 946771200, 1}; + CheckCase( + timestamp(TimeUnit::SECOND), v_second_nofail, is_valid, date64(), v_milli, options); + CheckCase( + timestamp(TimeUnit::SECOND), v_second_nofail, is_valid, date32(), v_day, options); +} + TEST_F(TestCast, TimeToTime) { CastOptions options; diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc index d52627ebfee12..0e775a0fb0e99 100644 --- a/cpp/src/arrow/python/builtin_convert.cc +++ b/cpp/src/arrow/python/builtin_convert.cc @@ -519,7 +519,26 @@ class UInt64Converter : public TypedConverterVisitor { +class Date32Converter : public TypedConverterVisitor { + public: + inline Status AppendItem(const OwnedRef& item) { + int32_t t; + if (PyDate_Check(item.obj())) { + auto pydate = reinterpret_cast(item.obj()); + t = static_cast(PyDate_to_s(pydate)); + } else { + int64_t casted_val = static_cast(PyLong_AsLongLong(item.obj())); + RETURN_IF_PYERROR(); + if (casted_val > std::numeric_limits::max()) { + return Status::Invalid("Integer as date32 larger than INT32_MAX"); + } + t = static_cast(casted_val); + } + return typed_builder_->Append(t); + } +}; + +class Date64Converter : public TypedConverterVisitor { public: inline Status AppendItem(const OwnedRef& item) { int64_t t; @@ -535,7 +554,7 @@ class DateConverter : public TypedConverterVisitor }; class TimestampConverter - : public TypedConverterVisitor { + : public TypedConverterVisitor { public: explicit TimestampConverter(TimeUnit::type unit) : unit_(unit) {} @@ -717,8 +736,10 @@ std::shared_ptr GetConverter(const std::shared_ptr& type return std::make_shared(); case Type::UINT64: return std::make_shared(); + case Type::DATE32: + return std::make_shared(); case Type::DATE64: - return std::make_shared(); + return std::make_shared(); case Type::TIMESTAMP: return std::make_shared( static_cast(*type).unit()); diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index ead3a04810121..c5aff2e4f2e3a 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -260,6 +260,7 @@ class NumPyConverter { : pool_(pool), type_(type), arr_(reinterpret_cast(ao)), + dtype_(PyArray_DESCR(arr_)), mask_(nullptr), use_pandas_null_sentinels_(use_pandas_null_sentinels) { if (mo != nullptr && mo != Py_None) { @@ -431,6 +432,7 @@ class NumPyConverter { MemoryPool* pool_; std::shared_ptr type_; PyArrayObject* arr_; + PyArray_Descr* dtype_; PyArrayObject* mask_; int64_t length_; int64_t stride_; @@ -450,7 +452,7 @@ Status NumPyConverter::Convert() { return Status::Invalid("only handle 1-dimensional arrays"); } - if (PyArray_DESCR(arr_)->type_num == NPY_OBJECT) { + if (dtype_->type_num == NPY_OBJECT) { return ConvertObjects(); } @@ -462,33 +464,12 @@ Status NumPyConverter::Convert() { return VisitTypeInline(*type_, this); } -template -void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) { - // Passing input_data as non-const is a concession to PyObject* - int64_t j = 0; - for (int64_t i = 0; i < length; ++i) { - output_data[i] = static_cast(input_data[j]); - j += stride; - } -} - -template <> -void CopyStrided(PyObject** input_data, int64_t length, - int64_t stride, PyObject** output_data) { - int64_t j = 0; - for (int64_t i = 0; i < length; ++i) { - output_data[i] = input_data[j]; - if (output_data[i] != nullptr) { - Py_INCREF(output_data[i]); - } - j += stride; - } -} +namespace { -static Status CastBuffer(const std::shared_ptr& input, const int64_t length, - const std::shared_ptr& in_type, - const std::shared_ptr& out_type, MemoryPool* pool, - std::shared_ptr* out) { +Status CastBuffer(const std::shared_ptr& input, const int64_t length, + const std::shared_ptr& in_type, + const std::shared_ptr& out_type, MemoryPool* pool, + std::shared_ptr* out) { // Must cast std::vector> buffers = {nullptr, input}; auto tmp_data = std::make_shared(in_type, length, buffers, 0); @@ -499,6 +480,7 @@ static Status CastBuffer(const std::shared_ptr& input, const int64_t len compute::FunctionContext context(pool); compute::CastOptions cast_options; cast_options.allow_int_overflow = false; + cast_options.allow_time_truncate = false; RETURN_NOT_OK( compute::Cast(&context, *tmp_array, out_type, cast_options, &casted_array)); @@ -506,29 +488,47 @@ static Status CastBuffer(const std::shared_ptr& input, const int64_t len return Status::OK(); } +template +void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) { + // Passing input_data as non-const is a concession to PyObject* + int64_t j = 0; + for (int64_t i = 0; i < length; ++i) { + output_data[i] = static_cast(input_data[j]); + j += stride; + } +} + template -inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { +Status CopyStridedArray(PyArrayObject* arr, const int64_t length, MemoryPool* pool, + std::shared_ptr* out) { using traits = internal::arrow_traits; using T = typename traits::T; + // Strided, must copy into new contiguous memory + const int64_t stride = PyArray_STRIDES(arr)[0]; + const int64_t stride_elements = stride / sizeof(T); + + auto new_buffer = std::make_shared(pool); + RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length)); + CopyStrided(reinterpret_cast(PyArray_DATA(arr)), length, stride_elements, + reinterpret_cast(new_buffer->mutable_data())); + *out = new_buffer; + return Status::OK(); +} + +} // namespace + +template +inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { if (is_strided()) { - // Strided, must copy into new contiguous memory - const int64_t stride = PyArray_STRIDES(arr_)[0]; - const int64_t stride_elements = stride / sizeof(T); - - auto new_buffer = std::make_shared(pool_); - RETURN_NOT_OK(new_buffer->Resize(sizeof(T) * length_)); - CopyStrided(reinterpret_cast(PyArray_DATA(arr_)), length_, stride_elements, - reinterpret_cast(new_buffer->mutable_data())); - *data = new_buffer; + RETURN_NOT_OK(CopyStridedArray(arr_, length_, pool_, data)); } else { // Can zero-copy *data = std::make_shared(reinterpret_cast(arr_)); } std::shared_ptr input_type; - RETURN_NOT_OK( - NumPyDtypeToArrow(reinterpret_cast(PyArray_DESCR(arr_)), &input_type)); + RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); if (!input_type->Equals(*type_)) { RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data)); @@ -537,45 +537,6 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { return Status::OK(); } -template <> -inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { - // Handle LONGLONG->INT64 and other fun things - int type_num_compat = cast_npy_type_compat(PyArray_DESCR(arr_)->type_num); - int type_size = NumPyTypeSize(type_num_compat); - - if (type_size == 4) { - // Source and target are INT32, so can refer to the main implementation. - return ConvertData(data); - } else if (type_size == 8) { - // We need to scale down from int64 to int32 - auto new_buffer = std::make_shared(pool_); - RETURN_NOT_OK(new_buffer->Resize(sizeof(int32_t) * length_)); - - auto input = reinterpret_cast(PyArray_DATA(arr_)); - auto output = reinterpret_cast(new_buffer->mutable_data()); - - if (is_strided()) { - // Strided, must copy into new contiguous memory - const int64_t stride = PyArray_STRIDES(arr_)[0]; - const int64_t stride_elements = stride / sizeof(int64_t); - CopyStrided(input, length_, stride_elements, output); - } else { - // TODO(wesm): int32 overflow checks - for (int64_t i = 0; i < length_; ++i) { - *output++ = static_cast(*input++); - } - } - *data = new_buffer; - } else { - std::stringstream ss; - ss << "Cannot convert NumPy array of element size "; - ss << type_size << " to a Date32 array"; - return Status::NotImplemented(ss.str()); - } - - return Status::OK(); -} - template <> inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { int64_t nbytes = BitUtil::BytesForBits(length_); @@ -597,6 +558,42 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* return Status::OK(); } +template <> +inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { + if (is_strided()) { + RETURN_NOT_OK(CopyStridedArray(arr_, length_, pool_, data)); + } else { + // Can zero-copy + *data = std::make_shared(reinterpret_cast(arr_)); + } + + // If we have inbound datetime64[D] data, this needs to be downcasted + // separately here from int64_t to int32_t, because this data is not + // supported in compute::Cast + auto date_dtype = reinterpret_cast(dtype_->c_metadata); + if (dtype_->type_num == NPY_DATETIME && date_dtype->meta.base == NPY_FR_D) { + auto date32_buffer = std::make_shared(pool_); + RETURN_NOT_OK(date32_buffer->Resize(sizeof(int32_t) * length_)); + + auto datetime64_values = reinterpret_cast((*data)->data()); + auto date32_values = reinterpret_cast(date32_buffer->mutable_data()); + for (int64_t i = 0; i < length_; ++i) { + // TODO(wesm): How pedantic do we really want to be about checking for int32 + // overflow here? + *date32_values++ = static_cast(*datetime64_values++); + } + *data = date32_buffer; + } else { + std::shared_ptr input_type; + RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); + if (!input_type->Equals(*type_)) { + RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data)); + } + } + + return Status::OK(); +} + template struct UnboxDate {}; diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h index c110bc64a2a2f..e76c2e0db4aea 100644 --- a/cpp/src/arrow/python/util/datetime.h +++ b/cpp/src/arrow/python/util/datetime.h @@ -235,6 +235,11 @@ static inline Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit, return Status::OK(); } +static inline int64_t PyDate_to_s(PyDateTime_Date* pydate) { + return get_days_from_date(PyDateTime_GET_YEAR(pydate), PyDateTime_GET_MONTH(pydate), + PyDateTime_GET_DAY(pydate)); +} + static inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) { int64_t total_seconds = 0; total_seconds += PyDateTime_DATE_GET_SECOND(pydate); diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py index 414266ddb14ed..c7a0d49b40db1 100644 --- a/python/pyarrow/tests/test_convert_builtin.py +++ b/python/pyarrow/tests/test_convert_builtin.py @@ -178,6 +178,25 @@ def test_date(self): assert arr[2].as_py() == datetime.date(1970, 1, 1) assert arr[3].as_py() == datetime.date(2040, 2, 26) + def test_date32(self): + data = [datetime.date(2000, 1, 1), None] + arr = pa.array(data, type=pa.date32()) + + data2 = [10957, None] + arr2 = pa.array(data2, type=pa.date32()) + + for x in [arr, arr2]: + assert len(x) == 2 + assert x.type == pa.date32() + assert x.null_count == 1 + assert x[0].as_py() == datetime.date(2000, 1, 1) + assert x[1] is pa.NA + + # Overflow + data3 = [2**32, None] + with pytest.raises(pa.ArrowException): + pa.array(data3, type=pa.date32()) + def test_timestamp(self): data = [ datetime.datetime(2007, 7, 13, 1, 23, 34, 123456), diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index d00bf1b28eddc..07ecf3010a32e 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -525,6 +525,16 @@ def test_timestamps_with_timezone(self): self._check_pandas_roundtrip(df) + def test_datetime64_to_date32(self): + # ARROW-1718 + arr = pa.array([date(2017, 10, 23), None]) + c = pa.Column.from_array("d", arr) + s = c.to_pandas() + + arr2 = pa.Array.from_pandas(s, type=pa.date32()) + + assert arr2.equals(arr.cast('date32')) + def test_date_infer(self): df = pd.DataFrame({ 'date': [date(2000, 1, 1), @@ -984,6 +994,7 @@ def test_numpy_datetime64_columns(self): dtype='datetime64[s]') self._check_array_from_pandas_roundtrip(datetime64_s) + def test_numpy_datetime64_day_unit(self): datetime64_d = np.array([ '2007-07-13', None, From 39243ffaf5eb1d1f2a748ea1ec2b36658ba7f3d7 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 30 Oct 2017 14:59:47 -0400 Subject: [PATCH 002/177] ARROW-1409: [Format] Remove page id from Buffer metadata, increment metadata version number This is a breaking metadata change per discussion on the mailing list. I expect this kind of truly breaking changes to be exceedingly rare going forward, and when we make a 1.0.0 release we should document expectations around metadata / memory format stability. This could be made backwards compatible with some effort (we would have to add `RecordBatchV3` and `BufferV3` types). Author: Wes McKinney Closes #1225 from wesm/ARROW-1409 and squashes the following commits: 582fad90 [Wes McKinney] Disable JS in Travis CI for now 845f290f [Wes McKinney] Bump metadata version in Java, add check for V4 e2150c19 [Wes McKinney] Remove page id from Buffer metadata, increment metadata version number --- .travis.yml | 15 +++++------ cpp/src/arrow/ipc/ipc-read-write-test.cc | 2 +- cpp/src/arrow/ipc/message.cc | 15 +---------- cpp/src/arrow/ipc/message.h | 14 ++++++++++- cpp/src/arrow/ipc/metadata-internal.cc | 25 +++++++++++++++++-- cpp/src/arrow/ipc/metadata-internal.h | 10 ++++---- cpp/src/arrow/ipc/reader.cc | 15 +---------- cpp/src/arrow/ipc/writer.cc | 12 +-------- format/Schema.fbs | 14 +++++++---- .../arrow/vector/schema/ArrowBuffer.java | 16 +++--------- .../arrow/vector/schema/ArrowRecordBatch.java | 2 +- .../vector/stream/MessageSerializer.java | 6 ++++- js/src/format/Schema_generated.ts | 20 +++------------ 13 files changed, 75 insertions(+), 91 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6419548a622f3..52d7a5f800505 100644 --- a/.travis.yml +++ b/.travis.yml @@ -112,13 +112,14 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh - - language: node_js - os: linux - node_js: node - before_script: - - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh + # TODO(wesm): Re-enable after issues in ARROW-1409 resolved + # - language: node_js + # os: linux + # node_js: node + # before_script: + # - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh + # script: + # - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh - compiler: gcc language: cpp os: linux diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc index adf34a9eb5422..6f2f5cf856055 100644 --- a/cpp/src/arrow/ipc/ipc-read-write-test.cc +++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc @@ -243,7 +243,7 @@ TEST_F(TestIpcRoundTrip, MetadataVersion) { std::unique_ptr message; ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message)); - ASSERT_EQ(MetadataVersion::V3, message->metadata_version()); + ASSERT_EQ(MetadataVersion::V4, message->metadata_version()); } TEST_P(TestIpcRoundTrip, SliceRoundTrip) { diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 0dd5c72e51980..21d6a69a28603 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -67,20 +67,7 @@ class Message::MessageImpl { } MetadataVersion version() const { - switch (message_->version()) { - case flatbuf::MetadataVersion_V1: - // Arrow 0.1 - return MetadataVersion::V1; - case flatbuf::MetadataVersion_V2: - // Arrow 0.2 - return MetadataVersion::V2; - case flatbuf::MetadataVersion_V3: - // Arrow >= 0.3 - return MetadataVersion::V3; - // Add cases as other versions become available - default: - return MetadataVersion::V3; - } + return internal::GetMetadataVersion(message_->version()); } const void* header() const { return message_->header(); } diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h index a1b6c07a43d0e..495474e505157 100644 --- a/cpp/src/arrow/ipc/message.h +++ b/cpp/src/arrow/ipc/message.h @@ -42,7 +42,19 @@ class RandomAccessFile; namespace ipc { -enum class MetadataVersion : char { V1, V2, V3 }; +enum class MetadataVersion : char { + /// 0.1.0 + V1, + + /// 0.2.0 + V2, + + /// 0.3.0 to 0.7.1 + V3, + + /// >= 0.8.0 + V4 +}; // ARROW-109: We set this number arbitrarily to help catch user mistakes. For // deeply nested schemas, it is expected the user will indicate explicitly the diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index ad00cfb6c09be..f04e9b05a01b8 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -33,6 +33,7 @@ #include "arrow/ipc/Message_generated.h" #include "arrow/ipc/Tensor_generated.h" #include "arrow/ipc/dictionary.h" +#include "arrow/ipc/message.h" #include "arrow/ipc/util.h" #include "arrow/status.h" #include "arrow/tensor.h" @@ -57,6 +58,26 @@ using VectorLayoutOffset = flatbuffers::Offset; using Offset = flatbuffers::Offset; using FBString = flatbuffers::Offset; +MetadataVersion GetMetadataVersion(flatbuf::MetadataVersion version) { + switch (version) { + case flatbuf::MetadataVersion_V1: + // Arrow 0.1 + return MetadataVersion::V1; + case flatbuf::MetadataVersion_V2: + // Arrow 0.2 + return MetadataVersion::V2; + case flatbuf::MetadataVersion_V3: + // Arrow 0.3 to 0.7.1 + return MetadataVersion::V4; + case flatbuf::MetadataVersion_V4: + // Arrow >= 0.8 + return MetadataVersion::V4; + // Add cases as other versions become available + default: + return MetadataVersion::V4; + } +} + static Status IntFromFlatbuffer(const flatbuf::Int* int_data, std::shared_ptr* out) { if (int_data->bitWidth() > 64) { @@ -700,7 +721,7 @@ static Status WriteBuffers(FBB& fbb, const std::vector& buffers, for (size_t i = 0; i < buffers.size(); ++i) { const BufferMetadata& buffer = buffers[i]; - fb_buffers.emplace_back(buffer.page, buffer.offset, buffer.length); + fb_buffers.emplace_back(buffer.offset, buffer.length); } *out = fbb.CreateVectorOfStructs(fb_buffers); return Status::OK(); @@ -751,7 +772,7 @@ Status WriteTensorMessage(const Tensor& tensor, int64_t buffer_start_offset, auto fb_shape = fbb.CreateVector(dims); auto fb_strides = fbb.CreateVector(tensor.strides()); int64_t body_length = tensor.data()->size(); - flatbuf::Buffer buffer(-1, buffer_start_offset, body_length); + flatbuf::Buffer buffer(buffer_start_offset, body_length); TensorOffset fb_tensor = flatbuf::CreateTensor(fbb, fb_type_type, fb_type, fb_shape, fb_strides, &buffer); diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h index 309e7587a754c..380f3c9eb1013 100644 --- a/cpp/src/arrow/ipc/metadata-internal.h +++ b/cpp/src/arrow/ipc/metadata-internal.h @@ -27,6 +27,7 @@ #include "arrow/ipc/Schema_generated.h" #include "arrow/ipc/dictionary.h" +#include "arrow/ipc/message.h" namespace arrow { @@ -48,10 +49,12 @@ namespace ipc { namespace internal { static constexpr flatbuf::MetadataVersion kCurrentMetadataVersion = - flatbuf::MetadataVersion_V3; + flatbuf::MetadataVersion_V4; static constexpr flatbuf::MetadataVersion kMinMetadataVersion = - flatbuf::MetadataVersion_V3; + flatbuf::MetadataVersion_V4; + +MetadataVersion GetMetadataVersion(flatbuf::MetadataVersion version); static constexpr const char* kArrowMagicBytes = "ARROW1"; @@ -62,9 +65,6 @@ struct FieldMetadata { }; struct BufferMetadata { - /// The shared memory page id where to find this. Set to -1 if unused - int32_t page; - /// The relative offset into the memory page to the starting byte of the buffer int64_t offset; diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 50eb9039c6ab6..8e10d7d66f907 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -550,20 +550,7 @@ class RecordBatchFileReader::RecordBatchFileReaderImpl { int num_record_batches() const { return footer_->recordBatches()->size(); } MetadataVersion version() const { - switch (footer_->version()) { - case flatbuf::MetadataVersion_V1: - // Arrow 0.1 - return MetadataVersion::V1; - case flatbuf::MetadataVersion_V2: - // Arrow 0.2 - return MetadataVersion::V2; - case flatbuf::MetadataVersion_V3: - // Arrow 0.3 - return MetadataVersion::V3; - // Add cases as other versions become available - default: - return MetadataVersion::V3; - } + return internal::GetMetadataVersion(footer_->version()); } FileBlock record_batch(int i) const { diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index 279a69544faf2..5598cc68296f7 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -149,8 +149,6 @@ class RecordBatchSerializer : public ArrayVisitor { buffer_meta_.reserve(buffers_.size()); - const int32_t kNoPageId = -1; - // Construct the buffer metadata for the record batch header for (size_t i = 0; i < buffers_.size(); ++i) { const Buffer* buffer = buffers_[i].get(); @@ -163,15 +161,7 @@ class RecordBatchSerializer : public ArrayVisitor { padding = BitUtil::RoundUpToMultipleOf8(size) - size; } - // TODO(wesm): We currently have no notion of shared memory page id's, - // but we've included it in the metadata IDL for when we have it in the - // future. Use page = -1 for now - // - // Note that page ids are a bespoke notion for Arrow and not a feature we - // are using from any OS-level shared memory. The thought is that systems - // may (in the future) associate integer page id's with physical memory - // pages (according to whatever is the desired shared memory mechanism) - buffer_meta_.push_back({kNoPageId, offset, size + padding}); + buffer_meta_.push_back({offset, size + padding}); offset += size + padding; } diff --git a/format/Schema.fbs b/format/Schema.fbs index 186f8e362bde2..6021e92b847e7 100644 --- a/format/Schema.fbs +++ b/format/Schema.fbs @@ -20,9 +20,17 @@ namespace org.apache.arrow.flatbuf; enum MetadataVersion:short { + /// 0.1.0 V1, + + /// 0.2.0 V2, - V3 + + /// 0.3.0 -> 0.7.1 + V3, + + /// >= 0.8.0 + V4 } /// These are stored in the flatbuffer in the Type union below @@ -293,10 +301,6 @@ enum Endianness:short { Little, Big } /// ---------------------------------------------------------------------- /// A Buffer represents a single contiguous memory segment struct Buffer { - /// The shared memory page id where this buffer is located. Currently this is - /// not used - page: int; - /// The relative offset into the shared memory page where the bytes for this /// buffer starts offset: long; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java index d8c9e3001d0a5..4e0187e791b5a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java @@ -24,21 +24,15 @@ public class ArrowBuffer implements FBSerializable { - private int page; private long offset; private long size; - public ArrowBuffer(int page, long offset, long size) { + public ArrowBuffer(long offset, long size) { super(); - this.page = page; this.offset = offset; this.size = size; } - public int getPage() { - return page; - } - public long getOffset() { return offset; } @@ -52,7 +46,6 @@ public int hashCode() { final int prime = 31; int result = 1; result = prime * result + (int) (offset ^ (offset >>> 32)); - result = prime * result + page; result = prime * result + (int) (size ^ (size >>> 32)); return result; } @@ -72,9 +65,6 @@ public boolean equals(Object obj) { if (offset != other.offset) { return false; } - if (page != other.page) { - return false; - } if (size != other.size) { return false; } @@ -83,12 +73,12 @@ public boolean equals(Object obj) { @Override public int writeTo(FlatBufferBuilder builder) { - return Buffer.createBuffer(builder, page, offset, size); + return Buffer.createBuffer(builder, offset, size); } @Override public String toString() { - return "ArrowBuffer [page=" + page + ", offset=" + offset + ", size=" + size + "]"; + return "ArrowBuffer [offset=" + offset + ", size=" + size + "]"; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java index c842d4c3f9a74..bf0967a2797fe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java @@ -72,7 +72,7 @@ public ArrowRecordBatch(int length, List nodes, List b for (ArrowBuf arrowBuf : buffers) { arrowBuf.retain(); long size = arrowBuf.readableBytes(); - arrowBuffers.add(new ArrowBuffer(0, offset, size)); + arrowBuffers.add(new ArrowBuffer(offset, size)); LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size)); offset += size; if (alignBuffers && offset % 8 != 0) { // align on 8 byte boundaries diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java index f69aa41e7f6bd..c397cec72f0ed 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java @@ -385,6 +385,10 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat throw new IOException("Cannot currently deserialize record batches over 2GB"); } + if (message.version() != MetadataVersion.V4) { + throw new IOException("Received metadata with an incompatible version number"); + } + switch (message.headerType()) { case MessageHeader.RecordBatch: return deserializeRecordBatch(in, message, alloc); @@ -409,7 +413,7 @@ public static ByteBuffer serializeMessage(FlatBufferBuilder builder, byte header Message.startMessage(builder); Message.addHeaderType(builder, headerType); Message.addHeader(builder, headerOffset); - Message.addVersion(builder, MetadataVersion.V3); + Message.addVersion(builder, MetadataVersion.V4); Message.addBodyLength(builder, bodyLength); builder.finish(Message.endMessage(builder)); return builder.dataBuffer(); diff --git a/js/src/format/Schema_generated.ts b/js/src/format/Schema_generated.ts index 65493b7f685ec..c5b3e5011d790 100644 --- a/js/src/format/Schema_generated.ts +++ b/js/src/format/Schema_generated.ts @@ -2027,16 +2027,6 @@ export namespace org.apache.arrow.flatbuf { return this; } - /** - * The shared memory page id where this buffer is located. Currently this is - * not used - * - * @returns {number} - */ - page(): number { - return this.bb.readInt32(this.bb_pos); - } - /** * The relative offset into the shared memory page where the bytes for this * buffer starts @@ -2044,7 +2034,7 @@ export namespace org.apache.arrow.flatbuf { * @returns {flatbuffers.Long} */ offset(): flatbuffers.Long { - return this.bb.readInt64(this.bb_pos + 8); + return this.bb.readInt64(this.bb_pos); } /** @@ -2054,7 +2044,7 @@ export namespace org.apache.arrow.flatbuf { * @returns {flatbuffers.Long} */ length(): flatbuffers.Long { - return this.bb.readInt64(this.bb_pos + 16); + return this.bb.readInt64(this.bb_pos + 8); } /** @@ -2064,12 +2054,10 @@ export namespace org.apache.arrow.flatbuf { * @param {flatbuffers.Long} length * @returns {flatbuffers.Offset} */ - static createBuffer(builder: flatbuffers.Builder, page: number, offset: flatbuffers.Long, length: flatbuffers.Long): flatbuffers.Offset { - builder.prep(8, 24); + static createBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Long, length: flatbuffers.Long): flatbuffers.Offset { + builder.prep(8, 16); builder.writeInt64(length); builder.writeInt64(offset); - builder.pad(4); - builder.writeInt32(page); return builder.offset(); } From 72b50bc597693f098b67489bd8da40862c9770a2 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 31 Oct 2017 11:36:28 -0400 Subject: [PATCH 003/177] [C++] Fix clang-format failure from ARROW-1409 Change-Id: Ia45874945e050de0aa6294cbbe01ec63e9b14235 --- cpp/src/arrow/ipc/metadata-internal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index f04e9b05a01b8..f0f0f675853b1 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -72,7 +72,7 @@ MetadataVersion GetMetadataVersion(flatbuf::MetadataVersion version) { case flatbuf::MetadataVersion_V4: // Arrow >= 0.8 return MetadataVersion::V4; - // Add cases as other versions become available + // Add cases as other versions become available default: return MetadataVersion::V4; } From 088055019837f3a8215425959fadb037e01d0b02 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Tue, 31 Oct 2017 12:51:06 -0400 Subject: [PATCH 004/177] ARROW-1754: [Python] Fix buggy Parquet roundtrip when an index name is the same as a column name Author: Phillip Cloud Closes #1271 from cpcloud/ARROW-1754 and squashes the following commits: 3ffbe541 [Phillip Cloud] ARROW-1754: [Python] Fix buggy Parquet roundtrip when an index name is the same as a column name --- python/pyarrow/pandas_compat.py | 52 ++++++++-------------------- python/pyarrow/tests/test_parquet.py | 43 +++++++++++++++++------ 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index d6c844c8490f5..1984598ff3533 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -18,7 +18,6 @@ import ast import collections import json -import re import numpy as np import pandas as pd @@ -29,13 +28,6 @@ from pyarrow.compat import PY2, zip_longest # noqa -INDEX_LEVEL_NAME_REGEX = re.compile(r'^__index_level_\d+__$') - - -def is_unnamed_index_level(name): - return INDEX_LEVEL_NAME_REGEX.match(name) is not None - - def infer_dtype(column): try: return pd.api.types.infer_dtype(column) @@ -143,7 +135,7 @@ def get_column_metadata(column, name, arrow_type): Parameters ---------- - column : pandas.Series + column : pandas.Series or pandas.Index name : str arrow_type : pyarrow.DataType @@ -161,7 +153,7 @@ def get_column_metadata(column, name, arrow_type): } string_dtype = 'object' - if not isinstance(name, six.string_types): + if name is not None and not isinstance(name, six.string_types): raise TypeError( 'Column name must be a string. Got column {} of type {}'.format( name, type(name).__name__ @@ -176,23 +168,7 @@ def get_column_metadata(column, name, arrow_type): } -def index_level_name(index, i): - """Return the name of an index level or a default name if `index.name` is - None. - - Parameters - ---------- - index : pandas.Index - i : int - - Returns - ------- - name : str - """ - if index.name is not None: - return index.name - else: - return '__index_level_{:d}__'.format(i) +index_level_name = '__index_level_{:d}__'.format def construct_metadata(df, column_names, index_levels, preserve_index, types): @@ -222,11 +198,11 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types): ] if preserve_index: - index_column_names = [index_level_name(level, i) - for i, level in enumerate(index_levels)] + index_column_names = list(map( + index_level_name, range(len(index_levels)) + )) index_column_metadata = [ - get_column_metadata(level, name=index_level_name(level, i), - arrow_type=arrow_type) + get_column_metadata(level, name=level.name, arrow_type=arrow_type) for i, (level, arrow_type) in enumerate( zip(index_levels, index_types) ) @@ -317,7 +293,7 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1): for i, column in enumerate(index_columns): columns_to_convert.append(column) convert_types.append(None) - names.append(index_level_name(column, i)) + names.append(index_level_name(i)) # NOTE(wesm): If nthreads=None, then we use a heuristic to decide whether # using a thread pool is worth it. Currently the heuristic is whether the @@ -378,6 +354,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): import pyarrow.lib as lib index_columns = [] + columns = [] column_indexes = [] index_arrays = [] index_names = [] @@ -390,6 +367,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): if has_pandas_metadata: pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8')) index_columns = pandas_metadata['index_columns'] + columns = pandas_metadata['columns'] column_indexes = pandas_metadata.get('column_indexes', []) table = _add_any_metadata(table, pandas_metadata) @@ -397,11 +375,11 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): # Build up a list of index columns and names while removing those columns # from the original table - for name in index_columns: - i = schema.get_field_index(name) + logical_index_names = [c['name'] for c in columns[-len(index_columns):]] + for raw_name, logical_name in zip(index_columns, logical_index_names): + i = schema.get_field_index(raw_name) if i != -1: col = table.column(i) - index_name = None if is_unnamed_index_level(name) else name col_pandas = col.to_pandas() values = col_pandas.values if not values.flags.writeable: @@ -410,9 +388,9 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): values = values.copy() index_arrays.append(pd.Series(values, dtype=col_pandas.dtype)) - index_names.append(index_name) + index_names.append(logical_name) block_table = block_table.remove_column( - block_table.schema.get_field_index(name) + block_table.schema.get_field_index(raw_name) ) # Convert an arrow table to Block from the internal pandas API diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index a7fe98ce71cd1..95dd6a471b6b3 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1171,7 +1171,8 @@ def test_dataset_read_pandas(tmpdir): @parquet -def test_dataset_read_pandas_common_metadata(tmpdir): +@pytest.mark.parametrize('preserve_index', [True, False]) +def test_dataset_read_pandas_common_metadata(tmpdir, preserve_index): # ARROW-1103 import pyarrow.parquet as pq @@ -1186,15 +1187,11 @@ def test_dataset_read_pandas_common_metadata(tmpdir): paths = [] for i in range(nfiles): df = _test_dataframe(size, seed=i) - df.index = pd.Index(np.arange(i * size, (i + 1) * size)) - df.index.name = 'index' + df.index = pd.Index(np.arange(i * size, (i + 1) * size), name='index') - path = pjoin(dirpath, '{0}.parquet'.format(i)) + path = pjoin(dirpath, '{:d}.parquet'.format(i)) - df_ex_index = df.reset_index(drop=True) - df_ex_index['index'] = df.index - table = pa.Table.from_pandas(df_ex_index, - preserve_index=False) + table = pa.Table.from_pandas(df, preserve_index=preserve_index) # Obliterate metadata table = table.replace_schema_metadata(None) @@ -1206,7 +1203,9 @@ def test_dataset_read_pandas_common_metadata(tmpdir): paths.append(path) # Write _metadata common file - table_for_metadata = pa.Table.from_pandas(df) + table_for_metadata = pa.Table.from_pandas( + df, preserve_index=preserve_index + ) pq.write_metadata(table_for_metadata.schema, pjoin(dirpath, '_metadata')) @@ -1214,7 +1213,7 @@ def test_dataset_read_pandas_common_metadata(tmpdir): columns = ['uint8', 'strings'] result = dataset.read_pandas(columns=columns).to_pandas() expected = pd.concat([x[columns] for x in frames]) - + expected.index.name = df.index.name if preserve_index else None tm.assert_frame_equal(result, expected) @@ -1387,3 +1386,27 @@ def test_large_table_int32_overflow(): table = pa.Table.from_arrays([parr], names=['one']) f = io.BytesIO() _write_table(table, f) + + +def test_index_column_name_duplicate(tmpdir): + data = { + 'close': { + pd.Timestamp('2017-06-30 01:31:00'): 154.99958999999998, + pd.Timestamp('2017-06-30 01:32:00'): 154.99958999999998, + }, + 'time': { + pd.Timestamp('2017-06-30 01:31:00'): pd.Timestamp( + '2017-06-30 01:31:00' + ), + pd.Timestamp('2017-06-30 01:32:00'): pd.Timestamp( + '2017-06-30 01:32:00' + ), + } + } + path = str(tmpdir / 'data.parquet') + dfx = pd.DataFrame(data).set_index('time', drop=False) + tdfx = pa.Table.from_pandas(dfx) + _write_table(tdfx, path) + arrow_table = _read_table(path) + result_df = arrow_table.to_pandas() + tm.assert_frame_equal(result_df, dfx) From eca992471b1b2230259509f74d7bc4af97922788 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 31 Oct 2017 12:54:06 -0400 Subject: [PATCH 005/177] ARROW-1658: [Python] Add boundschecking of dictionary indices when creating CategoricalBlock We should probably do this bounds-checking earlier and in the main Arrow C++ library when ingesting "untrusted" arrays. I will create a JIRA, but this is a stopgap in the meantime Author: Wes McKinney Closes #1270 from wesm/ARROW-1658 and squashes the following commits: 234a5685 [Wes McKinney] Add boundschecking of dictionary indices when creating CategoricalBlock as workaround for segfaults from invalid codes making their way into pandas --- cpp/src/arrow/python/arrow_to_pandas.cc | 38 ++++++++++++++------- python/pyarrow/array.pxi | 37 +++++++++++--------- python/pyarrow/tests/test_convert_pandas.py | 15 ++++++++ 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 7f1591213cec6..c92faede1347b 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -966,9 +966,10 @@ class CategoricalBlock : public PandasBlock { "CategoricalBlock allocation happens when calling Write"); } - template + template Status WriteIndices(const std::shared_ptr& col) { - using TRAITS = internal::arrow_traits; + using ArrayType = typename TypeTraits::ArrayType; + using TRAITS = internal::arrow_traits; using T = typename TRAITS::T; constexpr int npy_type = TRAITS::npy_type; @@ -977,10 +978,22 @@ class CategoricalBlock : public PandasBlock { // Sniff the first chunk const std::shared_ptr arr_first = data.chunk(0); const auto& dict_arr_first = static_cast(*arr_first); - const auto& indices_first = - static_cast(*dict_arr_first.indices()); + const auto& indices_first = static_cast(*dict_arr_first.indices()); + + auto CheckIndices = [](const ArrayType& arr, int64_t dict_length) { + const T* values = arr.raw_values(); + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsValid(i) && (values[i] < 0 || values[i] >= dict_length)) { + std::stringstream ss; + ss << "Out of bounds dictionary index: " << static_cast(values[i]); + return Status::Invalid(ss.str()); + } + } + return Status::OK(); + }; if (data.num_chunks() == 1 && indices_first.null_count() == 0) { + RETURN_NOT_OK(CheckIndices(indices_first, dict_arr_first.dictionary()->length())); RETURN_NOT_OK(AllocateNDArrayFromIndices(npy_type, indices_first)); } else { if (options_.zero_copy_only) { @@ -998,9 +1011,10 @@ class CategoricalBlock : public PandasBlock { const std::shared_ptr arr = data.chunk(c); const auto& dict_arr = static_cast(*arr); - const auto& indices = static_cast(*dict_arr.indices()); + const auto& indices = static_cast(*dict_arr.indices()); auto in_values = reinterpret_cast(indices.raw_values()); + RETURN_NOT_OK(CheckIndices(indices, dict_arr.dictionary()->length())); // Null is -1 in CategoricalBlock for (int i = 0; i < arr->length(); ++i) { *out_values++ = indices.IsNull(i) ? -1 : in_values[i]; @@ -1026,16 +1040,16 @@ class CategoricalBlock : public PandasBlock { switch (dict_type.index_type()->id()) { case Type::INT8: - RETURN_NOT_OK(WriteIndices(converted_col)); + RETURN_NOT_OK(WriteIndices(converted_col)); break; case Type::INT16: - RETURN_NOT_OK(WriteIndices(converted_col)); + RETURN_NOT_OK(WriteIndices(converted_col)); break; case Type::INT32: - RETURN_NOT_OK(WriteIndices(converted_col)); + RETURN_NOT_OK(WriteIndices(converted_col)); break; case Type::INT64: - RETURN_NOT_OK(WriteIndices(converted_col)); + RETURN_NOT_OK(WriteIndices(converted_col)); break; default: { std::stringstream ss; @@ -1091,13 +1105,11 @@ class CategoricalBlock : public PandasBlock { PyObject* block_arr = PyArray_NewFromDescr(&PyArray_Type, descr, 1, block_dims, nullptr, data, NPY_ARRAY_CARRAY, nullptr); + RETURN_IF_PYERROR(); npy_intp placement_dims[1] = {num_columns_}; PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - if (placement_arr == NULL) { - // TODO(wesm): propagating Python exception - return Status::OK(); - } + RETURN_IF_PYERROR(); block_arr_.reset(block_arr); placement_arr_.reset(placement_arr); diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 7da5c3caffdc2..7752d062a774c 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -162,6 +162,7 @@ def array(object obj, type=None, mask=None, return DictionaryArray.from_arrays( values.codes, values.categories.values, mask=mask, ordered=values.ordered, + from_pandas=from_pandas, memory_pool=memory_pool) else: values, type = pdcompat.get_datetimetz_type(values, obj.dtype, @@ -671,7 +672,7 @@ cdef class DictionaryArray(Array): @staticmethod def from_arrays(indices, dictionary, mask=None, ordered=False, - MemoryPool memory_pool=None): + from_pandas=False, MemoryPool memory_pool=None): """ Construct Arrow DictionaryArray from array of indices (must be non-negative integers) and corresponding array of dictionary values @@ -682,15 +683,20 @@ cdef class DictionaryArray(Array): dictionary : ndarray or pandas.Series mask : ndarray or pandas.Series, boolean type True values indicate that indices are actually null + from_pandas : boolean, default False + If True, the indices should be treated as though they originated in + a pandas.Categorical (null encoded as -1) ordered : boolean, default False Set to True if the category values are ordered + memory_pool : MemoryPool, default None + For memory allocations, if required, otherwise uses default pool Returns ------- dict_array : DictionaryArray """ cdef: - Array arrow_indices, arrow_dictionary + Array _indices, _dictionary DictionaryArray result shared_ptr[CDataType] c_type shared_ptr[CArray] c_result @@ -699,29 +705,28 @@ cdef class DictionaryArray(Array): if mask is not None: raise NotImplementedError( "mask not implemented with Arrow array inputs yet") - arrow_indices = indices + _indices = indices else: - if mask is None: - mask = indices == -1 - else: - mask = mask | (indices == -1) - arrow_indices = Array.from_pandas(indices, mask=mask, - memory_pool=memory_pool) + if from_pandas: + if mask is None: + mask = indices == -1 + else: + mask = mask | (indices == -1) + _indices = array(indices, mask=mask, memory_pool=memory_pool) if isinstance(dictionary, Array): - arrow_dictionary = dictionary + _dictionary = dictionary else: - arrow_dictionary = Array.from_pandas(dictionary, - memory_pool=memory_pool) + _dictionary = array(dictionary, memory_pool=memory_pool) - if not isinstance(arrow_indices, IntegerArray): + if not isinstance(_indices, IntegerArray): raise ValueError('Indices must be integer type') cdef c_bool c_ordered = ordered - c_type.reset(new CDictionaryType(arrow_indices.type.sp_type, - arrow_dictionary.sp_array, c_ordered)) - c_result.reset(new CDictionaryArray(c_type, arrow_indices.sp_array)) + c_type.reset(new CDictionaryType(_indices.type.sp_type, + _dictionary.sp_array, c_ordered)) + c_result.reset(new CDictionaryArray(c_type, _indices.sp_array)) result = DictionaryArray() result.init(c_result) diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index 07ecf3010a32e..dabccac37c3d8 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -217,6 +217,21 @@ def test_zero_copy_success(self): result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True) npt.assert_array_equal(result, [0, 1, 2]) + def test_dictionary_indices_boundscheck(self): + # ARROW-1658. No validation of indices leads to segfaults in pandas + indices = [[0, 1], [0, -1]] + + for inds in indices: + arr = pa.DictionaryArray.from_arrays(inds, ['a']) + batch = pa.RecordBatch.from_arrays([arr], ['foo']) + table = pa.Table.from_batches([batch, batch, batch]) + + with pytest.raises(pa.ArrowException): + arr.to_pandas() + + with pytest.raises(pa.ArrowException): + table.to_pandas() + def test_zero_copy_dictionaries(self): arr = pa.DictionaryArray.from_arrays( np.array([0, 0]), From 9dc4c58d57159edf24bcbe86b6220a6a3bcb09ef Mon Sep 17 00:00:00 2001 From: dhirschf Date: Tue, 31 Oct 2017 14:11:19 -0400 Subject: [PATCH 006/177] ARROW-1753: [Python] Provide for matching subclasses with register_type in serialization context https://issues.apache.org/jira/browse/ARROW-1753 Author: dhirschf Author: Philipp Moritz Closes #1272 from dhirschfeld/ARROW-1753 and squashes the following commits: bb7f041a [Philipp Moritz] fix subclass serialization tests cbc5e09c [dhirschf] Added a test that register_type will work for subclasses of the registered type 46b60f44 [dhirschf] Allow register_type to match subclasses in serialization context --- python/pyarrow/serialization.pxi | 17 ++++-- python/pyarrow/serialization.py | 2 + python/pyarrow/tests/test_serialization.py | 66 ++++++++++++++++++++++ python/requirements.txt | 2 +- 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index 4e9ab8eb3b374..6b7227797a836 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -88,17 +88,26 @@ cdef class SerializationContext: self.custom_deserializers[type_id] = custom_deserializer def _serialize_callback(self, obj): - if type(obj) not in self.type_to_type_id: + found = False + for type_ in type(obj).__mro__: + if type_ in self.type_to_type_id: + found = True + break + + if not found: raise SerializationCallbackError( "pyarrow does not know how to " - "serialize objects of type {}.".format(type(obj)), obj) - type_id = self.type_to_type_id[type(obj)] + "serialize objects of type {}.".format(type(obj)), obj + ) + + # use the closest match to type(obj) + type_id = self.type_to_type_id[type_] if type_id in self.types_to_pickle: serialized_obj = {"data": pickle.dumps(obj), "pickle": True} elif type_id in self.custom_serializers: serialized_obj = {"data": self.custom_serializers[type_id](obj)} else: - if is_named_tuple(type(obj)): + if is_named_tuple(type_): serialized_obj = {} serialized_obj["_pa_getnewargs_"] = obj.__getnewargs__() elif hasattr(obj, "__dict__"): diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py index 9dc8ee6dee9ad..2b47513fd1c85 100644 --- a/python/pyarrow/serialization.py +++ b/python/pyarrow/serialization.py @@ -69,6 +69,8 @@ def _deserialize_default_dict(data): type(lambda: 0), "function", pickle=True) + serialization_context.register_type(type, "type", pickle=True) + # ---------------------------------------------------------------------- # Set up serialization for numpy with dtype object (primitive types are # handled efficiently with Arrow's Tensor facilities, see diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index 7878a09228d06..b0c5bc49e6a58 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -416,3 +416,69 @@ class TempClass(object): with pytest.raises(pa.DeserializationCallbackError) as err: serialized_object.deserialize(deserialization_context) assert err.value.type_id == 20*b"\x00" + + +def test_fallback_to_subclasses(): + + class SubFoo(Foo): + def __init__(self): + Foo.__init__(self) + + # should be able to serialize/deserialize an instance + # if a base class has been registered + serialization_context = pa.SerializationContext() + serialization_context.register_type(Foo, "Foo") + + subfoo = SubFoo() + # should fallbact to Foo serializer + serialized_object = pa.serialize(subfoo, serialization_context) + + reconstructed_object = serialized_object.deserialize( + serialization_context + ) + assert type(reconstructed_object) == Foo + + +class Serializable(object): + pass + + +def serialize_serializable(obj): + return {"type": type(obj), "data": obj.__dict__} + + +def deserialize_serializable(obj): + val = obj["type"].__new__(obj["type"]) + val.__dict__.update(obj["data"]) + return val + + +class SerializableClass(Serializable): + def __init__(self): + self.value = 3 + + +def test_serialize_subclasses(): + + # This test shows how subclasses can be handled in an idiomatic way + # by having only a serializer for the base class + + # This technique should however be used with care, since pickling + # type(obj) with couldpickle will include the full class definition + # in the serialized representation. + # This means the class definition is part of every instance of the + # object, which in general is not desirable; registering all subclasses + # with register_type will result in faster and more memory + # efficient serialization. + + serialization_context.register_type( + Serializable, "Serializable", + custom_serializer=serialize_serializable, + custom_deserializer=deserialize_serializable) + + a = SerializableClass() + serialized = pa.serialize(a) + + deserialized = serialized.deserialize() + assert type(deserialized).__name__ == SerializableClass.__name__ + assert deserialized.value == 3 diff --git a/python/requirements.txt b/python/requirements.txt index d2e28a7747ba8..8d0c33afa69a6 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,4 +1,4 @@ pytest -cloudpickle +cloudpickle>=0.4.0 numpy>=1.10.0 six From 142e6ee69bd6a4dc316d00d9efd6d86d119df075 Mon Sep 17 00:00:00 2001 From: Heimir Sverrisson Date: Tue, 31 Oct 2017 22:54:09 -0400 Subject: [PATCH 007/177] ARROW-1455 [Python] Add Dockerfile for validating Dask integration A Docker container is created with all the dependencies needed to pull down the Dask code from Github and install it locally, together with Arrow, to run an integration test. Author: Heimir Sverrisson Closes #1249 from heimir-sverrisson/hs/dockerize_dask and squashes the following commits: d146185b [Heimir Sverrisson] ARROW-1455 [Python] Add Dockerfile for validating Dask integration --- dev/dask_integration.sh | 21 +++++ dev/dask_integration/Dockerfile | 88 +++++++++++++++++++ dev/dask_integration/dask_integration.sh | 49 +++++++++++ dev/docker-compose.yml | 5 ++ dev/run_docker_compose.sh | 2 +- python/testing/README.md | 24 ++++- .../dask_tests/test_dask_integration.py | 51 +++++++++++ 7 files changed, 238 insertions(+), 2 deletions(-) create mode 100755 dev/dask_integration.sh create mode 100644 dev/dask_integration/Dockerfile create mode 100755 dev/dask_integration/dask_integration.sh create mode 100644 python/testing/dask_tests/test_dask_integration.py diff --git a/dev/dask_integration.sh b/dev/dask_integration.sh new file mode 100755 index 0000000000000..d344328b6af1e --- /dev/null +++ b/dev/dask_integration.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pass the service name to run_docker_compose.sh +# Which validates environment and runs the service +exec "$(dirname ${BASH_SOURCE})"/run_docker_compose.sh dask_integration diff --git a/dev/dask_integration/Dockerfile b/dev/dask_integration/Dockerfile new file mode 100644 index 0000000000000..f72ef8ca0daab --- /dev/null +++ b/dev/dask_integration/Dockerfile @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM ubuntu:14.04 +ADD . /apache-arrow +WORKDIR /apache-arrow +# Basic OS utilities +RUN apt-get update && apt-get install -y \ + wget \ + git \ + gcc \ + g++ +# This will install conda in /home/ubuntu/miniconda +RUN wget -O /tmp/miniconda.sh \ + https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \ + rm /tmp/miniconda.sh +# Create Conda environment +ENV PATH="/home/ubuntu/miniconda/bin:${PATH}" +RUN conda create -y -q -n test-environment \ + python=3.6 +# Install dependencies +RUN conda install -c conda-forge \ + numpy \ + pandas \ + bcolz \ + blosc \ + bokeh \ + boto3 \ + chest \ + cloudpickle \ + coverage \ + cytoolz \ + distributed \ + graphviz \ + h5py \ + ipython \ + partd \ + psutil \ + "pytest<=3.1.1" \ + scikit-image \ + scikit-learn \ + scipy \ + sqlalchemy \ + toolz +# install pytables from defaults for now +RUN conda install pytables + +RUN pip install -q git+https://github.com/dask/partd --upgrade --no-deps +RUN pip install -q git+https://github.com/dask/zict --upgrade --no-deps +RUN pip install -q git+https://github.com/dask/distributed --upgrade --no-deps +RUN pip install -q git+https://github.com/mrocklin/sparse --upgrade --no-deps +RUN pip install -q git+https://github.com/dask/s3fs --upgrade --no-deps + +RUN conda install -q -c conda-forge numba cython +RUN pip install -q git+https://github.com/dask/fastparquet + +RUN pip install -q \ + cachey \ + graphviz \ + moto \ + pyarrow \ + --upgrade --no-deps + +RUN pip install -q \ + cityhash \ + flake8 \ + mmh3 \ + pandas_datareader \ + pytest-xdist \ + xxhash \ + pycodestyle + +CMD arrow/dev/dask_integration/dask_integration.sh + diff --git a/dev/dask_integration/dask_integration.sh b/dev/dask_integration/dask_integration.sh new file mode 100755 index 0000000000000..f5a24e462b742 --- /dev/null +++ b/dev/dask_integration/dask_integration.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set up environment and working directory +cd /apache-arrow + +export ARROW_BUILD_TYPE=release +export ARROW_HOME=$(pwd)/dist +export PARQUET_HOME=$(pwd)/dist +CONDA_BASE=/home/ubuntu/miniconda +export LD_LIBRARY_PATH=$(pwd)/dist/lib:${CONDA_BASE}/lib:${LD_LIBRARY_PATH} + +# Allow for --user Python installation inside Docker +export HOME=$(pwd) + +# Clean up and get the dask master branch from github +rm -rf dask .local +export GIT_COMMITTER_NAME="Nobody" +export GIT_COMMITTER_EMAIL="nobody@nowhere.com" +git clone https://github.com/dask/dask.git +pushd dask +pip install --user -e .[complete] +# Verify integrity of the installed dask dataframe code +py.test dask/dataframe/tests/test_dataframe.py +popd + +# Run the integration test +pushd arrow/python/testing +py.test dask_tests +popd + +pushd dask/dask/dataframe/io +py.test tests/test_parquet.py +popd diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 7bd2cd4412cec..4b9014894003b 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -28,3 +28,8 @@ services: - "4000:4000" volumes: - ../..:/apache-arrow + dask_integration: + build: + context: dask_integration + volumes: + - ../..:/apache-arrow diff --git a/dev/run_docker_compose.sh b/dev/run_docker_compose.sh index f46879ed1e436..681a3a75ffe20 100755 --- a/dev/run_docker_compose.sh +++ b/dev/run_docker_compose.sh @@ -37,4 +37,4 @@ fi GID=$(id -g ${USERNAME}) docker-compose -f arrow/dev/docker-compose.yml run \ - -u "${UID}:${GID}" "${1}" + --rm -u "${UID}:${GID}" "${1}" diff --git a/python/testing/README.md b/python/testing/README.md index 07970a231b54b..0ebeec4a1c3e7 100644 --- a/python/testing/README.md +++ b/python/testing/README.md @@ -23,4 +23,26 @@ ```shell ./test_hdfs.sh -``` \ No newline at end of file +``` + +## Testing Dask integration + +Initial integration testing with Dask has been Dockerized. +To invoke the test run the following command in the `arrow` +root-directory: + +```shell +bash dev/dask_integration.sh +``` + +This script will create a `dask` directory on the same level as +`arrow`. It will clone the Dask project from Github into `dask` +and do a Python `--user` install. The Docker code will use the parent +directory of `arrow` as `$HOME` and that's where Python will +install `dask` into a `.local` directory. + +The output of the Docker session will contain the results of tests +of the Dask dataframe followed by the single integration test that +now exists for Arrow. That test creates a set of `csv`-files and then +does parallel reading of `csv`-files into a Dask dataframe. The code +for this test resides here in the `dask_test` directory. diff --git a/python/testing/dask_tests/test_dask_integration.py b/python/testing/dask_tests/test_dask_integration.py new file mode 100644 index 0000000000000..e678348780cd8 --- /dev/null +++ b/python/testing/dask_tests/test_dask_integration.py @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from datetime import date, timedelta +import csv +from random import randint +import dask.dataframe as dd +import pyarrow as pa + +def make_datafiles(tmpdir, prefix='data', num_files=20): + rowcount = 5000 + fieldnames = ['date', 'temperature', 'dewpoint'] + start_date = date(1900, 1, 1) + for i in range(num_files): + filename = '{0}/{1}-{2}.csv'.format(tmpdir, prefix, i) + with open(filename, 'w') as outcsv: + writer = csv.DictWriter(outcsv, fieldnames) + writer.writeheader() + the_date = start_date + for _ in range(rowcount): + temperature = randint(-10, 35) + dewpoint = temperature - randint(0, 10) + writer.writerow({'date': the_date, 'temperature': temperature, + 'dewpoint': dewpoint}) + the_date += timedelta(days=1) + +def test_dask_file_read(tmpdir): + prefix = 'data' + make_datafiles(tmpdir, prefix) + # Read all datafiles in parallel + datafiles = '{0}/{1}-*.csv'.format(tmpdir, prefix) + dask_df = dd.read_csv(datafiles) + # Convert Dask dataframe to Arrow table + table = pa.Table.from_pandas(dask_df.compute()) + # Second column (1) is temperature + dask_temp = int(1000 * dask_df['temperature'].mean().compute()) + arrow_temp = int(1000 * table[1].to_pandas().mean()) + assert dask_temp == arrow_temp From 0373541e2e9cd3510d6e4c8ac5b580f11eb675ec Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Fri, 3 Nov 2017 17:10:30 +0100 Subject: [PATCH 008/177] ARROW-1766: [GLib] Fix failing builds on OSX Author: Phillip Cloud Closes #1279 from cpcloud/ARROW-1766 and squashes the following commits: ed3b22b [Phillip Cloud] Only set ARCHFLAGS on osx d9c1604 [Phillip Cloud] [GLib] Fix failing builds on OSX --- ci/travis_before_script_c_glib.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh index a63945e1745d0..a22ecd3753fd8 100755 --- a/ci/travis_before_script_c_glib.sh +++ b/ci/travis_before_script_c_glib.sh @@ -100,7 +100,12 @@ if [ $BUILD_SYSTEM = "autotools" ]; then ./configure $CONFIGURE_OPTIONS - make -j4 + if [ "$TRAVIS_OS_NAME" = "osx" ]; then + ARCHFLAGS="-arch x86_64" make -j4 + else + make -j4 + fi + make install else MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL" From 527af63cde4ac1df53e82d6f8ced064738f5c1a3 Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Fri, 3 Nov 2017 17:44:52 -0400 Subject: [PATCH 009/177] ARROW-1652: [JS] housekeeping, vector cleanup This PR addresses the first few issues in the [JS roadmap doc](https://docs.google.com/document/d/142dek89oM2TVI2Yql106Zo8IB1Ff_9zDg_EG6jPWS0M) I sent out a week or so ago. Sorry for the big PR, the housekeeping and vector cleanup work were pretty co-dependent. JIRA issues addressed by this PR: [ARROW-1032](https://issues.apache.org/jira/browse/ARROW-1032) - Support custom_metadata [ARROW-1651](https://issues.apache.org/jira/browse/ARROW-1651) - Lazy row accessor in Table [ARROW-1652](https://issues.apache.org/jira/browse/ARROW-1652) - Separate Vector into BatchVector and CompositeVector Tasks from the roadmap (some not in JIRA): ##### Housekeeping 1. Enable the strict-mode tsc compiler settings in the build 2. Compile mjs files for node 8.x ESModules 3. Compile ES6 UMD target with native iterators/generators ##### Vector 1. Refactor Vector types to primitive forms representing the portion of a column in a single RecordBatch 2. Add Column Vector that represents primitive Vectors across RecordBatches as an entire column 3. Refactor linear column-to-batch-index lookup in `Vector.get(i)` 4. Simplify inheritance hierarchy/generic types with Traits (e.g. Nullable, Iterable, and Typed numeric variants) ##### Table 1. Implement lazy row accessor 2. Share API/row logic with StructVector cc: @wesm @TheNeuralBit Author: Paul Taylor Closes #1273 from trxcllnt/vector-cleanup and squashes the following commits: c53d6de3 [Paul Taylor] refactor: rename vector mixins 2c83c823 [Paul Taylor] update to typescript@2.6.1 48c6ca48 [Paul Taylor] refactor: StructVector/Table#get always take numeric index, Table extends StructVector 18671edc [Paul Taylor] fix lint 04e9941d [Paul Taylor] refactor: use new compilation targets in perf tests bd7a8373 [Paul Taylor] refactor: update test's Arrow imports for new types 37b7f615 [Paul Taylor] refactor: update vector tests for new types 15ab8d4c [Paul Taylor] refactor: update table tests for new types db04a0b0 [Paul Taylor] refactor: export new Arrow types 84233dec [Paul Taylor] refactor reader to use new arrow types, fix strict TS compilation errors af4845d6 [Paul Taylor] refactor: add Arrow vector mixins 54fa2fd3 [Paul Taylor] refactor: break out virtual vector, move to types folder 2121bf1a [Paul Taylor] refactor: break out table, add Row type, move to types folder abc93310 [Paul Taylor] refactor: move struct to types folder 2a4127ce [Paul Taylor] refactor: move dictionary to types folder 607be424 [Paul Taylor] refactor: break out list/fixedsizelist/utf8, move to types folder b8a68665 [Paul Taylor] refactor: break out Typed vectors, move into types folder 0f8de75a [Paul Taylor] refactor: rename vector folder to types, move vector base class d2def198 [Paul Taylor] clean up build scripts, add ES2015 UMD and mjs targets 84b2c505 [Paul Taylor] use strict typescript compiler settings --- js/.gitignore | 9 +- js/closure-compiler-scripts/text-encoding.js | 2 +- js/gulp/argv.js | 36 ++ js/gulp/arrow-task.js | 57 +++ js/gulp/build-task.js | 35 ++ js/gulp/clean-task.js | 31 ++ js/gulp/closure-task.js | 91 +++++ js/gulp/memoize-task.js | 30 ++ js/gulp/package-task.js | 75 ++++ js/gulp/test-task.js | 44 +++ js/gulp/typescript-task.js | 43 +++ js/gulp/uglify-task.js | 113 ++++++ js/gulp/util.js | 152 ++++++++ js/gulpfile.js | 350 ++++-------------- js/package.json | 39 +- js/perf/index.js | 10 +- js/src/Arrow.externs.ts | 45 ++- js/src/Arrow.ts | 73 ++-- js/src/reader/arrow.ts | 28 +- js/src/reader/dictionary.ts | 16 +- js/src/reader/file.ts | 9 +- js/src/reader/message.ts | 8 +- js/src/reader/stream.ts | 4 +- js/src/reader/vector.ts | 312 +++++++++------- js/src/table.ts | 143 ------- js/src/text-encoding-utf-8.d.ts | 4 + js/src/types/arrow.ts | 88 +++++ js/src/types/dictionary.ts | 58 +++ .../struct.ts => types/fixedsizelist.ts} | 26 +- js/src/types/list.ts | 35 ++ js/src/types/table/from.ts | 34 ++ js/src/types/table/row.ts | 61 +++ js/src/types/table/struct.ts | 63 ++++ js/src/types/table/table.ts | 30 ++ js/src/types/table/toString.ts | 40 ++ js/src/types/types.ts | 98 +++++ .../{vector/dictionary.ts => types/utf8.ts} | 41 +- js/src/types/vector/bool.ts | 55 +++ js/src/types/vector/date.ts | 29 ++ js/src/types/vector/long.ts | 35 ++ js/src/types/vector/traits.ts | 69 ++++ js/src/types/vector/typed.ts | 57 +++ js/src/types/vector/virtual.ts | 129 +++++++ js/src/vector/list.ts | 108 ------ js/src/vector/typed.ts | 326 ---------------- js/src/vector/vector.ts | 91 ----- js/test/Arrow.ts | 30 +- js/test/__snapshots__/table-tests.ts.snap | 200 +++++----- js/test/table-tests.ts | 17 +- js/test/test-config.ts | 2 +- js/test/vector-tests.ts | 75 ++-- js/tsconfig/tsconfig.base.json | 30 +- js/tsconfig/tsconfig.es2015.cls.json | 3 +- js/tsconfig/tsconfig.es5.cls.json | 1 + js/tsconfig/tsconfig.esnext.cls.json | 3 +- 55 files changed, 2195 insertions(+), 1398 deletions(-) create mode 100644 js/gulp/argv.js create mode 100644 js/gulp/arrow-task.js create mode 100644 js/gulp/build-task.js create mode 100644 js/gulp/clean-task.js create mode 100644 js/gulp/closure-task.js create mode 100644 js/gulp/memoize-task.js create mode 100644 js/gulp/package-task.js create mode 100644 js/gulp/test-task.js create mode 100644 js/gulp/typescript-task.js create mode 100644 js/gulp/uglify-task.js create mode 100644 js/gulp/util.js delete mode 100644 js/src/table.ts create mode 100644 js/src/text-encoding-utf-8.d.ts create mode 100644 js/src/types/arrow.ts create mode 100644 js/src/types/dictionary.ts rename js/src/{vector/struct.ts => types/fixedsizelist.ts} (53%) create mode 100644 js/src/types/list.ts create mode 100644 js/src/types/table/from.ts create mode 100644 js/src/types/table/row.ts create mode 100644 js/src/types/table/struct.ts create mode 100644 js/src/types/table/table.ts create mode 100644 js/src/types/table/toString.ts create mode 100644 js/src/types/types.ts rename js/src/{vector/dictionary.ts => types/utf8.ts} (50%) create mode 100644 js/src/types/vector/bool.ts create mode 100644 js/src/types/vector/date.ts create mode 100644 js/src/types/vector/long.ts create mode 100644 js/src/types/vector/traits.ts create mode 100644 js/src/types/vector/typed.ts create mode 100644 js/src/types/vector/virtual.ts delete mode 100644 js/src/vector/list.ts delete mode 100644 js/src/vector/typed.ts delete mode 100644 js/src/vector/vector.ts diff --git a/js/.gitignore b/js/.gitignore index 6d0f88d191cb0..88c612d8faf37 100644 --- a/js/.gitignore +++ b/js/.gitignore @@ -18,6 +18,7 @@ # Logs logs *.log +.esm-cache npm-debug.log* yarn-debug.log* yarn-error.log* @@ -57,10 +58,6 @@ build/Release node_modules/ jspm_packages/ -# Typescript declaration files -types/ -typings/ - # Optional npm cache directory .npm @@ -85,6 +82,4 @@ package-lock.json # compilation targets dist -targets/es5 -targets/es2015 -targets/esnext +targets diff --git a/js/closure-compiler-scripts/text-encoding.js b/js/closure-compiler-scripts/text-encoding.js index ca9154f88ecba..398883ab9b4be 100644 --- a/js/closure-compiler-scripts/text-encoding.js +++ b/js/closure-compiler-scripts/text-encoding.js @@ -11,7 +11,7 @@ // Utilities // -goog.module("module$text_encoding"); +goog.module("module$text_encoding_utf_8"); goog.module.declareLegacyNamespace(); /** * @param {number} a The number to test. diff --git a/js/gulp/argv.js b/js/gulp/argv.js new file mode 100644 index 0000000000000..33553704eec25 --- /dev/null +++ b/js/gulp/argv.js @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const argv = require(`command-line-args`)([ + { name: `all`, alias: `a`, type: Boolean }, + { name: 'update', alias: 'u', type: Boolean }, + { name: 'verbose', alias: 'v', type: Boolean }, + { name: `target`, type: String, defaultValue: `` }, + { name: `module`, type: String, defaultValue: `` }, + { name: `coverage`, type: Boolean, defaultValue: false }, + { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] }, + { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] } +]); + +const { targets, modules } = argv; + +argv.target && !targets.length && targets.push(argv.target); +argv.module && !modules.length && modules.push(argv.module); +(argv.all || !targets.length) && targets.push(`all`); +(argv.all || !modules.length) && modules.push(`all`); + +module.exports = { argv, targets, modules }; \ No newline at end of file diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js new file mode 100644 index 0000000000000..d160ecb0e5de4 --- /dev/null +++ b/js/gulp/arrow-task.js @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + mainExport, gCCLanguageNames, + targetDir, observableFromStreams +} = require('./util'); + +const gulp = require('gulp'); +const path = require('path'); +const gulpRename = require(`gulp-rename`); +const { memoizeTask } = require('./memoize-task'); +const { Observable, ReplaySubject } = require('rxjs'); + +const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, format) { + const out = targetDir(target), srcGlob = `src/**/*`; + const es5Glob = `${targetDir(`es5`, `cjs`)}/**/*.js`; + const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`; + const es5UmdGlob = `${targetDir(`es5`, `umd`)}/**/*.js`; + const es5UmdMaps = `${targetDir(`es5`, `umd`)}/**/*.map`; + const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/**/*.js`; + const es2015UmdMaps = `${targetDir(`es2015`, `umd`)}/**/*.map`; + const ch_ext = (ext) => gulpRename((p) => { p.extname = ext; }); + const append = (ap) => gulpRename((p) => { p.basename += ap; }); + return Observable.forkJoin( + observableFromStreams(gulp.src(srcGlob), gulp.dest(out)), // copy src ts files + observableFromStreams(gulp.src(es5Glob), gulp.dest(out)), // copy es5 cjs files + observableFromStreams(gulp.src(esmGlob), ch_ext(`.mjs`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs` + observableFromStreams(gulp.src(es5UmdGlob), append(`.es5.min`), gulp.dest(out)), // copy es5 umd files and add `.min` + observableFromStreams(gulp.src(es5UmdMaps), gulp.dest(out)), // copy es5 umd sourcemap files, but don't rename + observableFromStreams(gulp.src(es2015UmdGlob), append(`.es2015.min`), gulp.dest(out)), // copy es2015 umd files and add `.es6.min` + observableFromStreams(gulp.src(es2015UmdMaps), gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename + ).publish(new ReplaySubject()).refCount(); +}))({}); + +const arrowTSTask = ((cache) => memoizeTask(cache, function copyTS(target, format) { + return observableFromStreams(gulp.src(`src/**/*`), gulp.dest(targetDir(target, format))); +}))({}); + + +module.exports = arrowTask; +module.exports.arrowTask = arrowTask; +module.exports.arrowTSTask = arrowTSTask; \ No newline at end of file diff --git a/js/gulp/build-task.js b/js/gulp/build-task.js new file mode 100644 index 0000000000000..01152e662fcec --- /dev/null +++ b/js/gulp/build-task.js @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { npmPkgName } = require('./util'); +const { memoizeTask } = require('./memoize-task'); + +const uglifyTask = require('./uglify-task'); +const closureTask = require('./closure-task'); +const typescriptTask = require('./typescript-task'); +const { arrowTask, arrowTSTask } = require('./arrow-task'); + +const buildTask = ((cache) => memoizeTask(cache, function build(target, format, ...args) { + return target === npmPkgName ? arrowTask(target, format, ...args)() + : target === `ts` ? arrowTSTask(target, format, ...args)() + : format === `umd` ? target === `es5` ? closureTask(target, format, ...args)() + : uglifyTask(target, format, ...args)() + : typescriptTask(target, format, ...args)(); +}))({}); + +module.exports = buildTask; +module.exports.buildTask = buildTask; \ No newline at end of file diff --git a/js/gulp/clean-task.js b/js/gulp/clean-task.js new file mode 100644 index 0000000000000..d6c90f4637c8b --- /dev/null +++ b/js/gulp/clean-task.js @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const del = require('del'); +const { targetDir } = require('./util'); +const { memoizeTask } = require('./memoize-task'); +const { Observable, ReplaySubject } = require('rxjs'); + +const cleanTask = ((cache) => memoizeTask(cache, function clean(target, format) { + return Observable + .from(del(`${targetDir(target, format)}/**`)) + .catch((e) => Observable.empty()) + .multicast(new ReplaySubject()).refCount(); +}))({}); + +module.exports = cleanTask; +module.exports.cleanTask = cleanTask; \ No newline at end of file diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js new file mode 100644 index 0000000000000..950bf40e22a39 --- /dev/null +++ b/js/gulp/closure-task.js @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, + mainExport, + gCCLanguageNames, + UMDSourceTargets, + observableFromStreams +} = require('./util'); + +const gulp = require('gulp'); +const path = require('path'); +const sourcemaps = require('gulp-sourcemaps'); +const { memoizeTask } = require('./memoize-task'); +const { Observable, ReplaySubject } = require('rxjs'); +const closureCompiler = require('google-closure-compiler').gulp(); + +const closureTask = ((cache) => memoizeTask(cache, function closure(target, format) { + const src = targetDir(target, `cls`); + const out = targetDir(target, format); + const entry = path.join(src, mainExport); + const externs = path.join(src, `${mainExport}.externs`); + return observableFromStreams( + gulp.src([ +/* external libs first --> */ `closure-compiler-scripts/*.js`, +/* then sources glob --> */ `${src}/**/*.js`, +/* and exclusions last --> */ `!${src}/format/*.js`, + `!${src}/Arrow.externs.js`, + ], { base: `./` }), + sourcemaps.init(), + closureCompiler(createClosureArgs(entry, externs)), + // rename the sourcemaps from *.js.map files to *.min.js.map + sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }), + gulp.dest(out) + ).publish(new ReplaySubject()).refCount(); +}))({}); + +const createClosureArgs = (entry, externs) => ({ + third_party: true, + warning_level: `QUIET`, + dependency_mode: `LOOSE`, + rewrite_polyfills: false, + externs: `${externs}.js`, + entry_point: `${entry}.js`, + // formatting: `PRETTY_PRINT`, + compilation_level: `ADVANCED`, + assume_function_wrapper: true, + js_output_file: `${mainExport}.js`, + language_in: gCCLanguageNames[`es2015`], + language_out: gCCLanguageNames[`es5`], + output_wrapper: +`// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (factory(global.Arrow = global.Arrow || {})); +}(this, (function (exports) {%output%}.bind(this))));` +}); + +module.exports = closureTask; +module.exports.closureTask = closureTask; diff --git a/js/gulp/memoize-task.js b/js/gulp/memoize-task.js new file mode 100644 index 0000000000000..0b0fc843c451a --- /dev/null +++ b/js/gulp/memoize-task.js @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { taskName } = require('./util'); + +const memoizeTask = ((cache, taskFn) => ((target, format, ...args) => { + // Give the memoized fn a displayName so gulp's output is easier to follow. + const fn = () => ( + cache[taskName(target, format)] || ( + cache[taskName(target, format)] = taskFn(target, format, ...args))); + fn.displayName = `${taskFn.name || ``}:${taskName(target, format, ...args)}:task`; + return fn; +})); + +module.exports = memoizeTask; +module.exports.memoizeTask = memoizeTask; \ No newline at end of file diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js new file mode 100644 index 0000000000000..7b4b15a33e6ef --- /dev/null +++ b/js/gulp/package-task.js @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + metadataFiles, packageJSONFields, + mainExport, npmPkgName, npmOrgName, + targetDir, packageName, observableFromStreams +} = require('./util'); + +const gulp = require('gulp'); +const { memoizeTask } = require('./memoize-task'); +const { Observable, ReplaySubject } = require('rxjs'); +const gulpJsonTransform = require('gulp-json-transform'); + +const packageTask = ((cache) => memoizeTask(cache, function bundle(target, format) { + const out = targetDir(target, format); + const jsonTransform = gulpJsonTransform(target === npmPkgName ? createMainPackageJson(target, format) : + target === `ts` ? createTypeScriptPackageJson(target, format) + : createScopedPackageJSON(target, format), + 2); + return Observable.forkJoin( + observableFromStreams(gulp.src(metadataFiles), gulp.dest(out)), // copy metadata files + observableFromStreams(gulp.src(`package.json`), jsonTransform, gulp.dest(out)) // write packageJSONs + ).publish(new ReplaySubject()).refCount(); +}))({}); + +module.exports = packageTask; +module.exports.packageTask = packageTask; + +const createMainPackageJson = (target, format) => (orig) => ({ + ...createTypeScriptPackageJson(target, format)(orig), + name: npmPkgName, + main: mainExport, + module: `${mainExport}.mjs`, + browser: `${mainExport}.es5.min.js`, + [`browser:es2015`]: `${mainExport}.es2015.min.js`, + [`@std/esm`]: { esm: `mjs` }, +}); + +const createTypeScriptPackageJson = (target, format) => (orig) => ({ + ...createScopedPackageJSON(target, format)(orig), + main: `${mainExport}.ts`, types: `${mainExport}.ts` +}); + +const createScopedPackageJSON = (target, format) => (({ name, ...orig }) => + conditionallyAddStandardESMEntry(target, format)( + packageJSONFields.reduce( + (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }), + { name: `${npmOrgName}/${packageName(target, format)}`, + version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`, + browser: undefined, [`browser:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined } + ) + ) +); + +const conditionallyAddStandardESMEntry = (target, format) => (packageJSON) => ( + format !== `esm` + ? packageJSON + : { ...packageJSON, [`@std/esm`]: { esm: `js` } } +); + \ No newline at end of file diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js new file mode 100644 index 0000000000000..b46b2bb14edcb --- /dev/null +++ b/js/gulp/test-task.js @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const path = require('path'); +const child_process = require(`child_process`); +const { argv } = require('./argv'); +const { memoizeTask } = require('./memoize-task'); + +const jestArgv = []; +argv.update && jestArgv.push(`-u`); +argv.verbose && jestArgv.push(`--verbose`); +argv.coverage && jestArgv.push(`--coverage`); + +const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`]; +const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`)); + +const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) { + const opts = Object.assign({}, testOptions); + const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv]; + opts.env = Object.assign({}, opts.env, { TEST_TARGET: target, TEST_MODULE: format }); + return !debug ? + child_process.spawn(jest, args, opts) : + child_process.exec(`node --inspect-brk ${jest} ${args.join(` `)}`, opts); +}))({}, jestArgv, { + env: Object.assign({}, process.env), + stdio: [`ignore`, `inherit`, `inherit`], +}); + +module.exports = testTask; +module.exports.testTask = testTask; diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js new file mode 100644 index 0000000000000..2c66846667cd3 --- /dev/null +++ b/js/gulp/typescript-task.js @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, tsconfigName, observableFromStreams +} = require('./util'); + +const gulp = require('gulp'); +const path = require('path'); +const ts = require(`gulp-typescript`); +const sourcemaps = require('gulp-sourcemaps'); +const { memoizeTask } = require('./memoize-task'); +const { Observable, ReplaySubject } = require('rxjs'); + +const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target, format) { + const out = targetDir(target, format); + const tsconfigFile = `tsconfig.${tsconfigName(target, format)}.json`; + const tsProject = ts.createProject(path.join(`tsconfig`, tsconfigFile), { typescript: require(`typescript`) }); + const { stream: { js, dts } } = observableFromStreams( + tsProject.src(), sourcemaps.init(), + tsProject(ts.reporter.fullReporter(true)) + ); + const writeDTypes = observableFromStreams(dts, gulp.dest(out)); + const writeJS = observableFromStreams(js, sourcemaps.write(), gulp.dest(out)); + return Observable.forkJoin(writeDTypes, writeJS).publish(new ReplaySubject()).refCount(); +}))({}); + +module.exports = typescriptTask; +module.exports.typescriptTask = typescriptTask; \ No newline at end of file diff --git a/js/gulp/uglify-task.js b/js/gulp/uglify-task.js new file mode 100644 index 0000000000000..804d450453644 --- /dev/null +++ b/js/gulp/uglify-task.js @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, + mainExport, + ESKeywords, + UMDSourceTargets, + uglifyLanguageNames, + observableFromStreams +} = require('./util'); + +const path = require('path'); +const webpack = require(`webpack`); +const { memoizeTask } = require('./memoize-task'); +const { Observable, ReplaySubject } = require('rxjs'); +const UglifyJSPlugin = require(`uglifyjs-webpack-plugin`); +const esmRequire = require(`@std/esm`)(module, { cjs: true, esm: `js` }); + +const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJS(target, format) { + + const sourceTarget = UMDSourceTargets[target]; + const PublicNames = reservePublicNames(sourceTarget, `cls`); + const out = targetDir(target, format), src = targetDir(sourceTarget, `cls`); + + const targetConfig = { ...commonConfig, + output: { ...commonConfig.output, + path: path.resolve(`./${out}`) } }; + + const webpackConfigs = [ + [mainExport, PublicNames] + ].map(([entry, reserved]) => ({ + ...targetConfig, + name: entry, + entry: { [entry]: path.resolve(`${src}/${entry}.js`) }, + plugins: [ + ...(targetConfig.plugins || []), + new webpack.SourceMapDevToolPlugin({ + filename: `[name].${target}.min.js.map`, + moduleFilenameTemplate: ({ resourcePath }) => + resourcePath + .replace(/\s/, `_`) + .replace(/\.\/node_modules\//, ``) + }), + new UglifyJSPlugin({ + sourceMap: true, + uglifyOptions: { + ecma: uglifyLanguageNames[target], + compress: { unsafe: true }, + output: { comments: false, beautify: false }, + mangle: { eval: true, safari10: true, // <-- Works around a Safari 10 bug: // https://github.com/mishoo/UglifyJS2/issues/1753 + properties: { reserved, keep_quoted: true } + } + }, + }) + ] + })); + + const compilers = webpack(webpackConfigs); + return Observable + .bindNodeCallback(compilers.run.bind(compilers))() + .multicast(new ReplaySubject()).refCount(); +}))({}, { + resolve: { mainFields: [`module`, `main`] }, + module: { rules: [{ test: /\.js$/, enforce: `pre`, use: [`source-map-loader`] }] }, + output: { filename: '[name].js', library: mainExport, libraryTarget: `umd`, umdNamedDefine: true }, +}); + +module.exports = uglifyTask; +module.exports.uglifyTask = uglifyTask; + +const reservePublicNames = ((ESKeywords) => function reservePublicNames(target, format) { + const publicModulePath = `../${targetDir(target, format)}/${mainExport}.js`; + return [ + ...ESKeywords, + ...reserveExportedNames(esmRequire(publicModulePath)) + ]; +})(ESKeywords); + +// Reflect on the Arrow modules to come up with a list of keys to save from Uglify's +// mangler. Assume all the non-inherited static and prototype members of the Arrow +// module and its direct exports are public, and should be preserved through minification. +const reserveExportedNames = (entryModule) => ( + Object + .getOwnPropertyNames(entryModule) + .filter((name) => ( + typeof entryModule[name] === `object` || + typeof entryModule[name] === `function` + )) + .map((name) => [name, entryModule[name]]) + .reduce((reserved, [name, value]) => { + const fn = function() {}; + const ownKeys = value && Object.getOwnPropertyNames(value) || []; + const protoKeys = typeof value === `function` && Object.getOwnPropertyNames(value.prototype) || []; + const publicNames = [...ownKeys, ...protoKeys].filter((x) => x !== `default` && x !== `undefined` && !(x in fn)); + return [...reserved, name, ...publicNames]; + }, [] + ) +); diff --git a/js/gulp/util.js b/js/gulp/util.js new file mode 100644 index 0000000000000..21ffc3127339c --- /dev/null +++ b/js/gulp/util.js @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const path = require(`path`); +const pump = require(`pump`); +const { Observable, ReplaySubject } = require('rxjs'); + +const mainExport = `Arrow`; +const npmPkgName = `apache-arrow`; +const npmOrgName = `@${npmPkgName}`; + +const releasesRootDir = `targets`; +const knownTargets = [`es5`, `es2015`, `esnext`]; +const knownModules = [`cjs`, `esm`, `cls`, `umd`]; +const moduleFormatsToSkipCombosOf = { cls: true }; +const metadataFiles = [`LICENSE`, `README.md`]; +const packageJSONFields = [ + `version`, `license`, `description`, + `author`, `homepage`, `repository`, + `bugs`, `keywords`, `dependencies` +]; + +// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988 +const gCCLanguageNames = { + es5: `ECMASCRIPT5`, + es2015: `ECMASCRIPT_2015`, + es2016: `ECMASCRIPT_2016`, + es2017: `ECMASCRIPT_2017`, + esnext: `ECMASCRIPT_NEXT` +}; + +const UMDSourceTargets = { + es5: `es5`, + es2015: `es2015`, + es2016: `es2015`, + es2017: `es2015`, + esnext: `es2015` +}; + +const uglifyLanguageNames = { + es5: 5, es2015: 6, + es2016: 7, es2017: 8, + esnext: 8 // <--- ? +}; + +// ES7+ keywords Uglify shouldn't mangle +// Hardcoded here since some are from ES7+, others are +// only defined in interfaces, so difficult to get by reflection. +const ESKeywords = [ + // PropertyDescriptors + `configurable`, `enumerable`, + // IteratorResult, Symbol.asyncIterator + `done`, `value`, `Symbol.asyncIterator`, `asyncIterator`, + // AsyncObserver + `values`, `hasError`, `hasCompleted`,`errorValue`, `closed`, + // Observable/Subscription/Scheduler + `next`, `error`, `complete`, `subscribe`, `unsubscribe`, `isUnsubscribed`, + // EventTarget + `addListener`, `removeListener`, `addEventListener`, `removeEventListener`, + // Arrow properties + `low`, `high`, `data`, `index`, `field`, `validity`, `columns`, `fieldNode`, `subarray`, +]; + +function taskName(target, format) { + return !format ? target : `${target}:${format}`; +} + +function packageName(target, format) { + return !format ? target : `${target}-${format}`; +} + +function tsconfigName(target, format) { + return !format ? target : `${target}.${format}`; +} + +function targetDir(target, format) { + return path.join(releasesRootDir, ...(!format ? [target] : [target, format])); +} + +function logAndDie(e) { + if (e) { + console.error(e); + process.exit(1); + } +} + +function observableFromStreams(...streams) { + const pumped = streams.length <= 1 ? streams[0] + : pump(...streams, logAndDie); + const fromEvent = Observable.fromEvent.bind(null, pumped); + const streamObs = fromEvent(`data`) + .merge(fromEvent(`error`).flatMap((e) => Observable.throw(e))) + .takeUntil(fromEvent(`end`).merge(fromEvent(`close`))) + .defaultIfEmpty(`empty stream`) + .multicast(new ReplaySubject()).refCount(); + streamObs.stream = pumped; + streamObs.observable = streamObs; + return streamObs; +} + +function* combinations(_targets, _modules) { + + const targets = known(knownTargets, _targets || [`all`]); + const modules = known(knownModules, _modules || [`all`]); + + if (_targets[0] === `all` && _modules[0] === `all`) { + yield [`ts`, ``]; + yield [npmPkgName, ``]; + } + + for (const format of modules) { + for (const target of targets) { + yield [target, format]; + } + } + + function known(known, values) { + return ~values.indexOf(`all`) + ? known + : Object.keys( + values.reduce((map, arg) => (( + (known.indexOf(arg) !== -1) && + (map[arg.toLowerCase()] = true) + || true) && map + ), {}) + ).sort((a, b) => known.indexOf(a) - known.indexOf(b)); + } +} + +module.exports = { + + mainExport, npmPkgName, npmOrgName, metadataFiles, packageJSONFields, + + knownTargets, knownModules, moduleFormatsToSkipCombosOf, + ESKeywords, gCCLanguageNames, UMDSourceTargets, uglifyLanguageNames, + + taskName, packageName, tsconfigName, targetDir, combinations, observableFromStreams, +}; \ No newline at end of file diff --git a/js/gulpfile.js b/js/gulpfile.js index 9f8e564bd9e3a..4cf0342c3be78 100644 --- a/js/gulpfile.js +++ b/js/gulpfile.js @@ -15,278 +15,92 @@ // specific language governing permissions and limitations // under the License. -const del = require(`del`); -const gulp = require(`gulp`); -const path = require(`path`); -const pump = require(`pump`); -const ts = require(`gulp-typescript`); -const streamMerge = require(`merge2`); -const sourcemaps = require(`gulp-sourcemaps`); -const child_process = require(`child_process`); -const gulpJsonTransform = require(`gulp-json-transform`); -const closureCompiler = require(`google-closure-compiler`).gulp(); - -const knownTargets = [`es5`, `es2015`, `esnext`]; -const knownModules = [`cjs`, `esm`, `cls`, `umd`]; - -// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988 -const gCCTargets = { - es5: `ECMASCRIPT5`, - es2015: `ECMASCRIPT_2015`, - es2016: `ECMASCRIPT_2016`, - es2017: `ECMASCRIPT_2017`, - esnext: `ECMASCRIPT_NEXT` -}; - -const tsProjects = []; -const argv = require(`command-line-args`)([ - { name: `all`, alias: `a`, type: Boolean }, - { name: 'update', alias: 'u', type: Boolean }, - { name: 'verbose', alias: 'v', type: Boolean }, - { name: `target`, type: String, defaultValue: `` }, - { name: `module`, type: String, defaultValue: `` }, - { name: `coverage`, type: Boolean, defaultValue: false }, - { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] }, - { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] } -]); - -const { targets, modules } = argv; - -argv.target && !targets.length && targets.push(argv.target); -argv.module && !modules.length && modules.push(argv.module); -(argv.all || !targets.length) && targets.push(`all`); -(argv.all || !modules.length) && modules.push(`all`); - -for (const [target, format] of combinations([`all`, `all`])) { - const combo = `${target}:${format}`; - gulp.task(`test:${combo}`, gulp.series(testTask(target, format, combo, `targets/${target}/${format}`))); - gulp.task(`clean:${combo}`, gulp.series(cleanTask(target, format, combo, `targets/${target}/${format}`))); - gulp.task(`build:${combo}`, gulp.series(buildTask(target, format, combo, `targets/${target}/${format}`))); - gulp.task(`bundle:${combo}`, gulp.series(bundleTask(target, format, combo, `targets/${target}/${format}`))); - gulp.task(`package:${combo}`, gulp.series(packageTask(target, format, combo, `targets/${target}/${format}`))); - gulp.task(`test:debug:${combo}`, gulp.series(testTask(target, format, combo, `targets/${target}/${format}`, true))); +const del = require('del'); +const gulp = require('gulp'); +const path = require('path'); +const { Observable } = require('rxjs'); +const testsTask = require('./gulp/test-task'); +const buildTask = require('./gulp/build-task'); +const cleanTask = require('./gulp/clean-task'); +const packageTask = require('./gulp/package-task'); +const { targets, modules } = require('./gulp/argv'); +const { + targetDir, + taskName, combinations, + knownTargets, knownModules, + npmPkgName, UMDSourceTargets, + moduleFormatsToSkipCombosOf +} = require('./gulp/util'); + +for (const [target, format] of combinations([`all`], [`all`])) { + const task = taskName(target, format); + gulp.task(`clean:${task}`, cleanTask(target, format)); + gulp.task( `test:${task}`, testsTask(target, format)); + gulp.task(`debug:${task}`, testsTask(target, format, true)); + gulp.task(`build:${task}`, gulp.series(`clean:${task}`, + buildTask(target, format), + packageTask(target, format))); } -gulp.task(`test`, gulp.series(runTaskCombos(`test`))); -gulp.task(`clean`, gulp.parallel(runTaskCombos(`clean`))); -gulp.task(`build`, gulp.parallel(runTaskCombos(`build`))); -gulp.task(`bundle`, gulp.parallel(runTaskCombos(`bundle`))); -gulp.task(`package`, gulp.parallel(runTaskCombos(`package`))); -gulp.task(`test:debug`, gulp.series(runTaskCombos(`test:debug`))); -gulp.task(`default`, gulp.task(`package`)); - -function runTaskCombos(name) { - const combos = []; +// The UMD bundles build temporary es5/6/next targets via TS, +// then run the TS source through either closure-compiler or +// uglify, so we special case that here. +knownTargets.forEach((target) => + gulp.task(`build:${target}:umd`, + gulp.series( + gulp.parallel( + cleanTask(target, `umd`), + cleanTask(UMDSourceTargets[target], `cls`), + ), + buildTask(UMDSourceTargets[target], `cls`), + buildTask(target, `umd`), packageTask(target, `umd`) + ) + ) +); + +// The main "apache-arrow" module builds the es5/cjs, es5/umd, +// es2015/esm, es2015/umd, and ts targets, then copies and +// renames the compiled output into the apache-arrow folder +gulp.task(`build:${npmPkgName}`, + gulp.series( + cleanTask(npmPkgName), + gulp.parallel( + `build:${taskName(`es5`, `cjs`)}`, + `build:${taskName(`es5`, `umd`)}`, + `build:${taskName(`es2015`, `esm`)}`, + `build:${taskName(`es2015`, `umd`)}` + ), + buildTask(npmPkgName), packageTask(npmPkgName) + ) +); + + +function gulpConcurrent(tasks) { + return () => Observable.bindCallback((tasks, cb) => gulp.parallel(tasks)(cb))(tasks); +} + +const buildConcurrent = (tasks) => () => + gulpConcurrent(tasks)() + .concat(Observable + .defer(() => Observable + .merge(...knownTargets.map((target) => + del(`${targetDir(target, `cls`)}/**`))))); + +gulp.task( `test`, gulp.series(getTasks(`test`))); +gulp.task(`debug`, gulp.series(getTasks(`debug`))); +gulp.task(`clean`, gulp.parallel(getTasks(`clean`))); +gulp.task(`build`, buildConcurrent(getTasks(`build`))); +gulp.task(`default`, gulp.series(`build`, `test`)); + +function getTasks(name) { + const tasks = []; + if (targets.indexOf(`ts`) !== -1) tasks.push(`${name}:ts`); + if (targets.indexOf(npmPkgName) !== -1) tasks.push(`${name}:${npmPkgName}`); for (const [target, format] of combinations(targets, modules)) { - if (format === `cls`) { + if (moduleFormatsToSkipCombosOf[format] && name === `test`) { continue; } - combos.push(`${name}:${target}:${format}`); - } - return combos; -} - -function cleanTask(target, format, taskName, outDir) { - return function cleanTask() { - const globs = [`${outDir}/**`]; - if (target === `es5` && format === `cjs`) { - globs.push(`types`, `typings`); - } - return del(globs); - }; -} - -function buildTask(target, format, taskName, outDir) { - return format === `umd` - ? closureTask(target, format, taskName, outDir) - : typescriptTask(target, format, taskName, outDir); -} - -function bundleTask(target, format, taskName, outDir) { - return function bundleTask() { - return streamMerge([ - pump(gulp.src([`LICENSE`, `README.md`]), gulp.dest(outDir), onError), - pump( - gulp.src(`package.json`), - gulpJsonTransform((orig) => [ - `version`, `description`, `keywords`, - `repository`, `author`, `homepage`, `bugs`, `license`, - `dependencies`, `peerDependencies` - ].reduce((copy, key) => ( - (copy[key] = orig[key]) && copy || copy - ), { - main: `Arrow.js`, - types: `Arrow.d.ts`, - typings: `Arrow.d.ts`, - name: `@apache-arrow/${target}-${format}` - }), 2), - gulp.dest(outDir), - onError - ) - ]); - } -} - -function packageTask(target, format, taskName, outDir) { - return [`build:${taskName}`, `bundle:${taskName}`]; -} - -function testTask(target, format, taskName, outDir, debug) { - const jestOptions = !debug ? [] : [ - `--runInBand`, `--env`, `jest-environment-node-debug`]; - argv.update && jestOptions.unshift(`-u`); - argv.verbose && jestOptions.unshift(`--verbose`); - argv.coverage && jestOptions.unshift(`--coverage`); - const jestPath = `./node_modules/.bin/jest`; - const debugOpts = jestOptions.join(' '); - const spawnOptions = { - stdio: [`ignore`, `inherit`, `inherit`], - env: Object.assign({}, process.env, { - TEST_TARGET: target, TEST_MODULE: format - }) - }; - return function testTask() { - return !debug ? - child_process.spawn(jestPath, jestOptions, spawnOptions) : - child_process.exec(`node --inspect-brk ${jestPath} ${debugOpts}`, spawnOptions); - } -} - -function closureTask(target, format, taskName, outDir) { - const clsTarget = `es5`; - const googleRoot = `targets/${clsTarget}/cls`; - const languageIn = clsTarget === `es5` ? `es2015` : clsTarget; - return [ - [`clean:${taskName}`, `build:${clsTarget}:cls`], - function closureTask() { - return closureStream( - closureSrcs(), - closureCompiler(closureArgs()) - ).on('end', () => del([`targets/${target}/cls/**`])); - } - ]; - function closureSrcs() { - return gulp.src([ - `closure-compiler-scripts/*.js`, - `${googleRoot}/**/*.js`, - `!${googleRoot}/format/*.js`, - `!${googleRoot}/Arrow.externs.js`, - ], { base: `./` }); - } - function closureStream(sources, compiler) { - const streams = [ - sources, - sourcemaps.init(), - compiler, - sourcemaps.write('.'), - gulp.dest(outDir) - ]; - // copy the ES5 UMD bundle to dist - if (target === `es5`) { - streams.push(gulp.dest(`dist`)); - } - return pump(...streams, onError); - } - function closureArgs() { - return { - third_party: true, - externs: `${googleRoot}/Arrow.externs.js`, - warning_level: `QUIET`, - dependency_mode: `LOOSE`, - rewrite_polyfills: false, - // formatting: `PRETTY_PRINT`, - compilation_level: `ADVANCED`, - assume_function_wrapper: true, - js_output_file: `Arrow.js`, - language_in: gCCTargets[languageIn], - language_out: gCCTargets[clsTarget], - entry_point: `${googleRoot}/Arrow.js`, - output_wrapper: -`// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -(function (global, factory) { - typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : - typeof define === 'function' && define.amd ? define(['exports'], factory) : - (factory(global.Arrow = global.Arrow || {})); -}(this, (function (exports) {%output%}.bind(this))));` - }; - } -} - -function typescriptTask(target, format, taskName, outDir) { - return [ - [`clean:${taskName}`], - function typescriptTask() { - const tsconfigPath = `tsconfig/tsconfig.${target}.${format}.json`; - let { js, dts } = tsProjects.find((p) => p.target === target && p.format === format) || {}; - if (!js || !dts) { - let tsProject = ts.createProject(tsconfigPath); - ({ js, dts } = pump( - tsProject.src(), - sourcemaps.init(), - tsProject(ts.reporter.fullReporter(true)), - onError - )); - dts = [dts, gulp.dest(outDir)]; - js = [js, sourcemaps.write(), gulp.dest(outDir)]; - // copy types to the root - if (target === `es5` && format === `cjs`) { - dts.push(gulp.dest(`types`)); - } - tsProjects.push({ - target, format, - js: js = pump(...js, onError), - dts: dts = pump(...dts, onError) - }); - } - return streamMerge([ dts, js ]); - } - ]; -} - -function* combinations(_targets, _modules) { - - const targets = known(knownTargets, _targets || [`all`]); - const modules = known(knownModules, _modules || [`all`]); - - for (const format of modules) { - for (const target of targets) { - yield [target, format]; - } - } - - function known(known, values) { - return ~values.indexOf(`all`) - ? known - : Object.keys( - values.reduce((map, arg) => (( - (known.indexOf(arg) !== -1) && - (map[arg.toLowerCase()] = true) - || true) && map - ), {}) - ).sort((a, b) => known.indexOf(a) - known.indexOf(b)); + tasks.push(`${name}:${taskName(target, format)}`); } + return tasks.length && tasks || [(done) => done()]; } - -function onError(err) { - if (typeof err === 'number') { - process.exit(err); - } else if (err) { - console.error(err.stack || err.toString()); - process.exit(1); - } -} \ No newline at end of file diff --git a/js/package.json b/js/package.json index 03687a8b25ca2..ba93a34686288 100644 --- a/js/package.json +++ b/js/package.json @@ -14,18 +14,18 @@ "test": "gulp test", "build": "gulp build", "clean": "gulp clean", + "debug": "gulp debug", "bundle": "gulp bundle", "package": "gulp package", "perf": "node ./perf/index.js", - "test:debug": "gulp test:debug", "test:coverage": "gulp test -t esnext -m esm --coverage", "validate": "npm-run-all clean lint build test bundle", "lerna:publish": "lerna exec --bail=false npm publish", "prepublishOnly": "sh ./prepublish.sh", "doc": "shx rm -rf ./doc && esdoc", "lint": "npm-run-all -p lint:*", - "lint:src": "tslint --fix --type-check -p tsconfig.json -c tslint.json \"src/**/*.ts\"", - "lint:test": "tslint --fix --type-check -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"" + "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"", + "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"" }, "repository": { "type": "git", @@ -50,7 +50,6 @@ "README.md" ], "peerDependencies": { - "tslib": "~1.7.1", "command-line-usage": "4.0.1" }, "dependencies": { @@ -58,38 +57,44 @@ "text-encoding": "0.6.4" }, "devDependencies": { - "@types/flatbuffers": "1.6.4", - "@types/jest": "20.0.8", - "@types/node": "^8.0.24", + "@std/esm": "0.12.5", + "@types/flatbuffers": "1.6.5", + "@types/jest": "21.1.5", + "@types/node": "8.0.47", "@types/text-encoding": "0.0.32", "benchmark": "2.1.4", - "coveralls": "2.13.1", "command-line-args": "4.0.7", + "coveralls": "3.0.0", "del": "3.0.0", "esdoc": "1.0.3", "esdoc-standard-plugin": "1.0.0", "google-closure-compiler": "20170910.0.0", "gulp": "github:gulpjs/gulp#4.0", - "gulp-json-transform": "0.4.2", + "gulp-json-transform": "0.4.5", + "gulp-rename": "1.2.2", "gulp-sourcemaps": "2.6.1", - "gulp-typescript": "3.2.2", - "jest": "21.1.0", + "gulp-typescript": "3.2.3", + "jest": "21.2.1", "jest-environment-node-debug": "2.0.0", "json": "9.0.6", - "lerna": "2.2.0", - "lint-staged": "4.2.1", + "lerna": "2.5.0", + "lint-staged": "4.3.0", "merge2": "1.2.0", "mkdirp": "0.5.1", "npm-run-all": "4.1.1", "pump": "1.0.2", "rimraf": "2.6.2", + "rxjs": "5.5.2", "shx": "0.2.2", + "source-map-loader": "0.2.3", "text-encoding-utf-8": "1.0.1", "trash": "4.1.0", - "ts-jest": "21.0.1", - "tslib": "1.7.1", - "tslint": "5.7.0", - "typescript": "2.5.2" + "ts-jest": "21.1.4", + "tslib": "1.8.0", + "tslint": "5.8.0", + "typescript": "2.6.1", + "uglifyjs-webpack-plugin": "1.0.1", + "webpack": "3.8.1" }, "lint-staged": { "*.@(ts)": [ diff --git a/js/perf/index.js b/js/perf/index.js index 669f690122d10..3a2ed96772330 100644 --- a/js/perf/index.js +++ b/js/perf/index.js @@ -16,12 +16,10 @@ // under the License. // Use the ES5 UMD target as perf baseline -// ES6/7 iterators are faster in turbofan, but something about the -// ES5 transpilation (rewriting let and const to var?) JITs better -const { Table, readBuffers } = require('../dist/Arrow'); +// const { Table, readBuffers } = require('../targets/es5/umd'); // const { Table, readBuffers } = require('../targets/es5/cjs'); +const { Table, readBuffers } = require('../targets/es2015/umd'); // const { Table, readBuffers } = require('../targets/es2015/cjs'); -// const { Table, readBuffers } = require('../targets/esnext/cjs'); const Benchmark = require('benchmark'); const arrowTestConfigurations = require('./config'); @@ -35,12 +33,12 @@ for (let [name, ...buffers] of arrowTestConfigurations) { const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true }); parseSuite.add(createFromTableTest(name, buffers)); parseSuite.add(createReadBuffersTest(name, buffers)); - for (const vector of Table.from(...buffers).cols()) { + for (const vector of Table.from(...buffers).columns) { sliceSuite.add(createSliceTest(vector)); iterateSuite.add(createIterateTest(vector)); getByIndexSuite.add(createGetByIndexTest(vector)); } - suites.push(parseSuite, sliceSuite, getByIndexSuite, iterateSuite); + suites.push(getByIndexSuite, iterateSuite, sliceSuite, parseSuite); } console.log('Running apache-arrow performance tests...\n'); diff --git a/js/src/Arrow.externs.ts b/js/src/Arrow.externs.ts index 7289d6d2732b6..c23930271183d 100644 --- a/js/src/Arrow.externs.ts +++ b/js/src/Arrow.externs.ts @@ -24,23 +24,30 @@ Symbol.iterator; /** @type {symbol} */ Symbol.asyncIterator; -let Table = function() {}; + +let RowVector = function() {}; /** @type {?} */ -Table.prototype.length; +RowVector.prototype.toJSON; /** @type {?} */ -Table.prototype.rows; +RowVector.prototype.toArray; /** @type {?} */ -Table.prototype.cols; +RowVector.prototype.toObject; /** @type {?} */ -Table.prototype.getRow; +RowVector.prototype.toString; + +let Table = function() {}; +/** @type {?} */ +( Table).from; /** @type {?} */ -Table.prototype.getCell; +Table.prototype.columns; /** @type {?} */ -Table.prototype.getCellAt; +Table.prototype.length; /** @type {?} */ -Table.prototype.getColumn; +Table.prototype.col; /** @type {?} */ -Table.prototype.getColumnAt; +Table.prototype.key; +/** @type {?} */ +Table.prototype.select; /** @type {?} */ Table.prototype.toString; @@ -52,24 +59,26 @@ Vector.prototype.name; /** @type {?} */ Vector.prototype.type; /** @type {?} */ -Vector.prototype.props; -/** @type {?} */ Vector.prototype.get; /** @type {?} */ Vector.prototype.concat; /** @type {?} */ Vector.prototype.slice; - -let TypedVector = function() {}; /** @type {?} */ -TypedVector.prototype.arrayType; +Vector.prototype.metadata; +/** @type {?} */ +Vector.prototype.nullable; +/** @type {?} */ +Vector.prototype.nullCount; -let ValidityVector = function() {}; +let BoolVector = function() {}; +/** @type {?} */ +( BoolVector).pack; /** @type {?} */ -( ValidityVector).pack; +BoolVector.prototype.set; let DictionaryVector = function() {}; /** @type {?} */ -DictionaryVector.prototype.index; +DictionaryVector.prototype.getKey; /** @type {?} */ -DictionaryVector.prototype.value; +DictionaryVector.prototype.getValue; diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts index ea8a5c3e1d9bb..3196550884dbf 100644 --- a/js/src/Arrow.ts +++ b/js/src/Arrow.ts @@ -15,44 +15,45 @@ // specific language governing permissions and limitations // under the License. -import { Table } from './table'; import { readBuffers } from './reader/arrow'; -import { Vector } from './vector/vector'; -import { StructVector } from './vector/struct'; -import { DictionaryVector } from './vector/dictionary'; -import { ListVector, Utf8Vector, FixedSizeListVector } from './vector/list'; -import { - TypedVector, BitVector, - DateVector, IndexVector, - Int8Vector, Int16Vector, - Int32Vector, Int64Vector, - Uint8Vector, Uint16Vector, - Uint32Vector, Uint64Vector, - Float32Vector, Float64Vector, -} from './vector/typed'; -export { - Table, readBuffers, - Vector, - BitVector, - ListVector, - Utf8Vector, - DateVector, - IndexVector, +import { Vector } from './types/types'; +import { ListVector } from './types/list'; +import { Utf8Vector } from './types/utf8'; +import { BoolVector } from './types/vector/bool'; +import { DateVector } from './types/vector/date'; +import { RowVector } from './types/table/row'; +import { TableVector } from './types/table/table'; +import { StructVector } from './types/table/struct'; +import { DictionaryVector } from './types/dictionary'; +import { FixedSizeListVector } from './types/fixedsizelist'; +import { LongVector, Int64Vector, Uint64Vector, } from './types/vector/long'; +import { TypedVector, Int8Vector, Int16Vector, Int32Vector, - Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, - Uint64Vector, Float32Vector, - Float64Vector, - StructVector, + Float64Vector +} from './types/vector/typed'; + +import './types/table/from'; + +export { + Vector, + readBuffers, DictionaryVector, - FixedSizeListVector, + RowVector as Row, + TableVector as Table, + StructVector, Utf8Vector, + ListVector, FixedSizeListVector, + BoolVector, TypedVector, LongVector, + DateVector, Float32Vector, Float64Vector, + Int8Vector, Int16Vector, Int32Vector, Int64Vector, + Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, }; /* These exports are needed for the closure umd targets */ @@ -60,15 +61,18 @@ try { const Arrow = eval('exports'); if (typeof Arrow === 'object') { // string indexers tell closure compiler not to rename these properties - Arrow['Table'] = Table; - Arrow['readBuffers'] = readBuffers; Arrow['Vector'] = Vector; - Arrow['BitVector'] = BitVector; - Arrow['ListVector'] = ListVector; + Arrow['Table'] = TableVector; + Arrow['readBuffers'] = readBuffers; + Arrow['BoolVector'] = BoolVector; Arrow['Utf8Vector'] = Utf8Vector; - Arrow['DateVector'] = DateVector; - Arrow['IndexVector'] = IndexVector; + Arrow['ListVector'] = ListVector; + Arrow['StructVector'] = StructVector; + Arrow['DictionaryVector'] = DictionaryVector; + Arrow['FixedSizeListVector'] = FixedSizeListVector; + Arrow['LongVector'] = LongVector; Arrow['TypedVector'] = TypedVector; + Arrow['DateVector'] = DateVector; Arrow['Int8Vector'] = Int8Vector; Arrow['Int16Vector'] = Int16Vector; Arrow['Int32Vector'] = Int32Vector; @@ -79,9 +83,6 @@ try { Arrow['Uint64Vector'] = Uint64Vector; Arrow['Float32Vector'] = Float32Vector; Arrow['Float64Vector'] = Float64Vector; - Arrow['StructVector'] = StructVector; - Arrow['DictionaryVector'] = DictionaryVector; - Arrow['FixedSizeListVector'] = FixedSizeListVector; } } catch (e) { /* not the UMD bundle */ } /* end closure exports */ diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts index dbb6acd0e79e8..033bfecae61dd 100644 --- a/js/src/reader/arrow.ts +++ b/js/src/reader/arrow.ts @@ -18,17 +18,18 @@ import { flatbuffers } from 'flatbuffers'; import * as Schema_ from '../format/Schema_generated'; import * as Message_ from '../format/Message_generated'; +export import Schema = Schema_.org.apache.arrow.flatbuf.Schema; +export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; import { readFile } from './file'; import { readStream } from './stream'; import { readVector } from './vector'; -import { Vector } from '../vector/vector'; import { readDictionary } from './dictionary'; +import { Vector, Column } from '../types/types'; import ByteBuffer = flatbuffers.ByteBuffer; -export import Schema = Schema_.org.apache.arrow.flatbuf.Schema; -export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; -export type Dictionaries = { [k: string]: Vector }; +import Field = Schema_.org.apache.arrow.flatbuf.Field; +export type Dictionaries = { [k: string]: Vector } | null; export type IteratorState = { nodeIndex: number; bufferIndex: number }; export function* readRecords(...bytes: ByteBuffer[]) { @@ -47,22 +48,27 @@ export function* readBuffers(...bytes: Array) { const dictionaries: Dictionaries = {}; const byteBuffers = bytes.map(toByteBuffer); for (let { schema, batch } of readRecords(...byteBuffers)) { - let vectors: Vector[] = []; + let vectors: Column[] = []; let state = { nodeIndex: 0, bufferIndex: 0 }; - let index = -1, fieldsLength = schema.fieldsLength(); + let fieldsLength = schema.fieldsLength(); + let index = -1, field: Field, vector: Vector; if (batch.id) { // A dictionary batch only contain a single vector. Traverse each // field and its children until we find one that uses this dictionary while (++index < fieldsLength) { - let vector = readDictionary(schema.fields(index), batch, state, dictionaries); - if (vector) { - dictionaries[batch.id] = dictionaries[batch.id] && dictionaries[batch.id].concat(vector) || vector; - break; + if (field = schema.fields(index)!) { + if (vector = readDictionary(field, batch, state, dictionaries)!) { + dictionaries[batch.id] = dictionaries[batch.id] && dictionaries[batch.id].concat(vector) || vector; + break; + } } } } else { while (++index < fieldsLength) { - vectors[index] = readVector(schema.fields(index), batch, state, dictionaries); + if ((field = schema.fields(index)!) && + (vector = readVector(field, batch, state, dictionaries)!)) { + vectors[index] = vector as Column; + } } yield vectors; } diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts index 93a9ba76bba3a..0c58ace3b0dca 100644 --- a/js/src/reader/dictionary.ts +++ b/js/src/reader/dictionary.ts @@ -17,22 +17,20 @@ import { readVector } from './vector'; import { MessageBatch } from './message'; +import { DictionaryVector } from '../types/dictionary'; import * as Schema_ from '../format/Schema_generated'; import { IteratorState, Dictionaries } from './arrow'; import Field = Schema_.org.apache.arrow.flatbuf.Field; -export function readDictionary(field: Field | null, - batch: MessageBatch, - iterator: IteratorState, - dictionaries: Dictionaries) { - let id: string, encoding = field && field.dictionary(); +export function readDictionary(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector | null { + let vector: DictionaryVector | null, id, encoding = field.dictionary(); if (encoding && batch.id === (id = encoding.id().toFloat64().toString())) { - return readVector(field, batch, iterator, null); + return readVector(field, batch, iterator, null) as DictionaryVector; } - for (let i = -1, n = field && field.childrenLength() || 0; ++i < n;) { - let vector = readDictionary(field.children(i), batch, iterator, dictionaries); - if (vector) { + for (let i = -1, n = field.childrenLength() | 0; ++i < n;) { + if (vector = readDictionary(field.children(i)!, batch, iterator, dictionaries)) { return vector; } } + return null; } diff --git a/js/src/reader/file.ts b/js/src/reader/file.ts index b05b99a5e6dcf..bd60b476312a0 100644 --- a/js/src/reader/file.ts +++ b/js/src/reader/file.ts @@ -58,9 +58,12 @@ export function* readFile(...bbs: ByteBuffer[]) { throw new Error('Invalid file'); } bb.setPosition(footerOffset - footerLength); - let footer = Footer.getRootAsFooter(bb), schema = footer.schema(); + let schema, footer = Footer.getRootAsFooter(bb); + if (!(schema = footer.schema()!)) { + return; + } for (let i = -1, n = footer.dictionariesLength(); ++i < n;) { - let block = footer.dictionaries(i); + let block = footer.dictionaries(i)!; bb.setPosition(block.offset().low); for (let batch of readMessageBatches(bb)) { yield { schema, batch }; @@ -68,7 +71,7 @@ export function* readFile(...bbs: ByteBuffer[]) { } } for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) { - const block = footer.recordBatches(i); + const block = footer.recordBatches(i)!; bb.setPosition(block.offset().low); for (let batch of readMessageBatches(bb)) { yield { schema, batch }; diff --git a/js/src/reader/message.ts b/js/src/reader/message.ts index 5472f10833878..6c8a969021f90 100644 --- a/js/src/reader/message.ts +++ b/js/src/reader/message.ts @@ -46,13 +46,13 @@ export function* readMessageBatches(bb: ByteBuffer) { let bytes = bb.bytes(); for (let message of readMessages(bb)) { let type = message.headerType(); - let id: string, data: RecordBatch; + let id: string | void, data: RecordBatch; if (type === MessageHeader.RecordBatch) { - data = message.header(new RecordBatch()); + data = message.header(new RecordBatch())!; } else if (type === MessageHeader.DictionaryBatch) { - let header = message.header(new DictionaryBatch()); + let header = message.header(new DictionaryBatch())!; id = header.id().toFloat64().toString(); - data = header.data(); + data = header.data()!; } else { continue; } diff --git a/js/src/reader/stream.ts b/js/src/reader/stream.ts index 9869f633d08f4..2062b1a8c4c10 100644 --- a/js/src/reader/stream.ts +++ b/js/src/reader/stream.ts @@ -30,8 +30,8 @@ export function* readStream(...bbs: ByteBuffer[]) { throw new Error('Invalid Arrow Stream'); } for (const message of readMessages(bbs[0])) { - if (message.headerType() === MessageHeader.Schema) { - const schema = message.header(new Schema()); + let schema: Schema; + if (message.headerType() === MessageHeader.Schema && (schema = message.header(new Schema())!)) { for (const bb of bbs) { for (const batch of readMessageBatches(bb)) { yield { schema, batch }; diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts index 3b6663be89bdc..4d3321833209d 100644 --- a/js/src/reader/vector.ts +++ b/js/src/reader/vector.ts @@ -17,22 +17,28 @@ import { flatbuffers } from 'flatbuffers'; import { MessageBatch } from './message'; -import { Vector } from '../vector/vector'; import * as Schema_ from '../format/Schema_generated'; -import { StructVector } from '../vector/struct'; +import * as Message_ from '../format/Message_generated'; import { IteratorState, Dictionaries } from './arrow'; -import { DictionaryVector } from '../vector/dictionary'; -import { Utf8Vector, ListVector, FixedSizeListVector } from '../vector/list'; import { - TypedArray, TypedArrayCtor, IntArray, FloatArray, + Vector, Column, + IntArray, FloatArray, + TypedArray, TypedArrayConstructor, +} from '../types/types'; + +import { + DictionaryVector, + Utf8Vector, StructVector, + ListVector, FixedSizeListVector, + DateVector, Float32Vector, Float64Vector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, - Float32Vector, Float64Vector, IndexVector, DateVector, -} from '../vector/typed'; +} from '../types/arrow'; import Int = Schema_.org.apache.arrow.flatbuf.Int; import Type = Schema_.org.apache.arrow.flatbuf.Type; import Field = Schema_.org.apache.arrow.flatbuf.Field; +import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; import Precision = Schema_.org.apache.arrow.flatbuf.Precision; import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType; import VectorLayout = Schema_.org.apache.arrow.flatbuf.VectorLayout; @@ -40,31 +46,33 @@ import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList; import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint; import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding; -export function readVector(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries) { - return readDictionaryVector(field, batch, state, dictionaries) || - readTypedVector(field, batch, state, dictionaries); +export function readVector(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries): Column | DictionaryVector | null { + return readDictionaryVector(field, batch, state, dictionaries) || + readTypedVector(field, batch, state, dictionaries); } -function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) { +function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): Column | DictionaryVector | null { let typeType = field.typeType(), readTyped = typedVectorReaders[typeType]; if (!readTyped) { throw new Error('Unrecognized vector name "' + Type[typeType] + '" type "' + typeType + '"'); } - return readTyped(field, batch, iterator, dictionaries); + return readTyped(field, batch, iterator, dictionaries) as Column; } -function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) { - let encoding: DictionaryEncoding | null; - if (dictionaries && (encoding = field.dictionary())) { - let id = encoding.id().toFloat64().toString(); - let fieldType = encoding.indexType() || +function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector | null { + let data: Vector, encoding: DictionaryEncoding; + if (dictionaries && + (encoding = field.dictionary()!) && + (data = dictionaries[encoding.id().toFloat64().toString()])) { + let indexType = encoding.indexType() || /* a dictionary index defaults to signed 32 bit int if unspecified */ { bitWidth: () => 32, isSigned: () => true }; // workaround for https://issues.apache.org/jira/browse/ARROW-1363 - let indexField = createSyntheticDictionaryIndexField(field, fieldType); - let index = readIntVector(indexField, batch, iterator, null, fieldType); - return DictionaryVector.create(field, index.length, index, dictionaries[id]); + let indexField = createSyntheticDictionaryIndexField(field, indexType); + let keys = readIntVector(indexField, batch, iterator, null, indexType)!; + return new DictionaryVector({ data, keys: keys! }); } + return null; } const IntViews = [Int8Array, Int16Array, Int32Array, Int32Array ]; @@ -74,39 +82,54 @@ const Uint8Views = [Uint8Array, Uint8Array, Uint8Array, Uint8Array ]; const Uint32Views = [Uint32Array, Uint32Array, Uint32Array, Uint32Array ]; const FloatViews = [Int8Array, Int16Array, Float32Array, Float64Array]; -const createIntDataViews = createDataView.bind(null, IntViews, null); -const createUintDataViews = createDataView.bind(null, UintViews, null); -const createDateDataViews = createDataView.bind(null, Uint32Views, null); -const createFloatDataViews = createDataView.bind(null, FloatViews, null); -const createNestedDataViews = createDataView.bind(null, Uint32Views, null); -const createValidityDataViews = createDataView.bind(null, Uint8Views, null); -const createUtf8DataViews = createDataView.bind(null, Uint8Views, Int32Views); +const createIntDataViews = createTypedArray.bind(null, IntViews, null); +const createUintDataViews = createTypedArray.bind(null, UintViews, null); +const createDateDataViews = createTypedArray.bind(null, Uint32Views, null); +const createFloatDataViews = createTypedArray.bind(null, FloatViews, null); +const createNestedDataViews = createTypedArray.bind(null, Uint32Views, null); +const createValidityDataViews = createTypedArray.bind(null, Uint8Views, null); +const createUtf8DataViews = createTypedArray.bind(null, Uint8Views, Int32Views); +// Define as computed properties for closure-compiler const floatVectors = { + [Precision.HALF]: Float32Vector, [Precision.SINGLE]: Float32Vector, - [Precision.DOUBLE]: Float64Vector -}; + [Precision.DOUBLE]: Float64Vector, +} as { [k: number]: any }; + +// and again as string-indexed keys for Uglify... +floatVectors[Precision['HALF']] = Float32Vector; +floatVectors[Precision['SINGLE']] = Float32Vector; +floatVectors[Precision['DOUBLE']] = Float64Vector; + const intVectors = [ [/* unsigned */ Uint8Vector, /* signed */ Int8Vector ], [/* unsigned */ Uint16Vector, /* signed */ Int16Vector], [/* unsigned */ Uint32Vector, /* signed */ Int32Vector], [/* unsigned */ Uint64Vector, /* signed */ Int64Vector] -]; +] as any[][]; -function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, fieldType?: FieldType) { - let type = (fieldType || field.type(new Int())); +function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { + let type = (primitiveType || field.type(new Int())!); return type.isSigned() ? read_IntVector(field, batch, iterator, dictionaries, type) : readUintVector(field, batch, iterator, dictionaries, type); } -const read_IntVector = readVectorLayout(createIntDataViews, createIntVector); -const readUintVector = readVectorLayout(createUintDataViews, createIntVector); -function createIntVector(field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) { - let type = fieldType || field.type(new Int()), bitWidth = type.bitWidth(); - let Vector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()]; - return Vector.create(field, length, validity, data || offsets); - // ----------------------------------------------- 👆: +function read_IntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { + return readVectorLayout(createIntDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType); +} + +function readUintVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { + return readVectorLayout(createUintDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType); +} + +function createIntVector(argv: VectorFactoryArgv) { + let { field, fieldNode, data, validity, offsets, primitiveType } = argv; + let type = primitiveType || field.type(new Int())!, bitWidth = type.bitWidth(); + let IntVector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()]; + return new IntVector({ fieldNode, field, validity, data: data! || offsets! }); + // ---------------------------------------------------- 👆: // Workaround for https://issues.apache.org/jira/browse/ARROW-1363 // This bug causes dictionary encoded vector indicies' IntVector data // buffers to be tagged as VectorType.OFFSET (0) in the field metadata @@ -118,68 +141,58 @@ function createIntVector(field, length, data, validity, offsets, fieldType, batc // the offset buffer is the data, because IntVectors don't have offsets. } -const readFloatVector = readVectorLayout( - createFloatDataViews, - (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { - let type = field.type(new FloatingPoint()); - let Vector = floatVectors[type.precision()]; - return Vector.create(field, length, validity, data); - } -); +function bindVectorReader(createBufferView: BufferViewFactory, createVector: VectorFactory) { + return function readVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { + return readVectorLayout(createBufferView, createVector, field, batch, iterator, dictionaries, primitiveType); + }; +} -const readDateVector = readVectorLayout( - createDateDataViews, - (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { - return DateVector.create(field, length, validity, data); - } -); +const readFloatVector = bindVectorReader(createFloatDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv) => { + const type = field.type(new FloatingPoint())!; + const FloatVector = floatVectors[type.precision()]; + return new FloatVector({ field, fieldNode, validity, data: data! }); +}); -const readUtf8Vector = readVectorLayout( - createUtf8DataViews, - (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { - let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1); - return Utf8Vector.create( - field, length, validity, - Uint8Vector.create(field, data.length, null, data), - IndexVector.create(field, length + 1, null, offsetsAdjusted) - ); - } -); +const readDateVector = bindVectorReader(createDateDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv) => { + return new DateVector({ field, fieldNode, validity, data: data! }); +}); -const readListVector = readVectorLayout( - createNestedDataViews, - (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { - let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1); - return ListVector.create( - field, length, validity, - readVector(field.children(0), batch, iterator, dictionaries), - IndexVector.create(field, length + 1, null, offsetsAdjusted) - ); - } -); +const readUtf8Vector = bindVectorReader(createUtf8DataViews, ({ field, fieldNode, data, offsets, validity }: VectorFactoryArgv) => { + return new Utf8Vector({ + field, fieldNode, + values: new ListVector({ + validity, + offsets: offsets as Int32Array, + values: new Uint8Vector({ data: data! }) + }) as any as Vector + }); +}); -const readFixedSizeListVector = readVectorLayout( - createNestedDataViews, - (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { - let size = field.type(new FixedSizeList()).listSize(); - return FixedSizeListVector.create( - field, length, size, validity, - readVector(field.children(0), batch, iterator, dictionaries) - ); - } -); +const readListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, offsets, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv) => { + return new ListVector({ + field, fieldNode, validity, + offsets: offsets! as Int32Array, + values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)! + }); +}); -const readStructVector = readVectorLayout>( - createNestedDataViews, - (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { - let vectors: Vector[] = []; - for (let i = -1, n = field.childrenLength(); ++i < n;) { - vectors[i] = readVector(field.children(i), batch, iterator, dictionaries); - } - return StructVector.create(field, length, validity, ...vectors); +const readFixedSizeListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv) => { + return new FixedSizeListVector({ + field, fieldNode, validity, + listSize: field.type(new FixedSizeList())!.listSize(), + values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)! + }); +}); + +const readStructVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv>) => { + let columns: Column[] = []; + for (let i = -1, n = field.childrenLength(); ++i < n;) { + columns[i] = readVector(field.children(i)!, messageBatch, iterator, dictionaries) as Column; } -); + return new StructVector({ field, fieldNode, validity, columns }); +}); +// Define as computed properties for closure-compiler const typedVectorReaders = { [Type.Int]: readIntVector, [Type.Date]: readDateVector, @@ -188,60 +201,73 @@ const typedVectorReaders = { [Type.Struct_]: readStructVector, [Type.FloatingPoint]: readFloatVector, [Type.FixedSizeList]: readFixedSizeListVector, -}; +} as { [k: number]: (...args: any[]) => Vector | null }; + +// and again as string-indexed keys for Uglify... +typedVectorReaders[Type['Int']] = readIntVector; +typedVectorReaders[Type['Date']] = readDateVector; +typedVectorReaders[Type['List']] = readListVector; +typedVectorReaders[Type['Utf8']] = readUtf8Vector; +typedVectorReaders[Type['Struct_']] = readStructVector; +typedVectorReaders[Type['FloatingPoint']] = readFloatVector; +typedVectorReaders[Type['FixedSizeList']] = readFixedSizeListVector; + +type VectorFactory = (argv: VectorFactoryArgv) => V; +type PrimitiveType = { bitWidth(): number; isSigned(): boolean }; +type BufferViewFactory = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => T; -type FieldType = { bitWidth(): number; isSigned(): boolean }; -type dataViewFactory = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => V; -type vectorFactory> = (field: Field, - length: number, - data: TList, - nulls: Uint8Array, - offsets: TypedArray, - fieldType: FieldType, - chunk: MessageBatch, - iterable: IteratorState, - dictionaries: Dictionaries) => V; +interface VectorFactoryArgv { + field: Field; + fieldNode: FieldNode; + iterator: IteratorState; + dictionaries: Dictionaries; + messageBatch: MessageBatch; + data?: T; + offsets?: TypedArray; + validity?: Uint8Array; + primitiveType?: PrimitiveType; +} -function readVectorLayout(createDataView: dataViewFactory, createVector: vectorFactory>) { - return function readLayout( - field: Field, - chunk: MessageBatch, - iterator: IteratorState, - dictionaries: Dictionaries, - integerFieldType?: FieldType - ) { - let batch = chunk.data; - let layoutLength = field.layoutLength(); - let node = batch.nodes(iterator.nodeIndex++); - let data: TList, offsets: any, validity: Uint8Array; - let type, bitWidth, bufferLength, nodeLength = node.length().low; - for (let i = -1; ++i < layoutLength;) { - let layout = field.layout(i); - let buffer = batch.buffers(iterator.bufferIndex++); - if ((type = layout.type()) === VectorType.TYPE || - (bufferLength = buffer.length().low) <= 0 || - (bitWidth = layout.bitWidth()) <= 0) { - continue; - } else if (type === VectorType.DATA) { - data = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength); - } else if (type === VectorType.OFFSET) { - offsets = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength); - } else if (node.nullCount().low > 0) { - validity = createValidityDataViews(chunk, type, bitWidth, buffer.offset().low, nodeLength); - } +function readVectorLayout( + createBufferView: BufferViewFactory, createVector: VectorFactory, + field: Field, messageBatch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType +) { + let fieldNode: FieldNode, recordBatch = messageBatch.data; + if (!(fieldNode = recordBatch.nodes(iterator.nodeIndex)!)) { + return null; + } + iterator.nodeIndex += 1; + let type, bitWidth, layout, buffer, bufferLength; + let data: T | undefined, offsets: TypedArray | undefined, validity: Uint8Array | undefined; + for (let i = -1, n = field.layoutLength(); ++i < n;) { + if (!(layout = field.layout(i)!) || + !(buffer = recordBatch.buffers(iterator.bufferIndex)!)) { + continue; } - return createVector(field, nodeLength, data, validity, offsets, integerFieldType, chunk, iterator, dictionaries); - }; + iterator.bufferIndex += 1; + if ((type = layout.type()) === VectorType.TYPE || + (bufferLength = buffer.length().low) <= 0 || + (bitWidth = layout.bitWidth()) <= 0) { + continue; + } else if (type === VectorType.DATA) { + data = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength); + } else if (type === VectorType.OFFSET) { + offsets = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength); + } else if (fieldNode.nullCount().low > 0) { + validity = createValidityDataViews(messageBatch, type, bitWidth, buffer.offset().low, fieldNode.length().low); + } + } + return createVector({ data, offsets, validity, field, fieldNode, iterator, messageBatch, dictionaries, primitiveType }); } -function createDataView( - dataViews: TypedArrayCtor[], offsetViews: TypedArrayCtor[] | null, +function createTypedArray( + bufferViews: TypedArrayConstructor[], offsetViews: TypedArrayConstructor[] | null, batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number ) { const buffer = batch.bytes.buffer; const byteLength = buffer.byteLength; const byteOffset = batch.offset + offset; - const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || dataViews); + const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || bufferViews); const dataViewLength = ((byteOffset + length) <= byteLength ? length : byteLength - byteOffset @@ -249,12 +275,12 @@ function createDataView( return new DataViewType(buffer, byteOffset, dataViewLength); } -function valueForBitWidth(bitWidth: number, values: any[]) { +function valueForBitWidth(bitWidth: number, values: T[]) { return values[bitWidth >> 4] || values[3]; } -function createSyntheticDictionaryIndexField(field: Field, type: FieldType) { - let layouts = []; +function createSyntheticDictionaryIndexField(field: Field, type: PrimitiveType) { + let layouts = [] as VectorLayout[]; let builder = new flatbuffers.Builder(); if (field.nullable()) { VectorLayout.startVectorLayout(builder); @@ -270,7 +296,7 @@ function createSyntheticDictionaryIndexField(field: Field, type: FieldType) { builder.finish(VectorLayout.endVectorLayout(builder)); layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer())); return Object.create(field, { - layout: { value(i) { return layouts[i]; } }, + layout: { value(i: number) { return layouts[i]; } }, layoutLength: { value() { return layouts.length; } } }); -} \ No newline at end of file +} diff --git a/js/src/table.ts b/js/src/table.ts deleted file mode 100644 index 5e781054daf31..0000000000000 --- a/js/src/table.ts +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { readBuffers } from './reader/arrow'; -import { StructVector } from './vector/struct'; -import { Vector, sliceToRangeArgs } from './vector/vector'; - -export type RowObject = { [k: string]: any }; - -export class Table implements Iterable> { - public length: number; - protected _columns: Vector[]; - protected _columnsMap: { [k: string]: Vector }; - static from(...bytes: Array) { - let columns: Vector[]; - for (let vectors of readBuffers(...bytes)) { - columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i])); - } - return new Table(columns); - } - static fromStruct(vector: StructVector) { - return new Table(( vector).vectors); - } - constructor(columns: Vector[]) { - this._columns = columns || []; - this.length = Math.max(...this._columns.map((v) => v.length)); - this._columnsMap = this._columns.reduce((map, vec) => { - return (map[vec.name] = vec) && map || map; - }, {}); - } - *[Symbol.iterator]() { - for (let cols = this._columns, i = -1, n = this.length; ++i < n;) { - yield rowAsMap(i, cols); - } - } - *rows(startRow?: number | boolean, endRow?: number | boolean, compact?: boolean) { - let start = startRow as number, end = endRow as number; - if (typeof startRow === 'boolean') { - compact = startRow; - start = end; - end = undefined; - } else if (typeof endRow === 'boolean') { - compact = endRow; - end = undefined; - } - let rowIndex = -1, { length } = this; - const [rowOffset, rowsTotal] = sliceToRangeArgs(length, start, end); - while (++rowIndex < rowsTotal) { - yield this.getRow((rowIndex + rowOffset) % length, compact); - } - } - *cols(startCol?: number, endCol?: number) { - for (const column of this._columns.slice(startCol, endCol)) { - yield column; - } - } - getRow(rowIndex: number): RowObject; - getRow(rowIndex: number, compact: boolean): Array; - getRow(rowIndex: number, compact?: boolean) { - return (compact && rowAsArray || rowAsObject)(rowIndex, this._columns); - } - getCell(columnName: string, rowIndex: number) { - return this.getColumn>(columnName).get(rowIndex); - } - getCellAt(columnIndex: number, rowIndex: number) { - return this.getColumnAt>(columnIndex).get(rowIndex); - } - getColumn>(columnName: string) { - return this._columnsMap[columnName] as T; - } - getColumnAt>(columnIndex: number) { - return this._columns[columnIndex] as T; - } - toString(): string; - toString(index: boolean): string; - toString(options: { index: boolean }): string; - toString(options?: any) { - const index = typeof options === 'object' ? options && !!options.index - : typeof options === 'boolean' ? !!options - : false; - const { length } = this; - if (length <= 0) { return ''; } - const maxColumnWidths = []; - const rows = new Array(length + 1); - rows[0] = this._columns.map((c) => c.name); - index && rows[0].unshift('Index'); - for (let i = -1, n = rows.length - 1; ++i < n;) { - rows[i + 1] = this.getRow(i, true); - index && rows[i + 1].unshift(i); - } - // Pass one to convert to strings and count max column widths - for (let i = -1, n = rows.length; ++i < n;) { - const row = rows[i]; - for (let j = -1, k = row.length; ++j < k;) { - const val = row[j] = `${row[j]}`; - maxColumnWidths[j] = !maxColumnWidths[j] - ? val.length - : Math.max(maxColumnWidths[j], val.length); - } - } - // Pass two to pad each one to max column width - for (let i = -1, n = rows.length; ++i < n;) { - const row = rows[i]; - for (let j = -1, k = row.length; ++j < k;) { - row[j] = leftPad(row[j], ' ', maxColumnWidths[j]); - } - rows[i] = row.join(', '); - } - return rows.join('\n'); - } -} - -Table.prototype.length = 0; - -function leftPad(str, fill, n) { - return (new Array(n + 1).join(fill) + str).slice(-1 * n); -} - -function rowAsMap(row: number, columns: Vector[]) { - return columns.reduce((map, vector) => map.set(vector.name, vector.get(row)), new Map()); -} - -function rowAsObject(rowIndex: number, columns: Vector[]) { - return columns.reduce((row, vector) => (row[vector.name] = vector.get(rowIndex)) && row || row, Object.create(null)); -} - -function rowAsArray(rowIndex: number, columns: Vector[]) { - return columns.reduce((row, vector, columnIndex) => (row[columnIndex] = vector.get(rowIndex)) && row || row, new Array(columns.length)); -} diff --git a/js/src/text-encoding-utf-8.d.ts b/js/src/text-encoding-utf-8.d.ts new file mode 100644 index 0000000000000..68ba4dfd9a346 --- /dev/null +++ b/js/src/text-encoding-utf-8.d.ts @@ -0,0 +1,4 @@ +declare module 'text-encoding-utf-8' { + import * as TextEncoding from 'text-encoding'; + export = TextEncoding; +} diff --git a/js/src/types/arrow.ts b/js/src/types/arrow.ts new file mode 100644 index 0000000000000..e18f5da4f1fd6 --- /dev/null +++ b/js/src/types/arrow.ts @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Schema_ from '../format/Schema_generated'; +import * as Message_ from '../format/Message_generated'; +import Field = Schema_.org.apache.arrow.flatbuf.Field; +import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; + +import { BoolVector } from './vector/bool'; +import { DictionaryVector } from './dictionary'; +import { nullableMixin, fieldMixin } from './vector/traits'; +import { ListVector as ListVectorBase } from './list'; +import { Utf8Vector as Utf8VectorBase } from './utf8'; +import { Vector, Column, TypedArray } from './types'; +import { DateVector as DateVectorBase } from './vector/date'; +import { TableVector as TableVectorBase } from './table/table'; +import { StructVector as StructVectorBase } from './table/struct'; +import { FixedSizeListVector as FixedSizeListVectorBase } from './fixedsizelist'; +import { + LongVector as LongVectorBase, + Int64Vector as Int64VectorBase, + Uint64Vector as Uint64VectorBase, +} from './vector/long'; +import { + TypedVector, + Int8Vector as Int8VectorBase, + Int16Vector as Int16VectorBase, + Int32Vector as Int32VectorBase, + Uint8Vector as Uint8VectorBase, + Uint16Vector as Uint16VectorBase, + Uint32Vector as Uint32VectorBase, + Float32Vector as Float32VectorBase, + Float64Vector as Float64VectorBase, +} from './vector/typed'; + +export { TypedArray, TypedVector }; +export { Column, BoolVector, DictionaryVector }; +export class ListVector extends MixinArrowTraits(ListVectorBase) {} +export class Utf8Vector extends MixinArrowTraits(Utf8VectorBase) {} +export class TableVector extends MixinArrowTraits(TableVectorBase) {} +export class StructVector extends MixinArrowTraits(StructVectorBase) {} +export class FixedSizeListVector extends MixinArrowTraits(FixedSizeListVectorBase) {} +export class DateVector extends MixinArrowTraits(DateVectorBase) {} +export class LongVector extends MixinArrowTraits(LongVectorBase) {} +export class Int8Vector extends MixinArrowTraits(Int8VectorBase) {} +export class Int16Vector extends MixinArrowTraits(Int16VectorBase) {} +export class Int32Vector extends MixinArrowTraits(Int32VectorBase) {} +export class Int64Vector extends MixinArrowTraits(Int64VectorBase) {} +export class Uint8Vector extends MixinArrowTraits(Uint8VectorBase) {} +export class Uint16Vector extends MixinArrowTraits(Uint16VectorBase) {} +export class Uint32Vector extends MixinArrowTraits(Uint32VectorBase) {} +export class Uint64Vector extends MixinArrowTraits(Uint64VectorBase) {} +export class Float32Vector extends MixinArrowTraits(Float32VectorBase) {} +export class Float64Vector extends MixinArrowTraits(Float64VectorBase) {} + +export function MixinArrowTraits, TArgv>(BaseVector: new (argv: TArgv) => T) { + const FieldVector = fieldMixin(BaseVector); + const NullableVector = nullableMixin(BaseVector); + const NullableFieldVector = nullableMixin(FieldVector); + return function(this: any, argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })) { + return new ((!isFieldArgv(argv) ? !isNullableArgv(argv) ? + BaseVector : NullableVector : !isNullableArgv(argv) ? + FieldVector : NullableFieldVector + ) as any)(argv); + } as any as { new (argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })): T }; +} + +function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } { + return x && x.field instanceof Field && x.fieldNode instanceof FieldNode; +} + +function isNullableArgv(x: any): x is { validity: Uint8Array } { + return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array; +} diff --git a/js/src/types/dictionary.ts b/js/src/types/dictionary.ts new file mode 100644 index 0000000000000..cafa753311d3b --- /dev/null +++ b/js/src/types/dictionary.ts @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector, Column } from './types'; +import { VirtualVector } from './vector/virtual'; + +export interface DictionaryVector extends Vector { + getValue(key: number): T; + getKey(index: number): number; +} + +export class DictionaryVector extends Vector implements Column, DictionaryVector { + readonly data: Vector; + readonly keys: Column; + constructor(argv: { data: Vector, keys: Vector }) { + super(); + this.data = argv.data; + this.keys = argv.keys as Column; + } + get name () { return this.keys.name; } + get type () { return this.keys.type; } + get length () { return this.keys.length; } + get metadata () { return this.keys.metadata; } + get nullable () { return this.keys.nullable; } + get nullCount () { return this.keys.nullCount; } + get(index: number) { + return this.getValue(this.getKey(index)!); + } + getKey(index: number) { + return this.keys.get(index); + } + getValue(key: number) { + return this.data.get(key); + } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(Array, this, ...vectors); + } + *[Symbol.iterator]() { + const { data, keys } = this; + for (let i = -1, n = keys.length; ++i < n;) { + yield data.get(keys.get(i)!); + } + } +} diff --git a/js/src/vector/struct.ts b/js/src/types/fixedsizelist.ts similarity index 53% rename from js/src/vector/struct.ts rename to js/src/types/fixedsizelist.ts index e59ac91e9cd08..6311d891d5058 100644 --- a/js/src/vector/struct.ts +++ b/js/src/types/fixedsizelist.ts @@ -15,25 +15,21 @@ // specific language governing permissions and limitations // under the License. -import { Vector } from './vector'; -import { BitVector, ValidityArgs } from './typed'; +import { List, Vector } from './types'; +import { VirtualVector } from './vector/virtual'; -export class StructVector extends Vector { - protected vectors: Vector[]; - constructor(validity: ValidityArgs, ...vectors: Vector[]) { +export class FixedSizeListVector> extends Vector { + readonly listSize: number; + readonly values: Vector; + constructor(argv: { listSize: number, values: Vector }) { super(); - this.vectors = vectors; - this.length = Math.max(0, ...vectors.map((v) => v.length)); - validity && (this.validity = BitVector.from(validity)); + this.values = argv.values; + this.listSize = argv.listSize; } get(index: number) { - return this.validity.get(index) ? this.vectors.map((v) => v.get(index)) : null; + return this.values.slice(this.listSize * index, this.listSize * (index + 1)); } - concat(vector: StructVector) { - return StructVector.from(this, - this.length + vector.length, - this.validity.concat(vector.validity), - ...this.vectors.map((v, i) => v.concat(vector.vectors[i])) - ); + concat(...vectors: Vector[]): Vector { + return new VirtualVector(Array, this, ...vectors); } } diff --git a/js/src/types/list.ts b/js/src/types/list.ts new file mode 100644 index 0000000000000..ca9170b5908d5 --- /dev/null +++ b/js/src/types/list.ts @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { List, Vector } from './types'; +import { VirtualVector } from './vector/virtual'; + +export class ListVector> extends Vector { + readonly offsets: Int32Array; + readonly values: Vector; + constructor(argv: { offsets: Int32Array, values: Vector }) { + super(); + this.values = argv.values; + this.offsets = argv.offsets; + } + get(index: number) { + return this.values.slice(this.offsets[index], this.offsets[index + 1]); + } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(Array, this, ...vectors); + } +} \ No newline at end of file diff --git a/js/src/types/table/from.ts b/js/src/types/table/from.ts new file mode 100644 index 0000000000000..ae0755961eb7d --- /dev/null +++ b/js/src/types/table/from.ts @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Column } from '../types'; +import { TableVector } from './table'; +import { readBuffers } from '../../reader/arrow'; + +export function fromBuffers(...bytes: Array) { + let columns: Column[] = null as any; + for (let vectors of readBuffers(...bytes)) { + columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i]) as Column); + } + return new TableVector({ columns }); +} + +TableVector.from = fromBuffers; + +declare module './table' { + namespace TableVector { export let from: typeof fromBuffers; } +} \ No newline at end of file diff --git a/js/src/types/table/row.ts b/js/src/types/table/row.ts new file mode 100644 index 0000000000000..432cfd7364ba3 --- /dev/null +++ b/js/src/types/table/row.ts @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Row, Vector, Struct } from '../types'; +import { VirtualVector } from '../vector/virtual'; + +export class RowVector extends Vector implements Row { + readonly row: number; + readonly length: number; + readonly table: Struct; + [Symbol.toStringTag]() { return 'Row'; } + constructor(table: Struct, row: number) { + super(); + this.row = row; + this.table = table; + this.length = table.columns.length; + } + get(index: number) { + const col = this.table.columns[index]; + return col ? col.get(this.row) as T : null; + } + col(key: string) { + const col = this.table.col(key); + return col ? col.get(this.row) as T : null; + } + *[Symbol.iterator]() { + const { row } = this; + for (const col of this.table.columns) { + yield col ? col.get(row) : null; + } + } + concat(...rows: Vector[]): Vector { + return new VirtualVector(Array, this, ...rows as any[]); + } + toArray() { return [...this]; } + toJSON() { return this.toArray(); } + toString() { return `Row [${this.length})` } + toObject(): Record { + const { row } = this, map = Object.create(null); + for (const col of this.table.columns) { + if (col && col.name) { + map[col.name] = col.get(row); + } + } + return map; + } +} diff --git a/js/src/types/table/struct.ts b/js/src/types/table/struct.ts new file mode 100644 index 0000000000000..de6a3a05692aa --- /dev/null +++ b/js/src/types/table/struct.ts @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { RowVector } from './row'; +import { toString } from './toString'; +import { VirtualVector } from '../vector/virtual'; +import { Row, Vector, Column, Struct } from '../types'; + +export interface StructVector { + toString(): string; + toString(index: boolean): string; + toString(options: { index: boolean }): string; +} + +export class StructVector extends Vector> implements Struct { + readonly length: number; + readonly columns: Column[]; + constructor(argv: { columns: Column[] }) { + super(); + this.columns = argv.columns || []; + if (!this.length) { + this.length = Math.max(...this.columns.map((col) => col.length)) | 0; + } + } + get(index: number): StructRow { + return new StructRow(this, index); + } + col(name: string) { + return this.columns.find((col) => col.name === name) || null; + } + key(index: number) { + return this.columns[index] ? this.columns[index].name : null; + } + select(...columns: string[]) { + return new StructVector({ columns: columns.map((name) => this.col(name)!) }); + } + concat(...structs: Vector>[]): Vector> { + return new VirtualVector(Array, this, ...structs as any[]); + } + toString(x?: any) { + return toString(this, x); + } +} + +export class StructRow extends RowVector { + toString() { + return JSON.stringify(this); + } +} \ No newline at end of file diff --git a/js/src/types/table/table.ts b/js/src/types/table/table.ts new file mode 100644 index 0000000000000..d9074dec2d382 --- /dev/null +++ b/js/src/types/table/table.ts @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { StructVector, StructRow } from './struct'; + +export class TableVector extends StructVector { + get(index: number): TableRow { + return new TableRow(this, index); + } +} + +export class TableRow extends StructRow { + toString() { + return this.toArray().map((x) => JSON.stringify(x)).join(', '); + } +} \ No newline at end of file diff --git a/js/src/types/table/toString.ts b/js/src/types/table/toString.ts new file mode 100644 index 0000000000000..85b23ca246058 --- /dev/null +++ b/js/src/types/table/toString.ts @@ -0,0 +1,40 @@ +import { Struct } from '../types'; + +export function toString(source: Struct, options?: any) { + const index = typeof options === 'object' ? options && !!options.index + : typeof options === 'boolean' ? !!options + : false; + const { length } = source; + if (length <= 0) { return ''; } + const rows = new Array(length + 1); + const maxColumnWidths = [] as number[]; + rows[0] = source.columns.map((_, i) => source.key(i)); + index && rows[0].unshift('Index'); + for (let i = -1, n = rows.length - 1; ++i < n;) { + rows[i + 1] = [...source.get(i)!]; + index && rows[i + 1].unshift(i); + } + // Pass one to convert to strings and count max column widths + for (let i = -1, n = rows.length; ++i < n;) { + const row = rows[i]; + for (let j = -1, k = row.length; ++j < k;) { + const val = row[j] = `${row[j]}`; + maxColumnWidths[j] = !maxColumnWidths[j] + ? val.length + : Math.max(maxColumnWidths[j], val.length); + } + } + // Pass two to pad each one to max column width + for (let i = -1, n = rows.length; ++i < n;) { + const row = rows[i]; + for (let j = -1, k = row.length; ++j < k;) { + row[j] = leftPad(row[j], ' ', maxColumnWidths[j]); + } + rows[i] = row.join(', '); + } + return rows.join('\n'); +} + +function leftPad(str: string, fill: string, n: number) { + return (new Array(n + 1).join(fill) + str).slice(-1 * n); +} diff --git a/js/src/types/types.ts b/js/src/types/types.ts new file mode 100644 index 0000000000000..f732bc0971f35 --- /dev/null +++ b/js/src/types/types.ts @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Schema_ from '../format/Schema_generated'; +import Type = Schema_.org.apache.arrow.flatbuf.Type; + +export interface TypedArrayConstructor { + readonly prototype: T; + readonly BYTES_PER_ELEMENT: number; + new (length: number): T; + new (elements: Iterable): T; + new (arrayOrArrayBuffer: ArrayLike | ArrayBufferLike): T; + new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T; +} + +export interface TypedArray extends Iterable { + [index: number]: number; + readonly length: number; + readonly byteLength: number; + readonly byteOffset: number; + readonly buffer: ArrayBufferLike; + readonly BYTES_PER_ELEMENT: number; + [Symbol.iterator](): IterableIterator; + slice(start?: number, end?: number): TypedArray; + subarray(begin: number, end?: number): TypedArray; + set(array: ArrayLike, offset?: number): void; +} + +export type FloatArray = Float32Array | Float64Array; +export type IntArray = Int8Array | Int16Array | Int32Array; +export type UintArray = Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array; + +export type List = T[] | TypedArray; + +export interface Vector extends Iterable { + readonly length: number; + get(index: number): T | null; + concat(...vectors: Vector[]): Vector; + slice(start?: number, end?: number): R; +} + +export interface Row extends Vector { + col(key: string): T | null; +} + +export interface Column extends Vector { + readonly name: string; + readonly type: string; + readonly nullable: boolean; + readonly nullCount: number; + readonly metadata: Map; +} + +export interface Struct extends Vector> { + readonly columns: Column[]; + key(key: number): string | null; + col(key: string): Column | null; + select(...columns: string[]): Struct; + concat(...structs: Vector>[]): Vector>; +} + +export class Vector implements Vector { + slice(start?: number, end?: number): R { + let { length } = this, from = start! | 0; + let to = end === undefined ? length : Math.max(end | 0, from); + let result = new Array(to - Math.min(from, to)); + for (let i = -1, n = result.length; ++i < n;) { + result[i] = this.get(i + from); + } + return result as any; + } + *[Symbol.iterator]() { + for (let i = -1, n = this.length; ++i < n;) { + yield this.get(i); + } + } +} + +(Vector.prototype as any).name = ''; +(Vector.prototype as any).type = Type[0]; +(Vector.prototype as any).stride = 1; +(Vector.prototype as any).nullable = !1; +(Vector.prototype as any).nullCount = 0; +(Vector.prototype as any).metadata = new Map(); diff --git a/js/src/vector/dictionary.ts b/js/src/types/utf8.ts similarity index 50% rename from js/src/vector/dictionary.ts rename to js/src/types/utf8.ts index de811eaf5b050..178704f6161f5 100644 --- a/js/src/vector/dictionary.ts +++ b/js/src/types/utf8.ts @@ -15,37 +15,26 @@ // specific language governing permissions and limitations // under the License. -import { Vector } from './vector'; +import { Vector } from './types'; +import { TextDecoder } from 'text-encoding-utf-8'; +import { VirtualVector } from './vector/virtual'; -export class DictionaryVector extends Vector { - protected data: Vector; - protected keys: Vector; - constructor(index: Vector, dictionary: Vector) { +const decoder = new TextDecoder('utf-8'); + +export class Utf8Vector extends Vector { + readonly values: Vector; + constructor(argv: { values: Vector }) { super(); - this.keys = index; - this.data = dictionary; - this.length = index && index.length || 0; - } - index(index: number) { - return this.keys.get(index); - } - value(index: number) { - return this.data.get(index); + this.values = argv.values; } get(index: number) { - return this.value(this.index(index)); + const chars = this.getCodePoints(index); + return chars ? decoder.decode(chars) : null; } - concat(vector: DictionaryVector) { - return DictionaryVector.from(this, - this.length + vector.length, - this.keys.concat(vector.keys), - this.data - ); + getCodePoints(index: number) { + return this.values.get(index); } - *[Symbol.iterator]() { - let { data } = this; - for (const loc of this.keys) { - yield data.get(loc); - } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(Array, this, ...vectors); } } diff --git a/js/src/types/vector/bool.ts b/js/src/types/vector/bool.ts new file mode 100644 index 0000000000000..b2eea81f87f05 --- /dev/null +++ b/js/src/types/vector/bool.ts @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { TypedVector } from './typed'; + +export interface BoolVector extends TypedVector { + set(index: number, value: boolean): void; +} + +export class BoolVector extends TypedVector { + static pack = pack; + get(index: number) { + return (this.data[index >> 3] & 1 << index % 8) !== 0; + } + set(index: number, value: boolean) { + if (index > -1 === false) { + return; + } else if (value) { + this.data[index >> 3] |= (1 << (index % 8)); + } else { + this.data[index >> 3] &= ~(1 << (index % 8)); + } + } +} + +export function pack(values: Iterable) { + let xs = [], n, i = 0; + let bit = 0, byte = 0; + for (const value of values) { + value && (byte |= 1 << bit); + if (++bit === 8) { + xs[i++] = byte; + byte = bit = 0; + } + } + if (i === 0 || bit > 0) { xs[i++] = byte; } + if (i % 8 && (n = i + 8 - i % 8)) { + do { xs[i] = 0; } while (++i < n); + } + return new Uint8Array(xs); +} diff --git a/js/src/types/vector/date.ts b/js/src/types/vector/date.ts new file mode 100644 index 0000000000000..82dc82e64021f --- /dev/null +++ b/js/src/types/vector/date.ts @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { TypedVector } from './typed'; + +export class DateVector extends TypedVector { + get(index: number): Date { + return new Date(4294967296 * /* 2^32 */ + (super.get(index * 2 + 1) as any) + /* high */ + (super.get(index * 2) as any) /* low */ + ); + } +} + +(DateVector.prototype as any).stride = 2; diff --git a/js/src/types/vector/long.ts b/js/src/types/vector/long.ts new file mode 100644 index 0000000000000..de8eb0c13710b --- /dev/null +++ b/js/src/types/vector/long.ts @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { TypedVector } from './typed'; +import { TypedArray } from '../types'; +import { flatbuffers } from 'flatbuffers'; +import Long = flatbuffers.Long; + +export class LongVector extends TypedVector { + get(index: number) { + return new Long( + super.get(index * 2) as any, /* low */ + super.get(index * 2 + 1) as any /* high */ + ); + } +} + +(LongVector.prototype as any).stride = 2; + +export class Int64Vector extends LongVector {} +export class Uint64Vector extends LongVector {} \ No newline at end of file diff --git a/js/src/types/vector/traits.ts b/js/src/types/vector/traits.ts new file mode 100644 index 0000000000000..872c40b64d559 --- /dev/null +++ b/js/src/types/vector/traits.ts @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BoolVector } from './bool'; +import { Vector, Column } from '../types'; +import * as Schema_ from '../../format/Schema_generated'; +import * as Message_ from '../../format/Message_generated'; +import Type = Schema_.org.apache.arrow.flatbuf.Type; +import Field = Schema_.org.apache.arrow.flatbuf.Field; +import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; + +type Ctor = new (argv: TArgv) => Vector; + +export const nullableMixin = (superclass: new (argv: TArgv) => T) => + class extends (superclass as Ctor) { + readonly validity: Vector; + constructor(argv: TArgv & { validity: Uint8Array }) { + super(argv); + this.validity = new BoolVector({ data: argv.validity }); + } + get(index: number) { + return this.validity.get(index) ? super.get(index) : null; + } + }; + +export const fieldMixin = (superclass: new (argv: TArgv) => T) => + class extends (superclass as Ctor) implements Column { + readonly field: Field; + readonly type: string; + readonly length: number; + readonly stride: number; + readonly nullable: boolean; + readonly nullCount: number; + readonly fieldNode: FieldNode; + constructor(argv: TArgv & { field: Field, fieldNode: FieldNode }) { + super(argv); + const { field, fieldNode } = argv; + this.field = field; + this.fieldNode = fieldNode; + this.nullable = field.nullable(); + this.type = Type[field.typeType()]; + this.nullCount = fieldNode.nullCount().low; + this.length = (fieldNode.length().low / this.stride) | 0; + } + get name() { return this.field.name()!; } + get metadata() { + const { field } = this, data = new Map(); + for (let entry, key, i = -1, n = field && field.customMetadataLength() | 0; ++i < n;) { + if ((entry = field.customMetadata(i)) && (key = entry.key()) != null) { + data.set(key, entry.value()!); + } + } + return data; + } + }; diff --git a/js/src/types/vector/typed.ts b/js/src/types/vector/typed.ts new file mode 100644 index 0000000000000..fc093f2cb5f2a --- /dev/null +++ b/js/src/types/vector/typed.ts @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../types'; +import { VirtualVector } from './virtual'; +import { TypedArray, TypedArrayConstructor } from '../types'; + +export interface TypedVector { + slice(start?: number, end?: number): TArray; +} + +export class TypedVector extends Vector { + readonly data: TArray; + readonly stride: number; + readonly length: number; + constructor(argv: { data: TArray } | TArray) { + super(); + const data = ArrayBuffer.isView(argv) ? argv : argv.data; + this.length = ((this.data = data).length / this.stride) | 0; + } + get(index: number): T | null { + return this.data[index] as any; + } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors); + } + slice(start?: number, end?: number) { + const { data, stride } = this, from = start! | 0; + const to = end === undefined ? data.length : Math.max(end | 0, from); + return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0); + } +} + +(TypedVector.prototype as any).stride = 1; + +export class Int8Vector extends TypedVector {} +export class Int16Vector extends TypedVector {} +export class Int32Vector extends TypedVector {} +export class Uint8Vector extends TypedVector {} +export class Uint16Vector extends TypedVector {} +export class Uint32Vector extends TypedVector {} +export class Float32Vector extends TypedVector {} +export class Float64Vector extends TypedVector {} diff --git a/js/src/types/vector/virtual.ts b/js/src/types/vector/virtual.ts new file mode 100644 index 0000000000000..7f56012dc7d4b --- /dev/null +++ b/js/src/types/vector/virtual.ts @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { TypedVector } from './typed'; +import { Vector, Column, TypedArray, TypedArrayConstructor } from '../types'; + +export class VirtualVector implements Column { + readonly name: string; + readonly type: string; + readonly length: number; + readonly vectors: Vector[]; + readonly offsets: Uint32Array; + readonly ArrayType: ArrayConstructor | TypedArrayConstructor; + constructor(ArrayType: ArrayConstructor | TypedArrayConstructor, ...vectors: Vector[]) { + this.vectors = vectors; + this.ArrayType = ArrayType; + this.name = (vectors[0] as any).name; + this.type = (vectors[0] as any).type; + this.length = vectors.reduce((sum, vec) => sum + vec.length, 0); + this.offsets = Uint32Array.from(vectors.reduce((sums, vector, index) => [...sums, vector.length + sums[index]], [0])); + } + *[Symbol.iterator]() { + for (const vector of this.vectors) { + yield* vector; + } + } + get nullable() { + return (this.vectors as Column[]).some((vec) => vec.nullable); + } + get nullCount() { + return (this.vectors as Column[]).reduce((sum, v) => sum + v.nullCount | 0, 0); + } + get metadata() { + return new Map( + (this.vectors as Column[]).reduce((entries, v) => [ + ...entries, ...v.metadata.entries() + ], [] as [string, string][]) + ); + } + get(index: number) { + return findIndex(this.offsets, index) ? this.vectors[_vector].get(_offset) : null; + } + concat(...vectors: Vector[]) { + return new VirtualVector(this.ArrayType, ...this.vectors, ...vectors); + } + slice(begin?: number, end?: number) { + const ArrayType = this.ArrayType as any; + // clamp begin and end values between the virtual length + clampRange(this.length, begin!, end); + const from = _from, total = _total; + // find the start vector index and adjusted value index offset + if (!findIndex(this.offsets, from)) { return new ArrayType(0); } + const set = ArrayType === Array ? arraySet : typedArraySet as any; + let index = _vector, vectors = this.vectors as TypedVector[]; + let vector = vectors[index], source = vector.slice(_offset, _offset + total), target = source; + // Perf optimization: if the first slice contains all the values we're looking for, + // we don't have to copy values to a target Array. If we're slicing a TypedArray, + // this is a significant improvement as we avoid the memcpy 🎉 + if ((source.length / vector.stride | 0) < total) { + let vectorsLength = vectors.length; + let count = 0, length = 0, sources = []; + do { + sources.push(source); + length += source.length; + count += (source.length / vector.stride | 0); + } while ( + (count < total) && + (vector = vectors[index = (++index % vectorsLength)]) && + (source = vector.slice(0, Math.min(vector.length, total - count))) + ); + target = new ArrayType(length); + for (let i = -1, j = 0, n = sources.length; ++i < n;) { + j = set(sources[i], target, j); + } + } + return target; + } +} + +let _from = -1, _total = -1; +function clampRange(length: number, start: number, end?: number) { + let total = length, from = start || 0; + let to = end === end && typeof end == 'number' ? end : total; + if (to < 0) { to = total + to; } + if (from < 0) { from = total - (from * -1) % total; } + if (to < from) { from = to; to = start; } + _from = from; + _total = !isFinite(total = (to - from)) || total < 0 ? 0 : total; +} + +let _offset = -1, _vector = -1; +function findIndex(offsets: Uint32Array, index: number) { + let offset = 0, left = 0, middle = 0, right = offsets.length - 1; + while (index < offsets[right] && index >= (offset = offsets[left])) { + if (left + 1 === right) { + _vector = left; + _offset = index - offset; + return true; + } + middle = left + ((right - left) / 2) | 0; + index >= offsets[middle] ? (left = middle) : (right = middle); + } + return false; +} + +function arraySet(source: T[], target: T[], index: number) { + for (let i = 0, n = source.length; i < n;) { + target[index++] = source[i++]; + } + return index; +} + +function typedArraySet(source: TypedArray, target: TypedArray, index: number) { + return target.set(source, index) || index + source.length; +} diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts deleted file mode 100644 index 7360d968b0250..0000000000000 --- a/js/src/vector/list.ts +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { Vector } from './vector'; -import { TextDecoder } from 'text-encoding'; -import { IndexVector, BitVector, ValidityArgs } from './typed'; - -export class ListVectorBase extends Vector { - protected values: Vector; - protected offsets: IndexVector; - constructor(validity: ValidityArgs, values: Vector, offsets: IndexVector) { - super(); - this.values = values; - this.offsets = offsets; - validity && (this.validity = BitVector.from(validity)); - } - get(index: number) { - let batch, from, to, { offsets } = this; - if (!this.validity.get(index) || - /* return null if `to` is null */ - ((to = offsets.get(index + 1)) === null) || !( - /* - return null if `batch` is less than than 0. this check is placed - second to avoid creating the [from, batch] tuple if `to` is null - */ - ([from, batch] = offsets.get(index, true) as number[]) && batch > -1)) { - return null; - } - return this.values.slice(from, to, batch) as any; - } - concat(vector: ListVectorBase) { - return (this.constructor as typeof ListVectorBase).from(this, - this.length + vector.length, - this.validity.concat(vector.validity), - this.values.concat(vector.values), - this.offsets.concat(vector.offsets) - ); - } - *[Symbol.iterator]() { - let v, r1, r2, { values } = this; - let it = this.offsets[Symbol.iterator](); - let iv = this.validity[Symbol.iterator](); - while (!(v = iv.next()).done && !(r1 = it.next()).done && !(r2 = it.next()).done) { - yield !v.value ? null : values.slice(r1.value[0], r2.value, r1.value[1]) as any; - } - } -} - -export class ListVector extends ListVectorBase {} -export class Utf8Vector extends ListVectorBase { - protected static decoder = new TextDecoder(`utf-8`); - get(index: number) { - let chars = super.get(index) as any; - return chars ? Utf8Vector.decoder.decode(chars) : null; - } - *[Symbol.iterator]() { - let decoder = Utf8Vector.decoder; - for (const chars of super[Symbol.iterator]()) { - yield !chars ? null : decoder.decode(chars); - } - } -} - -export class FixedSizeListVector extends Vector { - protected size: number; - protected values: Vector; - constructor(size: number, validity: ValidityArgs, values: Vector) { - super(); - this.values = values; - this.size = Math.abs(size | 0) || 1; - validity && (this.validity = BitVector.from(validity)); - } - get(index: number) { - return !this.validity.get(index) ? null : this.values.slice( - this.size * index, this.size * (index + 1) - ) as T[]; - } - concat(vector: FixedSizeListVector) { - return FixedSizeListVector.from(this, - this.length + vector.length, - this.size, - this.validity.concat(vector.validity), - this.values.concat(vector.values) - ); - } - *[Symbol.iterator]() { - let v, i = -1; - let { size, length, values } = this; - let iv = this.validity[Symbol.iterator](); - while (!(v = iv.next()).done && ++i < length) { - yield !v.value ? null : values.slice(size * i, size * (i + 1)) as T[]; - } - } -} diff --git a/js/src/vector/typed.ts b/js/src/vector/typed.ts deleted file mode 100644 index b38812e07d065..0000000000000 --- a/js/src/vector/typed.ts +++ /dev/null @@ -1,326 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { Vector } from './vector'; -import { flatbuffers } from 'flatbuffers'; - -import Long = flatbuffers.Long; - -export type VArray = { - [k: number]: T; length: number; - constructor: VArrayCtor>; -}; - -export type VArrayCtor = { - readonly prototype: VArray; - BYTES_PER_ELEMENT?: number; - new(...args: any[]): VArray; -}; - -export class VirtualVector> extends Vector { - protected lists: TArrayType[]; - protected _arrayType: VArrayCtor; - public get arrayType() { return this._arrayType; } - constructor(...lists: TArrayType[]) { - super(); - this.lists = lists.filter(Boolean); - } - get(index: number): T { - /* inlined `findVirtual` impl */ - let rows, length, lists = this.lists; - for (let batch = -1; - (rows = lists[++batch]) && - (length = rows.length) <= index && - 0 <= (index -= length);) {} - return rows && -1 < index ? rows[index] : null; - } - protected range(from: number, total: number, batch?: number) { - /* inlined `findVirtual` impl */ - let rows, local = from, length; - let { lists, _arrayType } = this; - for (batch = (batch || 0) - 1; - (rows = lists[++batch]) && - (length = rows.length) <= local && - 0 <= (local -= length);) {} - if (rows && local > -1) { - let index = 0, listsLength = lists.length; - let set: any = Array.isArray(rows) ? arraySet : typedArraySet; - let slice = _arrayType['prototype']['subarray'] || _arrayType['prototype']['slice']; - let source = slice.call(rows, local, local + total), target = source; - // Perf optimization: if the first slice contains all the values we're looking for, - // we don't have to copy values to a target Array. If we're slicing a TypedArray, - // this is a significant improvement as we avoid the memcpy 🎉 - if (source.length < total) { - target = new _arrayType(total); - while ((index = set(source, target, index)) < total) { - rows = lists[batch = ((batch + 1) % listsLength)]; - source = slice.call(rows, 0, Math.min(rows.length, total - index)); - } - } - return target as any; - } - return new _arrayType(0); - } - *[Symbol.iterator]() { - let index = -1, { lists, length } = this; - for (let outer = -1, n = lists.length; ++outer < n;) { - let list = lists[outer] as any; - for (let inner = -1, k = list.length; ++index < length && ++inner < k;) { - yield list[inner]; - } - } - } -} - -export type ValidityArgs = Vector | Uint8Array; -export class BitVector extends VirtualVector { - static constant: Vector = new (class ValidVector extends Vector { - get() { return true; } - *[Symbol.iterator]() { - do { yield true; } while (true); - } - })(); - static from(src: any) { - return src instanceof BitVector ? src - : src === BitVector.constant ? src - : src instanceof Uint8Array ? new BitVector(src) - : src instanceof Array ? new BitVector(BitVector.pack(src)) - : src instanceof Vector ? new BitVector(BitVector.pack(src)) - : BitVector.constant as Vector; - } - static pack(values: Iterable) { - let xs = [], n, i = 0; - let bit = 0, byte = 0; - for (const value of values) { - value && (byte |= 1 << bit); - if (++bit === 8) { - xs[i++] = byte; - byte = bit = 0; - } - } - if (i === 0 || bit > 0) { xs[i++] = byte; } - if (i % 8 && (n = n = i + 8 - i % 8)) { - do { xs[i] = 0; } while (++i < n); - } - return new Uint8Array(xs); - } - constructor(...lists: Uint8Array[]) { - super(...lists); - this.length = this.lists.reduce((l, xs) => l + xs['length'], 0); - } - get(index: number) { - /* inlined `findVirtual` impl */ - let rows, length, lists = this.lists; - for (let batch = -1; - (rows = lists[++batch]) && - (length = rows.length) <= index && - 0 <= (index -= length);) {} - return !(!rows || index < 0 || (rows[index >> 3 | 0] & 1 << index % 8) === 0); - } - set(index: number, value: boolean) { - /* inlined `findVirtual` impl */ - let rows, length, lists = this.lists; - for (let batch = -1; - (rows = lists[++batch]) && - (length = rows.length) <= index && - 0 <= (index -= length);) {} - if (rows && index > -1) { - value - ? (rows[index >> 3 | 0] |= (1 << (index % 8))) - : (rows[index >> 3 | 0] &= ~(1 << (index % 8))); - } - } - concat(vector: BitVector) { - return new BitVector(...this.lists, ...vector.lists); - } - *[Symbol.iterator]() { - for (const byte of super[Symbol.iterator]()) { - for (let i = -1; ++i < 8;) { - yield (byte & 1 << i) !== 0; - } - } - } -} - -export class TypedVector extends VirtualVector { - constructor(validity: ValidityArgs, ...lists: TArrayType[]) { - super(...lists); - validity && (this.validity = BitVector.from(validity)); - } - concat(vector: TypedVector) { - return (this.constructor as typeof TypedVector).from(this, - this.length + vector.length, - this.validity.concat(vector.validity), - ...this.lists, ...vector.lists - ); - } -} - -export class DateVector extends TypedVector { - get(index: number) { - return !this.validity.get(index) ? null : new Date( - Math.pow(2, 32) * - super.get(2 * index + 1) + - super.get(2 * index) - ); - } - *[Symbol.iterator]() { - let v, low, high; - let it = super[Symbol.iterator](); - let iv = this.validity[Symbol.iterator](); - while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) { - yield !v.value ? null : new Date(Math.pow(2, 32) * high.value + low.value); - } - } -} - -export class IndexVector extends TypedVector { - get(index: number, returnWithBatchIndex = false) { - /* inlined `findVirtual` impl */ - let rows, length, batch = -1, lists = this.lists; - for (; - (rows = lists[++batch]) && - (length = rows.length) <= index && - 0 <= (index -= length);) {} - return !returnWithBatchIndex - ? (rows && -1 < index ? rows[index + batch] : null) as number - : (rows && -1 < index ? [rows[index + batch], batch] : [0, -1]) as number[]; - } - *[Symbol.iterator]() { - // Alternate between iterating a tuple of [from, batch], and to. The from - // and to values are relative to the record batch they're defined in, so - // `ListVectorBase` needs to know the right batch to read. - let xs = new Int32Array(2), { lists } = this; - for (let i = -1, n = lists.length; ++i < n;) { - let list = lists[i] as any; - for (let j = -1, k = list.length - 1; ++j < k;) { - xs[1] = i; - xs[0] = list[j]; - yield xs; - yield list[j + 1]; - } - } - } -} - -export class ByteVector extends TypedVector { - get(index: number) { - return this.validity.get(index) ? super.get(index) : null; - } - *[Symbol.iterator]() { - let v, r, { validity } = this; - let it = super[Symbol.iterator](); - // fast path the case of no nulls - if (validity === BitVector.constant) { - yield* it; - } else { - let iv = validity[Symbol.iterator](); - while (!(v = iv.next()).done && !(r = it.next()).done) { - yield !v.value ? null : r.value; - } - } - } -} - -export class LongVector extends TypedVector { - get(index: number) { - return !this.validity.get(index) ? null : new Long( - super.get(index * 2), /* low */ - super.get(index * 2 + 1) /* high */ - ); - } - *[Symbol.iterator]() { - let v, low, high; - let it = super[Symbol.iterator](); - let iv = this.validity[Symbol.iterator](); - while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) { - yield !v.value ? null : new Long(low.value, high.value); - } - } -} - -export class Int8Vector extends ByteVector {} -export class Int16Vector extends ByteVector {} -export class Int32Vector extends ByteVector {} -export class Int64Vector extends LongVector {} -export class Uint8Vector extends ByteVector {} -export class Uint16Vector extends ByteVector {} -export class Uint32Vector extends ByteVector {} -export class Uint64Vector extends LongVector {} -export class Float32Vector extends ByteVector {} -export class Float64Vector extends ByteVector {} - -LongVector.prototype.stride = 2; -(Vector.prototype as any).lists = []; -(Vector.prototype as any).validity = BitVector.constant; -(VirtualVector.prototype as any)._arrayType = Array; -(BitVector.prototype as any)._arrayType = Uint8Array; -(Int8Vector.prototype as any)._arrayType = Int8Array; -(Int16Vector.prototype as any)._arrayType = Int16Array; -(Int32Vector.prototype as any)._arrayType = Int32Array; -(Int64Vector.prototype as any)._arrayType = Int32Array; -(Uint8Vector.prototype as any)._arrayType = Uint8Array; -(Uint16Vector.prototype as any)._arrayType = Uint16Array; -(Uint32Vector.prototype as any)._arrayType = Uint32Array; -(Uint64Vector.prototype as any)._arrayType = Uint32Array; -(DateVector.prototype as any)._arrayType = Uint32Array; -(IndexVector.prototype as any)._arrayType = Int32Array; -(Float32Vector.prototype as any)._arrayType = Float32Array; -(Float64Vector.prototype as any)._arrayType = Float64Array; - -function arraySet(source: Array, target: Array, index: number) { - for (let i = 0, n = source.length; i < n;) { - target[index++] = source[i++]; - } - return index; -} - -function typedArraySet(source: TypedArray, target: TypedArray, index: number) { - return target.set(source, index) || index + source.length; -} - -// Rather than eat the iterator cost, we've inlined this function into the relevant functions -// function* findVirtual(index: number, lists: TList[], batch?: number) { -// let rows, length; -// for (batch = (batch || 0) - 1; -// (rows = lists[++batch]) && -// (length = rows.length) <= index && -// 0 <= (index -= length);) {} -// return rows && -1 < index ? yield [rows, index, batch] : null; -// } - -export type TypedArrayCtor = { - readonly prototype: T; - readonly BYTES_PER_ELEMENT: number; - new(length: number): T; - new(array: ArrayLike): T; - new(buffer: ArrayBufferLike, byteOffset?: number, length?: number): T; -}; - -export type FloatArray = Float32Array | Float64Array; -export type IntArray = Int8Array | Int16Array | Int32Array | Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array; - -export type TypedArray = ( - Int8Array | - Uint8Array | - Int16Array | - Int32Array | - Uint16Array | - Uint32Array | - Float32Array | - Float64Array | - Uint8ClampedArray); diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts deleted file mode 100644 index 1f39f87cbefc8..0000000000000 --- a/js/src/vector/vector.ts +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import * as Schema_ from '../format/Schema_generated'; -export import Type = Schema_.org.apache.arrow.flatbuf.Type; -export import Field = Schema_.org.apache.arrow.flatbuf.Field; - -export function sliceToRangeArgs(length: number, start: number, end?: number) { - let total = length, from = start || 0; - let to = end === end && typeof end == 'number' ? end : total; - if (to < 0) { to = total + to; } - if (from < 0) { from = total - (from * -1) % total; } - if (to < from) { from = to; to = start; } - total = !isFinite(total = (to - from)) || total < 0 ? 0 : total; - return [from, total]; -} - -export class Vector implements Iterable { - static defaultName = ''; - static defaultProps = new Map(); - static defaultType = Type[Type.NONE]; - static create(field: Field, length: number, ...args: any[]) { - let vector = new this(...args), m; - vector.length = length; - vector.name = field.name(); - vector.type = Type[field.typeType()]; - if ((m = field.customMetadataLength()) > 0) { - let entry, i = 0, data = vector.props = new Map(); - do { - entry = field.customMetadata(i); - data[entry.key()] = entry.value(); - } while (++i < m); - } - return vector; - } - static from(source: Vector, length: number, ...args: any[]) { - let vector = new this(...args); - vector.length = length; - source.name !== Vector.defaultName && (vector.name = source.name); - source.type !== Vector.defaultType && (vector.type = source.type); - source.props !== Vector.defaultProps && (vector.props = source.props); - return vector; - } - public name: string; - public type: string; - public length: number; - public stride: number; - public props: Map; - protected validity: Vector; - get(index: number): T { return null; } - concat(vector: Vector) { return vector; } - slice(start?: number, end?: number, batch?: number) { - const { stride } = this; - const [offset, length] = sliceToRangeArgs( - stride * this.length, stride * (start || 0), stride * end - ); - return this.range(offset, length, batch); - } - protected range(index: number, length: number, batch?: number) { - const result = new Array(length); - for (let i = -1, n = this.length; ++i < length;) { - result[i] = this.get((i + index) % n) as any; - } - return result as Iterable; - } - *[Symbol.iterator]() { - for (let i = -1, n = this.length; ++i < n;) { - yield this.get(i); - } - } -} - -Vector.prototype.length = 0; -Vector.prototype.stride = 1; -Vector.prototype.name = Vector.defaultName; -Vector.prototype.type = Vector.defaultType; -Vector.prototype.props = Vector.defaultProps; diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts index 3f29c5409ab26..722781db658dc 100644 --- a/js/test/Arrow.ts +++ b/js/test/Arrow.ts @@ -18,9 +18,9 @@ /* tslint:disable */ // Dynamically load an Ix target build based on command line arguments -const target = process.env.TEST_TARGET; -const format = process.env.TEST_MODULE; -const resolve = require('path').resolve; +const path = require('path'); +const target = process.env.TEST_TARGET!; +const format = process.env.TEST_MODULE!; // these are duplicated in the gulpfile :< const targets = [`es5`, `es2015`, `esnext`]; @@ -30,21 +30,24 @@ function throwInvalidImportError(name: string, value: string, values: string[]) throw new Error('Unrecognized ' + name + ' \'' + value + '\'. Please run tests with \'--' + name + ' \''); } -if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets); -if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats); +let modulePath = ``; -let Arrow: any = require(resolve(`./targets/${target}/${format}/Arrow.js`)); +if (target === `ts` || target === `apache-arrow`) modulePath = target; +else if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets); +else if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats); +else modulePath = path.join(target, format); + +let Arrow: any = require(path.resolve(`./targets`, modulePath, `Arrow`)); import { Table as Table_, - readBuffers as readBuffers_, Vector as Vector_, - BitVector as BitVector_, + readBuffers as readBuffers_, + BoolVector as BoolVector_, + TypedVector as TypedVector_, ListVector as ListVector_, Utf8Vector as Utf8Vector_, DateVector as DateVector_, - IndexVector as IndexVector_, - TypedVector as TypedVector_, Int8Vector as Int8Vector_, Int16Vector as Int16Vector_, Int32Vector as Int32Vector_, @@ -61,14 +64,13 @@ import { } from '../src/Arrow'; export let Table = Arrow.Table as typeof Table_; -export let readBuffers = Arrow.readBuffers as typeof readBuffers_; export let Vector = Arrow.Vector as typeof Vector_; -export let BitVector = Arrow.BitVector as typeof BitVector_; +export let readBuffers = Arrow.readBuffers as typeof readBuffers_; +export let BoolVector = Arrow.BoolVector as typeof BoolVector_; +export let TypedVector = Arrow.TypedVector as typeof TypedVector_; export let ListVector = Arrow.ListVector as typeof ListVector_; export let Utf8Vector = Arrow.Utf8Vector as typeof Utf8Vector_; export let DateVector = Arrow.DateVector as typeof DateVector_; -export let IndexVector = Arrow.IndexVector as typeof IndexVector_; -export let TypedVector = Arrow.TypedVector as typeof TypedVector_; export let Int8Vector = Arrow.Int8Vector as typeof Int8Vector_; export let Int16Vector = Arrow.Int16Vector as typeof Int16Vector_; export let Int32Vector = Arrow.Int32Vector as typeof Int32Vector_; diff --git a/js/test/__snapshots__/table-tests.ts.snap b/js/test/__snapshots__/table-tests.ts.snap index a7fb9c5a64c24..401b992d9d7b5 100644 --- a/js/test/__snapshots__/table-tests.ts.snap +++ b/js/test/__snapshots__/table-tests.ts.snap @@ -118,19 +118,19 @@ Array [ `; exports[`dictionary file Arrow Table toString() prints a pretty Table 1`] = ` -" example-csv -Hermione,25,-53.235599517822266,40.231998443603516 - Severus,30,-62.22999954223633,3 - Harry,20,23,-100.23652648925781" +" example-csv +[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] + [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] + [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" `; exports[`dictionary file Arrow Table toString() prints an empty Table 1`] = `""`; exports[`dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, example-csv - 0, Hermione,25,-53.235599517822266,40.231998443603516 - 1, Severus,30,-62.22999954223633,3 - 2, Harry,20,23,-100.23652648925781" +"Index, example-csv + 0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] + 1, [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] + 2, [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" `; exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`; @@ -251,19 +251,19 @@ Array [ `; exports[`dictionary stream Arrow Table toString() prints a pretty Table 1`] = ` -" example-csv -Hermione,25,-53.235599517822266,40.231998443603516 - Severus,30,-62.22999954223633,3 - Harry,20,23,-100.23652648925781" +" example-csv +[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] + [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] + [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" `; exports[`dictionary stream Arrow Table toString() prints an empty Table 1`] = `""`; exports[`dictionary stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, example-csv - 0, Hermione,25,-53.235599517822266,40.231998443603516 - 1, Severus,30,-62.22999954223633,3 - 2, Harry,20,23,-100.23652648925781" +"Index, example-csv + 0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] + 1, [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] + 2, [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" `; exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`; @@ -353,17 +353,17 @@ Array [ `; exports[`dictionary2 file Arrow Table toString() prints a pretty Table 1`] = ` -" struct - a0fb47f9-f8fb-4403-a64a-786d7611f8ef,Airbus,1502880750,32.45663833618164,1.8712350130081177 -50fb46f4-fefa-42c1-919c-0121974cdd00,Boeing,1502880750,38.766666412353516,-4.181231498718262" +" struct + [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] +[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" `; exports[`dictionary2 file Arrow Table toString() prints an empty Table 1`] = `""`; exports[`dictionary2 file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct - 0, a0fb47f9-f8fb-4403-a64a-786d7611f8ef,Airbus,1502880750,32.45663833618164,1.8712350130081177 - 1, 50fb46f4-fefa-42c1-919c-0121974cdd00,Boeing,1502880750,38.766666412353516,-4.181231498718262" +"Index, struct + 0, [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] + 1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" `; exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`; @@ -459,17 +459,17 @@ Array [ `; exports[`multi_dictionary file Arrow Table toString() prints a pretty Table 1`] = ` -" struct - a0fb47f9-f8fb-4403-a64a-786d7611f8ef,12345,Airbus,1502880750,32.45663833618164,1.8712350130081177 -50fb46f4-fefa-42c1-919c-0121974cdd00,67890,Boeing,1502880750,38.766666412353516,-4.181231498718262" +" struct + [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] +[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" `; exports[`multi_dictionary file Arrow Table toString() prints an empty Table 1`] = `""`; exports[`multi_dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct - 0, a0fb47f9-f8fb-4403-a64a-786d7611f8ef,12345,Airbus,1502880750,32.45663833618164,1.8712350130081177 - 1, 50fb46f4-fefa-42c1-919c-0121974cdd00,67890,Boeing,1502880750,38.766666412353516,-4.181231498718262" +"Index, struct + 0, [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] + 1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" `; exports[`multipart count Arrow Table creates a Table from Arrow buffers 1`] = `"row_count"`; @@ -1373,47 +1373,47 @@ Array [ `; exports[`struct file Arrow Table toString() prints a pretty Table 1`] = ` -" struct_nullable - null - ,MhRNxD4 -137773603,3F9HBxK -410361374,aVd88fp - null - ,3loZrRf - null - null - , - , - null - ,78SLiRw - null - null - ,0ilsf82 - ,LjS9MbU - ," +" struct_nullable + null + [null,\\"MhRNxD4\\"] +[137773603,\\"3F9HBxK\\"] +[410361374,\\"aVd88fp\\"] + null + [null,\\"3loZrRf\\"] + null + null + [null,null] + [null,null] + null + [null,\\"78SLiRw\\"] + null + null + [null,\\"0ilsf82\\"] + [null,\\"LjS9MbU\\"] + [null,null]" `; exports[`struct file Arrow Table toString() prints an empty Table 1`] = `""`; exports[`struct file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct_nullable - 0, null - 1, ,MhRNxD4 - 2, 137773603,3F9HBxK - 3, 410361374,aVd88fp - 4, null - 5, ,3loZrRf - 6, null - 7, null - 8, , - 9, , - 10, null - 11, ,78SLiRw - 12, null - 13, null - 14, ,0ilsf82 - 15, ,LjS9MbU - 16, ," +"Index, struct_nullable + 0, null + 1, [null,\\"MhRNxD4\\"] + 2, [137773603,\\"3F9HBxK\\"] + 3, [410361374,\\"aVd88fp\\"] + 4, null + 5, [null,\\"3loZrRf\\"] + 6, null + 7, null + 8, [null,null] + 9, [null,null] + 10, null + 11, [null,\\"78SLiRw\\"] + 12, null + 13, null + 14, [null,\\"0ilsf82\\"] + 15, [null,\\"LjS9MbU\\"] + 16, [null,null]" `; exports[`struct stream Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`; @@ -1771,45 +1771,45 @@ Array [ `; exports[`struct stream Arrow Table toString() prints a pretty Table 1`] = ` -" struct_nullable - null - ,MhRNxD4 -137773603,3F9HBxK -410361374,aVd88fp - null - ,3loZrRf - null - null - , - , - null - ,78SLiRw - null - null - ,0ilsf82 - ,LjS9MbU - ," +" struct_nullable + null + [null,\\"MhRNxD4\\"] +[137773603,\\"3F9HBxK\\"] +[410361374,\\"aVd88fp\\"] + null + [null,\\"3loZrRf\\"] + null + null + [null,null] + [null,null] + null + [null,\\"78SLiRw\\"] + null + null + [null,\\"0ilsf82\\"] + [null,\\"LjS9MbU\\"] + [null,null]" `; exports[`struct stream Arrow Table toString() prints an empty Table 1`] = `""`; exports[`struct stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct_nullable - 0, null - 1, ,MhRNxD4 - 2, 137773603,3F9HBxK - 3, 410361374,aVd88fp - 4, null - 5, ,3loZrRf - 6, null - 7, null - 8, , - 9, , - 10, null - 11, ,78SLiRw - 12, null - 13, null - 14, ,0ilsf82 - 15, ,LjS9MbU - 16, ," +"Index, struct_nullable + 0, null + 1, [null,\\"MhRNxD4\\"] + 2, [137773603,\\"3F9HBxK\\"] + 3, [410361374,\\"aVd88fp\\"] + 4, null + 5, [null,\\"3loZrRf\\"] + 6, null + 7, null + 8, [null,null] + 9, [null,null] + 10, null + 11, [null,\\"78SLiRw\\"] + 12, null + 13, null + 14, [null,\\"0ilsf82\\"] + 15, [null,\\"LjS9MbU\\"] + 16, [null,null]" `; diff --git a/js/test/table-tests.ts b/js/test/table-tests.ts index c840299155af4..d0d70059e1561 100644 --- a/js/test/table-tests.ts +++ b/js/test/table-tests.ts @@ -23,7 +23,7 @@ for (let [name, ...buffers] of arrowTestConfigurations) { test(`creates a Table from Arrow buffers`, () => { expect.hasAssertions(); const table = Table.from(...buffers); - for (const vector of table.cols()) { + for (const vector of table.columns) { expect(vector.name).toMatchSnapshot(); expect(vector.type).toMatchSnapshot(); expect(vector.length).toMatchSnapshot(); @@ -35,7 +35,7 @@ for (let [name, ...buffers] of arrowTestConfigurations) { test(`vector iterators report the same values as get`, () => { expect.hasAssertions(); const table = Table.from(...buffers); - for (const vector of table.cols()) { + for (const vector of table.columns) { let i = -1, n = vector.length; for (let v of vector) { expect(++i).toBeLessThan(n); @@ -46,12 +46,13 @@ for (let [name, ...buffers] of arrowTestConfigurations) { }); test(`batch and Table Vectors report the same values`, () => { expect.hasAssertions(); - let rowsTotal = 0, table = Table.from(...buffers); + let rowsTotal = 0; + let table = Table.from(...buffers); for (let vectors of readBuffers(...buffers)) { let rowsNow = Math.max(...vectors.map((v) => v.length)); for (let vi = -1, vn = vectors.length; ++vi < vn;) { let v1 = vectors[vi]; - let v2 = table.getColumnAt(vi); + let v2 = table.columns[vi]; expect(v1.name).toEqual(v2.name); expect(v1.type).toEqual(v2.type); for (let i = -1, n = v1.length; ++i < n;) { @@ -64,15 +65,15 @@ for (let [name, ...buffers] of arrowTestConfigurations) { test(`enumerates Table rows`, () => { expect.hasAssertions(); const table = Table.from(...buffers); - for (const row of table.rows()) { - expect(row).toMatchSnapshot(); + for (const row of table) { + expect(row!.toObject()).toMatchSnapshot(); } }); test(`enumerates Table rows compact`, () => { expect.hasAssertions(); const table = Table.from(...buffers); - for (const row of table.rows(true)) { - expect(row).toMatchSnapshot(); + for (const row of table) { + expect(row!.toArray()).toMatchSnapshot(); } }); test(`toString() prints an empty Table`, () => { diff --git a/js/test/test-config.ts b/js/test/test-config.ts index b31ff11ad4173..89de1cc6c70c9 100644 --- a/js/test/test-config.ts +++ b/js/test/test-config.ts @@ -20,7 +20,7 @@ import * as path from 'path'; const arrowFormats = ['file', 'stream']; const arrowFileNames = ['simple', 'struct', 'dictionary', 'dictionary2', 'multi_dictionary']; const multipartArrows = ['count', 'latlong', 'origins']; -export let arrowTestConfigurations = []; +export let arrowTestConfigurations = [] as (string | Buffer)[][]; arrowTestConfigurations = arrowFormats.reduce((configs, format) => { return arrowFileNames.reduce((configs, name) => { diff --git a/js/test/vector-tests.ts b/js/test/vector-tests.ts index 0c9ef4404ed6a..0eca2327bff43 100644 --- a/js/test/vector-tests.ts +++ b/js/test/vector-tests.ts @@ -18,7 +18,7 @@ import { flatbuffers } from 'flatbuffers'; import Long = flatbuffers.Long; import { - BitVector, + BoolVector, TypedVector, Int64Vector, Uint64Vector, @@ -32,22 +32,34 @@ import { Float64Vector, } from './Arrow'; -const LongVectors = { Int64Vector, Uint64Vector }; -const ByteVectors = { Int8Vector, Int16Vector, Int32Vector, Uint8Vector, Uint16Vector, Uint32Vector, Float32Vector, Float64Vector }; +const LongVectors = { + Int64Vector: [Int64Vector, Int32Array], + Uint64Vector: [Uint64Vector, Uint32Array] +}; -const longVectors = toMap(LongVectors, Object.keys(LongVectors)); -const byteVectors = toMap(ByteVectors, Object.keys(ByteVectors)); +const TypedVectors = { + Int8Vector: [Int8Vector, Int8Array], + Int16Vector: [Int16Vector, Int16Array], + Int32Vector: [Int32Vector, Int32Array], + Uint8Vector: [Uint8Vector, Uint8Array], + Uint16Vector: [Uint16Vector, Uint16Array], + Uint32Vector: [Uint32Vector, Uint32Array], + Float32Vector: [Float32Vector, Float32Array], + Float64Vector: [Float64Vector, Float64Array] +}; + +const longVectors = toMap<[typeof TypedVector, any]>(LongVectors, Object.keys(LongVectors)); +const byteVectors = toMap<[typeof TypedVector, any]>(TypedVectors, Object.keys(TypedVectors)); const bytes = Array.from( { length: 5 }, () => Uint8Array.from( { length: 64 }, () => Math.random() * 255 | 0)); -describe(`BitVector`, () => { - const vector = new BitVector(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])); +describe(`BoolVector`, () => { + const vector = new BoolVector({ data: new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]) }); const values = [true, true, false, true, true, false, false, false]; const n = values.length; - vector.length = 1; test(`gets expected values`, () => { let i = -1; while (++i < n) { @@ -62,11 +74,11 @@ describe(`BitVector`, () => { } }); test(`can set values to true and false`, () => { - const v = new BitVector(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])); + const v = new BoolVector({ data: new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0]) }); const expected1 = [true, true, false, true, true, false, false, false]; const expected2 = [true, true, true, true, true, false, false, false]; const expected3 = [true, true, false, false, false, false, true, true]; - function validate(expected) { + function validate(expected: boolean[]) { for (let i = -1; ++i < n;) { expect(v.get(i)).toEqual(expected[i]); } @@ -88,40 +100,42 @@ describe(`BitVector`, () => { validate(expected1); }); test(`packs 0 values`, () => { - expect(BitVector.pack([])).toEqual( + expect(BoolVector.pack([])).toEqual( new Uint8Array([0, 0, 0, 0, 0, 0, 0, 0])); }); test(`packs 3 values`, () => { - expect(BitVector.pack([ + expect(BoolVector.pack([ true, false, true ])).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0])); }); test(`packs 8 values`, () => { - expect(BitVector.pack([ + expect(BoolVector.pack([ true, true, false, true, true, false, false, false ])).toEqual(new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])); }); test(`packs 25 values`, () => { - expect(BitVector.pack([ + expect(BoolVector.pack([ true, true, false, true, true, false, false, false, false, false, false, true, true, false, true, true, false ])).toEqual(new Uint8Array([27, 216, 0, 0, 0, 0, 0, 0])); }); test(`from with boolean Array packs values`, () => { - expect(BitVector.from([ - true, false, true - ]).slice()).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0])); + expect(new BoolVector({ + data: BoolVector.pack([true, false, true]) + }).slice()).toEqual(new Uint8Array([5, 0, 0, 0, 0, 0, 0, 0])); }); }); -for (const [VectorName, VectorType] of longVectors) { - const ArrayType = VectorType.prototype.arrayType; +for (const [VectorName, [VectorType, ArrayType]] of longVectors) { describe(`${VectorName}`, () => { const values = concatTyped(ArrayType, ...bytes); - const bLists = bytes.map((b) => new ArrayType(b.buffer)); - const vector = new VectorType(null, ...bLists); - const n = vector.length = values.length * 0.5; + const vector = bytes + .map((b) => new VectorType({ + data: new ArrayType(b.buffer) + })) + .reduce((v: any, v2) => v.concat(v2)); + const n = values.length * 0.5; test(`gets expected values`, () => { let i = -1; while (++i < n) { @@ -161,13 +175,16 @@ for (const [VectorName, VectorType] of longVectors) { }); } -for (const [VectorName, VectorType] of byteVectors) { - const ArrayType = VectorType.prototype.arrayType; +for (const [VectorName, [VectorType, ArrayType]] of byteVectors) { describe(`${VectorName}`, () => { const values = concatTyped(ArrayType, ...bytes); - const bLists = bytes.map((b) => new ArrayType(b.buffer)); - const vector = new VectorType(null, ...bLists); - const n = vector.length = values.length; + const vector = bytes + .map((b) => new VectorType({ + data: new ArrayType(b.buffer) + })) + .reduce((v: any, v2) => v.concat(v2)); + + const n = values.length; test(`gets expected values`, () => { let i = -1; while (++i < n) { @@ -212,9 +229,9 @@ function toMap(entries: any, keys: string[]) { } function concatTyped(ArrayType: any, ...bytes: any[]) { - const BPM = ArrayType.BYTES_PER_ELEMENT; + const BPE = ArrayType.BYTES_PER_ELEMENT; return bytes.reduce((v, bytes) => { - const l = bytes.byteLength / BPM; + const l = bytes.byteLength / BPE; const a = new ArrayType(v.length + l); const b = new ArrayType(bytes.buffer); a.set(v); diff --git a/js/tsconfig/tsconfig.base.json b/js/tsconfig/tsconfig.base.json index 4a46ed1f0af9b..8b8210198960a 100644 --- a/js/tsconfig/tsconfig.base.json +++ b/js/tsconfig/tsconfig.base.json @@ -3,23 +3,33 @@ "include": ["../src/**/*.ts"], "compileOnSave": false, "compilerOptions": { - "lib": ["dom", "esnext", "esnext.asynciterable"], + + /* Basic stuff */ "moduleResolution": "node", + "lib": ["dom", "esnext", "esnext.asynciterable"], + + /* Control what is emitted */ + "declaration": true, + "noEmitOnError": true, + "removeComments": false, + "downlevelIteration": true, + + /* Create inline sourcemaps with sources */ "sourceMap": false, "inlineSources": true, "inlineSourceMap": true, - "declaration": true, - "skipLibCheck": true, + + /* The most restrictive settings possible */ + "strict": true, + "skipLibCheck": false, "importHelpers": true, "noEmitHelpers": true, - "noImplicitAny": false, - "noEmitOnError": false, - "noImplicitThis": true, + "noImplicitAny": true, "noUnusedLocals": true, - "removeComments": false, - "downlevelIteration": true, - "noImplicitUseStrict": true, - "preserveConstEnums": false, + "noImplicitReturns": true, + "allowUnusedLabels": false, + "noUnusedParameters": true, + "allowUnreachableCode": false, "noFallthroughCasesInSwitch": true, "forceConsistentCasingInFileNames": true } diff --git a/js/tsconfig/tsconfig.es2015.cls.json b/js/tsconfig/tsconfig.es2015.cls.json index 11ccc04d58375..fccacb349d023 100644 --- a/js/tsconfig/tsconfig.es2015.cls.json +++ b/js/tsconfig/tsconfig.es2015.cls.json @@ -4,7 +4,8 @@ "compilerOptions": { "target": "ES2015", "module": "es2015", + "declaration": false, "noEmitHelpers": true, - "importHelpers": false + "importHelpers": true } } diff --git a/js/tsconfig/tsconfig.es5.cls.json b/js/tsconfig/tsconfig.es5.cls.json index 55f7ea52cf362..6e6f213b4d6e5 100644 --- a/js/tsconfig/tsconfig.es5.cls.json +++ b/js/tsconfig/tsconfig.es5.cls.json @@ -4,6 +4,7 @@ "compilerOptions": { "target": "ES5", "module": "es2015", + "declaration": false, "noEmitHelpers": true, "importHelpers": false } diff --git a/js/tsconfig/tsconfig.esnext.cls.json b/js/tsconfig/tsconfig.esnext.cls.json index 009a5ac10d644..03206c9d77d38 100644 --- a/js/tsconfig/tsconfig.esnext.cls.json +++ b/js/tsconfig/tsconfig.esnext.cls.json @@ -4,7 +4,8 @@ "compilerOptions": { "target": "ESNEXT", "module": "es2015", + "declaration": false, "noEmitHelpers": true, - "importHelpers": false + "importHelpers": true } } From 82cd6e5158169eb3c782dba8991473e88a113181 Mon Sep 17 00:00:00 2001 From: dhirschf Date: Fri, 3 Nov 2017 17:51:52 -0400 Subject: [PATCH 010/177] ARROW-1764: [Python] Add -c conda-forge for Windows dev installation instructions Author: dhirschf Author: Korn, Uwe Author: Uwe L. Korn Closes #1277 from xhochy/ARROW-1764 and squashes the following commits: 63143bba [Uwe L. Korn] Merge pull request #4 from dhirschfeld/gflags bdb6b3f1 [dhirschf] Clarification of gflags channel 27f08961 [Korn, Uwe] ARROW-1764: [Python] Add -c conda-forge for Windows dev installation instructions --- cpp/apidoc/Windows.md | 6 +++++- python/doc/source/development.rst | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md index 774482ea1c4f3..e7e83f1946b56 100644 --- a/cpp/apidoc/Windows.md +++ b/cpp/apidoc/Windows.md @@ -41,9 +41,13 @@ conda config --add channels conda-forge Now, you can bootstrap a build environment ```shell -conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd +conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd -c conda-forge ``` +***Note:*** +> *Make sure to get the `conda-forge` build of `gflags` as the + naming of the library differs from that in the `defaults` channel* + Activate just created conda environment with pre-installed packages from previous step: diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst index 3ca460d463a06..7ef6a722be1dc 100644 --- a/python/doc/source/development.rst +++ b/python/doc/source/development.rst @@ -257,7 +257,8 @@ First, starting from fresh clones of Apache Arrow and parquet-cpp: .. code-block:: shell conda create -n arrow-dev cmake git boost-cpp ^ - flatbuffers snappy zlib brotli thrift-cpp rapidjson + flatbuffers snappy zlib brotli thrift-cpp rapidjson ^ + -c conda-forge activate arrow-dev As one git housekeeping item, we must run this command in our Arrow clone: From 5d665762cd8c6ebbe94ce39b435a63ca4cf15967 Mon Sep 17 00:00:00 2001 From: Brian Hulette Date: Fri, 3 Nov 2017 20:55:27 -0400 Subject: [PATCH 011/177] ARROW-1727: [Format] Expand Arrow streaming format to permit deltas / additions to existing dictionaries Add an `isDelta` flag to the `DictionaryBatch` to allow for dictionary modifications mid-stream, update documentation. Author: Brian Hulette Closes #1257 from TheNeuralBit/ARROW-1727 and squashes the following commits: c69a5539 [Brian Hulette] Documentation tweaks 3dff0a9c [Brian Hulette] Add isDelta flag to DictionaryBatch, update documentation --- format/IPC.md | 45 ++++++++++++++++++++++++++++++++++++++++++++- format/Layout.md | 6 +++--- format/Message.fbs | 10 +++++++--- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/format/IPC.md b/format/IPC.md index 2f79031443b17..f3b48854c2072 100644 --- a/format/IPC.md +++ b/format/IPC.md @@ -67,7 +67,9 @@ We provide a streaming format for record batches. It is presented as a sequence of encapsulated messages, each of which follows the format above. The schema comes first in the stream, and it is the same for all of the record batches that follow. If any fields in the schema are dictionary-encoded, one or more -`DictionaryBatch` messages will follow the schema. +`DictionaryBatch` messages will be included. `DictionaryBatch` and +`RecordBatch` messages may be interleaved, but before any dictionary key is used +in a `RecordBatch` it should be defined in a `DictionaryBatch`. ``` @@ -76,6 +78,10 @@ that follow. If any fields in the schema are dictionary-encoded, one or more ... + +... + +... ``` @@ -109,6 +115,10 @@ Schematically we have: ``` +In the file format, there is no requirement that dictionary keys should be +defined in a `DictionaryBatch` before they are used in a `RecordBatch`, as long +as the keys are defined somewhere in the file. + ### RecordBatch body structure The `RecordBatch` metadata contains a depth-first (pre-order) flattened set of @@ -181,6 +191,7 @@ the dictionaries can be properly interpreted. table DictionaryBatch { id: long; data: RecordBatch; + isDelta: boolean = false; } ``` @@ -189,6 +200,38 @@ in the schema, so that dictionaries can even be used for multiple fields. See the [Physical Layout][4] document for more about the semantics of dictionary-encoded data. +The dictionary `isDelta` flag allows dictionary batches to be modified +mid-stream. A dictionary batch with `isDelta` set indicates that its vector +should be concatenated with those of any previous batches with the same `id`. A +stream which encodes one column, the list of strings +`["A", "B", "C", "B", "D", "C", "E", "A"]`, with a delta dictionary batch could +take the form: + +``` + + +(0) "A" +(1) "B" +(2) "C" + + +0 +1 +2 +1 + + +(3) "D" +(4) "E" + + +3 +2 +4 +0 +EOS +``` + ### Tensor (Multi-dimensional Array) Message Format The `Tensor` message types provides a way to write a multidimensional array of diff --git a/format/Layout.md b/format/Layout.md index ebf93821aab24..963202f9fb77a 100644 --- a/format/Layout.md +++ b/format/Layout.md @@ -615,9 +615,9 @@ the the types array indicates that a slot contains a different type at the index ## Dictionary encoding When a field is dictionary encoded, the values are represented by an array of Int32 representing the index of the value in the dictionary. -The Dictionary is received as a DictionaryBatch whose id is referenced by a dictionary attribute defined in the metadata ([Message.fbs][7]) in the Field table. -The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatch. -When a Schema references a Dictionary id, it must send a DictionaryBatch for this id before any RecordBatch. +The Dictionary is received as one or more DictionaryBatches with the id referenced by a dictionary attribute defined in the metadata ([Message.fbs][7]) in the Field table. +The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatches. +When a Schema references a Dictionary id, it must send at least one DictionaryBatch for this id. As an example, you could have the following data: ``` diff --git a/format/Message.fbs b/format/Message.fbs index f4a95713cea93..830718139d88c 100644 --- a/format/Message.fbs +++ b/format/Message.fbs @@ -61,16 +61,20 @@ table RecordBatch { buffers: [Buffer]; } -/// ---------------------------------------------------------------------- /// For sending dictionary encoding information. Any Field can be /// dictionary-encoded, but in this case none of its children may be /// dictionary-encoded. -/// There is one vector / column per dictionary -/// +/// There is one vector / column per dictionary, but that vector / column +/// may be spread across multiple dictionary batches by using the isDelta +/// flag table DictionaryBatch { id: long; data: RecordBatch; + + /// If isDelta is true the values in the dictionary are to be appended to a + /// dictionary with the indicated id + isDelta: bool = false; } /// ---------------------------------------------------------------------- From b9a2ce9b277c7938775d51b919b4a6464be7a66f Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Sat, 4 Nov 2017 14:24:12 +0100 Subject: [PATCH 012/177] ARROW-1765: [Doc] Use dependencies from conda in C++ docker build Author: Korn, Uwe Closes #1278 from xhochy/ARROW-1765 and squashes the following commits: f29ffaf [Korn, Uwe] ARROW-1765: [Doc] Use dependencies from conda in C++ docker build --- dev/gen_apidocs/Dockerfile | 15 +++++---------- dev/gen_apidocs/create_documents.sh | 8 ++++++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile index 0b2844cc8454b..eaeb548184d61 100644 --- a/dev/gen_apidocs/Dockerfile +++ b/dev/gen_apidocs/Dockerfile @@ -15,8 +15,6 @@ # limitations under the License. # FROM ubuntu:14.04 -ADD . /apache-arrow -WORKDIR /apache-arrow # Prerequsites for apt-add-repository RUN apt-get update && apt-get install -y \ software-properties-common python-software-properties @@ -34,14 +32,6 @@ RUN wget -O /tmp/miniconda.sh \ https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \ rm /tmp/miniconda.sh -# C++ dependencies -RUN /home/ubuntu/miniconda/bin/conda install -c conda-forge \ - boost-cpp \ - doxygen \ - maven \ - cmake \ - zlib \ - thrift-cpp # C_Glib dependencies RUN apt-get install -y \ libgtk2.0-dev \ @@ -69,6 +59,7 @@ RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \ six \ setuptools \ # C++ + boost-cpp \ cmake \ flatbuffers \ rapidjson \ @@ -79,5 +70,9 @@ RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \ jemalloc \ lz4-c \ zstd \ + doxygen \ + maven \ -c conda-forge +ADD . /apache-arrow +WORKDIR /apache-arrow CMD arrow/dev/gen_apidocs/create_documents.sh diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh index afbe041506d85..762b85222bd4d 100755 --- a/dev/gen_apidocs/create_documents.sh +++ b/dev/gen_apidocs/create_documents.sh @@ -16,6 +16,8 @@ # limitations under the License. # +set -ex + # Set up environment and output directory for C++ libraries cd /apache-arrow rm -rf dist @@ -25,8 +27,6 @@ export ARROW_HOME=$(pwd)/dist export PARQUET_HOME=$(pwd)/dist CONDA_BASE=/home/ubuntu/miniconda export LD_LIBRARY_PATH=$(pwd)/dist/lib:${CONDA_BASE}/lib:${LD_LIBRARY_PATH} -export THRIFT_HOME=${CONDA_BASE} -export BOOST_ROOT=${CONDA_BASE} export PATH=${CONDA_BASE}/bin:${PATH} # Prepare the asf-site before copying api docs @@ -41,6 +41,10 @@ popd # Make Python documentation (Depends on C++ ) # Build Arrow C++ source activate pyarrow-dev + +export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX +export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX + rm -rf arrow/cpp/build mkdir arrow/cpp/build pushd arrow/cpp/build From fc7104fc4f42d77525fe15bb221bf80c181fc1d4 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Sat, 4 Nov 2017 15:14:20 -0400 Subject: [PATCH 013/177] ARROW-1742: C++: clang-format is not detected correct on OSX anymore Author: Korn, Uwe Closes #1281 from xhochy/ARROW-1742 and squashes the following commits: 89e9a767 [Korn, Uwe] ARROW-1742: C++: clang-format is not detected correct on OSX anymore --- cpp/CMakeLists.txt | 7 ++-- cpp/build-support/run_clang_format.py | 2 +- cpp/cmake_modules/FindClangTools.cmake | 51 +++++++++++++++++++++----- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d8dc5df88b4a4..5f0c431d54aca 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -48,6 +48,7 @@ set(ARROW_ABI_VERSION "${ARROW_SO_VERSION}.0.0") set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") +set(CLANG_FORMAT_VERSION "4.0") find_package(ClangTools) if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND) # Generate a Clang compile_commands.json "compilation database" file for use @@ -435,11 +436,9 @@ endif (UNIX) # "make format" and "make check-format" targets ############################################################ -set(CLANG_FORMAT_VERSION 4.0) - # runs clang format and updates files in place. add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py - ${CLANG_FORMAT_VERSION} + ${CLANG_FORMAT_BIN} ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt ${CMAKE_CURRENT_SOURCE_DIR}/src) @@ -447,7 +446,7 @@ add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py # TODO(wesm): Make this work in run_clang_format.py add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py - ${CLANG_FORMAT_VERSION} + ${CLANG_FORMAT_BIN} ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt ${CMAKE_CURRENT_SOURCE_DIR}/src 1) diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py index fcf39ecc6a5f9..6dec34bd09afe 100755 --- a/cpp/build-support/run_clang_format.py +++ b/cpp/build-support/run_clang_format.py @@ -27,7 +27,7 @@ sys.argv[0]) sys.exit(1) -CLANG_FORMAT = 'clang-format-{0}'.format(sys.argv[1]) +CLANG_FORMAT = sys.argv[1] EXCLUDE_GLOBS_FILENAME = sys.argv[2] SOURCE_DIR = sys.argv[3] diff --git a/cpp/cmake_modules/FindClangTools.cmake b/cpp/cmake_modules/FindClangTools.cmake index 0e9430ba29195..e9221ff22dc1f 100644 --- a/cpp/cmake_modules/FindClangTools.cmake +++ b/cpp/cmake_modules/FindClangTools.cmake @@ -49,16 +49,47 @@ else() message("clang-tidy found at ${CLANG_TIDY_BIN}") endif() -find_program(CLANG_FORMAT_BIN - NAMES clang-format-4.0 - clang-format-3.9 - clang-format-3.8 - clang-format-3.7 - clang-format-3.6 - clang-format - PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin - NO_DEFAULT_PATH -) +if (CLANG_FORMAT_VERSION) + find_program(CLANG_FORMAT_BIN + NAMES clang-format-${CLANG_FORMAT_VERSION} + PATHS + ${ClangTools_PATH} + $ENV{CLANG_TOOLS_PATH} + /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) + + # If not found yet, search alternative locations + if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE) + # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/ + STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}") + STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}") + if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0") + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin + NO_DEFAULT_PATH + ) + else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin + NO_DEFAULT_PATH + ) + endif() + endif() +else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format-4.0 + clang-format-3.9 + clang-format-3.8 + clang-format-3.7 + clang-format-3.6 + clang-format + PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) +endif() if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) set(CLANG_FORMAT_FOUND 0) From 62190d7a5201cad6ae0b26d790942ffc8861eee9 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sat, 4 Nov 2017 16:56:27 -0400 Subject: [PATCH 014/177] ARROW-1756: [Python] Fix large file read/write error This is the part of [ARROW-1756](https://issues.apache.org/jira/browse/ARROW-1756). Author: Licht-T Author: Wes McKinney Closes #1276 from Licht-T/fix-large-file-read-write-error and squashes the following commits: e21964a3 [Wes McKinney] Break in read IO loop when reaching EOF 8a68756b [Wes McKinney] Minor code tweaks, fix clang documentation warnings 81c19721 [Licht-T] TST: Add test for the large file read/write 9b71afee [Licht-T] ENH: Convert errno to string error message fbb7eea6 [Licht-T] BUG: Fix large file read/write error --- cpp/src/arrow/buffer.h | 2 +- cpp/src/arrow/compare.h | 10 ++--- cpp/src/arrow/io/file.cc | 66 +++++++++++++++++++++++----- python/pyarrow/tests/conftest.py | 14 +++++- python/pyarrow/tests/test_feather.py | 9 +++- 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index 8e989064be4e1..7c5f6174fa6e9 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -340,7 +340,7 @@ Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size, #ifndef ARROW_NO_DEPRECATED_API /// \brief Create Buffer referencing std::string memory -/// \deprecated Since 0.8.0 +/// \note Deprecated since 0.8.0 /// /// Warning: string instance must stay alive /// diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h index 27176ed864cdd..df3386e4bfc19 100644 --- a/cpp/src/arrow/compare.h +++ b/cpp/src/arrow/compare.h @@ -33,27 +33,27 @@ class Tensor; #ifndef ARROW_NO_DEPRECATED_API /// Returns true if the arrays are exactly equal -/// \deprecated Since 0.8.0 +/// \note Deprecated since 0.8.0 Status ARROW_EXPORT ArrayEquals(const Array& left, const Array& right, bool* are_equal); -/// \deprecated Since 0.8.0 +/// \note Deprecated since 0.8.0 Status ARROW_EXPORT TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal); /// Returns true if the arrays are approximately equal. For non-floating point /// types, this is equivalent to ArrayEquals(left, right) -/// \deprecated Since 0.8.0 +/// \note Deprecated since 0.8.0 Status ARROW_EXPORT ArrayApproxEquals(const Array& left, const Array& right, bool* are_equal); /// Returns true if indicated equal-length segment of arrays is exactly equal -/// \deprecated Since 0.8.0 +/// \note Deprecated since 0.8.0 Status ARROW_EXPORT ArrayRangeEquals(const Array& left, const Array& right, int64_t start_idx, int64_t end_idx, int64_t other_start_idx, bool* are_equal); /// Returns true if the type metadata are exactly equal -/// \deprecated Since 0.8.0 +/// \note Deprecated since 0.8.0 Status ARROW_EXPORT TypeEquals(const DataType& left, const DataType& right, bool* are_equal); #endif diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc index 74c6c09e6219b..057cad1111685 100644 --- a/cpp/src/arrow/io/file.cc +++ b/cpp/src/arrow/io/file.cc @@ -22,6 +22,21 @@ #define _FILE_OFFSET_BITS 64 +// define max read/write count +#if defined(_MSC_VER) +#define ARROW_MAX_IO_CHUNKSIZE INT32_MAX +#else + +#ifdef __APPLE__ +// due to macOS bug, we need to set read/write max +#define ARROW_MAX_IO_CHUNKSIZE INT32_MAX +#else +// see notes on Linux read/write manpage +#define ARROW_MAX_IO_CHUNKSIZE 0x7ffff000 +#endif + +#endif + #include "arrow/io/file.h" #if _WIN32 || _WIN64 @@ -238,39 +253,68 @@ static inline Status FileSeek(int fd, int64_t pos) { return Status::OK(); } -static inline Status FileRead(int fd, uint8_t* buffer, int64_t nbytes, +static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbytes, int64_t* bytes_read) { #if defined(_MSC_VER) - if (nbytes > INT32_MAX) { + if (nbytes > ARROW_MAX_IO_CHUNKSIZE) { return Status::IOError("Unable to read > 2GB blocks yet"); } *bytes_read = static_cast(_read(fd, buffer, static_cast(nbytes))); #else - *bytes_read = static_cast(read(fd, buffer, static_cast(nbytes))); + *bytes_read = 0; + + while (*bytes_read != -1 && *bytes_read < nbytes) { + int64_t chunksize = + std::min(static_cast(ARROW_MAX_IO_CHUNKSIZE), nbytes - *bytes_read); + int64_t ret = static_cast( + read(fd, buffer + *bytes_read, static_cast(chunksize))); + + if (ret != -1) { + *bytes_read += ret; + if (ret < chunksize) { + // EOF + break; + } + } else { + *bytes_read = ret; + } + } #endif if (*bytes_read == -1) { - // TODO(wesm): errno to string - return Status::IOError("Error reading bytes from file"); + return Status::IOError(std::string("Error reading bytes from file: ") + + std::string(strerror(errno))); } return Status::OK(); } -static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) { - int ret; +static inline Status FileWrite(const int fd, const uint8_t* buffer, + const int64_t nbytes) { + int ret = 0; #if defined(_MSC_VER) - if (nbytes > INT32_MAX) { + if (nbytes > ARROW_MAX_IO_CHUNKSIZE) { return Status::IOError("Unable to write > 2GB blocks to file yet"); } ret = static_cast(_write(fd, buffer, static_cast(nbytes))); #else - ret = static_cast(write(fd, buffer, static_cast(nbytes))); + int64_t bytes_written = 0; + + while (ret != -1 && bytes_written < nbytes) { + int64_t chunksize = + std::min(static_cast(ARROW_MAX_IO_CHUNKSIZE), nbytes - bytes_written); + ret = static_cast( + write(fd, buffer + bytes_written, static_cast(chunksize))); + + if (ret != -1) { + bytes_written += ret; + } + } #endif if (ret == -1) { - // TODO(wesm): errno to string - return Status::IOError("Error writing bytes to file"); + return Status::IOError(std::string("Error writing bytes from file: ") + + std::string(strerror(errno))); } return Status::OK(); } diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index c6bd6c9b3a2d7..e27682232a22d 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from pytest import skip +from pytest import skip, mark groups = [ @@ -70,6 +70,18 @@ def pytest_addoption(parser): default=False, help=('Run only the {0} test group'.format(group))) + parser.addoption('--runslow', action='store_true', + default=False, help='run slow tests') + + +def pytest_collection_modifyitems(config, items): + if not config.getoption('--runslow'): + skip_slow = mark.skip(reason='need --runslow option to run') + + for item in items: + if 'slow' in item.keywords: + item.add_marker(skip_slow) + def pytest_runtest_setup(item): only_set = False diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py index 9e7fc8863e759..b0764fdec1768 100644 --- a/python/pyarrow/tests/test_feather.py +++ b/python/pyarrow/tests/test_feather.py @@ -50,7 +50,7 @@ def tearDown(self): pass def test_file_not_exist(self): - with self.assertRaises(pa.ArrowIOError): + with pytest.raises(pa.ArrowIOError): FeatherReader('test_invalid_file') def _get_null_counts(self, path, columns=None): @@ -98,7 +98,7 @@ def _assert_error_on_write(self, df, exc, path=None): def f(): write_feather(df, path) - self.assertRaises(exc, f) + pytest.raises(exc, f) def test_num_rows_attr(self): df = pd.DataFrame({'foo': [1, 2, 3, 4, 5]}) @@ -466,3 +466,8 @@ def test_unsupported(self): # non-strings df = pd.DataFrame({'a': ['a', 1, 2.0]}) self._assert_error_on_write(df, ValueError) + + @pytest.mark.slow + def test_large_dataframe(self): + df = pd.DataFrame({'A': np.arange(400000000)}) + self._check_pandas_roundtrip(df) From b513c8d2047c6ba7dea0530f436ca80f74809530 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sun, 5 Nov 2017 12:52:38 +0100 Subject: [PATCH 015/177] ARROW-1762: [C++] Add note to readme about need to set LC_ALL on some Linux systems Author: Wes McKinney Closes #1284 from wesm/ARROW-1762 and squashes the following commits: b71cf40 [Wes McKinney] Add note to readme about need to set LC_ALL on some Linux systems --- cpp/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/README.md b/cpp/README.md index 60383535b1596..2034d29fae324 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -69,6 +69,14 @@ Simple release build: Detailed unit test logs will be placed in the build directory under `build/test-logs`. +On some Linux distributions, running the test suite might require setting an +explicit locale. If you see any locale-related errors, try setting the +environment variable (which requires the `locales` package or equivalent): + +``` +export LC_ALL="en_US.UTF-8" +``` + ### Statically linking to Arrow on Windows The Arrow headers on Windows static library builds (enabled by the CMake From ea4a8f5a01c0c028bbec1e199ca70efaffbf068b Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sun, 5 Nov 2017 13:32:13 +0100 Subject: [PATCH 016/177] ARROW-1714: [Python] Fix invalid serialization/deserialization None name Series This closes [ARROW-1714](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1714). Author: Licht-T Author: Wes McKinney Closes #1263 from Licht-T/fix-invaid-conversion-none-column-name and squashes the following commits: 3afa60d [Wes McKinney] Be robust to pandas 0.21 conventions for null column labels b353260 [Wes McKinney] Don't use locals() fe6a075 [Licht-T] TST: Add test for None name Series serialization 3535dc4 [Licht-T] BUG: Fix invalid deserialization of None column name --- python/pyarrow/pandas_compat.py | 52 ++++++++++++++++++++++++-------- python/pyarrow/table.pxi | 5 ++- python/pyarrow/tests/test_ipc.py | 9 +++++- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 1984598ff3533..87b47b8a6bc13 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -263,6 +263,8 @@ def _column_name_to_strings(name): return tuple(map(_column_name_to_strings, name)) elif isinstance(name, collections.Sequence): raise TypeError("Unsupported type for MultiIndex level") + elif name is None: + return None return str(name) @@ -280,7 +282,9 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1): for name in df.columns: col = df[name] if not isinstance(name, six.string_types): - name = str(_column_name_to_strings(name)) + name = _column_name_to_strings(name) + if name is not None: + name = str(name) if schema is not None: field = schema.field_by_name(name) @@ -361,6 +365,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): schema = table.schema row_count = table.num_rows metadata = schema.metadata + columns_metadata = None has_pandas_metadata = metadata is not None and b'pandas' in metadata @@ -370,6 +375,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): columns = pandas_metadata['columns'] column_indexes = pandas_metadata.get('column_indexes', []) table = _add_any_metadata(table, pandas_metadata) + columns_metadata = pandas_metadata.get('columns', None) block_table = table @@ -428,6 +434,18 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): index = pd.RangeIndex(row_count) column_strings = [x.name for x in block_table.itercolumns()] + if columns_metadata is not None: + columns_name_dict = dict( + (str(x['name']), x['name']) + for x in columns_metadata + ) + columns_values = [ + columns_name_dict[y] + if y in columns_name_dict.keys() else y + for y in column_strings + ] + else: + columns_values = column_strings # If we're passed multiple column indexes then evaluate with # ast.literal_eval, since the column index values show up as a list of @@ -437,11 +455,11 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): # Create the column index # Construct the base index - if not column_strings: - columns = pd.Index(column_strings) + if not columns_values: + columns = pd.Index(columns_values) else: columns = pd.MultiIndex.from_tuples( - list(map(to_pair, column_strings)), + list(map(to_pair, columns_values)), names=[col_index['name'] for col_index in column_indexes] or None, ) @@ -466,25 +484,35 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): _level if _level.dtype == _dtype else _level.astype(_dtype) for _level, _dtype in levels_dtypes ] + columns = pd.MultiIndex( levels=new_levels, labels=labels, names=columns.names ) - # flatten a single level column MultiIndex for pandas 0.21.0 :( - if isinstance(columns, pd.MultiIndex) and columns.nlevels == 1: - levels, = columns.levels - labels, = columns.labels - - # Cheaply check that we do not somehow have duplicate column names - assert len(levels) == len(labels), 'Found non-unique column index' - columns = levels[labels] + # ARROW-1751: flatten a single level column MultiIndex for pandas 0.21.0 + columns = _flatten_single_level_multiindex(columns) axes = [columns, index] return _int.BlockManager(blocks, axes) +def _flatten_single_level_multiindex(index): + if isinstance(index, pd.MultiIndex) and index.nlevels == 1: + levels, = index.levels + labels, = index.labels + + # Cheaply check that we do not somehow have duplicate column names + if not index.is_unique: + raise ValueError('Found non-unique column index') + + return pd.Index([levels[_label] if _label != -1 else None + for _label in labels], + name=index.names[0]) + return index + + def _add_any_metadata(table, pandas_metadata): modified_columns = {} diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 6165a6622b836..5ba5f83d22f90 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -345,7 +345,10 @@ cdef _schema_from_arrays(arrays, names, dict metadata, else: raise TypeError(type(val)) - c_name = tobytes(names[i]) + if names[i] is None: + c_name = tobytes(u'None') + else: + c_name = tobytes(names[i]) fields[i].reset(new CField(c_name, type_, True)) schema.reset(new CSchema(fields, unbox_metadata(metadata))) diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py index 68c0c80aa6187..5033ea95783ab 100644 --- a/python/pyarrow/tests/test_ipc.py +++ b/python/pyarrow/tests/test_ipc.py @@ -432,16 +432,23 @@ def test_serialize_pandas_no_preserve_index(): def test_serialize_with_pandas_objects(): df = pd.DataFrame({'a': [1, 2, 3]}, index=[1, 2, 3]) + s = pd.Series([1, 2, 3, 4]) data = { 'a_series': df['a'], - 'a_frame': df + 'a_frame': df, + 's_series': s } serialized = pa.serialize(data).to_buffer() deserialized = pa.deserialize(serialized) assert_frame_equal(deserialized['a_frame'], df) + assert_series_equal(deserialized['a_series'], df['a']) + assert deserialized['a_series'].name == 'a' + + assert_series_equal(deserialized['s_series'], s) + assert deserialized['s_series'].name is None def test_schema_batch_serialize_methods(): From 1ee73ef4bedc5f63b909421b1ad247e086689a74 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 5 Nov 2017 18:26:06 -0500 Subject: [PATCH 017/177] ARROW-1770: [GLib] Fix GLib compiler warning Author: Phillip Cloud Closes #1287 from cpcloud/ARROW-1770 and squashes the following commits: bb406961 [Phillip Cloud] ARROW-1770: [GLib] Fix GLib compiler warning --- c_glib/arrow-glib/input-stream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp index d628baeeeae5b..a7a894b9d4f3e 100644 --- a/c_glib/arrow-glib/input-stream.cpp +++ b/c_glib/arrow-glib/input-stream.cpp @@ -437,13 +437,13 @@ namespace garrow { } arrow::Status ReadAt(int64_t position, int64_t n_bytes, - int64_t *n_read_bytes, uint8_t* out) { + int64_t *n_read_bytes, uint8_t* out) override { return arrow::io::RandomAccessFile::ReadAt( position, n_bytes, n_read_bytes, out); } arrow::Status ReadAt(int64_t position, int64_t n_bytes, - std::shared_ptr* out) { + std::shared_ptr* out) override { return arrow::io::RandomAccessFile::ReadAt(position, n_bytes, out); } From d7f1398a050b07dde3949e03792e5c477e4abfe4 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 5 Nov 2017 18:50:50 -0500 Subject: [PATCH 018/177] ARROW-1749: [C++] Handle range of Decimal128 values that require 39 digits to be displayed Author: Phillip Cloud Closes #1282 from cpcloud/ARROW-1749 and squashes the following commits: 858a6f1d [Phillip Cloud] Move _BitScanReverse to the correct file 71b0d88a [Phillip Cloud] Move bit utility to bitutil 439fd2e9 [Phillip Cloud] Fold the constant 2084065d [Phillip Cloud] Remove unused include 0c3df97c [Phillip Cloud] Try things f577612c [Phillip Cloud] Variable name 0eef9579 [Phillip Cloud] ARROW-1749: [C++] Handle range of Decimal128 values that require 39 digits to be displayed --- cpp/src/arrow/array.cc | 4 +- cpp/src/arrow/python/arrow_to_pandas.cc | 8 +- cpp/src/arrow/util/bit-util.h | 22 +++ cpp/src/arrow/util/decimal-test.cc | 39 ++++- cpp/src/arrow/util/decimal.cc | 213 ++++++++++++++---------- cpp/src/arrow/util/decimal.h | 24 ++- 6 files changed, 204 insertions(+), 106 deletions(-) diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index fc4b96e1b2bec..b523876bf0e4e 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -321,8 +321,8 @@ DecimalArray::DecimalArray(const std::shared_ptr& data) std::string DecimalArray::FormatValue(int64_t i) const { const auto& type_ = static_cast(*type()); - Decimal128 value(GetValue(i)); - return value.ToString(type_.precision(), type_.scale()); + const Decimal128 value(GetValue(i)); + return value.ToString(type_.scale()); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index c92faede1347b..3894772daa467 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -616,11 +616,10 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data, return Status::OK(); } -static Status RawDecimalToString(const uint8_t* bytes, int precision, int scale, - std::string* result) { +static Status RawDecimalToString(const uint8_t* bytes, int scale, std::string* result) { DCHECK_NE(result, nullptr); Decimal128 decimal(bytes); - *result = decimal.ToString(precision, scale); + *result = decimal.ToString(scale); return Status::OK(); } @@ -636,7 +635,6 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data, for (int c = 0; c < data.num_chunks(); c++) { auto* arr(static_cast(data.chunk(c).get())); auto type(std::dynamic_pointer_cast(arr->type())); - const int precision = type->precision(); const int scale = type->scale(); for (int64_t i = 0; i < arr->length(); ++i) { @@ -646,7 +644,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data, } else { const uint8_t* raw_value = arr->GetValue(i); std::string decimal_string; - RETURN_NOT_OK(RawDecimalToString(raw_value, precision, scale, &decimal_string)); + RETURN_NOT_OK(RawDecimalToString(raw_value, scale, &decimal_string)); *out_values++ = internal::DecimalFromString(Decimal, decimal_string); RETURN_IF_PYERROR(); } diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h index 8043f90ccdf6a..d6415f3c75b7b 100644 --- a/cpp/src/arrow/util/bit-util.h +++ b/cpp/src/arrow/util/bit-util.h @@ -43,6 +43,8 @@ #endif #if defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanReverse) #define ARROW_BYTE_SWAP64 _byteswap_uint64 #define ARROW_BYTE_SWAP32 _byteswap_ulong #else @@ -55,6 +57,7 @@ #include #include +#include "arrow/util/logging.h" #include "arrow/util/macros.h" #include "arrow/util/type_traits.h" #include "arrow/util/visibility.h" @@ -296,6 +299,25 @@ static inline int Log2(uint64_t x) { return result; } +/// \brief Count the number of leading zeros in a 32 bit integer. +static inline int64_t CountLeadingZeros(uint32_t value) { + DCHECK_NE(value, 0); +#if defined(__clang__) || defined(__GNUC__) + return static_cast(__builtin_clz(value)); +#elif defined(_MSC_VER) + unsigned long index; // NOLINT + _BitScanReverse(&index, static_cast(value)); // NOLINT + return 31LL - static_cast(index); +#else + int64_t bitpos = 0; + while (value != 0) { + value >>= 1; + ++bitpos; + } + return 32LL - bitpos; +#endif +} + /// Swaps the byte order (i.e. endianess) static inline int64_t ByteSwap(int64_t value) { return ARROW_BYTE_SWAP64(value); } static inline uint64_t ByteSwap(uint64_t value) { diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc index 565a1bbb939b6..b0271fff15ccf 100644 --- a/cpp/src/arrow/util/decimal-test.cc +++ b/cpp/src/arrow/util/decimal-test.cc @@ -36,9 +36,8 @@ class DecimalTestFixture : public ::testing::Test { TEST_F(DecimalTestFixture, TestToString) { Decimal128 decimal(this->integer_value_); - int precision = 8; int scale = 5; - std::string result = decimal.ToString(precision, scale); + std::string result = decimal.ToString(scale); ASSERT_EQ(result, this->string_value_); } @@ -256,4 +255,40 @@ TEST(Decimal128TestFalse, ConstructibleFromBool) { ASSERT_EQ(0, value.low_bits()); } +TEST(Decimal128Test, Division) { + const std::string expected_string_value("-23923094039234029"); + const Decimal128 value(expected_string_value); + const Decimal128 result(value / 3); + const Decimal128 expected_value("-7974364679744676"); + ASSERT_EQ(expected_value, result); +} + +TEST(Decimal128Test, PrintLargePositiveValue) { + const std::string string_value("99999999999999999999999999999999999999"); + const Decimal128 value(string_value); + const std::string printed_value = value.ToIntegerString(); + ASSERT_EQ(string_value, printed_value); +} + +TEST(Decimal128Test, PrintLargeNegativeValue) { + const std::string string_value("-99999999999999999999999999999999999999"); + const Decimal128 value(string_value); + const std::string printed_value = value.ToIntegerString(); + ASSERT_EQ(string_value, printed_value); +} + +TEST(Decimal128Test, PrintMaxValue) { + const std::string string_value("170141183460469231731687303715884105727"); + const Decimal128 value(string_value); + const std::string printed_value = value.ToIntegerString(); + ASSERT_EQ(string_value, printed_value); +} + +TEST(Decimal128Test, PrintMinValue) { + const std::string string_value("-170141183460469231731687303715884105728"); + const Decimal128 value(string_value); + const std::string printed_value = value.ToIntegerString(); + ASSERT_EQ(string_value, printed_value); +} + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc index 7196b252c5b60..9d94bef847fa1 100644 --- a/cpp/src/arrow/util/decimal.cc +++ b/cpp/src/arrow/util/decimal.cc @@ -19,14 +19,10 @@ #include #include #include +#include #include #include -#ifdef _MSC_VER -#include -#pragma intrinsic(_BitScanReverse) -#endif - #include "arrow/util/bit-util.h" #include "arrow/util/decimal.h" #include "arrow/util/logging.h" @@ -55,62 +51,116 @@ std::array Decimal128::ToBytes() const { return out; } -std::string Decimal128::ToString(int precision, int scale) const { - using std::size_t; +static constexpr Decimal128 kTenTo36(static_cast(0xC097CE7BC90715), + 0xB34B9F1000000000); +static constexpr Decimal128 kTenTo18(0xDE0B6B3A7640000); - const bool is_negative = *this < 0; - - // Decimal values are sent to clients as strings so in the interest of - // speed the string will be created without the using stringstream with the - // whole/fractional_part(). - size_t last_char_idx = precision + (scale > 0) // Add a space for decimal place - + (scale == precision) // Add a space for leading 0 - + is_negative; // Add a space for negative sign - - std::string str(last_char_idx, '0'); - - // Start filling in the values in reverse order by taking the last digit - // of the value. Use a positive value and worry about the sign later. At this - // point the last_char_idx points to the string terminator. - Decimal128 remaining_value(*this); - - const auto first_digit_idx = static_cast(is_negative); - if (is_negative) { - remaining_value.Negate(); - } - - if (scale > 0) { - int remaining_scale = scale; - do { - str[--last_char_idx] = - static_cast(remaining_value % 10 + '0'); // Ascii offset - remaining_value /= 10; - } while (--remaining_scale > 0); - str[--last_char_idx] = '.'; - DCHECK_GT(last_char_idx, first_digit_idx) << "Not enough space remaining"; - } - - do { - str[--last_char_idx] = static_cast(remaining_value % 10 + '0'); // Ascii offset - remaining_value /= 10; - if (remaining_value == 0) { - // Trim any extra leading 0's. - if (last_char_idx > first_digit_idx) { - str.erase(0, last_char_idx - first_digit_idx); - } +std::string Decimal128::ToIntegerString() const { + Decimal128 remainder; + std::stringstream buf; + bool need_fill = false; - break; + // get anything above 10 ** 36 and print it + Decimal128 top; + Status s = Divide(kTenTo36, &top, &remainder); + DCHECK(s.ok()) << s.message(); + + if (top != 0) { + buf << static_cast(top); + remainder.Abs(); + need_fill = true; + } + + // now get anything above 10 ** 18 and print it + Decimal128 tail; + s = remainder.Divide(kTenTo18, &top, &tail); + + if (need_fill || top != 0) { + if (need_fill) { + buf << std::setw(18) << std::setfill('0'); + } else { + need_fill = true; + tail.Abs(); + } + + buf << static_cast(top); + } + + // finally print the tail, which is less than 10**18 + if (need_fill) { + buf << std::setw(18) << std::setfill('0'); + } + buf << static_cast(tail); + return buf.str(); +} + +Decimal128::operator int64_t() const { + DCHECK(high_bits_ == 0 || high_bits_ == -1) + << "Trying to cast an Decimal128 greater than the value range of a " + "int64_t. high_bits_ must be equal to 0 or -1, got: " + << high_bits_; + return static_cast(low_bits_); +} + +std::string Decimal128::ToString(int32_t scale) const { + const std::string str(ToIntegerString()); + + if (scale == 0) { + return str; + } + + if (*this < 0) { + const auto len = static_cast(str.size()); + + if (len - 1 > scale) { + const auto n = static_cast(len - scale); + return str.substr(0, n) + "." + str.substr(n, static_cast(scale)); + } + + if (len - 1 == scale) { + return "-0." + str.substr(1, std::string::npos); } - // For safety, enforce string length independent of remaining_value. - } while (last_char_idx > first_digit_idx); - if (is_negative) { - str[0] = '-'; + std::string result("-0." + std::string(static_cast(scale - len + 1), '0')); + return result + str.substr(1, std::string::npos); + } + + const auto len = static_cast(str.size()); + + if (len > scale) { + const auto n = static_cast(len - scale); + return str.substr(0, n) + "." + str.substr(n, static_cast(scale)); + } + + if (len == scale) { + return "0." + str; } - return str; + return "0." + std::string(static_cast(scale - len), '0') + str; } +static constexpr auto kInt64DecimalDigits = + static_cast(std::numeric_limits::digits10); +static constexpr int64_t kPowersOfTen[kInt64DecimalDigits + 1] = {1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL}; + static void StringToInteger(const std::string& str, Decimal128* out) { using std::size_t; @@ -122,13 +172,10 @@ static void StringToInteger(const std::string& str, Decimal128* out) { DCHECK_GT(length, 0) << "length of parsed decimal string should be greater than 0"; - size_t posn = 0; - - while (posn < length) { - const size_t group = std::min(static_cast(18), length - posn); - const auto chunk = static_cast(std::stoll(str.substr(posn, group))); - const auto multiple = - static_cast(std::pow(10.0, static_cast(group))); + for (size_t posn = 0; posn < length;) { + const size_t group = std::min(kInt64DecimalDigits, length - posn); + const int64_t chunk = std::stoll(str.substr(posn, group)); + const int64_t multiple = kPowersOfTen[group]; *out *= multiple; *out += chunk; @@ -266,6 +313,8 @@ Decimal128& Decimal128::Negate() { return *this; } +Decimal128& Decimal128::Abs() { return *this < 0 ? Negate() : *this; } + Decimal128& Decimal128::operator+=(const Decimal128& right) { const uint64_t sum = low_bits_ + right.low_bits_; high_bits_ += right.high_bits_; @@ -288,20 +337,11 @@ Decimal128& Decimal128::operator-=(const Decimal128& right) { Decimal128& Decimal128::operator/=(const Decimal128& right) { Decimal128 remainder; - DCHECK(Divide(right, this, &remainder).ok()); + Status s = Divide(right, this, &remainder); + DCHECK(s.ok()); return *this; } -Decimal128::operator char() const { - DCHECK(high_bits_ == 0 || high_bits_ == -1) - << "Trying to cast an Decimal128 greater than the value range of a " - "char. high_bits_ must be equal to 0 or -1, got: " - << high_bits_; - DCHECK_LE(low_bits_, std::numeric_limits::max()) - << "low_bits_ too large for C type char, got: " << low_bits_; - return static_cast(low_bits_); -} - Decimal128& Decimal128::operator|=(const Decimal128& right) { low_bits_ |= right.low_bits_; high_bits_ |= right.high_bits_; @@ -440,18 +480,6 @@ static int64_t FillInArray(const Decimal128& value, uint32_t* array, bool& was_n return 1; } -/// \brief Find last set bit in a 32 bit integer. Bit 1 is the LSB and bit 32 is the MSB. -static int64_t FindLastSetBit(uint32_t value) { -#if defined(__clang__) || defined(__GNUC__) - // Count leading zeros - return __builtin_clz(value) + 1; -#elif defined(_MSC_VER) - unsigned long index; // NOLINT - _BitScanReverse(&index, static_cast(value)); // NOLINT - return static_cast(index + 1UL); -#endif -} - /// Shift the number in the array left by bits positions. /// \param array the number to shift, must have length elements /// \param length the number of entries in the array @@ -581,7 +609,7 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result, // Normalize by shifting both by a multiple of 2 so that // the digit guessing is better. The requirement is that // divisor_array[0] is greater than 2**31. - int64_t normalize_bits = 32 - FindLastSetBit(divisor_array[0]); + int64_t normalize_bits = BitUtil::CountLeadingZeros(divisor_array[0]); ShiftArrayLeft(divisor_array, divisor_length, normalize_bits); ShiftArrayLeft(dividend_array, dividend_length, normalize_bits); @@ -589,7 +617,7 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result, for (int64_t j = 0; j < result_length; ++j) { // Guess the next digit. At worst it is two too large uint32_t guess = std::numeric_limits::max(); - auto high_dividend = + const auto high_dividend = static_cast(dividend_array[j]) << 32 | dividend_array[j + 1]; if (dividend_array[j] != divisor_array[0]) { guess = static_cast(high_dividend / divisor_array[0]); @@ -625,10 +653,9 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result, // if guess was too big, we add back divisor if (dividend_array[j] > prev) { --guess; - uint32_t carry = 0; for (int64_t i = divisor_length - 1; i >= 0; --i) { - uint64_t sum = + const auto sum = static_cast(divisor_array[i]) + dividend_array[j + i + 1] + carry; dividend_array[j + i + 1] = static_cast(sum); carry = static_cast(sum >> 32); @@ -645,6 +672,7 @@ Status Decimal128::Divide(const Decimal128& divisor, Decimal128* result, // return result and remainder RETURN_NOT_OK(BuildFromArray(result, result_array, result_length)); RETURN_NOT_OK(BuildFromArray(remainder, dividend_array, dividend_length)); + FixDivisionSigns(result, remainder, dividend_was_negative, divisor_was_negative); return Status::OK(); } @@ -679,6 +707,11 @@ Decimal128 operator-(const Decimal128& operand) { return result.Negate(); } +Decimal128 operator~(const Decimal128& operand) { + Decimal128 result(~operand.high_bits(), ~operand.low_bits()); + return result; +} + Decimal128 operator+(const Decimal128& left, const Decimal128& right) { Decimal128 result(left.high_bits(), left.low_bits()); result += right; @@ -700,14 +733,16 @@ Decimal128 operator*(const Decimal128& left, const Decimal128& right) { Decimal128 operator/(const Decimal128& left, const Decimal128& right) { Decimal128 remainder; Decimal128 result; - DCHECK(left.Divide(right, &result, &remainder).ok()); + Status s = left.Divide(right, &result, &remainder); + DCHECK(s.ok()); return result; } Decimal128 operator%(const Decimal128& left, const Decimal128& right) { Decimal128 remainder; Decimal128 result; - DCHECK(left.Divide(right, &result, &remainder).ok()); + Status s = left.Divide(right, &result, &remainder); + DCHECK(s.ok()); return remainder; } diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h index 72da5547907db..487f222580201 100644 --- a/cpp/src/arrow/util/decimal.h +++ b/cpp/src/arrow/util/decimal.h @@ -39,15 +39,16 @@ namespace arrow { class ARROW_EXPORT Decimal128 { public: /// \brief Create an Decimal128 from the two's complement representation. - constexpr Decimal128(int64_t high, uint64_t low) : high_bits_(high), low_bits_(low) {} + constexpr Decimal128(int64_t high, uint64_t low) noexcept + : high_bits_(high), low_bits_(low) {} /// \brief Empty constructor creates an Decimal128 with a value of 0. - constexpr Decimal128() : Decimal128(0, 0) {} + constexpr Decimal128() noexcept : Decimal128(0, 0) {} /// \brief Convert any integer value into an Decimal128. template ::value, T>::type> - constexpr Decimal128(T value) + constexpr Decimal128(T value) noexcept : Decimal128(static_cast(value) >= 0 ? 0 : -1, static_cast(value)) {} @@ -61,6 +62,9 @@ class ARROW_EXPORT Decimal128 { /// \brief Negate the current value Decimal128& Negate(); + /// \brief Absolute value + Decimal128& Abs(); + /// \brief Add a number to this one. The result is truncated to 128 bits. Decimal128& operator+=(const Decimal128& right); @@ -85,9 +89,6 @@ class ARROW_EXPORT Decimal128 { /// \brief In-place division. Decimal128& operator/=(const Decimal128& right); - /// \brief Cast the value to char. This is used when converting the value a string. - explicit operator char() const; - /// \brief Bitwise or between two Decimal128. Decimal128& operator|=(const Decimal128& right); @@ -110,8 +111,14 @@ class ARROW_EXPORT Decimal128 { std::array ToBytes() const; /// \brief Convert the Decimal128 value to a base 10 decimal string with the given - /// precision and scale. - std::string ToString(int precision, int scale) const; + /// scale. + std::string ToString(int32_t scale) const; + + /// \brief Convert the value to an integer string + std::string ToIntegerString() const; + + /// \brief Cast this value to an int64_t. + explicit operator int64_t() const; /// \brief Convert a decimal string to an Decimal128 value, optionally including /// precision and scale if they're passed in and not null. @@ -131,6 +138,7 @@ ARROW_EXPORT bool operator>(const Decimal128& left, const Decimal128& right); ARROW_EXPORT bool operator>=(const Decimal128& left, const Decimal128& right); ARROW_EXPORT Decimal128 operator-(const Decimal128& operand); +ARROW_EXPORT Decimal128 operator~(const Decimal128& operand); ARROW_EXPORT Decimal128 operator+(const Decimal128& left, const Decimal128& right); ARROW_EXPORT Decimal128 operator-(const Decimal128& left, const Decimal128& right); ARROW_EXPORT Decimal128 operator*(const Decimal128& left, const Decimal128& right); From b25b2433d67d797e9cd461377eb3798e60de2727 Mon Sep 17 00:00:00 2001 From: Yuliya Feldman Date: Sun, 5 Nov 2017 18:57:05 -0500 Subject: [PATCH 019/177] =?UTF-8?q?ARROW-1663:=20[Java]=20use=20consistent?= =?UTF-8?q?=20name=20for=20null=20and=20not-null=20in=20FixedSizeLis?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …t, add backward compatibility while deserializing schema that was generated before this JIRA checkin Author: Yuliya Feldman Closes #1193 from yufeldman/ARROW-1663 and squashes the following commits: 7f9bd34f [Yuliya Feldman] ARROW-1663: Addressing code review comments 600d379b [Yuliya Feldman] ARROW-1663: Addressing code review comments 93f527b0 [Yuliya Feldman] ARROW-1663: Addressing code review comments 7cfa22b0 [Yuliya Feldman] ARROW-1663: use consistent name for null and not-null in FixedSizeListVector and ListVector, add backward compatibility while deserializing schema that was generated before this JIRA checkin 58d6e9c9 [Yuliya Feldman] ARROW-1663: use consistent name for null and not-null in FixedSizeList, add backward compatibility while deserializing schema that was generated before this JIRA checkin --- .../org/apache/arrow/vector/ZeroVector.java | 6 +-- .../arrow/vector/complex/ListVector.java | 5 --- .../apache/arrow/vector/types/pojo/Field.java | 27 ++++++++++++- .../arrow/vector/TestFixedSizeListVector.java | 14 +++++++ .../apache/arrow/vector/TestListVector.java | 7 +--- .../apache/arrow/vector/pojo/TestConvert.java | 40 +++++++++++++++++++ 6 files changed, 85 insertions(+), 14 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index b267b2087d05c..5ac00375f8317 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -18,6 +18,8 @@ package org.apache.arrow.vector; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; + import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -39,8 +41,6 @@ public class ZeroVector implements FieldVector { public final static ZeroVector INSTANCE = new ZeroVector(); - private final String name = "[DEFAULT]"; - private final TransferPair defaultPair = new TransferPair() { @Override public void transfer() { @@ -109,7 +109,7 @@ public void clear() { @Override public Field getField() { - return new Field(name, FieldType.nullable(new Null()), null); + return new Field(DATA_VECTOR_NAME, FieldType.nullable(new Null()), null); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 6511efcb7d513..ea28a60619209 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -39,7 +39,6 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; @@ -49,7 +48,6 @@ import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.ArrowType.Null; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; @@ -324,9 +322,6 @@ public int getBufferSize() { @Override public Field getField() { - if (getDataVector() instanceof ZeroVector) { - return new Field(name, fieldType, ImmutableList.of(new Field(DATA_VECTOR_NAME, FieldType.nullable(Null.INSTANCE), null))); - } return new Field(name, fieldType, ImmutableList.of(getDataVector().getField())); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java index 48e71a976c0e8..eba149bf79f65 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField; import java.util.Iterator; @@ -39,6 +40,7 @@ import com.google.flatbuffers.FlatBufferBuilder; import org.apache.arrow.flatbuf.KeyValue; +import org.apache.arrow.flatbuf.Type; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.schema.TypeLayout; @@ -121,7 +123,9 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) { } ImmutableList.Builder childrenBuilder = ImmutableList.builder(); for (int i = 0; i < field.childrenLength(); i++) { - childrenBuilder.add(convertField(field.children(i))); + Field childField = convertField(field.children(i)); + childField = mutateOriginalNameIfNeeded(field, childField); + childrenBuilder.add(childField); } List children = childrenBuilder.build(); ImmutableMap.Builder metadataBuilder = ImmutableMap.builder(); @@ -134,6 +138,27 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) { return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()), metadata); } + /** + * Helper method to ensure backward compatibility with schemas generated prior to ARROW-1347, ARROW-1663 + * @param field + * @param originalChildField original field which name might be mutated + * @return original or mutated field + */ + private static Field mutateOriginalNameIfNeeded(org.apache.arrow.flatbuf.Field field, Field originalChildField) { + if ((field.typeType() == Type.List || field.typeType() == Type.FixedSizeList) + && originalChildField.getName().equals("[DEFAULT]")) { + return + new Field(DATA_VECTOR_NAME, + originalChildField.isNullable(), + originalChildField.getType(), + originalChildField.getDictionary(), + originalChildField.getChildren(), + originalChildField.getTypeLayout(), + originalChildField.getMetadata()); + } + return originalChildField; + } + public void validate() { TypeLayout expectedLayout = TypeLayout.getTypeLayout(getType()); if (!expectedLayout.equals(typeLayout)) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 5677f2566797a..43d9387b106a4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -26,6 +26,7 @@ import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; @@ -220,4 +221,17 @@ public void testTransferPair() { } } } + + @Test + public void testConsistentChildName() throws Exception { + try (FixedSizeListVector listVector = FixedSizeListVector.empty("sourceVector", 2, allocator)) { + String emptyListStr = listVector.getField().toString(); + Assert.assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME)); + + listVector.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())); + String emptyVectorStr = listVector.getField().toString(); + Assert.assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME)); + } + } + } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1c9b574998018..59e1646e86e56 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -26,12 +26,9 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.impl.UnionListWriter; -import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.complex.writer.FieldWriter; -import org.apache.arrow.vector.holders.NullableBigIntHolder; import org.apache.arrow.vector.types.Types; -import org.apache.arrow.vector.types.Types.*; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; import org.junit.After; @@ -635,7 +632,7 @@ public void testConsistentChildName() throws Exception { String emptyListStr = listVector.getField().toString(); assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME)); - listVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + listVector.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())); String emptyVectorStr = listVector.getField().toString(); assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME)); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index f98aeac8c8196..f6f1ad221f3d1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -21,13 +21,19 @@ import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import com.google.common.collect.ImmutableList; import com.google.flatbuffers.FlatBufferBuilder; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.UnionMode; @@ -64,6 +70,40 @@ public void complex() { run(initialField); } + @Test + public void list() throws Exception { + ImmutableList.Builder childrenBuilder = ImmutableList.builder(); + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + ListVector writeVector = ListVector.empty("list", allocator); + FixedSizeListVector writeFixedVector = FixedSizeListVector.empty("fixedlist", 5, allocator)) { + Field listVectorField = writeVector.getField(); + childrenBuilder.add(listVectorField); + Field listFixedVectorField = writeFixedVector.getField(); + childrenBuilder.add(listFixedVectorField); + } + + Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), childrenBuilder.build()); + ImmutableList.Builder parentBuilder = ImmutableList.builder(); + parentBuilder.add(initialField); + FlatBufferBuilder builder = new FlatBufferBuilder(); + builder.finish(initialField.getField(builder)); + org.apache.arrow.flatbuf.Field flatBufField = org.apache.arrow.flatbuf.Field.getRootAsField(builder.dataBuffer()); + Field finalField = Field.convertField(flatBufField); + assertEquals(initialField, finalField); + assertFalse(finalField.toString().contains("[DEFAULT]")); + + Schema initialSchema = new Schema(parentBuilder.build()); + String jsonSchema = initialSchema.toJson(); + String modifiedSchema = jsonSchema.replace("$data$", "[DEFAULT]"); + + Schema tempSchema = Schema.fromJSON(modifiedSchema); + FlatBufferBuilder schemaBuilder = new FlatBufferBuilder(); + org.apache.arrow.vector.types.pojo.Schema schema = new org.apache.arrow.vector.types.pojo.Schema(tempSchema.getFields()); + schemaBuilder.finish(schema.getSchema(schemaBuilder)); + Schema finalSchema = Schema.deserialize(ByteBuffer.wrap(schemaBuilder.sizedByteArray())); + assertFalse(finalSchema.toString().contains("[DEFAULT]")); + } + @Test public void schema() { ImmutableList.Builder childrenBuilder = ImmutableList.builder(); From 972193035802c47fa9ee8cb0962cda14c3f77847 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Mon, 6 Nov 2017 00:02:31 -0500 Subject: [PATCH 020/177] ARROW-480: [Python] Implement RowGroupMetaData.ColumnChunk This is the patch for [ARROW-480](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-480). Author: Licht-T Closes #1215 from Licht-T/feature-column-metadata and squashes the following commits: b89c98c1 [Licht-T] Fix string format to get Python 3 compatible 2de9ca8a [Licht-T] Merge remote-tracking branch 'arrow/master' into feature-column-metadata 367761b7 [Licht-T] TST: Check every row-groups and columns d4dc4d1a [Licht-T] TST: Add test for column statistics 06397f4b [Licht-T] Refactoring c5dbef39 [Licht-T] Fix RowGroupStatistics.__repr__ e3b0c6a4 [Licht-T] Convert std::string to Python str by frombytes 5ab032ae [Licht-T] Change EncodeMin/EncodeMax to decoded min/max c29fb4e4 [Licht-T] Rename ColumnChunk to column 6a71795b [Licht-T] Rename HasMinMax to snake case 8f605cfb [Licht-T] Use bint as C++ boolean type for existing checks 9cc55cb9 [Licht-T] Remove unnecessary class and methods 33c087a4 [Licht-T] TST: Fix Parquet MetaData test bdc0bdca [Licht-T] ENH: Implement RowGroupMetaData.ColumnChunk --- python/pyarrow/_parquet.pxd | 49 ++++++ python/pyarrow/_parquet.pyx | 224 +++++++++++++++++++++++++++ python/pyarrow/compat.py | 6 + python/pyarrow/tests/test_parquet.py | 54 ++++++- 4 files changed, 330 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 5094232bdc7b4..04a5b1368ce45 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -105,6 +105,11 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: ParquetVersion_V1" parquet::ParquetVersion::PARQUET_1_0" ParquetVersion_V2" parquet::ParquetVersion::PARQUET_2_0" + enum ParquetSortOrder" parquet::SortOrder::type": + ParquetSortOrder_SIGNED" parquet::SortOrder::SIGNED" + ParquetSortOrder_UNSIGNED" parquet::SortOrder::UNSIGNED" + ParquetSortOrder_UNKNOWN" parquet::SortOrder::UNKNOWN" + cdef cppclass ColumnDescriptor: c_bool Equals(const ColumnDescriptor& other) @@ -126,6 +131,8 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: c_bool Equals(const SchemaDescriptor& other) int num_columns() + cdef c_string FormatStatValue(ParquetType parquet_type, const char* val) + cdef extern from "parquet/api/reader.h" namespace "parquet" nogil: cdef cppclass ColumnReader: @@ -155,10 +162,52 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil: cdef cppclass RowGroupReader: pass + cdef cppclass CEncodedStatistics" parquet::EncodedStatistics": + const c_string& max() const + const c_string& min() const + int64_t null_count + int64_t distinct_count + bint has_min + bint has_max + bint has_null_count + bint has_distinct_count + + cdef cppclass CRowGroupStatistics" parquet::RowGroupStatistics": + int64_t null_count() const + int64_t distinct_count() const + int64_t num_values() const + bint HasMinMax() + void Reset() + c_string EncodeMin() + c_string EncodeMax() + CEncodedStatistics Encode() + void SetComparator() + ParquetType physical_type() const + + cdef cppclass CColumnChunkMetaData" parquet::ColumnChunkMetaData": + int64_t file_offset() const + const c_string& file_path() const + + ParquetType type() const + int64_t num_values() const + shared_ptr[ColumnPath] path_in_schema() const + bint is_stats_set() const + shared_ptr[CRowGroupStatistics] statistics() const; + ParquetCompression compression() const + const vector[ParquetEncoding]& encodings() const + + bint has_dictionary_page() const + int64_t dictionary_page_offset() const + int64_t data_page_offset() const + int64_t index_page_offset() const + int64_t total_compressed_size() const + int64_t total_uncompressed_size() const + cdef cppclass CRowGroupMetaData" parquet::RowGroupMetaData": int num_columns() int64_t num_rows() int64_t total_byte_size() + unique_ptr[CColumnChunkMetaData] ColumnChunk(int i) const cdef cppclass CFileMetaData" parquet::FileMetaData": uint32_t size() diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index b096fa1b4d337..eca6b201b3ad0 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -35,6 +35,212 @@ from pyarrow.lib import ArrowException, NativeFile import six +try: + from textwrap import indent +except ImportError: + def indent(text, prefix): + lines = [prefix + line for line in text.splitlines(True)] + return ''.join(lines) + + +cdef class RowGroupStatistics: + cdef: + shared_ptr[CRowGroupStatistics] statistics + + def __cinit__(self): + pass + + cdef init(self, const shared_ptr[CRowGroupStatistics]& statistics): + self.statistics = statistics + + def __repr__(self): + return """{0} + has_min_max: {1} + min: {2} + max: {3} + null_count: {4} + distinct_count: {5} + num_values: {6} + physical_type: {7}""".format(object.__repr__(self), + self.has_min_max, + self.min, + self.max, + self.null_count, + self.distinct_count, + self.num_values, + self.physical_type) + + property has_min_max: + + def __get__(self): + return self.statistics.get().HasMinMax() + + property min: + + def __get__(self): + raw_physical_type = self.statistics.get().physical_type() + encode_min = self.statistics.get().EncodeMin() + + min_value = FormatStatValue(raw_physical_type, encode_min.c_str()) + return frombytes(min_value) + + property max: + + def __get__(self): + raw_physical_type = self.statistics.get().physical_type() + encode_max = self.statistics.get().EncodeMax() + + max_value = FormatStatValue(raw_physical_type, encode_max.c_str()) + return frombytes(max_value) + + property null_count: + + def __get__(self): + return self.statistics.get().null_count() + + property distinct_count: + + def __get__(self): + return self.statistics.get().distinct_count() + + property num_values: + + def __get__(self): + return self.statistics.get().num_values() + + property physical_type: + + def __get__(self): + physical_type = self.statistics.get().physical_type() + return physical_type_name_from_enum(physical_type) + + +cdef class ColumnChunkMetaData: + cdef: + unique_ptr[CColumnChunkMetaData] up_metadata + CColumnChunkMetaData* metadata + + def __cinit__(self): + pass + + cdef init(self, const CRowGroupMetaData& row_group_metadata, int i): + self.up_metadata = row_group_metadata.ColumnChunk(i) + self.metadata = self.up_metadata.get() + + def __repr__(self): + statistics = indent(repr(self.statistics), 4 * ' ') + return """{0} + file_offset: {1} + file_path: {2} + type: {3} + num_values: {4} + path_in_schema: {5} + is_stats_set: {6} + statistics: +{7} + compression: {8} + encodings: {9} + has_dictionary_page: {10} + dictionary_page_offset: {11} + data_page_offset: {12} + index_page_offset: {13} + total_compressed_size: {14} + total_uncompressed_size: {15}""".format(object.__repr__(self), + self.file_offset, + self.file_path, + self.type, + self.num_values, + self.path_in_schema, + self.is_stats_set, + statistics, + self.compression, + self.encodings, + self.has_dictionary_page, + self.dictionary_page_offset, + self.data_page_offset, + self.index_page_offset, + self.total_compressed_size, + self.total_uncompressed_size) + + property file_offset: + + def __get__(self): + return self.metadata.file_offset() + + property file_path: + + def __get__(self): + return frombytes(self.metadata.file_path()) + + property type: + + def __get__(self): + return physical_type_name_from_enum(self.metadata.type()) + + property num_values: + + def __get__(self): + return self.metadata.num_values() + + property path_in_schema: + + def __get__(self): + path = self.metadata.path_in_schema().get().ToDotString() + return frombytes(path) + + property is_stats_set: + + def __get__(self): + return self.metadata.is_stats_set() + + property statistics: + + def __get__(self): + statistics = RowGroupStatistics() + statistics.init(self.metadata.statistics()) + return statistics + + property compression: + + def __get__(self): + return self.metadata.compression() + + property encodings: + + def __get__(self): + return map(encoding_name_from_enum, + self.metadata.encodings()) + + property has_dictionary_page: + + def __get__(self): + return self.metadata.has_dictionary_page() + + property dictionary_page_offset: + + def __get__(self): + return self.metadata.dictionary_page_offset() + + property data_page_offset: + + def __get__(self): + return self.metadata.data_page_offset() + + property index_page_offset: + + def __get__(self): + return self.metadata.index_page_offset() + + property total_compressed_size: + + def __get__(self): + return self.metadata.total_compressed_size() + + property total_uncompressed_size: + + def __get__(self): + return self.metadata.total_uncompressed_size() + cdef class RowGroupMetaData: cdef: @@ -52,6 +258,11 @@ cdef class RowGroupMetaData: self.metadata = self.up_metadata.get() self.parent = parent + def column(self, int i): + chunk = ColumnChunkMetaData() + chunk.init(deref(self.metadata), i) + return chunk + def __repr__(self): return """{0} num_columns: {1} @@ -371,6 +582,19 @@ cdef logical_type_name_from_enum(ParquetLogicalType type_): }.get(type_, 'UNKNOWN') +cdef encoding_name_from_enum (ParquetEncoding encoding_): + return { + ParquetEncoding_PLAIN: "PLAIN", + ParquetEncoding_PLAIN_DICTIONARY: "PLAIN_DICTIONARY", + ParquetEncoding_RLE: "RLE", + ParquetEncoding_BIT_PACKED: "BIT_PACKED", + ParquetEncoding_DELTA_BINARY_PACKED: "DELTA_BINARY_PACKED", + ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY: "DELTA_LENGTH_BYTE_ARRAY", + ParquetEncoding_DELTA_BYTE_ARRAY: "DELTA_BYTE_ARRAY", + ParquetEncoding_RLE_DICTIONARY: "RLE_DICTIONARY", + }.get(encoding_, 'UNKNOWN') + + cdef class ParquetReader: cdef: object source diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py index f9c148b14e368..866cbdd96d063 100644 --- a/python/pyarrow/compat.py +++ b/python/pyarrow/compat.py @@ -103,6 +103,9 @@ def tobytes(o): def frombytes(o): return o + + def unichar(s): + return unichr(s) else: unicode_type = str def lzip(*x): @@ -131,6 +134,9 @@ def tobytes(o): def frombytes(o): return o.decode('utf8') + def unichar(s): + return chr(s) + try: import cloudpickle as pickle except ImportError: diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 95dd6a471b6b3..e2e6863c4748f 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -22,7 +22,7 @@ import json import pytest -from pyarrow.compat import guid, u, BytesIO +from pyarrow.compat import guid, u, BytesIO, unichar, frombytes from pyarrow.filesystem import LocalFileSystem import pyarrow as pa from .pandas_examples import dataframe_with_arrays, dataframe_with_lists @@ -469,13 +469,61 @@ def test_parquet_metadata_api(): schema[-1] # Row group - rg_meta = meta.row_group(0) - repr(rg_meta) + for rg in range(meta.num_row_groups): + rg_meta = meta.row_group(rg) + repr(rg_meta) + + for col in range(rg_meta.num_columns): + col_meta = rg_meta.column(col) + repr(col_meta) assert rg_meta.num_rows == len(df) assert rg_meta.num_columns == ncols + 1 # +1 for index +@parquet +@pytest.mark.parametrize( + 'data, dtype, min_value, max_value, null_count, num_values', + [ + ([1, 2, 2, None, 4], np.uint8, u'1', u'4', 1, 4), + ([1, 2, 2, None, 4], np.uint16, u'1', u'4', 1, 4), + ([1, 2, 2, None, 4], np.uint32, u'1', u'4', 1, 4), + ([1, 2, 2, None, 4], np.uint64, u'1', u'4', 1, 4), + ([-1, 2, 2, None, 4], np.int16, u'-1', u'4', 1, 4), + ([-1, 2, 2, None, 4], np.int32, u'-1', u'4', 1, 4), + ([-1, 2, 2, None, 4], np.int64, u'-1', u'4', 1, 4), + ([-1.1, 2.2, 2.3, None, 4.4], np.float32, u'-1.1', u'4.4', 1, 4), + ([-1.1, 2.2, 2.3, None, 4.4], np.float64, u'-1.1', u'4.4', 1, 4), + ( + [u'', u'b', unichar(1000), None, u'aaa'], + str, u' ', frombytes((unichar(1000) + u' ').encode('utf-8')), 1, 4 + ), + ([True, False, False, True, True], np.bool, u'0', u'1', 0, 5), + ] +) +def test_parquet_column_statistics_api( + data, + dtype, + min_value, + max_value, + null_count, + num_values): + df = pd.DataFrame({'data': data}, dtype=dtype) + + fileh = make_sample_file(df) + + meta = fileh.metadata + + rg_meta = meta.row_group(0) + col_meta = rg_meta.column(0) + + stat = col_meta.statistics + assert stat.min == min_value + assert stat.max == max_value + assert stat.null_count == null_count + assert stat.num_values == num_values + + @parquet def test_compare_schemas(): df = alltypes_sample(size=10000) From 0106f531c04477b1c8bd088d097624ff43b44658 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 6 Nov 2017 12:55:49 -0500 Subject: [PATCH 021/177] ARROW-1750: [C++] Remove the need for arrow/util/random.h Author: Phillip Cloud Closes #1283 from cpcloud/ARROW-1750 and squashes the following commits: 3f6af737 [Phillip Cloud] ARROW-1750: [C++] Remove the need for arrow/util/random.h --- cpp/src/arrow/array-test.cc | 93 ++++++++----- cpp/src/arrow/ipc/ipc-json-test.cc | 4 +- cpp/src/arrow/test-util.h | 214 ++++++++++++++++++++++------- cpp/src/arrow/util/CMakeLists.txt | 1 - cpp/src/arrow/util/decimal.cc | 11 +- cpp/src/arrow/util/decimal.h | 5 +- cpp/src/arrow/util/random.h | 126 ----------------- 7 files changed, 236 insertions(+), 218 deletions(-) delete mode 100644 cpp/src/arrow/util/random.h diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc index 168ef10573e77..9f248cdbbb754 100644 --- a/cpp/src/arrow/array-test.cc +++ b/cpp/src/arrow/array-test.cc @@ -263,6 +263,8 @@ class TestPrimitiveBuilder : public TestBuilder { ASSERT_TRUE(result->Equals(*expected)); } + int64_t FlipValue(int64_t value) const { return ~value; } + protected: std::shared_ptr type_; std::unique_ptr builder_; @@ -272,44 +274,64 @@ class TestPrimitiveBuilder : public TestBuilder { vector valid_bytes_; }; -#define PTYPE_DECL(CapType, c_type) \ - typedef CapType##Array ArrayType; \ - typedef CapType##Builder BuilderType; \ - typedef CapType##Type Type; \ - typedef c_type T; \ - \ - static std::shared_ptr type() { \ - return std::shared_ptr(new Type()); \ - } +/// \brief uint8_t isn't a valid template parameter to uniform_int_distribution, so +/// we use SampleType to determine which kind of integer to use to sample. +template ::value, T>::type> +struct UniformIntSampleType { + using type = T; +}; + +template <> +struct UniformIntSampleType { + using type = uint16_t; +}; + +template <> +struct UniformIntSampleType { + using type = int16_t; +}; -#define PINT_DECL(CapType, c_type, LOWER, UPPER) \ +#define PTYPE_DECL(CapType, c_type) \ + typedef CapType##Array ArrayType; \ + typedef CapType##Builder BuilderType; \ + typedef CapType##Type Type; \ + typedef c_type T; \ + \ + static std::shared_ptr type() { return std::make_shared(); } + +#define PINT_DECL(CapType, c_type) \ + struct P##CapType { \ + PTYPE_DECL(CapType, c_type) \ + static void draw(int64_t N, vector* draws) { \ + using sample_type = typename UniformIntSampleType::type; \ + const T lower = std::numeric_limits::min(); \ + const T upper = std::numeric_limits::max(); \ + test::randint(N, static_cast(lower), static_cast(upper), \ + draws); \ + } \ + } + +#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER) \ struct P##CapType { \ PTYPE_DECL(CapType, c_type) \ static void draw(int64_t N, vector* draws) { \ - test::randint(N, LOWER, UPPER, draws); \ + test::random_real(N, 0, LOWER, UPPER, draws); \ } \ } -#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER) \ - struct P##CapType { \ - PTYPE_DECL(CapType, c_type) \ - static void draw(int64_t N, vector* draws) { \ - test::random_real(N, 0, LOWER, UPPER, draws); \ - } \ - } - -PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX); -PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX); -PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX); -PINT_DECL(UInt64, uint64_t, 0, UINT64_MAX); +PINT_DECL(UInt8, uint8_t); +PINT_DECL(UInt16, uint16_t); +PINT_DECL(UInt32, uint32_t); +PINT_DECL(UInt64, uint64_t); -PINT_DECL(Int8, int8_t, INT8_MIN, INT8_MAX); -PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX); -PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX); -PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX); +PINT_DECL(Int8, int8_t); +PINT_DECL(Int16, int16_t); +PINT_DECL(Int32, int32_t); +PINT_DECL(Int64, int64_t); -PFLOAT_DECL(Float, float, -1000, 1000); -PFLOAT_DECL(Double, double, -1000, 1000); +PFLOAT_DECL(Float, float, -1000.0f, 1000.0f); +PFLOAT_DECL(Double, double, -1000.0, 1000.0); struct PBoolean { PTYPE_DECL(Boolean, uint8_t) @@ -324,6 +346,11 @@ void TestPrimitiveBuilder::RandomData(int64_t N, double pct_null) { test::random_null_bytes(N, pct_null, valid_bytes_.data()); } +template <> +int64_t TestPrimitiveBuilder::FlipValue(int64_t value) const { + return !value; +} + template <> void TestPrimitiveBuilder::Check(const std::unique_ptr& builder, bool nullable) { @@ -454,8 +481,8 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) { const int64_t first_valid_idx = std::distance(valid_bytes.begin(), first_valid); // This should be true with a very high probability, but might introduce flakiness ASSERT_LT(first_valid_idx, size - 1); - draws[first_valid_idx] = - static_cast(~*reinterpret_cast(&draws[first_valid_idx])); + draws[first_valid_idx] = static_cast( + this->FlipValue(*reinterpret_cast(&draws[first_valid_idx]))); ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array)); // test normal equality @@ -724,8 +751,8 @@ void CheckSliceApproxEquals() { vector draws2; const uint32_t kSeed = 0; - test::random_real(kSize, kSeed, 0, 100, &draws1); - test::random_real(kSize, kSeed + 1, 0, 100, &draws2); + test::random_real(kSize, kSeed, 0.0, 100.0, &draws1); + test::random_real(kSize, kSeed + 1, 0.0, 100.0, &draws2); // Make the draws equal in the sliced segment, but unequal elsewhere (to // catch not using the slice offset) diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc index f2dd9e74e335d..a560f09d6fdb1 100644 --- a/cpp/src/arrow/ipc/ipc-json-test.cc +++ b/cpp/src/arrow/ipc/ipc-json-test.cc @@ -222,8 +222,8 @@ void MakeBatchArrays(const std::shared_ptr& schema, const int num_rows, std::vector v1_values; std::vector v2_values; - test::randint(num_rows, 0, 100, &v1_values); - test::randint(num_rows, 0, 100, &v2_values); + test::randint(num_rows, 0, 100, &v1_values); + test::randint(num_rows, 0, 100, &v2_values); std::shared_ptr v1; ArrayFromVector(is_valid, v1_values, &v1); diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h index 044fb9476ca73..7306f577a36e0 100644 --- a/cpp/src/arrow/test-util.h +++ b/cpp/src/arrow/test-util.h @@ -18,6 +18,7 @@ #ifndef ARROW_TEST_UTIL_H_ #define ARROW_TEST_UTIL_H_ +#include #include #include #include @@ -38,8 +39,8 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit-util.h" +#include "arrow/util/decimal.h" #include "arrow/util/logging.h" -#include "arrow/util/random.h" #define ASSERT_RAISES(ENUM, expr) \ do { \ @@ -47,7 +48,7 @@ if (!s.Is##ENUM()) { \ FAIL() << s.ToString(); \ } \ - } while (0) + } while (false) #define ASSERT_OK(expr) \ do { \ @@ -55,7 +56,7 @@ if (!s.ok()) { \ FAIL() << s.ToString(); \ } \ - } while (0) + } while (false) #define ASSERT_OK_NO_THROW(expr) ASSERT_NO_THROW(ASSERT_OK(expr)) @@ -63,15 +64,15 @@ do { \ ::arrow::Status s = (expr); \ EXPECT_TRUE(s.ok()); \ - } while (0) + } while (false) #define ABORT_NOT_OK(s) \ do { \ ::arrow::Status _s = (s); \ if (ARROW_PREDICT_FALSE(!_s.ok())) { \ - exit(-1); \ + exit(EXIT_FAILURE); \ } \ - } while (0); + } while (false); namespace arrow { @@ -79,27 +80,22 @@ using ArrayVector = std::vector>; namespace test { -template -void randint(int64_t N, T lower, T upper, std::vector* out) { - Random rng(random_seed()); - uint64_t draw; - uint64_t span = upper - lower; - T val; - for (int64_t i = 0; i < N; ++i) { - draw = rng.Uniform64(span); - val = static_cast(draw + lower); - out->push_back(val); - } +template +void randint(int64_t N, T lower, T upper, std::vector* out) { + const int random_seed = 0; + std::mt19937 gen(random_seed); + std::uniform_int_distribution d(lower, upper); + out->resize(N, static_cast(0)); + std::generate(out->begin(), out->end(), [&d, &gen] { return static_cast(d(gen)); }); } -template +template void random_real(int64_t n, uint32_t seed, T min_value, T max_value, - std::vector* out) { + std::vector* out) { std::mt19937 gen(seed); std::uniform_real_distribution d(min_value, max_value); - for (int64_t i = 0; i < n; ++i) { - out->push_back(d(gen)); - } + out->resize(n, static_cast(0)); + std::generate(out->begin(), out->end(), [&d, &gen] { return static_cast(d(gen)); }); } template @@ -115,7 +111,8 @@ inline Status CopyBufferFromVector(const std::vector& values, MemoryPool* poo auto buffer = std::make_shared(pool); RETURN_NOT_OK(buffer->Resize(nbytes)); - memcpy(buffer->mutable_data(), values.data(), nbytes); + auto immutable_data = reinterpret_cast(values.data()); + std::copy(immutable_data, immutable_data + nbytes, buffer->mutable_data()); *result = buffer; return Status::OK(); @@ -143,56 +140,173 @@ static inline Status GetBitmapFromVector(const std::vector& is_valid, // Sets approximately pct_null of the first n bytes in null_bytes to zero // and the rest to non-zero (true) values. static inline void random_null_bytes(int64_t n, double pct_null, uint8_t* null_bytes) { - Random rng(random_seed()); - for (int64_t i = 0; i < n; ++i) { - null_bytes[i] = rng.NextDoubleFraction() > pct_null; - } + const int random_seed = 0; + std::mt19937 gen(random_seed); + std::uniform_real_distribution d(0.0, 1.0); + std::generate(null_bytes, null_bytes + n, + [&d, &gen, &pct_null] { return d(gen) > pct_null; }); } static inline void random_is_valid(int64_t n, double pct_null, std::vector* is_valid) { - Random rng(random_seed()); - for (int64_t i = 0; i < n; ++i) { - is_valid->push_back(rng.NextDoubleFraction() > pct_null); - } + const int random_seed = 0; + std::mt19937 gen(random_seed); + std::uniform_real_distribution d(0.0, 1.0); + is_valid->resize(n, false); + std::generate(is_valid->begin(), is_valid->end(), + [&d, &gen, &pct_null] { return d(gen) > pct_null; }); } static inline void random_bytes(int64_t n, uint32_t seed, uint8_t* out) { std::mt19937 gen(seed); - std::uniform_int_distribution d(0, 255); + std::uniform_int_distribution d(0, std::numeric_limits::max()); + std::generate(out, out + n, [&d, &gen] { return static_cast(d(gen) & 0xFF); }); +} - for (int64_t i = 0; i < n; ++i) { - out[i] = static_cast(d(gen) & 0xFF); +static void DecimalRange(int32_t precision, Decimal128* min_decimal, + Decimal128* max_decimal) { + DCHECK_GE(precision, 1) << "decimal precision must be greater than or equal to 1, got " + << precision; + DCHECK_LE(precision, 38) << "decimal precision must be less than or equal to 38, got " + << precision; + + switch (precision) { + case 1: + case 2: + *max_decimal = std::numeric_limits::max(); + break; + case 3: + case 4: + *max_decimal = std::numeric_limits::max(); + break; + case 5: + case 6: + *max_decimal = 8388607; + break; + case 7: + case 8: + case 9: + *max_decimal = std::numeric_limits::max(); + break; + case 10: + case 11: + *max_decimal = 549755813887; + break; + case 12: + case 13: + case 14: + *max_decimal = 140737488355327; + break; + case 15: + case 16: + *max_decimal = 36028797018963967; + break; + case 17: + case 18: + *max_decimal = std::numeric_limits::max(); + break; + case 19: + case 20: + case 21: + *max_decimal = Decimal128("2361183241434822606847"); + break; + case 22: + case 23: + *max_decimal = Decimal128("604462909807314587353087"); + break; + case 24: + case 25: + case 26: + *max_decimal = Decimal128("154742504910672534362390527"); + break; + case 27: + case 28: + *max_decimal = Decimal128("39614081257132168796771975167"); + break; + case 29: + case 30: + case 31: + *max_decimal = Decimal128("10141204801825835211973625643007"); + break; + case 32: + case 33: + *max_decimal = Decimal128("2596148429267413814265248164610047"); + break; + case 34: + case 35: + *max_decimal = Decimal128("664613997892457936451903530140172287"); + break; + case 36: + case 37: + case 38: + *max_decimal = Decimal128("170141183460469231731687303715884105727"); + break; + default: + DCHECK(false); + break; } + + *min_decimal = ~(*max_decimal); } -static inline void random_ascii(int64_t n, uint32_t seed, uint8_t* out) { +class UniformDecimalDistribution { + public: + explicit UniformDecimalDistribution(int32_t precision) { + Decimal128 max_decimal; + Decimal128 min_decimal; + DecimalRange(precision, &min_decimal, &max_decimal); + + const auto min_low = static_cast(min_decimal.low_bits()); + const auto max_low = static_cast(max_decimal.low_bits()); + + const int64_t min_high = min_decimal.high_bits(); + const int64_t max_high = max_decimal.high_bits(); + + using param_type = std::uniform_int_distribution::param_type; + + lower_dist_.param(param_type(min_low, max_low)); + upper_dist_.param(param_type(min_high, max_high)); + } + + template + Decimal128 operator()(Generator& gen) { + return Decimal128(upper_dist_(gen), static_cast(lower_dist_(gen))); + } + + private: + // The lower bits distribution is intentionally int64_t. + // If it were uint64_t then the size of the interval [min_high, max_high] would be 0 + // because min_high > max_high due to 2's complement. + // So, we generate the same range of bits using int64_t and then cast to uint64_t. + std::uniform_int_distribution lower_dist_; + std::uniform_int_distribution upper_dist_; +}; + +static inline void random_decimals(int64_t n, uint32_t seed, int32_t precision, + uint8_t* out) { std::mt19937 gen(seed); - std::uniform_int_distribution d(65, 122); + UniformDecimalDistribution dist(precision); - for (int64_t i = 0; i < n; ++i) { - out[i] = static_cast(d(gen) & 0xFF); + for (int64_t i = 0; i < n; ++i, out += 16) { + const Decimal128 value(dist(gen)); + value.ToBytes(out); } } -template -void rand_uniform_int(int64_t n, uint32_t seed, T min_value, T max_value, T* out) { +template +void rand_uniform_int(int64_t n, uint32_t seed, T min_value, T max_value, U* out) { DCHECK(out || (n == 0)); std::mt19937 gen(seed); std::uniform_int_distribution d(min_value, max_value); - for (int64_t i = 0; i < n; ++i) { - out[i] = static_cast(d(gen)); - } + std::generate(out, out + n, [&d, &gen] { return static_cast(d(gen)); }); +} + +static inline void random_ascii(int64_t n, uint32_t seed, uint8_t* out) { + rand_uniform_int(n, seed, static_cast('A'), static_cast('z'), out); } static inline int64_t null_count(const std::vector& valid_bytes) { - int64_t result = 0; - for (size_t i = 0; i < valid_bytes.size(); ++i) { - if (valid_bytes[i] == 0) { - ++result; - } - } - return result; + return static_cast(std::count(valid_bytes.cbegin(), valid_bytes.cend(), '\0')); } Status MakeRandomInt32PoolBuffer(int64_t length, MemoryPool* pool, diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 5df5e748f39e5..7810a3be46da5 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -38,7 +38,6 @@ install(FILES logging.h macros.h parallel.h - random.h rle-encoding.h sse-util.h stl.h diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc index 9d94bef847fa1..cc180258aa4df 100644 --- a/cpp/src/arrow/util/decimal.cc +++ b/cpp/src/arrow/util/decimal.cc @@ -43,14 +43,17 @@ Decimal128::Decimal128(const uint8_t* bytes) } std::array Decimal128::ToBytes() const { - const uint64_t raw[] = {BitUtil::ToLittleEndian(low_bits_), - BitUtil::ToLittleEndian(static_cast(high_bits_))}; - const auto* raw_data = reinterpret_cast(raw); std::array out{{0}}; - std::copy(raw_data, raw_data + out.size(), out.begin()); + ToBytes(out.data()); return out; } +void Decimal128::ToBytes(uint8_t* out) const { + DCHECK_NE(out, NULLPTR); + reinterpret_cast(out)[0] = BitUtil::ToLittleEndian(low_bits_); + reinterpret_cast(out)[1] = BitUtil::ToLittleEndian(high_bits_); +} + static constexpr Decimal128 kTenTo36(static_cast(0xC097CE7BC90715), 0xB34B9F1000000000); static constexpr Decimal128 kTenTo18(0xDE0B6B3A7640000); diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h index 487f222580201..a0423e9fce49c 100644 --- a/cpp/src/arrow/util/decimal.h +++ b/cpp/src/arrow/util/decimal.h @@ -102,13 +102,14 @@ class ARROW_EXPORT Decimal128 { Decimal128& operator>>=(uint32_t bits); /// \brief Get the high bits of the two's complement representation of the number. - int64_t high_bits() const { return high_bits_; } + inline int64_t high_bits() const { return high_bits_; } /// \brief Get the low bits of the two's complement representation of the number. - uint64_t low_bits() const { return low_bits_; } + inline uint64_t low_bits() const { return low_bits_; } /// \brief Return the raw bytes of the value in little-endian byte order. std::array ToBytes() const; + void ToBytes(uint8_t* out) const; /// \brief Convert the Decimal128 value to a base 10 decimal string with the given /// scale. diff --git a/cpp/src/arrow/util/random.h b/cpp/src/arrow/util/random.h deleted file mode 100644 index 2e05a73033d0f..0000000000000 --- a/cpp/src/arrow/util/random.h +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -// Moved from Kudu http://github.com/cloudera/kudu - -#ifndef ARROW_UTIL_RANDOM_H_ -#define ARROW_UTIL_RANDOM_H_ - -#include - -#include - -namespace arrow { -namespace internal { -namespace random { - -static const uint32_t M = 2147483647L; // 2^31-1 -const double kTwoPi = 6.283185307179586476925286; - -} // namespace random -} // namespace internal - -// A very simple random number generator. Not especially good at -// generating truly random bits, but good enough for our needs in this -// package. This implementation is not thread-safe. -class Random { - public: - explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { - // Avoid bad seeds. - if (seed_ == 0 || seed_ == internal::random::M) { - seed_ = 1; - } - } - - // Next pseudo-random 32-bit unsigned integer. - // FIXME: This currently only generates 31 bits of randomness. - // The MSB will always be zero. - uint32_t Next() { - static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 - // We are computing - // seed_ = (seed_ * A) % M, where M = 2^31-1 - // - // seed_ must not be zero or M, or else all subsequent computed values - // will be zero or M respectively. For all other values, seed_ will end - // up cycling through every number in [1,M-1] - uint64_t product = seed_ * A; - - // Compute (product % M) using the fact that ((x << 31) % M) == x. - seed_ = static_cast((product >> 31) + (product & internal::random::M)); - // The first reduction may overflow by 1 bit, so we may need to - // repeat. mod == M is not possible; using > allows the faster - // sign-bit-based test. - if (seed_ > internal::random::M) { - seed_ -= internal::random::M; - } - return seed_; - } - - // Alias for consistency with Next64 - uint32_t Next32() { return Next(); } - - // Next pseudo-random 64-bit unsigned integer. - // FIXME: This currently only generates 62 bits of randomness due to Next() - // only giving 31 bits of randomness. The 2 most significant bits will always - // be zero. - uint64_t Next64() { - uint64_t large = Next(); - // Only shift by 31 bits so we end up with zeros in MSB and not scattered - // throughout the 64-bit word. This is due to the weakness in Next() noted - // above. - large <<= 31; - large |= Next(); - return large; - } - - // Returns a uniformly distributed value in the range [0..n-1] - // REQUIRES: n > 0 - uint32_t Uniform(uint32_t n) { return Next() % n; } - - // Alias for consistency with Uniform64 - uint32_t Uniform32(uint32_t n) { return Uniform(n); } - - // Returns a uniformly distributed 64-bit value in the range [0..n-1] - // REQUIRES: n > 0 - uint64_t Uniform64(uint64_t n) { return Next64() % n; } - - // Randomly returns true ~"1/n" of the time, and false otherwise. - // REQUIRES: n > 0 - bool OneIn(int n) { return (Next() % n) == 0; } - - // Skewed: pick "base" uniformly from range [0,max_log] and then - // return "base" random bits. The effect is to pick a number in the - // range [0,2^max_log-1] with exponential bias towards smaller numbers. - uint32_t Skewed(int max_log) { return Uniform(1 << Uniform(max_log + 1)); } - - // Creates a normal distribution variable using the - // Box-Muller transform. See: - // http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform - // Adapted from WebRTC source code at: - // webrtc/trunk/modules/video_coding/main/test/test_util.cc - double Normal(double mean, double std_dev) { - double uniform1 = (Next() + 1.0) / (internal::random::M + 1.0); - double uniform2 = (Next() + 1.0) / (internal::random::M + 1.0); - return (mean + - std_dev * sqrt(-2 * ::log(uniform1)) * - cos(internal::random::kTwoPi * uniform2)); - } - - // Return a random number between 0.0 and 1.0 inclusive. - double NextDoubleFraction() { - return Next() / static_cast(internal::random::M + 1.0); - } - - private: - uint32_t seed_; -}; - -uint32_t random_seed() { - // TODO(wesm): use system time to get a reasonably random seed - return 0; -} - -} // namespace arrow - -#endif // ARROW_UTIL_RANDOM_H_ From 99ea353dbaf15b8db1ad7d8d4419643abe99189d Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 6 Nov 2017 22:24:19 -0500 Subject: [PATCH 022/177] ARROW-1771: [C++] ARROW-1749 Breaks Public API test in parquet-cpp Author: Phillip Cloud Closes #1288 from cpcloud/ARROW-1771 and squashes the following commits: dc002568 [Phillip Cloud] Formatting again ac2367b4 [Phillip Cloud] Fix formatting 35e67e50 [Phillip Cloud] Add public API test 659e5165 [Phillip Cloud] Fix formatting de4632ac [Phillip Cloud] ARROW-1771: [C++] ARROW-1749 Breaks Public API test in parquet-cpp --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/public-api-test.cc | 26 ++++++++++++++++++++++++++ cpp/src/arrow/util/bit-util.h | 3 +-- 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 cpp/src/arrow/public-api-test.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 22b475146da7c..69d5052330c3a 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -171,6 +171,7 @@ ADD_ARROW_TEST(array-test) ADD_ARROW_TEST(buffer-test) ADD_ARROW_TEST(memory_pool-test) ADD_ARROW_TEST(pretty_print-test) +ADD_ARROW_TEST(public-api-test) ADD_ARROW_TEST(status-test) ADD_ARROW_TEST(type-test) ADD_ARROW_TEST(table-test) diff --git a/cpp/src/arrow/public-api-test.cc b/cpp/src/arrow/public-api-test.cc new file mode 100644 index 0000000000000..8298d748fe8fd --- /dev/null +++ b/cpp/src/arrow/public-api-test.cc @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/api.h" + +#ifdef DCHECK +#error "DCHECK should not be visible from Arrow public headers." +#endif + +#include + +TEST(_, _) {} diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h index d6415f3c75b7b..cab3c9ee703c9 100644 --- a/cpp/src/arrow/util/bit-util.h +++ b/cpp/src/arrow/util/bit-util.h @@ -57,7 +57,6 @@ #include #include -#include "arrow/util/logging.h" #include "arrow/util/macros.h" #include "arrow/util/type_traits.h" #include "arrow/util/visibility.h" @@ -301,7 +300,7 @@ static inline int Log2(uint64_t x) { /// \brief Count the number of leading zeros in a 32 bit integer. static inline int64_t CountLeadingZeros(uint32_t value) { - DCHECK_NE(value, 0); +// DCHECK_NE(value, 0); #if defined(__clang__) || defined(__GNUC__) return static_cast(__builtin_clz(value)); #elif defined(_MSC_VER) From 3995eb3c1d94ab8fd151c32a5d1994a6b085deec Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 6 Nov 2017 22:24:51 -0500 Subject: [PATCH 023/177] ARROW-1768: [Python] Fix suppressed exception in ParquetWriter.__del__ This closes [ARROW-1768](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1768). Author: Wes McKinney Author: Licht-T Closes #1286 from Licht-T/fix-suppressed-exception-in-parquetwriter-del and squashes the following commits: ccd7344d [Wes McKinney] Use getattr to be robust to is_open attribute not existing 4a42683d [Licht-T] Fix suppressed exception in ParquetWriter.__del__ --- python/pyarrow/parquet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index 9dcc30c8af479..9e0749bb35c9e 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -260,7 +260,7 @@ def __init__(self, where, schema, flavor=None, self.is_open = True def __del__(self): - if self.is_open: + if getattr(self, 'is_open', False): self.close() def write_table(self, table, row_group_size=None): From e631119d853a3182fa4e2d8cd980440f59c7a679 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Tue, 7 Nov 2017 11:57:43 -0700 Subject: [PATCH 024/177] [Format] Fix link to Flatbuffers project in IPC.md Close #1291 Change-Id: Ibae55c472b08cedcf22bd783c7a7fce1449051fe --- format/IPC.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/format/IPC.md b/format/IPC.md index f3b48854c2072..5a5d3aef62be7 100644 --- a/format/IPC.md +++ b/format/IPC.md @@ -252,5 +252,5 @@ shared memory region) to be a multiple of 8: [1]: https://github.com/apache/arrow/blob/master/format/File.fbs [2]: https://github.com/apache/arrow/blob/master/format/Message.fbs -[3]: https://github.com/google]/flatbuffers +[3]: https://github.com/google/flatbuffers [4]: https://github.com/apache/arrow/blob/master/format/Layout.md From 3188d70202795d8e0a8092ec5685d859b02e366d Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Tue, 7 Nov 2017 16:22:20 -0500 Subject: [PATCH 025/177] ARROW-1716: [Format/JSON] Use string integer value for Decimals in JSON Author: Phillip Cloud Closes #1267 from cpcloud/ARROW-1716 and squashes the following commits: b4f3aed2 [Phillip Cloud] Add cases for every valid precision f8d4391f [Phillip Cloud] Use the full range of decimal values in integration tests 6fef5f71 [Phillip Cloud] ARROW-1716: [Format/JSON] Use string integer value for Decimals in JSON --- cpp/src/arrow/ipc/ipc-read-write-test.cc | 4 +- cpp/src/arrow/ipc/json-internal.cc | 69 +++++++++--- cpp/src/arrow/ipc/test-common.h | 13 ++- integration/integration_test.py | 105 ++++++++++-------- .../vector/file/json/JsonFileReader.java | 9 +- .../vector/file/json/JsonFileWriter.java | 8 +- .../arrow/vector/util/DecimalUtility.java | 32 +++++- 7 files changed, 165 insertions(+), 75 deletions(-) diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc index 6f2f5cf856055..40cd3f0eef0b8 100644 --- a/cpp/src/arrow/ipc/ipc-read-write-test.cc +++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc @@ -727,7 +727,7 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) { int64_t size = 24; std::vector values; - test::randint(size, 0, 100, &values); + test::randint(size, 0, 100, &values); auto data = test::GetBufferFromVector(values); @@ -748,7 +748,7 @@ TEST_F(TestTensorRoundTrip, NonContiguous) { ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_)); std::vector values; - test::randint(24, 0, 100, &values); + test::randint(24, 0, 100, &values); auto data = test::GetBufferFromVector(values); Tensor tensor(int64(), data, {4, 3}, {48, 16}); diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index 025f6c276541e..c1c0661d6ad35 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -33,6 +33,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit-util.h" +#include "arrow/util/decimal.h" #include "arrow/util/logging.h" #include "arrow/util/string.h" #include "arrow/visitor_inline.h" @@ -448,7 +449,8 @@ class ArrayWriter { } void WriteDataValues(const FixedSizeBinaryArray& arr) { - int32_t width = arr.byte_width(); + const int32_t width = arr.byte_width(); + for (int64_t i = 0; i < arr.length(); ++i) { const uint8_t* buf = arr.GetValue(i); std::string encoded = HexEncode(buf, width); @@ -456,6 +458,13 @@ class ArrayWriter { } } + void WriteDataValues(const DecimalArray& arr) { + for (int64_t i = 0; i < arr.length(); ++i) { + const Decimal128 value(arr.GetValue(i)); + writer_->String(value.ToIntegerString()); + } + } + void WriteDataValues(const BooleanArray& arr) { for (int i = 0; i < arr.length(); ++i) { writer_->Bool(arr.Value(i)); @@ -1053,7 +1062,9 @@ class ArrayReader { } template - typename std::enable_if::value, Status>::type + typename std::enable_if::value && + !std::is_base_of::value, + Status>::type Visit(const T& type) { typename TypeTraits::BuilderType builder(type_, pool_); @@ -1073,22 +1084,52 @@ class ArrayReader { for (int i = 0; i < length_; ++i) { if (!is_valid_[i]) { RETURN_NOT_OK(builder.AppendNull()); - continue; - } + } else { + const rj::Value& val = json_data_arr[i]; + DCHECK(val.IsString()) + << "Found non-string JSON value when parsing FixedSizeBinary value"; + std::string hex_string = val.GetString(); + if (static_cast(hex_string.size()) != byte_width * 2) { + DCHECK(false) << "Expected size: " << byte_width * 2 + << " got: " << hex_string.size(); + } + const char* hex_data = hex_string.c_str(); - const rj::Value& val = json_data_arr[i]; - DCHECK(val.IsString()); - std::string hex_string = val.GetString(); - if (static_cast(hex_string.size()) != byte_width * 2) { - DCHECK(false) << "Expected size: " << byte_width * 2 - << " got: " << hex_string.size(); + for (int32_t j = 0; j < byte_width; ++j) { + RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, &byte_buffer_data[j])); + } + RETURN_NOT_OK(builder.Append(byte_buffer_data)); } - const char* hex_data = hex_string.c_str(); + } + return builder.Finish(&result_); + } + + template + typename std::enable_if::value, Status>::type Visit( + const T& type) { + typename TypeTraits::BuilderType builder(type_, pool_); + + const auto& json_data = obj_->FindMember("DATA"); + RETURN_NOT_ARRAY("DATA", json_data, *obj_); - for (int32_t j = 0; j < byte_width; ++j) { - RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, &byte_buffer_data[j])); + const auto& json_data_arr = json_data->value.GetArray(); + + DCHECK_EQ(static_cast(json_data_arr.Size()), length_); + + for (int i = 0; i < length_; ++i) { + if (!is_valid_[i]) { + RETURN_NOT_OK(builder.AppendNull()); + } else { + const rj::Value& val = json_data_arr[i]; + DCHECK(val.IsString()) + << "Found non-string JSON value when parsing Decimal128 value"; + DCHECK_GT(val.GetStringLength(), 0) + << "Empty string found when parsing Decimal128 value"; + + Decimal128 value; + RETURN_NOT_OK(Decimal128::FromString(val.GetString(), &value)); + RETURN_NOT_OK(builder.Append(value)); } - RETURN_NOT_OK(builder.Append(byte_buffer_data)); } return builder.Finish(&result_); } diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h index b2137b7dbef6a..91023db489852 100644 --- a/cpp/src/arrow/ipc/test-common.h +++ b/cpp/src/arrow/ipc/test-common.h @@ -671,8 +671,11 @@ Status MakeFWBinary(std::shared_ptr* out) { } Status MakeDecimal(std::shared_ptr* out) { - auto f0 = field("f0", decimal(19, 4)); - auto schema = ::arrow::schema({f0, f0}); + constexpr int kDecimalPrecision = 38; + auto type = decimal(kDecimalPrecision, 4); + auto f0 = field("f0", type); + auto f1 = field("f1", type); + auto schema = ::arrow::schema({f0, f1}); constexpr int kDecimalSize = 16; constexpr int length = 10; @@ -682,7 +685,7 @@ Status MakeDecimal(std::shared_ptr* out) { RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), kDecimalSize * length, &data)); - test::random_bytes(kDecimalSize * length, 0, data->mutable_data()); + test::random_decimals(length, 1, kDecimalPrecision, data->mutable_data()); test::random_null_bytes(length, 0.1, is_valid_bytes.data()); RETURN_NOT_OK(BitUtil::BytesToBits(is_valid_bytes, default_memory_pool(), &is_valid)); @@ -690,10 +693,10 @@ Status MakeDecimal(std::shared_ptr* out) { auto a1 = std::make_shared(f0->type(), length, data, is_valid, kUnknownNullCount); - auto a2 = std::make_shared(f0->type(), length, data); + auto a2 = std::make_shared(f1->type(), length, data); ArrayVector arrays = {a1, a2}; - *out = std::make_shared(schema, a1->length(), arrays); + *out = std::make_shared(schema, length, arrays); return Status::OK(); } diff --git a/integration/integration_test.py b/integration/integration_test.py index 59a1de5a4639d..205176eccc11a 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -65,24 +65,16 @@ def rands(nchars): return ''.join(np.random.choice(RANDS_CHARS, nchars)) -if six.PY2: - def frombytes(o): - return o +def tobytes(o): + if isinstance(o, six.text_type): + return o.encode('utf8') + return o - def tobytes(o): - if isinstance(o, unicode): - return o.encode('utf8') - else: - return o -else: - def tobytes(o): - if isinstance(o, str): - return o.encode('utf8') - else: - return o - def frombytes(o): +def frombytes(o): + if isinstance(o, six.binary_type): return o.decode('utf8') + return o # from the merge_arrow_pr.py script @@ -177,7 +169,7 @@ def _get_type_layout(self): class PrimitiveColumn(Column): def __init__(self, name, count, is_valid, values): - Column.__init__(self, name, count) + super(PrimitiveColumn, self).__init__(name, count) self.is_valid = is_valid self.values = values @@ -191,15 +183,16 @@ def _get_buffers(self): ] -TEST_INT_MIN = - 2**31 + 1 -TEST_INT_MAX = 2**31 - 1 +TEST_INT_MAX = 2 ** 31 - 1 +TEST_INT_MIN = ~TEST_INT_MAX + class IntegerType(PrimitiveType): def __init__(self, name, is_signed, bit_width, nullable=True, min_value=TEST_INT_MIN, max_value=TEST_INT_MAX): - PrimitiveType.__init__(self, name, nullable=nullable) + super(IntegerType, self).__init__(name, nullable=nullable) self.is_signed = is_signed self.bit_width = bit_width self.min_value = min_value @@ -239,9 +232,11 @@ class DateType(IntegerType): MILLISECOND = 1 def __init__(self, name, unit, nullable=True): - self.unit = unit bit_width = 32 if unit == self.DAY else 64 - IntegerType.__init__(self, name, True, bit_width, nullable=nullable) + super(DateType, self).__init__( + name, True, bit_width, nullable=nullable + ) + self.unit = unit def _get_type(self): return OrderedDict([ @@ -268,9 +263,10 @@ class TimeType(IntegerType): } def __init__(self, name, unit='s', nullable=True): + super(TimeType, self).__init__( + name, True, self.BIT_WIDTHS[unit], nullable=nullable + ) self.unit = unit - IntegerType.__init__(self, name, True, self.BIT_WIDTHS[unit], - nullable=nullable) def _get_type(self): return OrderedDict([ @@ -283,9 +279,9 @@ def _get_type(self): class TimestampType(IntegerType): def __init__(self, name, unit='s', tz=None, nullable=True): + super(TimestampType, self).__init__(name, True, 64, nullable=nullable) self.unit = unit self.tz = tz - IntegerType.__init__(self, name, True, 64, nullable=nullable) def _get_type(self): fields = [ @@ -302,7 +298,7 @@ def _get_type(self): class FloatingPointType(PrimitiveType): def __init__(self, name, bit_width, nullable=True): - PrimitiveType.__init__(self, name, nullable=nullable) + super(FloatingPointType, self).__init__(name, nullable=nullable) self.bit_width = bit_width self.precision = { @@ -331,13 +327,30 @@ def generate_column(self, size, name=None): return PrimitiveColumn(name, size, is_valid, values) -class DecimalType(PrimitiveType): - def __init__(self, name, bit_width, precision, scale, nullable=True): - PrimitiveType.__init__(self, name, nullable=True) +DECIMAL_PRECISION_TO_VALUE = { + key: (1 << (8 * i - 1)) - 1 for i, key in enumerate( + [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36], + start=1, + ) +} - self.bit_width = bit_width + +def decimal_range_from_precision(precision): + assert 1 <= precision <= 38 + try: + max_value = DECIMAL_PRECISION_TO_VALUE[precision] + except KeyError: + return decimal_range_from_precision(precision - 1) + else: + return ~max_value, max_value + + +class DecimalType(PrimitiveType): + def __init__(self, name, precision, scale, bit_width=128, nullable=True): + super(DecimalType, self).__init__(name, nullable=True) self.precision = precision self.scale = scale + self.bit_width = bit_width @property def numpy_type(self): @@ -359,7 +372,8 @@ def _get_type_layout(self): ('typeBitWidth', self.bit_width)])])]) def generate_column(self, size, name=None): - values = [random.randint(0, 2**self.bit_width - 1) for x in range(size)] + min_value, max_value = decimal_range_from_precision(self.precision) + values = [random.randint(min_value, max_value) for _ in range(size)] is_valid = self._make_is_valid(size) if name is None: @@ -369,14 +383,12 @@ def generate_column(self, size, name=None): class DecimalColumn(PrimitiveColumn): - def __init__(self, name, count, is_valid, values, bit_width): - PrimitiveColumn.__init__(self, name, count, is_valid, values) + def __init__(self, name, count, is_valid, values, bit_width=128): + super(DecimalColumn, self).__init__(name, count, is_valid, values) self.bit_width = bit_width - self.hex_width = bit_width / 4 def _encode_value(self, x): - hex_format_str = '%%0%dx' % self.hex_width - return (hex_format_str % x).upper() + return str(x) class BooleanType(PrimitiveType): @@ -510,7 +522,7 @@ def _encode_value(self, x): class ListType(DataType): def __init__(self, name, value_type, nullable=True): - DataType.__init__(self, name, nullable=nullable) + super(ListType, self).__init__(name, nullable=nullable) self.value_type = value_type def _get_type(self): @@ -553,7 +565,7 @@ def generate_column(self, size, name=None): class ListColumn(Column): def __init__(self, name, count, is_valid, offsets, values): - Column.__init__(self, name, count) + super(ListColumn, self).__init__(name, count) self.is_valid = is_valid self.offsets = offsets self.values = values @@ -571,7 +583,7 @@ def _get_children(self): class StructType(DataType): def __init__(self, name, field_types, nullable=True): - DataType.__init__(self, name, nullable=nullable) + super(StructType, self).__init__(name, nullable=nullable) self.field_types = field_types def _get_type(self): @@ -620,7 +632,7 @@ def get_json(self): class DictionaryType(DataType): def __init__(self, name, index_type, dictionary, nullable=True): - DataType.__init__(self, name, nullable=nullable) + super(DictionaryType, self).__init__(name, nullable=nullable) assert isinstance(index_type, IntegerType) assert isinstance(dictionary, Dictionary) @@ -655,7 +667,7 @@ def generate_column(self, size, name=None): class StructColumn(Column): def __init__(self, name, count, is_valid, field_values): - Column.__init__(self, name, count) + super(StructColumn, self).__init__(name, count) self.is_valid = is_valid self.field_values = field_values @@ -758,11 +770,12 @@ def generate_primitive_case(batch_sizes): def generate_decimal_case(): fields = [ - DecimalType('f1', 128, 24, 10, True), - DecimalType('f2', 128, 32, -10, True) + DecimalType(name='f{}'.format(i), precision=precision, scale=2) + for i, precision in enumerate(range(3, 39)) ] - batch_sizes = [7, 10] + possible_batch_sizes = 7, 10 + batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))] return _generate_file('decimal', fields, batch_sizes) @@ -867,8 +880,9 @@ def run(self): def _compare_implementations(self, producer, consumer): print('##########################################################') - print('{0} producing, {1} consuming'.format(producer.name, - consumer.name)) + print( + '{0} producing, {1} consuming'.format(producer.name, consumer.name) + ) print('##########################################################') for json_path in self.json_files: @@ -1033,6 +1047,7 @@ def run_all_tests(debug=False): runner.run() print('-- All tests passed!') + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Arrow integration test CLI') parser.add_argument('--debug', dest='debug', action='store_true', diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index c6ebd61aa07b9..e1c7c909f10be 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -27,6 +27,8 @@ import java.io.File; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -332,9 +334,10 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class)); break; case DECIMAL: { - DecimalVector decimalVector = ((DecimalVector) valueVector); - byte[] value = decodeHexSafe(parser.readValueAs(String.class)); - DecimalUtility.writeByteArrayToArrowBuf(value, decimalVector.getBuffer(), i); + DecimalVector decimalVector = (DecimalVector) valueVector; + // Here we assume the decimal value is the unscaled integer value as a string + BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); + DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, decimalVector.getBuffer(), i); } break; case VARBINARY: diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java index 04e44379e5dfa..05341bec44ea3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -48,6 +49,7 @@ import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -242,9 +244,9 @@ private void writeValueToGenerator(ValueVector valueVector, int i) throws IOExce } break; case DECIMAL: { - ArrowBuf bytebuf = valueVector.getDataBuffer(); - String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(bytebuf, i)); - generator.writeString(hexString); + BigDecimal decimalValue = ((DecimalVector) valueVector).getAccessor().getObject(i); + // We write the unscaled value, because the scale is stored in the type metadata. + generator.writeString(decimalValue.unscaledValue().toString()); } break; default: diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index 033ae6c09914d..acf7c58a1337f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -142,8 +142,18 @@ public static StringBuilder toStringWithZeroes(long number, int desiredLength) { */ public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) { byte[] value = new byte[DECIMAL_BYTE_LENGTH]; + byte temp; final int startIndex = index * DECIMAL_BYTE_LENGTH; + + // Decimal stored as little endian, need to swap bytes to make BigDecimal bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH); + int stop = DECIMAL_BYTE_LENGTH / 2; + for (int i = 0, j; i < stop; i++) { + temp = value[i]; + j = (DECIMAL_BYTE_LENGTH - 1) - i; + value[i] = value[j]; + value[j] = temp; + } BigInteger unscaledValue = new BigInteger(value); return new BigDecimal(unscaledValue, scale); } @@ -212,10 +222,26 @@ private static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int if (bytes.length > DECIMAL_BYTE_LENGTH) { throw new UnsupportedOperationException("Decimal size greater than 16 bytes"); } - final int padLength = DECIMAL_BYTE_LENGTH - bytes.length; - for (int i = 0; i < padLength; i++) { + + // Decimal stored as little endian, need to swap data bytes before writing to ArrowBuf + byte[] bytesLE = new byte[bytes.length]; + int stop = bytes.length / 2; + for (int i = 0, j; i < stop; i++) { + j = (bytes.length - 1) - i; + bytesLE[i] = bytes[j]; + bytesLE[j] = bytes[i]; + } + if (bytes.length % 2 != 0) { + int i = (bytes.length / 2); + bytesLE[i] = bytes[i]; + } + + // Write LE data + bytebuf.setBytes(startIndex, bytesLE, 0, bytes.length); + + // Write padding after data + for (int i = bytes.length; i < DECIMAL_BYTE_LENGTH; i++) { bytebuf.setByte(startIndex + i, padValue); } - bytebuf.setBytes(startIndex + padLength, bytes, 0, bytes.length); } } From bfc0f24fcc58f3885c2175c64864cc12af95f938 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Wed, 8 Nov 2017 14:44:26 +0100 Subject: [PATCH 026/177] ARROW-1776: [C++] Define arrow::gpu::CudaContext::bytes_allocated() Author: Kouhei Sutou Closes #1293 from kou/cpp-gpu-cuda-context-bytes-allocated and squashes the following commits: 5221887 [Kouhei Sutou] [C++] Define arrow::gpu::CudaContext::bytes_allocated() --- cpp/src/arrow/gpu/cuda-test.cc | 1 + cpp/src/arrow/gpu/cuda_context.cc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/cpp/src/arrow/gpu/cuda-test.cc b/cpp/src/arrow/gpu/cuda-test.cc index afdc3020e8581..7595f8bec7912 100644 --- a/cpp/src/arrow/gpu/cuda-test.cc +++ b/cpp/src/arrow/gpu/cuda-test.cc @@ -55,6 +55,7 @@ TEST_F(TestCudaBuffer, Allocate) { std::shared_ptr buffer; ASSERT_OK(context_->Allocate(kSize, &buffer)); ASSERT_EQ(kSize, buffer->size()); + ASSERT_EQ(kSize, context_->bytes_allocated()); } void AssertCudaBufferEquals(const CudaBuffer& buffer, const uint8_t* host_data, diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc index 42315cd509c3c..fff8ece6c1cff 100644 --- a/cpp/src/arrow/gpu/cuda_context.cc +++ b/cpp/src/arrow/gpu/cuda_context.cc @@ -64,6 +64,7 @@ class CudaContext::CudaContextImpl { CUdeviceptr data; CU_RETURN_NOT_OK(cuMemAlloc(&data, static_cast(nbytes))); + bytes_allocated_ += nbytes; *out = reinterpret_cast(data); return Status::OK(); } @@ -85,6 +86,7 @@ class CudaContext::CudaContextImpl { Status Free(uint8_t* device_ptr, int64_t nbytes) { CU_RETURN_NOT_OK(cuMemFree(reinterpret_cast(device_ptr))); + bytes_allocated_ -= nbytes; return Status::OK(); } @@ -273,5 +275,7 @@ Status CudaContext::OpenIpcBuffer(const CudaIpcMemHandle& ipc_handle, return Status::OK(); } +int64_t CudaContext::bytes_allocated() const { return impl_->bytes_allocated(); } + } // namespace gpu } // namespace arrow From 252a2a55fc8b80a6412922987b42d64fe41119ef Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Thu, 9 Nov 2017 00:24:59 +0900 Subject: [PATCH 027/177] [GLib] Fix a typo in document --- c_glib/arrow-glib/array-builder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp index 86e7f985be4a4..a5df681421c25 100644 --- a/c_glib/arrow-glib/array-builder.cpp +++ b/c_glib/arrow-glib/array-builder.cpp @@ -316,7 +316,7 @@ garrow_array_builder_new(const std::shared_ptr &type, * * Release ownership of `arrow::ArrayBuilder` in `builder`. * - * Since: 0.8.8 + * Since: 0.8.0 */ void garrow_array_builder_release_ownership(GArrowArrayBuilder *builder) From 78872a1be263e61d7901eb36663a184c2b04effb Mon Sep 17 00:00:00 2001 From: Stephanie Date: Wed, 8 Nov 2017 14:13:28 -0800 Subject: [PATCH 028/177] ARROW-1775: Ability to abort created but unsealed Plasma objects Author: Stephanie Author: Philipp Moritz Closes #1289 from stephanie-wang/abort-objects and squashes the following commits: 38c42b9 [Stephanie] TODO for PascalCase 08d4040 [Stephanie] Move documentation dd5b29e [Stephanie] Fix memory error e6934ac [Philipp Moritz] fix linting 2b8e385 [Stephanie] Return status code when unmapping object fe20b3b [Stephanie] Add test case for PlasmaClient::Abort 646190c [Stephanie] Abort objects that were not sealed when client disconnects 5fc44c5 [Stephanie] Implement PlasmaClient::Abort --- cpp/src/plasma/client.cc | 97 +++++++++++++++++++++++------ cpp/src/plasma/client.h | 19 ++++++ cpp/src/plasma/format/plasma.fbs | 12 ++++ cpp/src/plasma/protocol.cc | 28 +++++++++ cpp/src/plasma/protocol.h | 8 +++ cpp/src/plasma/store.cc | 23 ++++++- cpp/src/plasma/store.h | 3 + cpp/src/plasma/test/client_tests.cc | 44 +++++++++++++ 8 files changed, 213 insertions(+), 21 deletions(-) diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc index e57a2a6f3008c..dd32bdc8149a3 100644 --- a/cpp/src/plasma/client.cc +++ b/cpp/src/plasma/client.cc @@ -278,6 +278,39 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects, return Status::OK(); } +Status PlasmaClient::UnmapObject(const ObjectID& object_id) { + auto object_entry = objects_in_use_.find(object_id); + ARROW_CHECK(object_entry != objects_in_use_.end()); + ARROW_CHECK(object_entry->second->count == 0); + + // Decrement the count of the number of objects in this memory-mapped file + // that the client is using. The corresponding increment should have + // happened in plasma_get. + int fd = object_entry->second->object.handle.store_fd; + auto entry = mmap_table_.find(fd); + ARROW_CHECK(entry != mmap_table_.end()); + ARROW_CHECK(entry->second.count >= 1); + if (entry->second.count == 1) { + // If no other objects are being used, then unmap the file. + int err = munmap(entry->second.pointer, entry->second.length); + if (err == -1) { + return Status::IOError("Error during munmap"); + } + // Remove the corresponding entry from the hash table. + mmap_table_.erase(fd); + } else { + // If there are other objects being used, decrement the reference count. + entry->second.count -= 1; + } + // Update the in_use_object_bytes_. + in_use_object_bytes_ -= (object_entry->second->object.data_size + + object_entry->second->object.metadata_size); + DCHECK_GE(in_use_object_bytes_, 0); + // Remove the entry from the hash table of objects currently in use. + objects_in_use_.erase(object_id); + return Status::OK(); +} + /// This is a helper method for implementing plasma_release. We maintain a /// buffer /// of release calls and only perform them once the buffer becomes full (as @@ -297,28 +330,9 @@ Status PlasmaClient::PerformRelease(const ObjectID& object_id) { ARROW_CHECK(object_entry->second->count >= 0); // Check if the client is no longer using this object. if (object_entry->second->count == 0) { - // Decrement the count of the number of objects in this memory-mapped file - // that the client is using. The corresponding increment should have - // happened in plasma_get. - int fd = object_entry->second->object.handle.store_fd; - auto entry = mmap_table_.find(fd); - ARROW_CHECK(entry != mmap_table_.end()); - entry->second.count -= 1; - ARROW_CHECK(entry->second.count >= 0); - // If none are being used then unmap the file. - if (entry->second.count == 0) { - munmap(entry->second.pointer, entry->second.length); - // Remove the corresponding entry from the hash table. - mmap_table_.erase(fd); - } // Tell the store that the client no longer needs the object. + RETURN_NOT_OK(UnmapObject(object_id)); RETURN_NOT_OK(SendReleaseRequest(store_conn_, object_id)); - // Update the in_use_object_bytes_. - in_use_object_bytes_ -= (object_entry->second->object.data_size + - object_entry->second->object.metadata_size); - DCHECK_GE(in_use_object_bytes_, 0); - // Remove the entry from the hash table of objects currently in use. - objects_in_use_.erase(object_id); } return Status::OK(); } @@ -344,6 +358,20 @@ Status PlasmaClient::Release(const ObjectID& object_id) { return Status::OK(); } +Status PlasmaClient::FlushReleaseHistory() { + // If the client is already disconnected, ignore the flush. + if (store_conn_ < 0) { + return Status::OK(); + } + while (release_history_.size() > 0) { + // Perform a release for the object ID for the first pending release. + RETURN_NOT_OK(PerformRelease(release_history_.back())); + // Remove the last entry from the release history. + release_history_.pop_back(); + } + return Status::OK(); +} + // This method is used to query whether the plasma store contains an object. Status PlasmaClient::Contains(const ObjectID& object_id, bool* has_object) { // Check if we already have a reference to the object. @@ -443,6 +471,35 @@ Status PlasmaClient::Seal(const ObjectID& object_id) { return Release(object_id); } +Status PlasmaClient::Abort(const ObjectID& object_id) { + auto object_entry = objects_in_use_.find(object_id); + ARROW_CHECK(object_entry != objects_in_use_.end()) + << "Plasma client called abort on an object without a reference to it"; + ARROW_CHECK(!object_entry->second->is_sealed) + << "Plasma client called abort on a sealed object"; + + // Flush the release history. + RETURN_NOT_OK(FlushReleaseHistory()); + // Make sure that the Plasma client only has one reference to the object. If + // it has more, then the client needs to release the buffer before calling + // abort. + if (object_entry->second->count > 1) { + return Status::Invalid("Plasma client cannot have a reference to the buffer."); + } + + // Send the abort request. + RETURN_NOT_OK(SendAbortRequest(store_conn_, object_id)); + // Decrease the reference count to zero, then remove the object. + object_entry->second->count--; + RETURN_NOT_OK(UnmapObject(object_id)); + + std::vector buffer; + ObjectID id; + int64_t type; + RETURN_NOT_OK(ReadMessage(store_conn_, &type, &buffer)); + return ReadAbortReply(buffer.data(), buffer.size(), &id); +} + Status PlasmaClient::Delete(const ObjectID& object_id) { // TODO(rkn): In the future, we can use this method to give hints to the // eviction policy about when an object will no longer be needed. diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h index 145942441c9f1..89df2b0b0039f 100644 --- a/cpp/src/plasma/client.h +++ b/cpp/src/plasma/client.h @@ -152,6 +152,15 @@ class ARROW_EXPORT PlasmaClient { /// \return The return status. Status Contains(const ObjectID& object_id, bool* has_object); + /// Abort an unsealed object in the object store. If the abort succeeds, then + /// it will be as if the object was never created at all. The unsealed object + /// must have only a single reference (the one that would have been removed by + /// calling Seal). + /// + /// \param object_id The ID of the object to abort. + /// \return The return status. + Status Abort(const ObjectID& object_id); + /// Seal an object in the object store. The object will be immutable after /// this /// call. @@ -307,6 +316,16 @@ class ARROW_EXPORT PlasmaClient { int get_manager_fd(); private: + /// This is a helper method for unmapping objects for which all references have + /// gone out of scope, either by calling Release or Abort. + /// + /// @param object_id The object ID whose data we should unmap. + Status UnmapObject(const ObjectID& object_id); + + /// This is a helper method that flushes all pending release calls to the + /// store. + Status FlushReleaseHistory(); + Status PerformRelease(const ObjectID& object_id); uint8_t* lookup_or_mmap(int fd, int store_fd_val, int64_t map_size); diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs index 23782ade539d4..b6d03b8a3c10d 100644 --- a/cpp/src/plasma/format/plasma.fbs +++ b/cpp/src/plasma/format/plasma.fbs @@ -21,6 +21,8 @@ enum MessageType:int { // Create a new object. PlasmaCreateRequest = 1, PlasmaCreateReply, + PlasmaAbortRequest, + PlasmaAbortReply, // Seal an object. PlasmaSealRequest, PlasmaSealReply, @@ -113,6 +115,16 @@ table PlasmaCreateReply { error: PlasmaError; } +table PlasmaAbortRequest { + // ID of the object to be aborted. + object_id: string; +} + +table PlasmaAbortReply { + // ID of the object that was aborted. + object_id: string; +} + table PlasmaSealRequest { // ID of the object to be sealed. object_id: string; diff --git a/cpp/src/plasma/protocol.cc b/cpp/src/plasma/protocol.cc index 2261b6a624a8c..c0ebb88fe5019 100644 --- a/cpp/src/plasma/protocol.cc +++ b/cpp/src/plasma/protocol.cc @@ -100,6 +100,34 @@ Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id, return plasma_error_status(message->error()); } +Status SendAbortRequest(int sock, ObjectID object_id) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaAbortRequest(fbb, fbb.CreateString(object_id.binary())); + return PlasmaSend(sock, MessageType_PlasmaAbortRequest, &fbb, message); +} + +Status ReadAbortRequest(uint8_t* data, size_t size, ObjectID* object_id) { + DCHECK(data); + auto message = flatbuffers::GetRoot(data); + DCHECK(verify_flatbuffer(message, data, size)); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return Status::OK(); +} + +Status SendAbortReply(int sock, ObjectID object_id) { + flatbuffers::FlatBufferBuilder fbb; + auto message = CreatePlasmaAbortReply(fbb, fbb.CreateString(object_id.binary())); + return PlasmaSend(sock, MessageType_PlasmaAbortReply, &fbb, message); +} + +Status ReadAbortReply(uint8_t* data, size_t size, ObjectID* object_id) { + DCHECK(data); + auto message = flatbuffers::GetRoot(data); + DCHECK(verify_flatbuffer(message, data, size)); + *object_id = ObjectID::from_binary(message->object_id()->str()); + return Status::OK(); +} + // Seal messages. Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest) { diff --git a/cpp/src/plasma/protocol.h b/cpp/src/plasma/protocol.h index af4b13978c697..e8c334f9181fc 100644 --- a/cpp/src/plasma/protocol.h +++ b/cpp/src/plasma/protocol.h @@ -51,6 +51,14 @@ Status SendCreateReply(int sock, ObjectID object_id, PlasmaObject* object, int e Status ReadCreateReply(uint8_t* data, size_t size, ObjectID* object_id, PlasmaObject* object); +Status SendAbortRequest(int sock, ObjectID object_id); + +Status ReadAbortRequest(uint8_t* data, size_t size, ObjectID* object_id); + +Status SendAbortReply(int sock, ObjectID object_id); + +Status ReadAbortReply(uint8_t* data, size_t size, ObjectID* object_id); + /* Plasma Seal message functions. */ Status SendSealRequest(int sock, ObjectID object_id, unsigned char* digest); diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc index 210cce16238f8..5dbdebc237ce6 100644 --- a/cpp/src/plasma/store.cc +++ b/cpp/src/plasma/store.cc @@ -393,6 +393,18 @@ void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[]) update_object_get_requests(object_id); } +void PlasmaStore::abort_object(const ObjectID& object_id) { + auto entry = get_object_table_entry(&store_info_, object_id); + ARROW_CHECK(entry != NULL) << "To abort an object it must be in the object table."; + ARROW_CHECK(entry->state != PLASMA_SEALED) + << "To abort an object it must not have been sealed."; + ARROW_CHECK(entry->clients.size() == 1) + << "To abort an object, the only client currently using it is the creator."; + + dlfree(entry->pointer); + store_info_.objects.erase(object_id); +} + void PlasmaStore::delete_objects(const std::vector& object_ids) { for (const auto& object_id : object_ids) { ARROW_LOG(DEBUG) << "deleting object " << object_id.hex(); @@ -443,7 +455,11 @@ void PlasmaStore::disconnect_client(int client_fd) { // If this client was using any objects, remove it from the appropriate // lists. for (const auto& entry : store_info_.objects) { - remove_client_from_object_clients(entry.second.get(), it->second.get()); + if (entry.second->state == PLASMA_SEALED) { + remove_client_from_object_clients(entry.second.get(), it->second.get()); + } else { + abort_object(entry.first); + } } // Note, the store may still attempt to send a message to the disconnected @@ -582,6 +598,11 @@ Status PlasmaStore::process_message(Client* client) { warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd); } } break; + case MessageType_PlasmaAbortRequest: { + RETURN_NOT_OK(ReadAbortRequest(input, input_size, &object_id)); + abort_object(object_id); + HANDLE_SIGPIPE(SendAbortReply(client->fd, object_id), client->fd); + } break; case MessageType_PlasmaGetRequest: { std::vector object_ids_to_get; int64_t timeout_ms; diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h index d03d11f4ef0c4..0d08d8a67ffaa 100644 --- a/cpp/src/plasma/store.h +++ b/cpp/src/plasma/store.h @@ -48,6 +48,7 @@ struct Client { class PlasmaStore { public: + // TODO: PascalCase PlasmaStore methods. PlasmaStore(EventLoop* loop, int64_t system_memory, std::string directory, bool hugetlbfs_enabled); @@ -73,6 +74,8 @@ class PlasmaStore { int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size, Client* client, PlasmaObject* result); + void abort_object(const ObjectID& object_id); + /// Delete objects that have been created in the hash table. This should only /// be called on objects that are returned by the eviction policy to evict. /// diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc index 0f19da5f72342..5c0cee4c071ad 100644 --- a/cpp/src/plasma/test/client_tests.cc +++ b/cpp/src/plasma/test/client_tests.cc @@ -127,6 +127,50 @@ TEST_F(TestPlasmaStore, MultipleGetTest) { ASSERT_EQ(object_buffer[1].data[0], 2); } +TEST_F(TestPlasmaStore, AbortTest) { + ObjectID object_id = ObjectID::from_random(); + ObjectBuffer object_buffer; + + // Test for object non-existence. + ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer)); + ASSERT_EQ(object_buffer.data_size, -1); + + // Test object abort. + // First create object. + int64_t data_size = 4; + uint8_t metadata[] = {5}; + int64_t metadata_size = sizeof(metadata); + uint8_t* data; + ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); + // Write some data. + for (int64_t i = 0; i < data_size / 2; i++) { + data[i] = static_cast(i % 4); + } + // Attempt to abort. Test that this fails before the first release. + Status status = client_.Abort(object_id); + ASSERT_TRUE(status.IsInvalid()); + // Release, then abort. + ARROW_CHECK_OK(client_.Release(object_id)); + ARROW_CHECK_OK(client_.Abort(object_id)); + + // Test for object non-existence after the abort. + ARROW_CHECK_OK(client_.Get(&object_id, 1, 0, &object_buffer)); + ASSERT_EQ(object_buffer.data_size, -1); + + // Create the object successfully this time. + ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); + for (int64_t i = 0; i < data_size; i++) { + data[i] = static_cast(i % 4); + } + ARROW_CHECK_OK(client_.Seal(object_id)); + + // Test that we can get the object. + ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer)); + for (int64_t i = 0; i < data_size; i++) { + ASSERT_EQ(data[i], object_buffer.data[i]); + } +} + } // namespace plasma int main(int argc, char** argv) { From dffa486c86d1d09c16e4c52ad0ff78bbee22c4e1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Wed, 8 Nov 2017 20:50:51 -0500 Subject: [PATCH 029/177] ARROW-1709: [C++] Decimal.ToString is incorrect for negative scale This is on top of ARROW-1716. Will rebase when that's merged. Author: Phillip Cloud Closes #1292 from cpcloud/ARROW-1709 and squashes the following commits: 13ef9a32 [Phillip Cloud] Fix formatting e87e88f0 [Phillip Cloud] Cleanup 3431b667 [Phillip Cloud] Skip leading negative sign in dcheck 9874d84d [Phillip Cloud] Use lambda 3a5e3f6f [Phillip Cloud] DCHECK and format 2dde6d2d [Phillip Cloud] ARROW-1709: [C++] Decimal.ToString is incorrect for negative scale --- cpp/src/arrow/util/decimal-test.cc | 75 +++++++++++++++++++++ cpp/src/arrow/util/decimal.cc | 104 +++++++++++++++++++++++------ 2 files changed, 159 insertions(+), 20 deletions(-) diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc index b0271fff15ccf..0d0c08cc4551f 100644 --- a/cpp/src/arrow/util/decimal-test.cc +++ b/cpp/src/arrow/util/decimal-test.cc @@ -18,6 +18,7 @@ #include #include +#include #include @@ -291,4 +292,78 @@ TEST(Decimal128Test, PrintMinValue) { ASSERT_EQ(string_value, printed_value); } +class Decimal128PrintingTest + : public ::testing::TestWithParam> {}; + +TEST_P(Decimal128PrintingTest, Print) { + int32_t test_value; + int32_t scale; + std::string expected_string; + std::tie(test_value, scale, expected_string) = GetParam(); + const Decimal128 value(test_value); + const std::string printed_value = value.ToString(scale); + ASSERT_EQ(expected_string, printed_value); +} + +INSTANTIATE_TEST_CASE_P(Decimal128PrintingTest, Decimal128PrintingTest, + ::testing::Values(std::make_tuple(123, 1, "12.3"), + std::make_tuple(123, 5, "0.00123"), + std::make_tuple(123, 10, "1.23E-8"), + std::make_tuple(123, -1, "1.23E+3"), + std::make_tuple(-123, -1, "-1.23E+3"), + std::make_tuple(123, -3, "1.23E+5"), + std::make_tuple(-123, -3, "-1.23E+5"), + std::make_tuple(12345, -3, "1.2345E+7"))); + +class Decimal128ParsingTest + : public ::testing::TestWithParam> {}; + +TEST_P(Decimal128ParsingTest, Parse) { + std::string test_string; + uint64_t expected_low_bits; + int32_t expected_scale; + std::tie(test_string, expected_low_bits, expected_scale) = GetParam(); + Decimal128 value; + int32_t scale; + ASSERT_OK(Decimal128::FromString(test_string, &value, NULLPTR, &scale)); + ASSERT_EQ(value.low_bits(), expected_low_bits); + ASSERT_EQ(expected_scale, scale); +} + +INSTANTIATE_TEST_CASE_P(Decimal128ParsingTest, Decimal128ParsingTest, + ::testing::Values(std::make_tuple("12.3", 123ULL, 1), + std::make_tuple("0.00123", 123ULL, 5), + std::make_tuple("1.23E-8", 123ULL, 10), + std::make_tuple("-1.23E-8", -123LL, 10), + std::make_tuple("1.23E+3", 123ULL, -1), + std::make_tuple("-1.23E+3", -123LL, -1), + std::make_tuple("1.23E+5", 123ULL, -3), + std::make_tuple("1.2345E+7", 12345ULL, -3), + std::make_tuple("1.23e-8", 123ULL, 10), + std::make_tuple("-1.23e-8", -123LL, 10), + std::make_tuple("1.23e+3", 123ULL, -1), + std::make_tuple("-1.23e+3", -123LL, -1), + std::make_tuple("1.23e+5", 123ULL, -3), + std::make_tuple("1.2345e+7", 12345ULL, -3))); + +class Decimal128ParsingTestInvalid : public ::testing::TestWithParam {}; + +TEST_P(Decimal128ParsingTestInvalid, Parse) { + std::string test_string = GetParam(); + Decimal128 value; + ASSERT_RAISES(Invalid, Decimal128::FromString(test_string, &value)); +} + +INSTANTIATE_TEST_CASE_P(Decimal128ParsingTestInvalid, Decimal128ParsingTestInvalid, + ::testing::Values("0.00123D/3", "1.23eA8", "1.23E+3A", + "-1.23E--5", "1.2345E+++07")); + +TEST(Decimal128ParseTest, WithExponentAndNullptrScale) { + Decimal128 value; + ASSERT_OK(Decimal128::FromString("1.23E-8", &value)); + + const Decimal128 expected_value(123); + ASSERT_EQ(expected_value, value); +} + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc index cc180258aa4df..447cae5c54654 100644 --- a/cpp/src/arrow/util/decimal.cc +++ b/cpp/src/arrow/util/decimal.cc @@ -105,6 +105,22 @@ Decimal128::operator int64_t() const { return static_cast(low_bits_); } +static std::string ToStringNegativeScale(const std::string& str, + int32_t adjusted_exponent, bool is_negative) { + std::stringstream buf; + + size_t offset = 0; + buf << str[offset++]; + + if (is_negative) { + buf << str[offset++]; + } + + buf << '.' << str.substr(offset, std::string::npos) << 'E' << std::showpos + << adjusted_exponent; + return buf.str(); +} + std::string Decimal128::ToString(int32_t scale) const { const std::string str(ToIntegerString()); @@ -112,9 +128,18 @@ std::string Decimal128::ToString(int32_t scale) const { return str; } - if (*this < 0) { - const auto len = static_cast(str.size()); + const bool is_negative = *this < 0; + const auto len = static_cast(str.size()); + const auto is_negative_offset = static_cast(is_negative); + const int32_t adjusted_exponent = -scale + (len - 1 - is_negative_offset); + + /// Note that the -6 is taken from the Java BigDecimal documentation. + if (scale < 0 || adjusted_exponent < -6) { + return ToStringNegativeScale(str, adjusted_exponent, is_negative); + } + + if (is_negative) { if (len - 1 > scale) { const auto n = static_cast(len - scale); return str.substr(0, n) + "." + str.substr(n, static_cast(scale)); @@ -128,8 +153,6 @@ std::string Decimal128::ToString(int32_t scale) const { return result + str.substr(1, std::string::npos); } - const auto len = static_cast(str.size()); - if (len > scale) { const auto n = static_cast(len - scale); return str.substr(0, n) + "." + str.substr(n, static_cast(scale)); @@ -164,10 +187,12 @@ static constexpr int64_t kPowersOfTen[kInt64DecimalDigits + 1] = {1LL, 100000000000000000LL, 1000000000000000000LL}; +static inline bool isdigit(char value) { return std::isdigit(value) != 0; } + static void StringToInteger(const std::string& str, Decimal128* out) { using std::size_t; - DCHECK_NE(out, nullptr) << "Decimal128 output variable cannot be nullptr"; + DCHECK_NE(out, NULLPTR) << "Decimal128 output variable cannot be NULLPTR"; DCHECK_EQ(*out, 0) << "When converting a string to Decimal128 the initial output must be 0"; @@ -189,7 +214,7 @@ static void StringToInteger(const std::string& str, Decimal128* out) { Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precision, int* scale) { - // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?"; + // Implements this regex: "(\\+?|-?)((0*)(\\d*))(\\.(\\d+))?((E|e)(\\+|-)?\\d+)?"; if (s.empty()) { return Status::Invalid("Empty string cannot be converted to decimal"); } @@ -215,21 +240,21 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis DCHECK_LT(charp, end); // skip leading zeros - charp = std::find_if_not(charp, end, [](char c) { return c == '0'; }); + charp = std::find_if_not(charp, end, [](char value) { return value == '0'; }); // all zeros and no decimal point if (charp == end) { - if (out != nullptr) { + if (out != NULLPTR) { *out = 0; } // Not sure what other libraries assign precision to for this case (this case of // a string consisting only of one or more zeros) - if (precision != nullptr) { + if (precision != NULLPTR) { *precision = static_cast(charp - numeric_string_start); } - if (scale != nullptr) { + if (scale != NULLPTR) { *scale = 0; } @@ -238,7 +263,7 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis std::string::const_iterator whole_part_start = charp; - charp = std::find_if_not(charp, end, [](char c) { return std::isdigit(c) != 0; }); + charp = std::find_if_not(charp, end, isdigit); std::string::const_iterator whole_part_end = charp; std::string whole_part(whole_part_start, whole_part_end); @@ -269,14 +294,13 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis std::string::const_iterator fractional_part_start = charp; - // The rest must be digits, because if we have a decimal point it must be followed by - // digits + // The rest must be digits or an exponent if (charp != end) { - charp = std::find_if_not(charp, end, [](char c) { return std::isdigit(c) != 0; }); + charp = std::find_if_not(charp, end, isdigit); // The while loop has ended before the end of the string which means we've hit a - // character that isn't a base ten digit - if (charp != end) { + // character that isn't a base ten digit or "E" for exponent + if (charp != end && *charp != 'E' && *charp != 'e') { std::stringstream ss; ss << "Found non base ten digit character '" << *charp << "' before the end of the string"; @@ -287,15 +311,55 @@ Status Decimal128::FromString(const std::string& s, Decimal128* out, int* precis std::string::const_iterator fractional_part_end = charp; std::string fractional_part(fractional_part_start, fractional_part_end); - if (precision != nullptr) { + if (precision != NULLPTR) { *precision = static_cast(whole_part.size() + fractional_part.size()); } - if (scale != nullptr) { - *scale = static_cast(fractional_part.size()); + if (charp != end) { + // we must have an exponent, if this aborts then we have somehow not caught this and + // raised a proper error + DCHECK(*charp == 'E' || *charp == 'e'); + + ++charp; + + const char value = *charp; + const bool starts_with_plus_or_minus = value == '+' || value == '-'; + + // we use this to construct the adjusted exponent integer later + std::string::const_iterator digit_start = charp; + + // skip plus or minus + charp += starts_with_plus_or_minus; + + // confirm that the rest of the characters are digits + charp = std::find_if_not(charp, end, isdigit); + + if (charp != end) { + // we have something other than digits here + std::stringstream ss; + ss << "Found non decimal digit exponent value '" << *charp << "'"; + return Status::Invalid(ss.str()); + } + + if (scale != NULLPTR) { + // compute the scale from the adjusted exponent + std::string adjusted_exponent_string(digit_start, end); + DCHECK(std::all_of(adjusted_exponent_string.cbegin() + starts_with_plus_or_minus, + adjusted_exponent_string.cend(), isdigit)) + << "Non decimal digit character found in " << adjusted_exponent_string; + const auto adjusted_exponent = + static_cast(std::stol(adjusted_exponent_string)); + const auto len = static_cast(whole_part.size() + fractional_part.size()); + + *scale = -adjusted_exponent + len - 1; + } + } else { + if (scale != NULLPTR) { + *scale = static_cast(fractional_part.size()); + } } - if (out != nullptr) { + if (out != NULLPTR) { // zero out in case we've passed in a previously used value *out = 0; StringToInteger(whole_part + fractional_part, out); From 65a9055c705e5f09c949c12365d12839ace063f5 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 9 Nov 2017 11:31:36 -0500 Subject: [PATCH 030/177] ARROW-972: UnionArray in pyarrow This is taking a stab at exposing UnionArray to pyarrow. Tasks to be done: - [x] Support UnionType::SPARSE - [x] Add doc strings Author: Philipp Moritz Author: Wes McKinney Closes #1216 from pcmoritz/pyarrow-union-array and squashes the following commits: 7f3ca313 [Wes McKinney] Fix flakes 9f33076b [Wes McKinney] Change UnionMode to scoped enumeration 9e602a8d [Philipp Moritz] wrap UnionType in pyarrow eeef7226 [Philipp Moritz] linting 502c335a [Philipp Moritz] fixes c6c85491 [Philipp Moritz] add doc strings 9068bbb5 [Philipp Moritz] linting d8da0170 [Philipp Moritz] implement dense and sparse UnionArrays cbdedc7a [Philipp Moritz] make fields in UnionArray unique to be compatiable with Java b796ce64 [Philipp Moritz] Implement UnionArray in pyarrow --- cpp/src/arrow/array.cc | 56 ++++++++++++++++++++++++++ cpp/src/arrow/array.h | 33 ++++++++++++++- cpp/src/arrow/compare.cc | 2 +- cpp/src/arrow/ipc/json-internal.cc | 2 +- cpp/src/arrow/ipc/metadata-internal.cc | 5 ++- cpp/src/arrow/type.cc | 18 ++++++++- cpp/src/arrow/type.h | 18 ++++++--- python/pyarrow/__init__.py | 4 +- python/pyarrow/_parquet.pxd | 2 +- python/pyarrow/array.pxi | 53 ++++++++++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 25 ++++++++++++ python/pyarrow/lib.pxd | 16 ++++++++ python/pyarrow/lib.pyx | 2 + python/pyarrow/public-api.pxi | 2 +- python/pyarrow/scalar.pxi | 19 +++++++++ python/pyarrow/tests/test_array.py | 22 ++++++++++ python/pyarrow/tests/test_schema.py | 8 ++++ python/pyarrow/tests/test_types.py | 13 +++--- python/pyarrow/types.pxi | 49 ++++++++++++++++++++++ 19 files changed, 327 insertions(+), 22 deletions(-) diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index b523876bf0e4e..9c91d619cc7e8 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -393,6 +393,62 @@ UnionArray::UnionArray(const std::shared_ptr& type, int64_t length, SetData(internal_data); } +Status UnionArray::MakeDense(const Array& type_ids, const Array& value_offsets, + const std::vector>& children, + std::shared_ptr* out) { + if (value_offsets.length() == 0) { + return Status::Invalid("UnionArray offsets must have non-zero length"); + } + + if (value_offsets.type_id() != Type::INT32) { + return Status::Invalid("UnionArray offsets must be signed int32"); + } + + if (type_ids.type_id() != Type::INT8) { + return Status::Invalid("UnionArray type_ids must be signed int8"); + } + + if (value_offsets.null_count() != 0) { + return Status::Invalid("MakeDense does not allow NAs in value_offsets"); + } + + BufferVector buffers = {type_ids.null_bitmap(), + static_cast(type_ids).values(), + static_cast(value_offsets).values()}; + auto union_type = union_(children, UnionMode::DENSE); + auto internal_data = + std::make_shared(union_type, type_ids.length(), std::move(buffers), + type_ids.null_count(), type_ids.offset()); + for (const auto& child : children) { + internal_data->child_data.push_back(child->data()); + } + *out = std::make_shared(internal_data); + return Status::OK(); +} + +Status UnionArray::MakeSparse(const Array& type_ids, + const std::vector>& children, + std::shared_ptr* out) { + if (type_ids.type_id() != Type::INT8) { + return Status::Invalid("UnionArray type_ids must be signed int8"); + } + BufferVector buffers = {type_ids.null_bitmap(), + static_cast(type_ids).values(), nullptr}; + auto union_type = union_(children, UnionMode::SPARSE); + auto internal_data = + std::make_shared(union_type, type_ids.length(), std::move(buffers), + type_ids.null_count(), type_ids.offset()); + for (const auto& child : children) { + internal_data->child_data.push_back(child->data()); + if (child->length() != type_ids.length()) { + return Status::Invalid( + "Sparse UnionArray must have len(child) == len(type_ids) for all children"); + } + } + *out = std::make_shared(internal_data); + return Status::OK(); +} + std::shared_ptr UnionArray::child(int i) const { if (!boxed_fields_[i]) { boxed_fields_[i] = MakeArray(data_->child_data[i]); diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index afbd780dd3ad5..f7762ce104398 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -612,16 +612,47 @@ class ARROW_EXPORT UnionArray : public Array { const std::shared_ptr& null_bitmap = NULLPTR, int64_t null_count = 0, int64_t offset = 0); + /// \brief Construct Dense UnionArray from types_ids, value_offsets and children + /// + /// This function does the bare minimum of validation of the offsets and + /// input types. The value_offsets are assumed to be well-formed. + /// + /// \param[in] type_ids An array of 8-bit signed integers, enumerated from + /// 0 corresponding to each type. + /// \param[in] value_offsets An array of signed int32 values indicating the + /// relative offset into the respective child array for the type in a given slot. + /// The respective offsets for each child value array must be in order / increasing. + /// \param[in] children Vector of children Arrays containing the data for each type. + /// \param[out] out Will have length equal to value_offsets.length() + static Status MakeDense(const Array& type_ids, const Array& value_offsets, + const std::vector>& children, + std::shared_ptr* out); + + /// \brief Construct Sparse UnionArray from type_ids and children + /// + /// This function does the bare minimum of validation of the offsets and + /// input types. + /// + /// \param[in] type_ids An array of 8-bit signed integers, enumerated from + /// 0 corresponding to each type. + /// \param[in] children Vector of children Arrays containing the data for each type. + /// \param[out] out Will have length equal to type_ids.length() + static Status MakeSparse(const Array& type_ids, + const std::vector>& children, + std::shared_ptr* out); + /// Note that this buffer does not account for any slice offset std::shared_ptr type_ids() const { return data_->buffers[1]; } /// Note that this buffer does not account for any slice offset std::shared_ptr value_offsets() const { return data_->buffers[2]; } + int32_t value_offset(int64_t i) const { return raw_value_offsets_[i + data_->offset]; } + const type_id_t* raw_type_ids() const { return raw_type_ids_ + data_->offset; } const int32_t* raw_value_offsets() const { return raw_value_offsets_ + data_->offset; } - UnionMode mode() const { return static_cast(*type()).mode(); } + UnionMode::type mode() const { return static_cast(*type()).mode(); } std::shared_ptr child(int pos) const; diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index 2ec86c3695aa5..a2d4de7b73afb 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -152,7 +152,7 @@ class RangeEqualsVisitor { bool CompareUnions(const UnionArray& left) const { const auto& right = static_cast(right_); - const UnionMode union_mode = left.mode(); + const UnionMode::type union_mode = left.mode(); if (union_mode != right.mode()) { return false; } diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index c1c0661d6ad35..1b9baee7dafef 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -774,7 +774,7 @@ static Status GetUnion(const RjObject& json_type, RETURN_NOT_STRING("mode", it_mode, json_type); std::string mode_str = it_mode->value.GetString(); - UnionMode mode; + UnionMode::type mode; if (mode_str == "SPARSE") { mode = UnionMode::SPARSE; diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index f0f0f675853b1..63ef8a549f236 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -163,8 +163,9 @@ static Status StructToFlatbuffer(FBB& fbb, const DataType& type, static Status UnionFromFlatbuffer(const flatbuf::Union* union_data, const std::vector>& children, std::shared_ptr* out) { - UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE - : UnionMode::DENSE; + UnionMode::type mode = + (union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE + : UnionMode::DENSE); std::vector type_codes; diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index a9bf591918558..0d1985fb2d914 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -190,7 +190,7 @@ std::string TimestampType::ToString() const { // Union type UnionType::UnionType(const std::vector>& fields, - const std::vector& type_codes, UnionMode mode) + const std::vector& type_codes, UnionMode::type mode) : NestedType(Type::UNION), mode_(mode), type_codes_(type_codes) { children_ = fields; } @@ -440,10 +440,24 @@ std::shared_ptr struct_(const std::vector>& fie } std::shared_ptr union_(const std::vector>& child_fields, - const std::vector& type_codes, UnionMode mode) { + const std::vector& type_codes, + UnionMode::type mode) { return std::make_shared(child_fields, type_codes, mode); } +std::shared_ptr union_(const std::vector>& children, + UnionMode::type mode) { + std::vector> types; + std::vector type_codes; + uint8_t counter = 0; + for (const auto& child : children) { + types.push_back(field(std::to_string(counter), child->type())); + type_codes.push_back(counter); + counter++; + } + return union_(types, type_codes, mode); +} + std::shared_ptr dictionary(const std::shared_ptr& index_type, const std::shared_ptr& dict_values, bool ordered) { diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 446f4d3a0b33f..9e11a034420e5 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -517,14 +517,17 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType { int32_t scale_; }; -enum class UnionMode : char { SPARSE, DENSE }; +struct UnionMode { + enum type { SPARSE, DENSE }; +}; class ARROW_EXPORT UnionType : public NestedType { public: static constexpr Type::type type_id = Type::UNION; UnionType(const std::vector>& fields, - const std::vector& type_codes, UnionMode mode = UnionMode::SPARSE); + const std::vector& type_codes, + UnionMode::type mode = UnionMode::SPARSE); std::string ToString() const override; std::string name() const override { return "union"; } @@ -534,10 +537,10 @@ class ARROW_EXPORT UnionType : public NestedType { const std::vector& type_codes() const { return type_codes_; } - UnionMode mode() const { return mode_; } + UnionMode::type mode() const { return mode_; } private: - UnionMode mode_; + UnionMode::type mode_; // The type id used in the data to indicate each data type in the union. For // example, the first type in the union might be denoted by the id 5 (instead @@ -842,7 +845,12 @@ struct_(const std::vector>& fields); /// \brief Create an instance of Union type std::shared_ptr ARROW_EXPORT union_(const std::vector>& child_fields, - const std::vector& type_codes, UnionMode mode = UnionMode::SPARSE); + const std::vector& type_codes, UnionMode::type mode = UnionMode::SPARSE); + +/// \brief Create and instance of Union type +std::shared_ptr ARROW_EXPORT +union_(const std::vector>& children, + UnionMode::type mode = UnionMode::SPARSE); /// \brief Create an instance of Dictionary type std::shared_ptr ARROW_EXPORT diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 1215c822d2e47..2d7d7288b3835 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -36,7 +36,7 @@ time32, time64, timestamp, date32, date64, float16, float32, float64, binary, string, decimal, - list_, struct, dictionary, field, + list_, struct, union, dictionary, field, type_for_alias, DataType, NAType, Field, @@ -52,7 +52,7 @@ Int16Array, UInt16Array, Int32Array, UInt32Array, Int64Array, UInt64Array, - ListArray, + ListArray, UnionArray, BinaryArray, StringArray, FixedSizeBinaryArray, DictionaryArray, diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 04a5b1368ce45..7e5e575096ddd 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -192,7 +192,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil: int64_t num_values() const shared_ptr[ColumnPath] path_in_schema() const bint is_stats_set() const - shared_ptr[CRowGroupStatistics] statistics() const; + shared_ptr[CRowGroupStatistics] statistics() const ParquetCompression compression() const const vector[ParquetEncoding]& encodings() const diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 7752d062a774c..9991411e55dfe 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -631,6 +631,58 @@ cdef class ListArray(Array): return pyarrow_wrap_array(out) +cdef class UnionArray(Array): + + @staticmethod + def from_dense(Array types, Array value_offsets, list children): + """ + Construct dense UnionArray from arrays of int8 types, int32 offsets and + children arrays + + Parameters + ---------- + types : Array (int8 type) + value_offsets : Array (int32 type) + children : list + + Returns + ------- + union_array : UnionArray + """ + cdef shared_ptr[CArray] out + cdef vector[shared_ptr[CArray]] c + cdef Array child + for child in children: + c.push_back(child.sp_array) + with nogil: + check_status(CUnionArray.MakeDense( + deref(types.ap), deref(value_offsets.ap), c, &out)) + return pyarrow_wrap_array(out) + + @staticmethod + def from_sparse(Array types, list children): + """ + Construct sparse UnionArray from arrays of int8 types and children + arrays + + Parameters + ---------- + types : Array (int8 type) + children : list + + Returns + ------- + union_array : UnionArray + """ + cdef shared_ptr[CArray] out + cdef vector[shared_ptr[CArray]] c + cdef Array child + for child in children: + c.push_back(child.sp_array) + with nogil: + check_status(CUnionArray.MakeSparse(deref(types.ap), c, &out)) + return pyarrow_wrap_array(out) + cdef class StringArray(Array): pass @@ -789,6 +841,7 @@ cdef dict _array_classes = { _Type_FLOAT: FloatArray, _Type_DOUBLE: DoubleArray, _Type_LIST: ListArray, + _Type_UNION: UnionArray, _Type_BINARY: BinaryArray, _Type_STRING: StringArray, _Type_DICTIONARY: DictionaryArray, diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 731ef94971da0..dfafd371b2857 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -67,6 +67,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: _Type_DICTIONARY" arrow::Type::DICTIONARY" _Type_MAP" arrow::Type::MAP" + enum UnionMode" arrow::UnionMode::type": + _UnionMode_SPARSE" arrow::UnionMode::SPARSE" + _UnionMode_DENSE" arrow::UnionMode::DENSE" + enum TimeUnit" arrow::TimeUnit::type": TimeUnit_SECOND" arrow::TimeUnit::SECOND" TimeUnit_MILLI" arrow::TimeUnit::MILLI" @@ -222,6 +226,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef cppclass CStructType" arrow::StructType"(CDataType): CStructType(const vector[shared_ptr[CField]]& fields) + cdef cppclass CUnionType" arrow::UnionType"(CDataType): + CUnionType(const vector[shared_ptr[CField]]& fields, + const vector[uint8_t]& type_codes, UnionMode mode) + UnionMode mode() + cdef cppclass CSchema" arrow::Schema": CSchema(const vector[shared_ptr[CField]]& fields) CSchema(const vector[shared_ptr[CField]]& fields, @@ -317,6 +326,22 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CArray] values() shared_ptr[CDataType] value_type() + cdef cppclass CUnionArray" arrow::UnionArray"(CArray): + @staticmethod + CStatus MakeSparse(const CArray& type_ids, + const vector[shared_ptr[CArray]]& children, + shared_ptr[CArray]* out) + + @staticmethod + CStatus MakeDense(const CArray& type_ids, const CArray& value_offsets, + const vector[shared_ptr[CArray]]& children, + shared_ptr[CArray]* out) + uint8_t* raw_type_ids() + int32_t value_offset(int i) + shared_ptr[CArray] child(int pos) + const CArray* UnsafeChild(int pos) + UnionMode mode() + cdef cppclass CBinaryArray" arrow::BinaryArray"(CListArray): const uint8_t* GetValue(int i, int32_t* length) diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 8fdcf553c13fc..531489490754e 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -56,6 +56,11 @@ cdef class DictionaryType(DataType): const CDictionaryType* dict_type +cdef class UnionType(DataType): + cdef: + list child_types + + cdef class TimestampType(DataType): cdef: const CTimestampType* ts_type @@ -139,6 +144,13 @@ cdef class ListValue(ArrayValue): cdef getitem(self, int64_t i) +cdef class UnionValue(ArrayValue): + cdef: + CUnionArray* ap + list value_types + + cdef getitem(self, int64_t i) + cdef class StringValue(ArrayValue): pass @@ -242,6 +254,10 @@ cdef class ListArray(Array): pass +cdef class UnionArray(Array): + pass + + cdef class StringArray(Array): pass diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 6f4451e3f5a41..b4ca49cafe160 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -92,6 +92,8 @@ Type_UNION = _Type_UNION Type_DICTIONARY = _Type_DICTIONARY Type_MAP = _Type_MAP +UnionMode_SPARSE = _UnionMode_SPARSE +UnionMode_DENSE = _UnionMode_DENSE # Exception types include "error.pxi" diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi index 9f1051228047a..90aff9e936d95 100644 --- a/python/pyarrow/public-api.pxi +++ b/python/pyarrow/public-api.pxi @@ -72,7 +72,7 @@ cdef public api object pyarrow_wrap_data_type( elif type.get().id() == _Type_STRUCT: out = StructType() elif type.get().id() == _Type_UNION: - out = StructType() + out = UnionType() elif type.get().id() == _Type_TIMESTAMP: out = TimestampType() elif type.get().id() == _Type_FIXED_SIZE_BINARY: diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index c37ed3b200ea3..a396fa763c8c8 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -315,6 +315,24 @@ cdef class ListValue(ArrayValue): return result +cdef class UnionValue(ArrayValue): + + cdef void _set_array(self, const shared_ptr[CArray]& sp_array): + self.sp_array = sp_array + self.ap = sp_array.get() + + cdef getitem(self, int64_t i): + cdef int8_t type_id = self.ap.raw_type_ids()[i] + cdef shared_ptr[CArray] child = self.ap.child(type_id) + if self.ap.mode() == _UnionMode_SPARSE: + return box_scalar(self.type[type_id], child, i) + else: + return box_scalar(self.type[type_id], child, + self.ap.value_offset(i)) + + def as_py(self): + return self.getitem(self.index).as_py() + cdef class FixedSizeBinaryValue(ArrayValue): def as_py(self): @@ -364,6 +382,7 @@ cdef dict _scalar_classes = { _Type_FLOAT: FloatValue, _Type_DOUBLE: DoubleValue, _Type_LIST: ListValue, + _Type_UNION: UnionValue, _Type_BINARY: BinaryValue, _Type_STRING: StringValue, _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue, diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index e3a4c97567ee6..7dc93c28ea7a4 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -235,6 +235,28 @@ def test_list_from_arrays(): assert result.equals(expected) +def test_union_from_dense(): + binary = pa.array([b'a', b'b', b'c', b'd'], type='binary') + int64 = pa.array([1, 2, 3], type='int64') + types = pa.array([0, 1, 0, 0, 1, 1, 0], type='int8') + value_offsets = pa.array([0, 0, 2, 1, 1, 2, 3], type='int32') + + result = pa.UnionArray.from_dense(types, value_offsets, [binary, int64]) + + assert result.to_pylist() == [b'a', 1, b'c', b'b', 2, 3, b'd'] + + +def test_union_from_sparse(): + binary = pa.array([b'a', b' ', b'b', b'c', b' ', b' ', b'd'], + type='binary') + int64 = pa.array([0, 1, 0, 0, 2, 3, 0], type='int64') + types = pa.array([0, 1, 0, 0, 1, 1, 0], type='int8') + + result = pa.UnionArray.from_sparse(types, [binary, int64]) + + assert result.to_pylist() == [b'a', 1, b'b', b'c', 2, 3, b'd'] + + def _check_cast_case(case, safe=True): in_data, in_type, out_data, out_type = case diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py index d6b2655b7c6a0..116f3978333a8 100644 --- a/python/pyarrow/tests/test_schema.py +++ b/python/pyarrow/tests/test_schema.py @@ -319,6 +319,14 @@ def test_type_schema_pickling(): pa.field('a', 'int8'), pa.field('b', 'string') ]), + pa.union([ + pa.field('a', pa.int8()), + pa.field('b', pa.int16()) + ], pa.lib.UnionMode_SPARSE), + pa.union([ + pa.field('a', pa.int8()), + pa.field('b', pa.int16()) + ], pa.lib.UnionMode_DENSE), pa.time32('s'), pa.time64('us'), pa.date32(), diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index e6ff5b1560c1d..0e3ea1fd40bf5 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -85,16 +85,17 @@ def test_is_nested_or_struct(): assert not types.is_nested(pa.int32()) -# TODO(wesm): Union types not yet implemented in pyarrow +def test_is_union(): + assert types.is_union(pa.union([pa.field('a', pa.int32()), + pa.field('b', pa.int8()), + pa.field('c', pa.string())], + pa.lib.UnionMode_SPARSE)) + assert not types.is_union(pa.list_(pa.int32())) -# def test_is_union(): -# assert types.is_union(pa.union([pa.field('a', pa.int32()), -# pa.field('b', pa.int8()), -# pa.field('c', pa.string())])) -# assert not types.is_union(pa.list_(pa.int32())) # TODO(wesm): is_map, once implemented + def test_is_binary_string(): assert types.is_binary(pa.binary()) assert not types.is_binary(pa.string()) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index c9a490960ec38..d2e68ff79a524 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -186,7 +186,32 @@ cdef class UnionType(DataType): cdef void init(self, const shared_ptr[CDataType]& type): DataType.init(self, type) + self.child_types = [ + pyarrow_wrap_data_type(type.get().child(i).get().type()) + for i in range(self.num_children)] + property num_children: + + def __get__(self): + return self.type.num_children() + + property mode: + + def __get__(self): + cdef CUnionType* type = self.sp_type.get() + return type.mode() + + def __getitem__(self, i): + return self.child_types[i] + + def __getstate__(self): + children = [pyarrow_wrap_field(self.type.child(i)) + for i in range(self.num_children)] + return children, self.mode + + def __setstate__(self, state): + cdef DataType reconstituted = union(*state) + self.init(reconstituted.sp_type) cdef class TimestampType(DataType): @@ -1056,6 +1081,30 @@ def struct(fields): return pyarrow_wrap_data_type(struct_type) +def union(children_fields, mode): + """ + Create UnionType from children fields. + """ + cdef: + Field child_field + vector[shared_ptr[CField]] c_fields + vector[uint8_t] type_codes + shared_ptr[CDataType] union_type + int i + + for i, child_field in enumerate(children_fields): + type_codes.push_back(i) + c_fields.push_back(child_field.sp_field) + + if mode == UnionMode_SPARSE: + union_type.reset(new CUnionType(c_fields, type_codes, + _UnionMode_SPARSE)) + else: + union_type.reset(new CUnionType(c_fields, type_codes, + _UnionMode_DENSE)) + + return pyarrow_wrap_data_type(union_type) + cdef dict _type_aliases = { 'null': null, 'i1': int8, From ed8aef2c66169bca214f3a77cf6b6d797e172791 Mon Sep 17 00:00:00 2001 From: Lu Qi Date: Fri, 10 Nov 2017 14:23:42 +0100 Subject: [PATCH 031/177] ARROW-1793: fix a typo for README.md fix a typo for integration test README.md Author: Lu Qi Closes #1301 from luchy0120/master and squashes the following commits: 2cc86fe [Lu Qi] fix a typo --- integration/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/README.md b/integration/README.md index 6e5a6c15641ec..8d6df8e5772de 100644 --- a/integration/README.md +++ b/integration/README.md @@ -77,7 +77,7 @@ export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug Here `$ARROW_HOME` is the location of your Arrow git clone. The `$CPP_BUILD_DIR` may be different depending on how you built with CMake -(in-source of out-of-source). +(in-source or out-of-source). Once this is done, run the integration tests with (optionally adding `--debug` for additional output) @@ -88,4 +88,4 @@ python integration_test.py python integration_test.py --debug # additional output ``` -[1]: https://conda.io/miniconda.html \ No newline at end of file +[1]: https://conda.io/miniconda.html From 2d34f34dc81966f3e186055dc0b962699c98b236 Mon Sep 17 00:00:00 2001 From: Stephanie Date: Fri, 10 Nov 2017 09:36:12 -0500 Subject: [PATCH 032/177] ARROW-1788 Fix Plasma store abort bug on client disconnection Author: Stephanie Closes #1299 from stephanie-wang/plasma-client-disconnect-bug and squashes the following commits: 295144bd [Stephanie] Revert disconnect client check 8e24affd [Stephanie] Refactor abort_object to match remove_client_from_object_clients b41591d5 [Stephanie] When disconnecting a plasma client, only abort the objects that the client created 50932e53 [Stephanie] Add Plasma test for multiple clients --- cpp/src/plasma/store.cc | 27 +++++++++++++------- cpp/src/plasma/store.h | 9 ++++++- cpp/src/plasma/test/client_tests.cc | 38 +++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc index 5dbdebc237ce6..31033ccbb3202 100644 --- a/cpp/src/plasma/store.cc +++ b/cpp/src/plasma/store.cc @@ -393,16 +393,22 @@ void PlasmaStore::seal_object(const ObjectID& object_id, unsigned char digest[]) update_object_get_requests(object_id); } -void PlasmaStore::abort_object(const ObjectID& object_id) { +int PlasmaStore::abort_object(const ObjectID& object_id, Client* client) { auto entry = get_object_table_entry(&store_info_, object_id); ARROW_CHECK(entry != NULL) << "To abort an object it must be in the object table."; ARROW_CHECK(entry->state != PLASMA_SEALED) << "To abort an object it must not have been sealed."; - ARROW_CHECK(entry->clients.size() == 1) - << "To abort an object, the only client currently using it is the creator."; - - dlfree(entry->pointer); - store_info_.objects.erase(object_id); + auto it = entry->clients.find(client); + if (it == entry->clients.end()) { + // If the client requesting the abort is not the creator, do not + // perform the abort. + return 0; + } else { + // The client requesting the abort is the creator. Free the object. + dlfree(entry->pointer); + store_info_.objects.erase(object_id); + return 1; + } } void PlasmaStore::delete_objects(const std::vector& object_ids) { @@ -454,11 +460,12 @@ void PlasmaStore::disconnect_client(int client_fd) { ARROW_LOG(INFO) << "Disconnecting client on fd " << client_fd; // If this client was using any objects, remove it from the appropriate // lists. + auto client = it->second.get(); for (const auto& entry : store_info_.objects) { if (entry.second->state == PLASMA_SEALED) { - remove_client_from_object_clients(entry.second.get(), it->second.get()); + remove_client_from_object_clients(entry.second.get(), client); } else { - abort_object(entry.first); + abort_object(entry.first, client); } } @@ -600,7 +607,9 @@ Status PlasmaStore::process_message(Client* client) { } break; case MessageType_PlasmaAbortRequest: { RETURN_NOT_OK(ReadAbortRequest(input, input_size, &object_id)); - abort_object(object_id); + ARROW_CHECK(abort_object(object_id, client) == 1) << "To abort an object, the only " + "client currently using it " + "must be the creator."; HANDLE_SIGPIPE(SendAbortReply(client->fd, object_id), client->fd); } break; case MessageType_PlasmaGetRequest: { diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h index 0d08d8a67ffaa..a72c6259a9cea 100644 --- a/cpp/src/plasma/store.h +++ b/cpp/src/plasma/store.h @@ -74,7 +74,14 @@ class PlasmaStore { int create_object(const ObjectID& object_id, int64_t data_size, int64_t metadata_size, Client* client, PlasmaObject* result); - void abort_object(const ObjectID& object_id); + /// Abort a created but unsealed object. If the client is not the + /// creator, then the abort will fail. + /// + /// @param object_id Object ID of the object to be aborted. + /// @param client The client who created the object. If this does not + /// match the creator of the object, then the abort will fail. + /// @return 1 if the abort succeeds, else 0. + int abort_object(const ObjectID& object_id, Client* client); /// Delete objects that have been created in the hash table. This should only /// be called on objects that are returned by the eviction policy to evict. diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc index 5c0cee4c071ad..d4285f8988404 100644 --- a/cpp/src/plasma/test/client_tests.cc +++ b/cpp/src/plasma/test/client_tests.cc @@ -45,14 +45,17 @@ class TestPlasmaStore : public ::testing::Test { "/plasma_store -m 1000000000 -s /tmp/store 1> /dev/null 2> /dev/null &"; system(plasma_command.c_str()); ARROW_CHECK_OK(client_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY)); + ARROW_CHECK_OK(client2_.Connect("/tmp/store", "", PLASMA_DEFAULT_RELEASE_DELAY)); } virtual void Finish() { ARROW_CHECK_OK(client_.Disconnect()); + ARROW_CHECK_OK(client2_.Disconnect()); system("killall plasma_store &"); } protected: PlasmaClient client_; + PlasmaClient client2_; }; TEST_F(TestPlasmaStore, ContainsTest) { @@ -171,6 +174,41 @@ TEST_F(TestPlasmaStore, AbortTest) { } } +TEST_F(TestPlasmaStore, MultipleClientTest) { + ObjectID object_id = ObjectID::from_random(); + + // Test for object non-existence on the first client. + bool has_object; + ARROW_CHECK_OK(client_.Contains(object_id, &has_object)); + ASSERT_EQ(has_object, false); + + // Test for the object being in local Plasma store. + // First create and seal object on the second client. + int64_t data_size = 100; + uint8_t metadata[] = {5}; + int64_t metadata_size = sizeof(metadata); + uint8_t* data; + ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data)); + ARROW_CHECK_OK(client2_.Seal(object_id)); + // Test that the first client can get the object. + ObjectBuffer object_buffer; + ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer)); + ARROW_CHECK_OK(client_.Contains(object_id, &has_object)); + ASSERT_EQ(has_object, true); + + // Test that one client disconnecting does not interfere with the other. + // First create object on the second client. + object_id = ObjectID::from_random(); + ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data)); + // Disconnect the first client. + ARROW_CHECK_OK(client_.Disconnect()); + // Test that the second client can seal and get the created object. + ARROW_CHECK_OK(client2_.Seal(object_id)); + ARROW_CHECK_OK(client2_.Get(&object_id, 1, -1, &object_buffer)); + ARROW_CHECK_OK(client2_.Contains(object_id, &has_object)); + ASSERT_EQ(has_object, true); +} + } // namespace plasma int main(int argc, char** argv) { From 7c205b0337cd0364a7f9e1e19a9a6d5423abfe30 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 11 Nov 2017 17:09:50 -0500 Subject: [PATCH 033/177] ARROW-1787: [Python] Support reading parquet files into DataFrames in a backward compatible way Author: Phillip Cloud Closes #1298 from cpcloud/ARROW-1787 and squashes the following commits: 6f5fbd55 [Phillip Cloud] Add more index naming tests 8ba06256 [Phillip Cloud] Add test data 56e7fe58 [Phillip Cloud] Use BytesIO 5f50da38 [Phillip Cloud] Implement d61c43e6 [Phillip Cloud] Add test 9abad95f [Phillip Cloud] Add test data --- python/pyarrow/pandas_compat.py | 13 +++- .../tests/data/v0.7.1.all-named-index.parquet | Bin 0 -> 3948 bytes python/pyarrow/tests/data/v0.7.1.parquet | Bin 0 -> 4372 bytes .../data/v0.7.1.some-named-index.parquet | Bin 0 -> 4008 bytes python/pyarrow/tests/test_parquet.py | 73 ++++++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 python/pyarrow/tests/data/v0.7.1.all-named-index.parquet create mode 100644 python/pyarrow/tests/data/v0.7.1.parquet create mode 100644 python/pyarrow/tests/data/v0.7.1.some-named-index.parquet diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 87b47b8a6bc13..db28ee09e1e73 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -18,6 +18,7 @@ import ast import collections import json +import re import numpy as np import pandas as pd @@ -353,6 +354,14 @@ def make_datetimetz(tz): return DatetimeTZDtype('ns', tz=tz) +def backwards_compatible_index_name(raw_name, logical_name): + pattern = r'^__index_level_\d+__$' + if raw_name == logical_name and re.match(pattern, raw_name) is not None: + return None + else: + return logical_name + + def table_to_blockmanager(options, table, memory_pool, nthreads=1): import pandas.core.internals as _int import pyarrow.lib as lib @@ -394,7 +403,9 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): values = values.copy() index_arrays.append(pd.Series(values, dtype=col_pandas.dtype)) - index_names.append(logical_name) + index_names.append( + backwards_compatible_index_name(raw_name, logical_name) + ) block_table = block_table.remove_column( block_table.schema.get_field_index(raw_name) ) diff --git a/python/pyarrow/tests/data/v0.7.1.all-named-index.parquet b/python/pyarrow/tests/data/v0.7.1.all-named-index.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e9efd9b390ed4ceaa53c20ca1ac182101593a6bb GIT binary patch literal 3948 zcmcInU5FD`6uvi^$xhOkjN9Dlq^33mJ4_)iCc7@Bxa11D?$)>)U0DjX$e7I5l>C`Z z#9eDysM5t3tq;Xo5Yf^nDTS6IMT)egNLh>62SG|vD8AIEwifZJ=iZq~ve{9nYz8u! zIrq$Wzwh4jok?W=H7hd ziyilV&fxXgeWz|^K4Z~KH#1k(pZIh4m0vOhkr;_5Nd)kG#1~T~=Rs9+9#s$=p&<{P zBiJi0riX_S(iS~25{PaVkT0r=d82MLh)0PikFm1g(LTlsQluxK=`M2sHZZt=$@^Kx z1tjIr493CH938}1n$1!imS;YnqzGf-+Js*g?(g!)g~3U`B>X)Ar?22v*Cy!_FxW)a`yL11x|)4joLZgg*AMU71p=U#fab(R;^ylTMD4Ce#hGN04Vs)4+`Ai zbeE{5;4~&DahCM+^&gA>t24WV=%9!Nnc`4H-?x^Y`vl`;n7+)Mu|7<1EMZ)b=@_?x zOf7jb<)bIUE(OLN2so-U?kMyJ_9n>QmhOGZilI5{SbYy8wVtr9uiaAWXdfuC{ySa&@kN8S;k`O9P}rt zyV~EOZq8A6-lP4r7Sx@W9qY_22eYlSEZS1nv3Wq8N$}F6th3E~D3s#{AGI2vmttRV zBp^~Td~igt_tHy$c*r)~OReVSb=f?a2@W5~b6{7<+iHFYDcWPOrqlTFydw`b?>GT@ zcb<3L)dgm|YgrWFz+DGO!RZXCm;kX+qgcn%=^fQ-L4@1!f@zfc;38Ky%f)8d50}$s zeJ;)tR`|@{8d3!Y#AUSG#Msv#u{Lo$8Y4m5WZrz!>fQIT@&IdRRgiJ^h1f9CyRl$m z;H#5lmPp)5ku0KlGGlxiXu0J^oe#9Fs`K6THpv3(MGvx2+)L?dK1fJpE{* zVw6pIu+l7*Aj5**-yU`8;K_VHAvpVM82iPqQ39 z!GAn6q^>DuN>!sVJhXC-j)Cq;lJOSr1n)FGz~w!w2e`cZx!@KnuB*@rKWz<;xZrgx z-}1fd7@bCOYj)Q~ic79k-Cd2l(Uql8OFAZ#nvy^B zBkqbI1=~dgr4Pl5w4&mJ4ai4+kQ9_cgOVQrosu)Ch}Ij5Xkp0+j^0rY ze*M@Xh@;n?2&pHypwtuC$RlS!5t4dIuSlHim;c4`r<{5Q%SaYqfTr2Z0ocG`zm$$i zJ*K3{O#y6EDMQb?;3i%&a+zw5hueulA?1eaoKcz&(FbbWu4hUeTHJLPg{KZ}LKZLN zf*|Ca5R^PMA&z8~$#{(AXIQ<)gN&<=;26@{v0zRhtVx40Ni+%jV;F4P=+6}W*+G9M zfs-Nyo8pUiD0rJm#52eu$_eCw2dP(F4H`nSkS&zVcz%K9vsnGr2}DjiDG?I@pkem+ zrC3bzn6nQm4zSHm4ZB?^f?fRFtk*?`X=LW+X02{wC-W~4p#pXf!0Zp(rO?vZs##gPe?0B&p1LDdu~pXG+Z9^ls9&$H@pizTHKDa7bF$6h1Lc5AaY#)2AFGCUf6r7EE67 z4evd41~{loo~8s%8O6#0lle7HUgXsKuL1MhJ&>VxfSQe!$%;!`tqj$uD)HD(f?Xmy zHVtw}+%0No7XepRbZw&F5J2xZTFb~5&j;#wRVfZSGZ_cd{^==-yV$^8MAm$ACs z3*7(k0(Uz&&Bh9TkS;|jUT+xUc8mm!CLs3HaL&LP_llc$)1-xCPwm>Nz#35~f_b!M z4jH|^5ft1R2ui%w9*U(*k|}`p@DQu_`$56)K2Tr>r`cFxCQxu1ON%%r4VYOq$)EiS zW37*@p{;Fo9~lv_$4BliVZ6>q%>K5op9pIX9PcBOW-s%Rzo9AKdfrPIbDvFqS??*L z<=dHGAAJt*t0(2g8sWp{`dr}TuQ~Ng3DiAX1?ud;G#jg(x)f`~eT22{-nxdTyYYbN zBWY0EPp(ejz`8!d+P{woE^WM*RnLf>Iy2eR{g8j=egVEOQgj}*Q@5;ux+||+ zmD2`RtDF{VUEPVs+HGcx9pnhzuNF9hdwt_Q0)eF5>Lg6MgM3dnv0DxzyfZn@lc%uC z$DcTA>+P^vJhrS(kY~ZR%Uk=NVJ_ZhZ#KzWuomw;r=AXiyvJ`?#f4hIYQ@!JW%8ik zrJDMk7b6Tc#lh)-lZR7)(+Q^w>WtMWLUbRaM`xe2MruV4KJ8MhK6ftEsI>k>B9l)U zhZ9-jkdaMnNF=PD_YYPtuK{V7R)aJK*n^*n0zHgWo^KI6R!QXsh{SBUYip+XNZ4Bzlx|^@xEzmpaYoivzv-T$*X@S{O_&(Rkz(PY zmFMD-Fl#FCRa(vE%i(uJAA~y^P58sXP?&kJ9%Pt~N5XkMXTU4@YBmcV3^Y-hFH&Q; zTq$MpABSNeU(FTg>-NI_PmE-R+H*!lPw5rC26418O)Gxb{2PS6(KbaoThJ?Gqn&d! z4YW@(7^?FQLvyo1t>!byLdxhweKGSEQ2K%21z7fpzVg6k$F_lOiec*lHVwQixMuI~ z0*x+XJwy97zzl`f)8@wi&ucUvz3z!RC!0asd&O)mPu{9EPb9-!QCD&^8kWngU z3VGTQHVh4K7>b0CA_WFZ2dhS9eX>{#)jUFxp^>5C^|RAk;9T7 zhq!cVgiWF%}k)lQHgGEZIP<*LREtW(2(BI5vlU%a;U@r^V z%>HKfo9~yQ7r=ju;Mf=HM|Vk8K7F5(I+lJ%k@S+B|nF5j>K z_aJsSmlGo+2&v0Ln0P{)MdS)8&a7V5YebNP@?$J73F;?UMzZt-G|grXzy=2AF?m15 zxPYWIn#0&TmZn1(OY;N^JsHrVYwxx)NOVbdjkd`JX?voHr1ADaFEon=;C=5ktDaF`#Yzyz9jJ-K#m&v|u zTU_fl0Qaw?HtscjwqS7X_pp2itKatl_dnP!+reozQ5eTFE#1sCX%9wDjCQP(&=h>e z;4`^S+OijONR)1TcPp?)DAHJCOlM+^Twsay^>bk&T8UMu=Ch^(NUYzmdcy+>eszNa zJ2=fIYAQH|$qAey1AP6*BmZcuTzquMiNyfLzM!_hpPu~`o$+k}%1dG>K*1}(FznQsg~ila z=uc92ZJdpnM(wSQg)+n7-(Wbh#%>&|0f`cAm&bBB}NXHE>Y8sz| zVwZQ+fBevlVYMPtFX7gYsxLhF5g6)#G)%XyGsE?sEE#t$BmOR+J zV+7>gd)}(94zNabt%?F1RM#xYS?##=rcS*~GC>n0OmS=E&Af?|y*ghgdzMfQ&OQgh!Cp zi3Jk_Uya04MBwryE~K7*sYRnd zlgXEI#^FrCIAjzu@l1xV?P)BZ#p?AJfz(Qfk>X%54KR6=fG>GDlbGUw*vxdo-G}11 zPn@1``o^ax;%q%!5^g{$!pl)|2VhOc)z!I$xs?Ou}j{ zG+T?T<%O{hwt>icvRTJwUsC{!$iYxJbb0L$@*bR@Jvi2SaL%l;ud8coq-$(k&u$_A ztLmS%9_7CVH#kIwI1MdhqS@Xmuy#| z8GhOt9Jax0TfWtM*Vgeg%!M9cF7^Pkk^3E4RWIeU<($FhTdn@dncJRJww1KDQ&@G` zT@lZ~omE|f-XWtppD&kK^==wWY#NM5k0J#+s|V{wZDY1li8Mw;;)6qjiH*~fThr6y PQ~S`r@MA3v|Iz;gF;Sai literal 0 HcmV?d00001 diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index e2e6863c4748f..6ba4fd2fad8ea 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1458,3 +1458,76 @@ def test_index_column_name_duplicate(tmpdir): arrow_table = _read_table(path) result_df = arrow_table.to_pandas() tm.assert_frame_equal(result_df, dfx) + + +def test_backwards_compatible_index_naming(): + expected_string = b"""\ +carat cut color clarity depth table price x y z + 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43 + 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31 + 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31 + 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63 + 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75 + 0.24 Very Good J VVS2 62.8 57.0 336 3.94 3.96 2.48 + 0.24 Very Good I VVS1 62.3 57.0 336 3.95 3.98 2.47 + 0.26 Very Good H SI1 61.9 55.0 337 4.07 4.11 2.53 + 0.22 Fair E VS2 65.1 61.0 337 3.87 3.78 2.49 + 0.23 Very Good H VS1 59.4 61.0 338 4.00 4.05 2.39""" + expected = pd.read_csv( + io.BytesIO(expected_string), sep=r'\s{2,}', index_col=None, header=0 + ) + path = os.path.join(os.path.dirname(__file__), 'data', 'v0.7.1.parquet') + t = _read_table(path) + result = t.to_pandas() + tm.assert_frame_equal(result, expected) + + +def test_backwards_compatible_index_multi_level_named(): + expected_string = b"""\ +carat cut color clarity depth table price x y z + 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43 + 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31 + 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31 + 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63 + 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75 + 0.24 Very Good J VVS2 62.8 57.0 336 3.94 3.96 2.48 + 0.24 Very Good I VVS1 62.3 57.0 336 3.95 3.98 2.47 + 0.26 Very Good H SI1 61.9 55.0 337 4.07 4.11 2.53 + 0.22 Fair E VS2 65.1 61.0 337 3.87 3.78 2.49 + 0.23 Very Good H VS1 59.4 61.0 338 4.00 4.05 2.39""" + expected = pd.read_csv( + io.BytesIO(expected_string), + sep=r'\s{2,}', index_col=['cut', 'color', 'clarity'], header=0 + ).sort_index() + path = os.path.join( + os.path.dirname(__file__), 'data', 'v0.7.1.all-named-index.parquet' + ) + t = _read_table(path) + result = t.to_pandas() + tm.assert_frame_equal(result, expected) + + +def test_backwards_compatible_index_multi_level_some_named(): + expected_string = b"""\ +carat cut color clarity depth table price x y z + 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43 + 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31 + 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31 + 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63 + 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75 + 0.24 Very Good J VVS2 62.8 57.0 336 3.94 3.96 2.48 + 0.24 Very Good I VVS1 62.3 57.0 336 3.95 3.98 2.47 + 0.26 Very Good H SI1 61.9 55.0 337 4.07 4.11 2.53 + 0.22 Fair E VS2 65.1 61.0 337 3.87 3.78 2.49 + 0.23 Very Good H VS1 59.4 61.0 338 4.00 4.05 2.39""" + expected = pd.read_csv( + io.BytesIO(expected_string), + sep=r'\s{2,}', index_col=['cut', 'color', 'clarity'], header=0 + ).sort_index() + expected.index = expected.index.set_names(['cut', None, 'clarity']) + path = os.path.join( + os.path.dirname(__file__), 'data', 'v0.7.1.some-named-index.parquet' + ) + t = _read_table(path) + result = t.to_pandas() + tm.assert_frame_equal(result, expected) From 21112f85faa6e1328b3d59a54aa24becc50df4c1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 11 Nov 2017 18:58:01 -0500 Subject: [PATCH 034/177] ARROW-1800: [C++] Fix and simplify random_decimals Author: Phillip Cloud Closes #1306 from cpcloud/ARROW-1800 and squashes the following commits: d5b08ff0 [Phillip Cloud] ARROW-1800: [C++] Fix and simplify random_decimals --- cpp/src/arrow/test-util.h | 110 ++++++++++++-------------------------- 1 file changed, 34 insertions(+), 76 deletions(-) diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h index 7306f577a36e0..9b875ce116a29 100644 --- a/cpp/src/arrow/test-util.h +++ b/cpp/src/arrow/test-util.h @@ -159,12 +159,11 @@ static inline void random_is_valid(int64_t n, double pct_null, static inline void random_bytes(int64_t n, uint32_t seed, uint8_t* out) { std::mt19937 gen(seed); - std::uniform_int_distribution d(0, std::numeric_limits::max()); - std::generate(out, out + n, [&d, &gen] { return static_cast(d(gen) & 0xFF); }); + std::uniform_int_distribution d(0, std::numeric_limits::max()); + std::generate(out, out + n, [&d, &gen] { return static_cast(d(gen)); }); } -static void DecimalRange(int32_t precision, Decimal128* min_decimal, - Decimal128* max_decimal) { +static int32_t DecimalSize(int32_t precision) { DCHECK_GE(precision, 1) << "decimal precision must be greater than or equal to 1, got " << precision; DCHECK_LE(precision, 38) << "decimal precision must be less than or equal to 38, got " @@ -173,123 +172,82 @@ static void DecimalRange(int32_t precision, Decimal128* min_decimal, switch (precision) { case 1: case 2: - *max_decimal = std::numeric_limits::max(); - break; + return 1; // 127 case 3: case 4: - *max_decimal = std::numeric_limits::max(); - break; + return 2; // 32,767 case 5: case 6: - *max_decimal = 8388607; - break; + return 3; // 8,388,607 case 7: case 8: case 9: - *max_decimal = std::numeric_limits::max(); - break; + return 4; // 2,147,483,427 case 10: case 11: - *max_decimal = 549755813887; - break; + return 5; // 549,755,813,887 case 12: case 13: case 14: - *max_decimal = 140737488355327; - break; + return 6; // 140,737,488,355,327 case 15: case 16: - *max_decimal = 36028797018963967; - break; + return 7; // 36,028,797,018,963,967 case 17: case 18: - *max_decimal = std::numeric_limits::max(); - break; + return 8; // 9,223,372,036,854,775,807 case 19: case 20: case 21: - *max_decimal = Decimal128("2361183241434822606847"); - break; + return 9; // 2,361,183,241,434,822,606,847 case 22: case 23: - *max_decimal = Decimal128("604462909807314587353087"); - break; + return 10; // 604,462,909,807,314,587,353,087 case 24: case 25: case 26: - *max_decimal = Decimal128("154742504910672534362390527"); - break; + return 11; // 154,742,504,910,672,534,362,390,527 case 27: case 28: - *max_decimal = Decimal128("39614081257132168796771975167"); - break; + return 12; // 39,614,081,257,132,168,796,771,975,167 case 29: case 30: case 31: - *max_decimal = Decimal128("10141204801825835211973625643007"); - break; + return 13; // 10,141,204,801,825,835,211,973,625,643,007 case 32: case 33: - *max_decimal = Decimal128("2596148429267413814265248164610047"); - break; + return 14; // 2,596,148,429,267,413,814,265,248,164,610,047 case 34: case 35: - *max_decimal = Decimal128("664613997892457936451903530140172287"); - break; + return 15; // 664,613,997,892,457,936,451,903,530,140,172,287 case 36: case 37: case 38: - *max_decimal = Decimal128("170141183460469231731687303715884105727"); - break; + return 16; // 170,141,183,460,469,231,731,687,303,715,884,105,727 default: DCHECK(false); break; } - - *min_decimal = ~(*max_decimal); + return -1; } -class UniformDecimalDistribution { - public: - explicit UniformDecimalDistribution(int32_t precision) { - Decimal128 max_decimal; - Decimal128 min_decimal; - DecimalRange(precision, &min_decimal, &max_decimal); - - const auto min_low = static_cast(min_decimal.low_bits()); - const auto max_low = static_cast(max_decimal.low_bits()); - - const int64_t min_high = min_decimal.high_bits(); - const int64_t max_high = max_decimal.high_bits(); - - using param_type = std::uniform_int_distribution::param_type; - - lower_dist_.param(param_type(min_low, max_low)); - upper_dist_.param(param_type(min_high, max_high)); - } - - template - Decimal128 operator()(Generator& gen) { - return Decimal128(upper_dist_(gen), static_cast(lower_dist_(gen))); - } - - private: - // The lower bits distribution is intentionally int64_t. - // If it were uint64_t then the size of the interval [min_high, max_high] would be 0 - // because min_high > max_high due to 2's complement. - // So, we generate the same range of bits using int64_t and then cast to uint64_t. - std::uniform_int_distribution lower_dist_; - std::uniform_int_distribution upper_dist_; -}; - static inline void random_decimals(int64_t n, uint32_t seed, int32_t precision, uint8_t* out) { std::mt19937 gen(seed); - UniformDecimalDistribution dist(precision); - - for (int64_t i = 0; i < n; ++i, out += 16) { - const Decimal128 value(dist(gen)); - value.ToBytes(out); + std::uniform_int_distribution d(0, std::numeric_limits::max()); + const int32_t required_bytes = DecimalSize(precision); + constexpr int32_t byte_width = 16; + std::fill(out, out + byte_width * n, '\0'); + + for (int64_t i = 0; i < n; ++i, out += byte_width) { + std::generate(out, out + required_bytes, + [&d, &gen] { return static_cast(d(gen)); }); + + // sign extend if the sign bit is set for the last byte generated + // 0b10000000 == 0x80 == 128 + if ((out[required_bytes - 1] & '\x80') != 0) { + std::fill(out + required_bytes, out + byte_width, '\xFF'); + } } } From 357eedcbf4b06d4ab1a78005fe34244319d6b9b0 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Sat, 11 Nov 2017 19:02:53 -0500 Subject: [PATCH 035/177] ARROW-1781: Don't use brew when using the toolchain Author: Korn, Uwe Author: Uwe L. Korn Closes #1295 from xhochy/ARROW-1781 and squashes the following commits: 7de96d8a [Uwe L. Korn] Use a single Brewfile for c_glib 9fde4e91 [Korn, Uwe] Use gtest from conda-forge 8cc6898f [Korn, Uwe] ARROW-1781: Don't use brew when using the toolchain --- .travis.yml | 1 + c_glib/Brewfile | 11 +++++++---- ci/travis_before_script_c_glib.sh | 2 -- ci/travis_before_script_cpp.sh | 5 ++--- cpp/Brewfile | 21 --------------------- 5 files changed, 10 insertions(+), 30 deletions(-) delete mode 100644 cpp/Brewfile diff --git a/.travis.yml b/.travis.yml index 52d7a5f800505..9917a261451c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -152,6 +152,7 @@ matrix: rvm: 2.2 env: BUILD_SYSTEM=autotools before_script: + - brew update && brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh script: diff --git a/c_glib/Brewfile b/c_glib/Brewfile index 80d3c81dd6f82..9fe5c3b616317 100644 --- a/c_glib/Brewfile +++ b/c_glib/Brewfile @@ -15,11 +15,14 @@ # specific language governing permissions and limitations # under the License. -brew "gtk-doc" brew "autoconf-archive" -brew "gobject-introspection" -brew "git" +brew "boost" +brew "ccache" brew "cmake" -brew "wget" +brew "git" +brew "gobject-introspection" +brew "gtk-doc" +brew "jemalloc" brew "libtool" brew "lua" +brew "wget" diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh index a22ecd3753fd8..99d05397a9f23 100755 --- a/ci/travis_before_script_c_glib.sh +++ b/ci/travis_before_script_c_glib.sh @@ -22,8 +22,6 @@ set -ex source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh if [ $TRAVIS_OS_NAME = "osx" ]; then - brew update && brew bundle --file=c_glib/Brewfile - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig export XML_CATALOG_FILES=/usr/local/etc/xml/catalog fi diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index dbdcd33ed0d5b..4998f190f9891 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -38,8 +38,10 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then rapidjson \ flatbuffers \ gflags \ + gtest \ lz4-c \ snappy \ + ccache \ zstd \ brotli \ zlib \ @@ -53,9 +55,6 @@ if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then conda update -y -p $CPP_TOOLCHAIN ca-certificates -c defaults fi -if [ $TRAVIS_OS_NAME == "osx" ]; then - brew update && brew bundle --file=cpp/Brewfile -fi mkdir $ARROW_CPP_BUILD_DIR pushd $ARROW_CPP_BUILD_DIR diff --git a/cpp/Brewfile b/cpp/Brewfile deleted file mode 100644 index 5f82cacc55991..0000000000000 --- a/cpp/Brewfile +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -brew "jemalloc" -brew "ccache" -brew "boost" -brew "cmake" From 550a39ff23054ee8cce99380b1ee1f29035f9f67 Mon Sep 17 00:00:00 2001 From: rvernica Date: Sun, 12 Nov 2017 22:16:29 +0900 Subject: [PATCH 036/177] ARROW-1801: [Docs] Update install instructions to use red-data-tools repos * Update package install instructions to use red-data-tools.org repositories. * Instructions taken from https://github.com/red-data-tools/packages.red-data-tools.org/blob/master/README.md * Dropped support for Debian jessie and Ubuntu 16.10 * Add support for Debian stretch, Ubuntu 14.04, and CentOS 6 Author: rvernica Closes #1307 from rvernica/patch-3 and squashes the following commits: f15dac08 [rvernica] ARROW-1801 [Docs] Update install instructions to use red-data-tools repos --- site/install.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/site/install.md b/site/install.md index 1513fe819789c..0ef2008db9061 100644 --- a/site/install.md +++ b/site/install.md @@ -75,23 +75,25 @@ with the wheel. We have provided APT and Yum repositories for Apache Arrow C++ and Apache Arrow GLib (C). Here are supported platforms: -* Debian GNU/Linux Jessie +* Debian GNU/Linux stretch +* Ubuntu 14.04 LTS * Ubuntu 16.04 LTS -* Ubuntu 16.10 * Ubuntu 17.04 +* Ubuntu 17.10 +* CentOS 6 * CentOS 7 -Debian GNU/Linux Jessie: +Debian GNU/Linux: ```shell -sudo apt update sudo apt install -y -V apt-transport-https -cat < Date: Sun, 12 Nov 2017 15:16:35 +0100 Subject: [PATCH 037/177] ARROW-1763: [Python] Implement __hash__ for DataType Author: Wes McKinney Closes #1308 from wesm/ARROW-1763 and squashes the following commits: 4e379b9 [Wes McKinney] Implement __hash__ for DataType --- python/pyarrow/tests/test_types.py | 24 ++++++++++++++++++++++++ python/pyarrow/types.pxi | 3 +++ 2 files changed, 27 insertions(+) diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index 0e3ea1fd40bf5..9eefa33b66187 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -137,3 +137,27 @@ def test_is_temporal_date_time_timestamp(): def test_timestamp_type(): # See ARROW-1683 assert isinstance(pa.timestamp('ns'), pa.TimestampType) + + +def test_types_hashable(): + types = [ + pa.null(), + pa.int32(), + pa.time32('s'), + pa.time64('us'), + pa.date32(), + pa.timestamp('us'), + pa.string(), + pa.binary(), + pa.binary(10), + pa.list_(pa.int32()), + pa.struct([pa.field('a', pa.int32()), + pa.field('b', pa.int8()), + pa.field('c', pa.string())]) + ] + + in_dict = {} + for i, type_ in enumerate(types): + assert hash(type_) == hash(type_) + in_dict[type_] = i + assert in_dict[type_] == i diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index d2e68ff79a524..edf0d8a305a02 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -69,6 +69,9 @@ cdef class DataType: ) return frombytes(self.type.ToString()) + def __hash__(self): + return hash(str(self)) + def __reduce__(self): return self.__class__, (), self.__getstate__() From e8331f46f8b324271e694557789ea53b082fdc05 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 12 Nov 2017 23:47:47 -0500 Subject: [PATCH 038/177] ARROW-1794: [C++/Python] Rename DecimalArray to Decimal128Array Author: Phillip Cloud Closes #1312 from cpcloud/ARROW-1794 and squashes the following commits: 0b8ba5e0 [Phillip Cloud] Backward compat 4eb2a3ba [Phillip Cloud] ARROW-1794: [C++/Python] Rename DecimalArray to Decimal128Array --- cpp/src/arrow/array-test.cc | 2 +- cpp/src/arrow/array.cc | 6 +++--- cpp/src/arrow/array.h | 11 +++++++---- cpp/src/arrow/compare.cc | 2 +- cpp/src/arrow/ipc/json-internal.cc | 2 +- cpp/src/arrow/ipc/test-common.h | 6 +++--- cpp/src/arrow/ipc/writer.cc | 2 +- cpp/src/arrow/pretty_print.cc | 2 +- cpp/src/arrow/python/arrow_to_pandas.cc | 2 +- cpp/src/arrow/type_fwd.h | 2 +- cpp/src/arrow/type_traits.h | 2 +- cpp/src/arrow/visitor.cc | 2 +- cpp/src/arrow/visitor.h | 2 +- python/doc/source/api.rst | 2 +- python/pyarrow/__init__.py | 2 +- python/pyarrow/array.pxi | 4 ++-- python/pyarrow/includes/libarrow.pxd | 4 +++- python/pyarrow/lib.pxd | 2 +- python/pyarrow/scalar.pxi | 2 +- 19 files changed, 32 insertions(+), 27 deletions(-) diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc index 9f248cdbbb754..15c75534e53ef 100644 --- a/cpp/src/arrow/array-test.cc +++ b/cpp/src/arrow/array-test.cc @@ -2799,7 +2799,7 @@ class DecimalTest : public ::testing::TestWithParam { BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &expected_null_bitmap)); int64_t expected_null_count = test::null_count(valid_bytes); - auto expected = std::make_shared( + auto expected = std::make_shared( type, size, expected_data, expected_null_bitmap, expected_null_count); std::shared_ptr lhs = out->Slice(offset); diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index 9c91d619cc7e8..651fa26ba9026 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -314,12 +314,12 @@ const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const { // ---------------------------------------------------------------------- // Decimal -DecimalArray::DecimalArray(const std::shared_ptr& data) +Decimal128Array::Decimal128Array(const std::shared_ptr& data) : FixedSizeBinaryArray(data) { DCHECK_EQ(data->type->id(), Type::DECIMAL); } -std::string DecimalArray::FormatValue(int64_t i) const { +std::string Decimal128Array::FormatValue(int64_t i) const { const auto& type_ = static_cast(*type()); const Decimal128 value(GetValue(i)); return value.ToString(type_.scale()); @@ -515,7 +515,7 @@ struct ValidateVisitor { Status Visit(const PrimitiveArray&) { return Status::OK(); } - Status Visit(const DecimalArray&) { return Status::OK(); } + Status Visit(const Decimal128Array&) { return Status::OK(); } Status Visit(const BinaryArray&) { // TODO(wesm): what to do here? diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index f7762ce104398..3337e4b158267 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -559,19 +559,22 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray { }; // ---------------------------------------------------------------------- -// DecimalArray -class ARROW_EXPORT DecimalArray : public FixedSizeBinaryArray { +// Decimal128Array +class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray { public: using TypeClass = DecimalType; using FixedSizeBinaryArray::FixedSizeBinaryArray; - /// \brief Construct DecimalArray from ArrayData instance - explicit DecimalArray(const std::shared_ptr& data); + /// \brief Construct Decimal128Array from ArrayData instance + explicit Decimal128Array(const std::shared_ptr& data); std::string FormatValue(int64_t i) const; }; +// Backward compatibility +using DecimalArray = Decimal128Array; + // ---------------------------------------------------------------------- // Struct diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index a2d4de7b73afb..253c2e1feb801 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -255,7 +255,7 @@ class RangeEqualsVisitor { return Status::OK(); } - Status Visit(const DecimalArray& left) { + Status Visit(const Decimal128Array& left) { return Visit(static_cast(left)); } diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index 1b9baee7dafef..976f9660a1b78 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -458,7 +458,7 @@ class ArrayWriter { } } - void WriteDataValues(const DecimalArray& arr) { + void WriteDataValues(const Decimal128Array& arr) { for (int64_t i = 0; i < arr.length(); ++i) { const Decimal128 value(arr.GetValue(i)); writer_->String(value.ToIntegerString()); diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h index 91023db489852..7fc139381052c 100644 --- a/cpp/src/arrow/ipc/test-common.h +++ b/cpp/src/arrow/ipc/test-common.h @@ -690,10 +690,10 @@ Status MakeDecimal(std::shared_ptr* out) { RETURN_NOT_OK(BitUtil::BytesToBits(is_valid_bytes, default_memory_pool(), &is_valid)); - auto a1 = std::make_shared(f0->type(), length, data, is_valid, - kUnknownNullCount); + auto a1 = std::make_shared(f0->type(), length, data, is_valid, + kUnknownNullCount); - auto a2 = std::make_shared(f1->type(), length, data); + auto a2 = std::make_shared(f1->type(), length, data); ArrayVector arrays = {a1, a2}; *out = std::make_shared(schema, length, arrays); diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index 5598cc68296f7..323116f589b73 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -339,7 +339,7 @@ class RecordBatchSerializer : public ArrayVisitor { VISIT_FIXED_WIDTH(Time32Array) VISIT_FIXED_WIDTH(Time64Array) VISIT_FIXED_WIDTH(FixedSizeBinaryArray) - VISIT_FIXED_WIDTH(DecimalArray) + VISIT_FIXED_WIDTH(Decimal128Array) #undef VISIT_FIXED_WIDTH diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc index aaea34a51388c..cfbc30315fcd7 100644 --- a/cpp/src/arrow/pretty_print.cc +++ b/cpp/src/arrow/pretty_print.cc @@ -170,7 +170,7 @@ class ArrayPrinter : public PrettyPrinter { } template - inline typename std::enable_if::value, void>::type + inline typename std::enable_if::value, void>::type WriteDataValues(const T& array) { for (int i = 0; i < array.length(); ++i) { if (i > 0) { diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 3894772daa467..f966b2987b93f 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -633,7 +633,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data, PyObject* Decimal = Decimal_ref.obj(); for (int c = 0; c < data.num_chunks(); c++) { - auto* arr(static_cast(data.chunk(c).get())); + auto* arr(static_cast(data.chunk(c).get())); auto type(std::dynamic_pointer_cast(arr->type())); const int scale = type->scale(); diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h index 0d06b6f6cb86e..343487055c6fc 100644 --- a/cpp/src/arrow/type_fwd.h +++ b/cpp/src/arrow/type_fwd.h @@ -69,7 +69,7 @@ class StructArray; class StructBuilder; class DecimalType; -class DecimalArray; +class Decimal128Array; class DecimalBuilder; class UnionType; diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index fbd78398f4579..bc600372eb01f 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -231,7 +231,7 @@ struct TypeTraits { template <> struct TypeTraits { - using ArrayType = DecimalArray; + using ArrayType = Decimal128Array; using BuilderType = DecimalBuilder; constexpr static bool is_parameter_free = false; }; diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc index a7b01b0f6315a..3739e89f3d019 100644 --- a/cpp/src/arrow/visitor.cc +++ b/cpp/src/arrow/visitor.cc @@ -56,7 +56,7 @@ ARRAY_VISITOR_DEFAULT(ListArray); ARRAY_VISITOR_DEFAULT(StructArray); ARRAY_VISITOR_DEFAULT(UnionArray); ARRAY_VISITOR_DEFAULT(DictionaryArray); -ARRAY_VISITOR_DEFAULT(DecimalArray); +ARRAY_VISITOR_DEFAULT(Decimal128Array); #undef ARRAY_VISITOR_DEFAULT diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h index 6c36e465ec436..34679eb950d4b 100644 --- a/cpp/src/arrow/visitor.h +++ b/cpp/src/arrow/visitor.h @@ -50,7 +50,7 @@ class ARROW_EXPORT ArrayVisitor { virtual Status Visit(const Time64Array& array); virtual Status Visit(const TimestampArray& array); virtual Status Visit(const IntervalArray& array); - virtual Status Visit(const DecimalArray& array); + virtual Status Visit(const Decimal128Array& array); virtual Status Visit(const ListArray& array); virtual Status Visit(const StructArray& array); virtual Status Visit(const UnionArray& array); diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst index 6bceba3c650b6..c52024044a62d 100644 --- a/python/doc/source/api.rst +++ b/python/doc/source/api.rst @@ -155,7 +155,7 @@ Array Types Date32Array Date64Array TimestampArray - DecimalArray + Decimal128Array ListArray .. _api.table: diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 2d7d7288b3835..09bf6b35f396a 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -58,7 +58,7 @@ DictionaryArray, Date32Array, Date64Array, TimestampArray, Time32Array, Time64Array, - DecimalArray, StructArray, + Decimal128Array, StructArray, ArrayValue, Scalar, NA, BooleanValue, Int8Value, Int16Value, Int32Value, Int64Value, diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 9991411e55dfe..2ef592ff758fc 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -597,7 +597,7 @@ cdef class FixedSizeBinaryArray(Array): pass -cdef class DecimalArray(FixedSizeBinaryArray): +cdef class Decimal128Array(FixedSizeBinaryArray): pass @@ -846,7 +846,7 @@ cdef dict _array_classes = { _Type_STRING: StringArray, _Type_DICTIONARY: DictionaryArray, _Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray, - _Type_DECIMAL: DecimalArray, + _Type_DECIMAL: Decimal128Array, _Type_STRUCT: StructArray, } diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index dfafd371b2857..11cc6b3ff2664 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -312,7 +312,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef cppclass CFixedSizeBinaryArray" arrow::FixedSizeBinaryArray"(CArray): const uint8_t* GetValue(int i) - cdef cppclass CDecimalArray" arrow::DecimalArray"(CFixedSizeBinaryArray): + cdef cppclass CDecimal128Array" arrow::Decimal128Array"( + CFixedSizeBinaryArray + ): c_string FormatValue(int i) cdef cppclass CListArray" arrow::ListArray"(CArray): diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 531489490754e..6413b838f0595 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -246,7 +246,7 @@ cdef class FixedSizeBinaryArray(Array): pass -cdef class DecimalArray(FixedSizeBinaryArray): +cdef class Decimal128Array(FixedSizeBinaryArray): pass diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index a396fa763c8c8..1bc5ed7a372a8 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -258,7 +258,7 @@ cdef class DecimalValue(ArrayValue): def as_py(self): cdef: - CDecimalArray* ap = self.sp_array.get() + CDecimal128Array* ap = self.sp_array.get() c_string s = ap.FormatValue(self.index) return _pydecimal.Decimal(s.decode('utf8')) From 4a33bad5f28830812fe4d47dcfdfb184d5ee43c0 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sun, 12 Nov 2017 23:49:12 -0500 Subject: [PATCH 039/177] ARROW-1767: [C++] Support file reads and writes over 2GB on Windows This closes [ARROW-1767](https://issues.apache.org/jira/browse/ARROW-1767). Author: Licht-T Closes #1311 from Licht-T/feature-large-file-io-windows and squashes the following commits: 690d2801 [Licht-T] ENH: Support large file io on Windows --- cpp/src/arrow/io/file.cc | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc index 057cad1111685..1ec5e23e587d5 100644 --- a/cpp/src/arrow/io/file.cc +++ b/cpp/src/arrow/io/file.cc @@ -255,19 +255,18 @@ static inline Status FileSeek(int fd, int64_t pos) { static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbytes, int64_t* bytes_read) { -#if defined(_MSC_VER) - if (nbytes > ARROW_MAX_IO_CHUNKSIZE) { - return Status::IOError("Unable to read > 2GB blocks yet"); - } - *bytes_read = static_cast(_read(fd, buffer, static_cast(nbytes))); -#else *bytes_read = 0; while (*bytes_read != -1 && *bytes_read < nbytes) { int64_t chunksize = std::min(static_cast(ARROW_MAX_IO_CHUNKSIZE), nbytes - *bytes_read); +#if defined(_MSC_VER) + int64_t ret = static_cast( + _read(fd, buffer + *bytes_read, static_cast(chunksize))); +#else int64_t ret = static_cast( read(fd, buffer + *bytes_read, static_cast(chunksize))); +#endif if (ret != -1) { *bytes_read += ret; @@ -279,7 +278,6 @@ static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbyte *bytes_read = ret; } } -#endif if (*bytes_read == -1) { return Status::IOError(std::string("Error reading bytes from file: ") + @@ -292,25 +290,23 @@ static inline Status FileRead(const int fd, uint8_t* buffer, const int64_t nbyte static inline Status FileWrite(const int fd, const uint8_t* buffer, const int64_t nbytes) { int ret = 0; -#if defined(_MSC_VER) - if (nbytes > ARROW_MAX_IO_CHUNKSIZE) { - return Status::IOError("Unable to write > 2GB blocks to file yet"); - } - ret = static_cast(_write(fd, buffer, static_cast(nbytes))); -#else int64_t bytes_written = 0; while (ret != -1 && bytes_written < nbytes) { int64_t chunksize = std::min(static_cast(ARROW_MAX_IO_CHUNKSIZE), nbytes - bytes_written); +#if defined(_MSC_VER) + ret = static_cast( + _write(fd, buffer + bytes_written, static_cast(chunksize))); +#else ret = static_cast( write(fd, buffer + bytes_written, static_cast(chunksize))); +#endif if (ret != -1) { bytes_written += ret; } } -#endif if (ret == -1) { return Status::IOError(std::string("Error writing bytes from file: ") + From 6f8e2873c3931ff279b05a5e517a87637eb13d05 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Mon, 13 Nov 2017 22:47:22 -0500 Subject: [PATCH 040/177] ARROW-1743: [Python] Avoid non-array writeable-flag check This closes [ARROW-1743](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1743). Author: Licht-T Closes #1260 from Licht-T/fix-non-array-writable-check and squashes the following commits: 8a104f7c [Licht-T] TST: Add test_categorical_row_index 555acdeb [Licht-T] BUG: Avoid the writeable-flag check for non-array index --- python/pyarrow/pandas_compat.py | 2 +- python/pyarrow/tests/test_convert_pandas.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index db28ee09e1e73..41eaf0bac4525 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -397,7 +397,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1): col = table.column(i) col_pandas = col.to_pandas() values = col_pandas.values - if not values.flags.writeable: + if hasattr(values, 'flags') and not values.flags.writeable: # ARROW-1054: in pandas 0.19.2, factorize will reject # non-writeable arrays when calling MultiIndex.from_arrays values = values.copy() diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index dabccac37c3d8..b9c3a12213243 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -198,6 +198,13 @@ def test_datetimetz_column_index(self): md = column_indexes['metadata'] assert md['timezone'] == 'America/New_York' + def test_categorical_row_index(self): + df = pd.DataFrame({'a': [1, 2, 3], 'b': [1, 2, 3]}) + df['a'] = df.a.astype('category') + df = df.set_index('a') + + self._check_pandas_roundtrip(df, preserve_index=True) + def test_float_no_nulls(self): data = {} fields = [] From 8f2d15256d6b3a9da569797f363518abf50d23f5 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Mon, 13 Nov 2017 22:53:58 -0500 Subject: [PATCH 041/177] ARROW-1802: [GLib] Support arrow-gpu arrow-gpu isn't required. If `arrow-gpu.pc` isn't installed, GPU support is just ignored. Author: Kouhei Sutou Closes #1313 from kou/glib-gpu and squashes the following commits: b36d491a [Kouhei Sutou] [GLib] Support arrow-gpu --- c_glib/.gitignore | 1 + c_glib/Makefile.am | 1 + c_glib/arrow-glib/Makefile.am | 8 +- c_glib/arrow-glib/buffer.h | 82 +- c_glib/arrow-glib/input-stream.h | 136 +-- c_glib/arrow-glib/meson.build | 33 +- c_glib/arrow-glib/output-stream.h | 47 +- c_glib/arrow-glib/readable.cpp | 4 +- c_glib/arrow-glib/readable.hpp | 1 + c_glib/arrow-gpu-glib/Makefile.am | 109 +++ c_glib/arrow-gpu-glib/arrow-gpu-glib.h | 24 + c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp | 24 + c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in | 28 + c_glib/arrow-gpu-glib/cuda.cpp | 941 +++++++++++++++++++++ c_glib/arrow-gpu-glib/cuda.h | 181 ++++ c_glib/arrow-gpu-glib/cuda.hpp | 54 ++ c_glib/arrow-gpu-glib/meson.build | 80 ++ c_glib/configure.ac | 26 +- c_glib/doc/Makefile.am | 2 +- c_glib/doc/reference/Makefile.am | 11 + c_glib/doc/reference/arrow-glib-docs.sgml | 10 + c_glib/doc/reference/meson.build | 23 +- c_glib/meson.build | 9 +- c_glib/test/run-test.rb | 6 + c_glib/test/run-test.sh | 37 +- c_glib/test/test-gpu-cuda.rb | 144 ++++ 26 files changed, 1744 insertions(+), 278 deletions(-) create mode 100644 c_glib/arrow-gpu-glib/Makefile.am create mode 100644 c_glib/arrow-gpu-glib/arrow-gpu-glib.h create mode 100644 c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp create mode 100644 c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in create mode 100644 c_glib/arrow-gpu-glib/cuda.cpp create mode 100644 c_glib/arrow-gpu-glib/cuda.h create mode 100644 c_glib/arrow-gpu-glib/cuda.hpp create mode 100644 c_glib/arrow-gpu-glib/meson.build create mode 100644 c_glib/test/test-gpu-cuda.rb diff --git a/c_glib/.gitignore b/c_glib/.gitignore index 03bb0fe61892e..2719147405f71 100644 --- a/c_glib/.gitignore +++ b/c_glib/.gitignore @@ -41,6 +41,7 @@ Makefile.in /arrow-glib/enums.h /arrow-glib/stamp-* /arrow-glib/*.pc +/arrow-gpu-glib/*.pc /example/build /example/read-batch /example/read-stream diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am index 577b749fb38bc..4cc70e5a08870 100644 --- a/c_glib/Makefile.am +++ b/c_glib/Makefile.am @@ -19,6 +19,7 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS} SUBDIRS = \ arrow-glib \ + arrow-gpu-glib \ doc \ example \ tool diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am index bf68ec4910e77..5ecb1a6614268 100644 --- a/c_glib/arrow-glib/Makefile.am +++ b/c_glib/arrow-glib/Makefile.am @@ -203,20 +203,18 @@ pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = \ arrow-glib.pc -# GObject Introspection +if HAVE_INTROSPECTION -include $(INTROSPECTION_MAKEFILE) INTROSPECTION_GIRS = INTROSPECTION_SCANNER_ARGS = INTROSPECTION_COMPILER_ARGS = -if HAVE_INTROSPECTION Arrow-1.0.gir: libarrow-glib.la Arrow_1_0_gir_PACKAGES = \ - gobject-2.0 \ gio-2.0 -Arrow_1_0_gir_EXPORT_PACKAGES = arrow +Arrow_1_0_gir_EXPORT_PACKAGES = \ + arrow-glib Arrow_1_0_gir_INCLUDES = \ - GObject-2.0 \ Gio-2.0 Arrow_1_0_gir_CFLAGS = \ $(AM_CPPFLAGS) diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h index b3f3a2cdc5e9b..300bb4f4ea3ca 100644 --- a/c_glib/arrow-glib/buffer.h +++ b/c_glib/arrow-glib/buffer.h @@ -19,44 +19,21 @@ #pragma once -#include +#include G_BEGIN_DECLS -#define GARROW_TYPE_BUFFER \ - (garrow_buffer_get_type()) -#define GARROW_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_BUFFER, GArrowBuffer)) -#define GARROW_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_BUFFER, GArrowBufferClass)) -#define GARROW_IS_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_BUFFER)) -#define GARROW_IS_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_BUFFER)) -#define GARROW_BUFFER_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_BUFFER, GArrowBufferClass)) - -typedef struct _GArrowBuffer GArrowBuffer; -typedef struct _GArrowBufferClass GArrowBufferClass; - -/** - * GArrowBuffer: - * - * It wraps `arrow::Buffer`. - */ -struct _GArrowBuffer -{ - /*< private >*/ - GObject parent_instance; -}; - +#define GARROW_TYPE_BUFFER (garrow_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowBuffer, + garrow_buffer, + GARROW, + BUFFER, + GObject) struct _GArrowBufferClass { GObjectClass parent_class; }; -GType garrow_buffer_get_type (void) G_GNUC_CONST; - GArrowBuffer *garrow_buffer_new (const guint8 *data, gint64 size); gboolean garrow_buffer_equal (GArrowBuffer *buffer, @@ -80,49 +57,16 @@ GArrowBuffer *garrow_buffer_slice (GArrowBuffer *buffer, gint64 size); -#define GARROW_TYPE_MUTABLE_BUFFER \ - (garrow_mutable_buffer_get_type()) -#define GARROW_MUTABLE_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_MUTABLE_BUFFER, \ - GArrowMutableBuffer)) -#define GARROW_MUTABLE_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_MUTABLE_BUFFER, \ - GArrowMutableBufferClass)) -#define GARROW_IS_MUTABLE_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_MUTABLE_BUFFER)) -#define GARROW_IS_MUTABLE_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_MUTABLE_BUFFER)) -#define GARROW_MUTABLE_BUFFER_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_MUTABLE_BUFFER, \ - GArrowMutableBufferClass)) - -typedef struct _GArrowMutableBuffer GArrowMutableBuffer; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowMutableBufferClass GArrowMutableBufferClass; -#endif - -/** - * GArrowMutableBuffer: - * - * It wraps `arrow::MutableBuffer`. - */ -struct _GArrowMutableBuffer -{ - /*< private >*/ - GArrowBuffer parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +#define GARROW_TYPE_MUTABLE_BUFFER (garrow_mutable_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowMutableBuffer, + garrow_mutable_buffer, + GARROW, + MUTABLE_BUFFER, + GArrowBuffer) struct _GArrowMutableBufferClass { GArrowBufferClass parent_class; }; -#endif - -GType garrow_mutable_buffer_get_type(void) G_GNUC_CONST; GArrowMutableBuffer *garrow_mutable_buffer_new (guint8 *data, gint64 size); diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h index 12c7ae700f79d..c2068d6ac0e41 100644 --- a/c_glib/arrow-glib/input-stream.h +++ b/c_glib/arrow-glib/input-stream.h @@ -26,98 +26,28 @@ G_BEGIN_DECLS -#define GARROW_TYPE_INPUT_STREAM \ - (garrow_input_stream_get_type()) -#define GARROW_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_INPUT_STREAM, \ - GArrowInputStream)) -#define GARROW_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_INPUT_STREAM, \ - GArrowInputStreamClass)) -#define GARROW_IS_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_INPUT_STREAM)) -#define GARROW_IS_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_INPUT_STREAM)) -#define GARROW_INPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_INPUT_STREAM, \ - GArrowInputStreamClass)) - -typedef struct _GArrowInputStream GArrowInputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowInputStreamClass GArrowInputStreamClass; -#endif - -/** - * GArrowInputStream: - * - * It wraps `arrow::io::InputStream`. - */ -struct _GArrowInputStream -{ - /*< private >*/ - GObject parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +#define GARROW_TYPE_INPUT_STREAM (garrow_input_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowInputStream, + garrow_input_stream, + GARROW, + INPUT_STREAM, + GObject) struct _GArrowInputStreamClass { GObjectClass parent_class; }; -#endif - -GType garrow_input_stream_get_type(void) G_GNUC_CONST; - #define GARROW_TYPE_SEEKABLE_INPUT_STREAM \ (garrow_seekable_input_stream_get_type()) -#define GARROW_SEEKABLE_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM, \ - GArrowSeekableInputStream)) -#define GARROW_SEEKABLE_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM, \ - GArrowSeekableInputStreamClass)) -#define GARROW_IS_SEEKABLE_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM)) -#define GARROW_IS_SEEKABLE_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM)) -#define GARROW_SEEKABLE_INPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM, \ - GArrowSeekableInputStreamClass)) - -typedef struct _GArrowSeekableInputStream GArrowSeekableInputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowSeekableInputStreamClass GArrowSeekableInputStreamClass; -#endif - -/** - * GArrowSeekableInputStream: - * - * It wraps `arrow::io::RandomAccessFile`. - */ -struct _GArrowSeekableInputStream -{ - /*< private >*/ - GArrowInputStream parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +G_DECLARE_DERIVABLE_TYPE(GArrowSeekableInputStream, + garrow_seekable_input_stream, + GARROW, + SEEKABLE_INPUT_STREAM, + GArrowInputStream) struct _GArrowSeekableInputStreamClass { GArrowInputStreamClass parent_class; }; -#endif - -GType garrow_seekable_input_stream_get_type(void) G_GNUC_CONST; guint64 garrow_seekable_input_stream_get_size(GArrowSeekableInputStream *input_stream, GError **error); @@ -133,49 +63,15 @@ GArrowTensor *garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream #define GARROW_TYPE_BUFFER_INPUT_STREAM \ (garrow_buffer_input_stream_get_type()) -#define GARROW_BUFFER_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_BUFFER_INPUT_STREAM, \ - GArrowBufferInputStream)) -#define GARROW_BUFFER_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_BUFFER_INPUT_STREAM, \ - GArrowBufferInputStreamClass)) -#define GARROW_IS_BUFFER_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_BUFFER_INPUT_STREAM)) -#define GARROW_IS_BUFFER_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_BUFFER_INPUT_STREAM)) -#define GARROW_BUFFER_INPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_BUFFER_INPUT_STREAM, \ - GArrowBufferInputStreamClass)) - -typedef struct _GArrowBufferInputStream GArrowBufferInputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowBufferInputStreamClass GArrowBufferInputStreamClass; -#endif - -/** - * GArrowBufferInputStream: - * - * It wraps `arrow::io::BufferReader`. - */ -struct _GArrowBufferInputStream -{ - /*< private >*/ - GArrowSeekableInputStream parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +G_DECLARE_DERIVABLE_TYPE(GArrowBufferInputStream, + garrow_buffer_input_stream, + GARROW, + BUFFER_INPUT_STREAM, + GArrowSeekableInputStream) struct _GArrowBufferInputStreamClass { GArrowSeekableInputStreamClass parent_class; }; -#endif - -GType garrow_buffer_input_stream_get_type(void) G_GNUC_CONST; GArrowBufferInputStream *garrow_buffer_input_stream_new(GArrowBuffer *buffer); diff --git a/c_glib/arrow-glib/meson.build b/c_glib/arrow-glib/meson.build index 464a002e78b0c..aeec4172d6284 100644 --- a/c_glib/arrow-glib/meson.build +++ b/c_glib/arrow-glib/meson.build @@ -179,22 +179,23 @@ pkgconfig.generate(filebase: meson.project_name(), name: 'Apache Arrow GLib', description: 'C API for Apache Arrow based on GLib', version: version, - requires: ['gobject-2.0', 'arrow'], + requires: ['gio-2.0', 'arrow'], libraries: [libarrow_glib], subdirs: ['arrow-glib']) -gnome.generate_gir(libarrow_glib, - sources: sources + c_headers + enums, - namespace: 'Arrow', - nsversion: api_version, - identifier_prefix: 'GArrow', - symbol_prefix: 'garrow', - export_packages: 'arrow-glib', - includes: [ - 'GObject-2.0', - 'Gio-2.0', - ], - install: true, - extra_args: [ - '--warn-all', - ]) +arrow_glib_gir = gnome.generate_gir(libarrow_glib, + sources: sources + c_headers + enums, + namespace: 'Arrow', + nsversion: api_version, + identifier_prefix: 'GArrow', + symbol_prefix: 'garrow', + export_packages: 'arrow-glib', + includes: [ + 'GObject-2.0', + 'Gio-2.0', + ], + install: true, + extra_args: [ + '--warn-all', + ]) +arrow_glib_gir_dependency = declare_dependency(sources: arrow_glib_gir) diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h index e42ebcde47d6b..195a97ac9f053 100644 --- a/c_glib/arrow-glib/output-stream.h +++ b/c_glib/arrow-glib/output-stream.h @@ -26,51 +26,16 @@ G_BEGIN_DECLS -#define GARROW_TYPE_OUTPUT_STREAM \ - (garrow_output_stream_get_type()) -#define GARROW_OUTPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_OUTPUT_STREAM, \ - GArrowOutputStream)) -#define GARROW_OUTPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_OUTPUT_STREAM, \ - GArrowOutputStreamClass)) -#define GARROW_IS_OUTPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_OUTPUT_STREAM)) -#define GARROW_IS_OUTPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_OUTPUT_STREAM)) -#define GARROW_OUTPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_OUTPUT_STREAM, \ - GArrowOutputStreamClass)) - -typedef struct _GArrowOutputStream GArrowOutputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowOutputStreamClass GArrowOutputStreamClass; -#endif - -/** - * GArrowOutputStream: - * - * It wraps `arrow::io::OutputStream`. - */ -struct _GArrowOutputStream -{ - /*< private >*/ - GObject parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +#define GARROW_TYPE_OUTPUT_STREAM (garrow_output_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowOutputStream, + garrow_output_stream, + GARROW, + OUTPUT_STREAM, + GObject) struct _GArrowOutputStreamClass { GObjectClass parent_class; }; -#endif - -GType garrow_output_stream_get_type(void) G_GNUC_CONST; gint64 garrow_output_stream_write_tensor(GArrowOutputStream *stream, GArrowTensor *tensor, diff --git a/c_glib/arrow-glib/readable.cpp b/c_glib/arrow-glib/readable.cpp index 6a9023e6cddf0..33f98d98c88a4 100644 --- a/c_glib/arrow-glib/readable.cpp +++ b/c_glib/arrow-glib/readable.cpp @@ -45,6 +45,7 @@ G_DEFINE_INTERFACE(GArrowReadable, static void garrow_readable_default_init (GArrowReadableInterface *iface) { + iface->new_raw = garrow_buffer_new_raw; } /** @@ -66,7 +67,8 @@ garrow_readable_read(GArrowReadable *readable, std::shared_ptr arrow_buffer; auto status = arrow_readable->Read(n_bytes, &arrow_buffer); if (garrow_error_check(error, status, "[io][readable][read]")) { - return garrow_buffer_new_raw(&arrow_buffer); + auto *iface = GARROW_READABLE_GET_IFACE(readable); + return iface->new_raw(&arrow_buffer); } else { return NULL; } diff --git a/c_glib/arrow-glib/readable.hpp b/c_glib/arrow-glib/readable.hpp index c241c77aa0329..ce7770103aa1a 100644 --- a/c_glib/arrow-glib/readable.hpp +++ b/c_glib/arrow-glib/readable.hpp @@ -32,6 +32,7 @@ struct _GArrowReadableInterface { GTypeInterface parent_iface; + GArrowBuffer *(*new_raw)(std::shared_ptr *arrow_buffer); std::shared_ptr (*get_raw)(GArrowReadable *file); }; diff --git a/c_glib/arrow-gpu-glib/Makefile.am b/c_glib/arrow-gpu-glib/Makefile.am new file mode 100644 index 0000000000000..ec9615987ee5e --- /dev/null +++ b/c_glib/arrow-gpu-glib/Makefile.am @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +CLEANFILES = + +EXTRA_DIST = \ + meson.build + +AM_CPPFLAGS = \ + -I$(top_builddir) \ + -I$(top_srcdir) + +if HAVE_ARROW_GPU +lib_LTLIBRARIES = \ + libarrow-gpu-glib.la + +libarrow_gpu_glib_la_CXXFLAGS = \ + $(GLIB_CFLAGS) \ + $(ARROW_CFLAGS) \ + $(ARROW_GPU_CFLAGS) \ + $(GARROW_CXXFLAGS) + +libarrow_gpu_glib_la_LIBADD = \ + $(GLIB_LIBS) \ + $(ARROW_LIBS) \ + $(ARROW_GPU_LIBS) \ + ../arrow-glib/libarrow-glib.la + +libarrow_gpu_glib_la_headers = \ + arrow-gpu-glib.h \ + cuda.h + +libarrow_gpu_glib_la_sources = \ + cuda.cpp \ + $(libarrow_gpu_glib_la_headers) + +libarrow_gpu_glib_la_cpp_headers = \ + arrow-gpu-glib.hpp \ + cuda.hpp + +libarrow_gpu_glib_la_SOURCES = \ + $(libarrow_gpu_glib_la_sources) \ + $(libarrow_gpu_glib_la_cpp_headers) + +arrow_gpu_glib_includedir = \ + $(includedir)/arrow-gpu-glib +arrow_gpu_glib_include_HEADERS = \ + $(libarrow_gpu_glib_la_headers) \ + $(libarrow_gpu_glib_la_cpp_headers) + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = \ + arrow-gpu-glib.pc + +if HAVE_INTROSPECTION +-include $(INTROSPECTION_MAKEFILE) +INTROSPECTION_GIRS = +INTROSPECTION_SCANNER_ARGS = +INTROSPECTION_SCANNER_ENV = \ + PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH} +INTROSPECTION_COMPILER_ARGS = \ + --includedir=$(abs_builddir)/../arrow-glib + +ArrowGPU-1.0.gir: libarrow-gpu-glib.la +ArrowGPU_1_0_gir_PACKAGES = \ + arrow-glib +ArrowGPU_1_0_gir_EXPORT_PACKAGES = \ + arrow-gpu-glib +ArrowGPU_1_0_gir_INCLUDES = \ + Arrow-1.0 +ArrowGPU_1_0_gir_CFLAGS = \ + $(AM_CPPFLAGS) +ArrowGPU_1_0_gir_LIBS = \ + $(abs_builddir)/../arrow-glib/libarrow-glib.la \ + libarrow-gpu-glib.la +ArrowGPU_1_0_gir_FILES = \ + $(libarrow_gpu_glib_la_sources) +ArrowGPU_1_0_gir_SCANNERFLAGS = \ + --warn-all \ + --add-include-path=$(abs_builddir)/../arrow-glib \ + --identifier-prefix=GArrowGPU \ + --symbol-prefix=garrow_gpu +INTROSPECTION_GIRS += ArrowGPU-1.0.gir + +girdir = $(datadir)/gir-1.0 +gir_DATA = $(INTROSPECTION_GIRS) + +typelibdir = $(libdir)/girepository-1.0 +typelib_DATA = $(INTROSPECTION_GIRS:.gir=.typelib) + +CLEANFILES += \ + $(gir_DATA) \ + $(typelib_DATA) +endif +endif diff --git a/c_glib/arrow-gpu-glib/arrow-gpu-glib.h b/c_glib/arrow-gpu-glib/arrow-gpu-glib.h new file mode 100644 index 0000000000000..1538c9a1865ac --- /dev/null +++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.h @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include diff --git a/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp b/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp new file mode 100644 index 0000000000000..92017d8b67aab --- /dev/null +++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include diff --git a/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in b/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in new file mode 100644 index 0000000000000..38a6bae1a1298 --- /dev/null +++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Apache Arrow GPU GLib +Description: C API for Apache Arrow GPU based on GLib +Version: @VERSION@ +Libs: -L${libdir} -larrow-gpu-glib +Cflags: -I${includedir} +Requires: arrow-glib diff --git a/c_glib/arrow-gpu-glib/cuda.cpp b/c_glib/arrow-gpu-glib/cuda.cpp new file mode 100644 index 0000000000000..c2a9af54dda94 --- /dev/null +++ b/c_glib/arrow-gpu-glib/cuda.cpp @@ -0,0 +1,941 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +G_BEGIN_DECLS + +/** + * SECTION: cuda + * @section_id: cuda-classes + * @title: CUDA related classes + * @include: arrow-gpu-glib/arrow-gpu-glib.h + * + * The following classes provide CUDA support for Apache Arrow data. + * + * #GArrowGPUCUDADeviceManager is the starting point. You need at + * least one #GArrowGPUCUDAContext to process Apache Arrow data on + * NVIDIA GPU. + * + * #GArrowGPUCUDAContext is a class to keep context for one GPU. You + * need to create #GArrowGPUCUDAContext for each GPU that you want to + * use. You can create #GArrowGPUCUDAContext by + * garrow_gpu_cuda_device_manager_get_context(). + * + * #GArrowGPUCUDABuffer is a class for data on GPU. You can copy data + * on GPU to/from CPU by garrow_gpu_cuda_buffer_copy_to_host() and + * garrow_gpu_cuda_buffer_copy_from_host(). You can share data on GPU + * with other processes by garrow_gpu_cuda_buffer_export() and + * garrow_gpu_cuda_buffer_new_ipc(). + * + * #GArrowGPUCUDAHostBuffer is a class for data on CPU that is + * directly accessible from GPU. + * + * #GArrowGPUCUDAIPCMemoryHandle is a class to share data on GPU with + * other processes. You can export your data on GPU to other processes + * by garrow_gpu_cuda_buffer_export() and + * garrow_gpu_cuda_ipc_memory_handle_new(). You can import other + * process data on GPU by garrow_gpu_cuda_ipc_memory_handle_new() and + * garrow_gpu_cuda_buffer_new_ipc(). + * + * #GArrowGPUCUDABufferInputStream is a class to read data in + * #GArrowGPUCUDABuffer. + * + * #GArrowGPUCUDABufferOutputStream is a class to write data into + * #GArrowGPUCUDABuffer. + */ + +G_DEFINE_TYPE(GArrowGPUCUDADeviceManager, + garrow_gpu_cuda_device_manager, + G_TYPE_OBJECT) + +static void +garrow_gpu_cuda_device_manager_init(GArrowGPUCUDADeviceManager *object) +{ +} + +static void +garrow_gpu_cuda_device_manager_class_init(GArrowGPUCUDADeviceManagerClass *klass) +{ +} + +/** + * garrow_gpu_cuda_device_manager_new: + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: A newly created #GArrowGPUCUDADeviceManager on success, + * %NULL on error. + * + * Since: 0.8.0 + */ +GArrowGPUCUDADeviceManager * +garrow_gpu_cuda_device_manager_new(GError **error) +{ + arrow::gpu::CudaDeviceManager *manager; + auto status = arrow::gpu::CudaDeviceManager::GetInstance(&manager); + if (garrow_error_check(error, status, "[gpu][cuda][device-manager][new]")) { + auto manager = g_object_new(GARROW_GPU_TYPE_CUDA_DEVICE_MANAGER, + NULL); + return GARROW_GPU_CUDA_DEVICE_MANAGER(manager); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_device_manager_get_context: + * @manager: A #GArrowGPUCUDADeviceManager. + * @gpu_number: A GPU device number for the target context. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDAContext on + * success, %NULL on error. Contexts for the same GPU device number + * share the same data internally. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAContext * +garrow_gpu_cuda_device_manager_get_context(GArrowGPUCUDADeviceManager *manager, + gint gpu_number, + GError **error) +{ + arrow::gpu::CudaDeviceManager *arrow_manager; + arrow::gpu::CudaDeviceManager::GetInstance(&arrow_manager); + std::shared_ptr context; + auto status = arrow_manager->GetContext(gpu_number, &context); + if (garrow_error_check(error, status, + "[gpu][cuda][device-manager][get-context]]")) { + return garrow_gpu_cuda_context_new_raw(&context); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_device_manager_get_n_devices: + * @manager: A #GArrowGPUCUDADeviceManager. + * + * Returns: The number of GPU devices. + * + * Since: 0.8.0 + */ +gsize +garrow_gpu_cuda_device_manager_get_n_devices(GArrowGPUCUDADeviceManager *manager) +{ + arrow::gpu::CudaDeviceManager *arrow_manager; + arrow::gpu::CudaDeviceManager::GetInstance(&arrow_manager); + return arrow_manager->num_devices(); +} + + +typedef struct GArrowGPUCUDAContextPrivate_ { + std::shared_ptr context; +} GArrowGPUCUDAContextPrivate; + +enum { + PROP_CONTEXT = 1 +}; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowGPUCUDAContext, + garrow_gpu_cuda_context, + G_TYPE_OBJECT) + +#define GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object) \ + static_cast( \ + garrow_gpu_cuda_context_get_instance_private( \ + GARROW_GPU_CUDA_CONTEXT(object))) + +static void +garrow_gpu_cuda_context_finalize(GObject *object) +{ + auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object); + + priv->context = nullptr; + + G_OBJECT_CLASS(garrow_gpu_cuda_context_parent_class)->finalize(object); +} + +static void +garrow_gpu_cuda_context_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_CONTEXT: + priv->context = + *static_cast *>(g_value_get_pointer(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_context_get_property(GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_context_init(GArrowGPUCUDAContext *object) +{ +} + +static void +garrow_gpu_cuda_context_class_init(GArrowGPUCUDAContextClass *klass) +{ + GParamSpec *spec; + + auto gobject_class = G_OBJECT_CLASS(klass); + + gobject_class->finalize = garrow_gpu_cuda_context_finalize; + gobject_class->set_property = garrow_gpu_cuda_context_set_property; + gobject_class->get_property = garrow_gpu_cuda_context_get_property; + + /** + * GArrowGPUCUDAContext:context: + * + * Since: 0.8.0 + */ + spec = g_param_spec_pointer("context", + "Context", + "The raw std::shared_ptr *", + static_cast(G_PARAM_WRITABLE | + G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_CONTEXT, spec); +} + +/** + * garrow_gpu_cuda_context_get_allocated_size: + * @context: A #GArrowGPUCUDAContext. + * + * Returns: The allocated memory by this context in bytes. + * + * Since: 0.8.0 + */ +gint64 +garrow_gpu_cuda_context_get_allocated_size(GArrowGPUCUDAContext *context) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + return arrow_context->bytes_allocated(); +} + + +G_DEFINE_TYPE(GArrowGPUCUDABuffer, + garrow_gpu_cuda_buffer, + GARROW_TYPE_BUFFER) + +static void +garrow_gpu_cuda_buffer_init(GArrowGPUCUDABuffer *object) +{ +} + +static void +garrow_gpu_cuda_buffer_class_init(GArrowGPUCUDABufferClass *klass) +{ +} + +/** + * garrow_gpu_cuda_buffer_new: + * @context: A #GArrowGPUCUDAContext. + * @size: The number of bytes to be allocated on GPU device for this context. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on + * success, %NULL on error. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new(GArrowGPUCUDAContext *context, + gint64 size, + GError **error) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + std::shared_ptr arrow_buffer; + auto status = arrow_context->Allocate(size, &arrow_buffer); + if (garrow_error_check(error, status, "[gpu][cuda][buffer][new]")) { + return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_new_ipc: + * @context: A #GArrowGPUCUDAContext. + * @handle: A #GArrowGPUCUDAIPCMemoryHandle to be communicated. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on + * success, %NULL on error. The buffer has data from the IPC target. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_ipc(GArrowGPUCUDAContext *context, + GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + auto arrow_handle = garrow_gpu_cuda_ipc_memory_handle_get_raw(handle); + std::shared_ptr arrow_buffer; + auto status = arrow_context->OpenIpcBuffer(*arrow_handle, &arrow_buffer); + if (garrow_error_check(error, status, + "[gpu][cuda][buffer][new-ipc]")) { + return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_new_record_batch: + * @context: A #GArrowGPUCUDAContext. + * @record_batch: A #GArrowRecordBatch to be serialized. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on + * success, %NULL on error. The buffer has serialized record batch + * data. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_record_batch(GArrowGPUCUDAContext *context, + GArrowRecordBatch *record_batch, + GError **error) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + auto arrow_record_batch = garrow_record_batch_get_raw(record_batch); + std::shared_ptr arrow_buffer; + auto status = arrow::gpu::SerializeRecordBatch(*arrow_record_batch, + arrow_context.get(), + &arrow_buffer); + if (garrow_error_check(error, status, + "[gpu][cuda][buffer][new-record-batch]")) { + return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_copy_to_host: + * @buffer: A #GArrowGPUCUDABuffer. + * @position: The offset of memory on GPU device to be copied. + * @size: The size of memory on GPU device to be copied in bytes. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A #GBytes that have copied memory on CPU + * host on success, %NULL on error. + * + * Since: 0.8.0 + */ +GBytes * +garrow_gpu_cuda_buffer_copy_to_host(GArrowGPUCUDABuffer *buffer, + gint64 position, + gint64 size, + GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto data = static_cast(g_malloc(size)); + auto status = arrow_buffer->CopyToHost(position, size, data); + if (garrow_error_check(error, status, "[gpu][cuda][buffer][copy-to-host]")) { + return g_bytes_new_take(data, size); + } else { + g_free(data); + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_copy_from_host: + * @buffer: A #GArrowGPUCUDABuffer. + * @data: (array length=size): Data on CPU host to be copied. + * @size: The size of data on CPU host to be copied in bytes. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Since: 0.8.0 + */ +gboolean +garrow_gpu_cuda_buffer_copy_from_host(GArrowGPUCUDABuffer *buffer, + const guint8 *data, + gint64 size, + GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto status = arrow_buffer->CopyFromHost(0, data, size); + return garrow_error_check(error, + status, + "[gpu][cuda][buffer][copy-from-host]"); +} + +/** + * garrow_gpu_cuda_buffer_export: + * @buffer: A #GArrowGPUCUDABuffer. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created + * #GArrowGPUCUDAIPCMemoryHandle to handle the exported buffer on + * success, %NULL on error + * + * Since: 0.8.0 + */ +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_buffer_export(GArrowGPUCUDABuffer *buffer, GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + std::unique_ptr arrow_handle; + auto status = arrow_buffer->ExportForIpc(&arrow_handle); + if (garrow_error_check(error, status, "[gpu][cuda][buffer][export-for-ipc]")) { + return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release()); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_get_context: + * @buffer: A #GArrowGPUCUDABuffer. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDAContext for the + * buffer. Contexts for the same buffer share the same data internally. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAContext * +garrow_gpu_cuda_buffer_get_context(GArrowGPUCUDABuffer *buffer) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_context = arrow_buffer->context(); + return garrow_gpu_cuda_context_new_raw(&arrow_context); +} + +/** + * garrow_gpu_cuda_buffer_read_record_batch: + * @buffer: A #GArrowGPUCUDABuffer. + * @schema: A #GArrowSchema for record batch. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowRecordBatch on + * success, %NULL on error. The record batch data is located on GPU. + * + * Since: 0.8.0 + */ +GArrowRecordBatch * +garrow_gpu_cuda_buffer_read_record_batch(GArrowGPUCUDABuffer *buffer, + GArrowSchema *schema, + GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_schema = garrow_schema_get_raw(schema); + auto pool = arrow::default_memory_pool(); + std::shared_ptr arrow_record_batch; + auto status = arrow::gpu::ReadRecordBatch(arrow_schema, + arrow_buffer, + pool, + &arrow_record_batch); + if (garrow_error_check(error, status, + "[gpu][cuda][buffer][read-record-batch]")) { + return garrow_record_batch_new_raw(&arrow_record_batch); + } else { + return NULL; + } +} + + +G_DEFINE_TYPE(GArrowGPUCUDAHostBuffer, + garrow_gpu_cuda_host_buffer, + GARROW_TYPE_MUTABLE_BUFFER) + +static void +garrow_gpu_cuda_host_buffer_init(GArrowGPUCUDAHostBuffer *object) +{ +} + +static void +garrow_gpu_cuda_host_buffer_class_init(GArrowGPUCUDAHostBufferClass *klass) +{ +} + +/** + * garrow_gpu_cuda_host_buffer_new: + * @size: The number of bytes to be allocated on CPU host. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: A newly created #GArrowGPUCUDAHostBuffer on success, + * %NULL on error. The allocated memory is accessible from GPU + * device for the @context. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new(gint64 size, GError **error) +{ + arrow::gpu::CudaDeviceManager *manager; + auto status = arrow::gpu::CudaDeviceManager::GetInstance(&manager); + std::shared_ptr arrow_buffer; + status = manager->AllocateHost(size, &arrow_buffer); + if (garrow_error_check(error, status, "[gpu][cuda][host-buffer][new]")) { + return garrow_gpu_cuda_host_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + + +typedef struct GArrowGPUCUDAIPCMemoryHandlePrivate_ { + arrow::gpu::CudaIpcMemHandle *ipc_memory_handle; +} GArrowGPUCUDAIPCMemoryHandlePrivate; + +enum { + PROP_IPC_MEMORY_HANDLE = 1 +}; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowGPUCUDAIPCMemoryHandle, + garrow_gpu_cuda_ipc_memory_handle, + G_TYPE_OBJECT) + +#define GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object) \ + static_cast( \ + garrow_gpu_cuda_ipc_memory_handle_get_instance_private( \ + GARROW_GPU_CUDA_IPC_MEMORY_HANDLE(object))) + +static void +garrow_gpu_cuda_ipc_memory_handle_finalize(GObject *object) +{ + auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object); + + delete priv->ipc_memory_handle; + + G_OBJECT_CLASS(garrow_gpu_cuda_ipc_memory_handle_parent_class)->finalize(object); +} + +static void +garrow_gpu_cuda_ipc_memory_handle_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_IPC_MEMORY_HANDLE: + priv->ipc_memory_handle = + static_cast(g_value_get_pointer(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_ipc_memory_handle_get_property(GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_ipc_memory_handle_init(GArrowGPUCUDAIPCMemoryHandle *object) +{ +} + +static void +garrow_gpu_cuda_ipc_memory_handle_class_init(GArrowGPUCUDAIPCMemoryHandleClass *klass) +{ + GParamSpec *spec; + + auto gobject_class = G_OBJECT_CLASS(klass); + + gobject_class->finalize = garrow_gpu_cuda_ipc_memory_handle_finalize; + gobject_class->set_property = garrow_gpu_cuda_ipc_memory_handle_set_property; + gobject_class->get_property = garrow_gpu_cuda_ipc_memory_handle_get_property; + + /** + * GArrowGPUCUDAIPCMemoryHandle:ipc-memory-handle: + * + * Since: 0.8.0 + */ + spec = g_param_spec_pointer("ipc-memory-handle", + "IPC Memory Handle", + "The raw arrow::gpu::CudaIpcMemHandle *", + static_cast(G_PARAM_WRITABLE | + G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_IPC_MEMORY_HANDLE, spec); +} + +/** + * garrow_gpu_cuda_ipc_memory_handle_new: + * @data: (array length=size): A serialized #GArrowGPUCUDAIPCMemoryHandle. + * @size: The size of data. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDAIPCMemoryHandle + * on success, %NULL on error. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new(const guint8 *data, + gsize size, + GError **error) +{ + std::unique_ptr arrow_handle; + auto status = arrow::gpu::CudaIpcMemHandle::FromBuffer(data, &arrow_handle); + if (garrow_error_check(error, status, + "[gpu][cuda][ipc-memory-handle][new]")) { + return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release()); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_ipc_memory_handle_serialize: + * @handle: A #GArrowGPUCUDAIPCMemoryHandle. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowBuffer on success, + * %NULL on error. The buffer has serialized @handle. The serialized + * @handle can be deserialized by garrow_gpu_cuda_ipc_memory_handle_new() + * in other process. + * + * Since: 0.8.0 + */ +GArrowBuffer * +garrow_gpu_cuda_ipc_memory_handle_serialize(GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error) +{ + auto arrow_handle = garrow_gpu_cuda_ipc_memory_handle_get_raw(handle); + std::shared_ptr arrow_buffer; + auto status = arrow_handle->Serialize(arrow::default_memory_pool(), + &arrow_buffer); + if (garrow_error_check(error, status, + "[gpu][cuda][ipc-memory-handle][serialize]")) { + return garrow_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +GArrowBuffer * +garrow_gpu_cuda_buffer_input_stream_new_raw_readable_interface(std::shared_ptr *arrow_buffer) +{ + auto buffer = GARROW_BUFFER(g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER, + "buffer", arrow_buffer, + NULL)); + return buffer; +} + +static std::shared_ptr +garrow_gpu_cuda_buffer_input_stream_get_raw_readable_interface(GArrowReadable *readable) +{ + auto input_stream = GARROW_INPUT_STREAM(readable); + auto arrow_input_stream = garrow_input_stream_get_raw(input_stream); + return arrow_input_stream; +} + +static void +garrow_gpu_cuda_buffer_input_stream_readable_interface_init(GArrowReadableInterface *iface) +{ + iface->new_raw = + garrow_gpu_cuda_buffer_input_stream_new_raw_readable_interface; + iface->get_raw = + garrow_gpu_cuda_buffer_input_stream_get_raw_readable_interface; +} + +G_DEFINE_TYPE_WITH_CODE( + GArrowGPUCUDABufferInputStream, + garrow_gpu_cuda_buffer_input_stream, + GARROW_TYPE_BUFFER_INPUT_STREAM, + G_IMPLEMENT_INTERFACE( + GARROW_TYPE_READABLE, + garrow_gpu_cuda_buffer_input_stream_readable_interface_init)) + +static void +garrow_gpu_cuda_buffer_input_stream_init(GArrowGPUCUDABufferInputStream *object) +{ +} + +static void +garrow_gpu_cuda_buffer_input_stream_class_init(GArrowGPUCUDABufferInputStreamClass *klass) +{ +} + +/** + * garrow_gpu_cuda_buffer_input_stream_new: + * @buffer: A #GArrowGPUCUDABuffer. + * + * Returns: (transfer full): A newly created + * #GArrowGPUCUDABufferInputStream. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new(GArrowGPUCUDABuffer *buffer) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_reader = + std::make_shared(arrow_buffer); + return garrow_gpu_cuda_buffer_input_stream_new_raw(&arrow_reader); +} + + +G_DEFINE_TYPE(GArrowGPUCUDABufferOutputStream, + garrow_gpu_cuda_buffer_output_stream, + GARROW_TYPE_OUTPUT_STREAM) + +static void +garrow_gpu_cuda_buffer_output_stream_init(GArrowGPUCUDABufferOutputStream *object) +{ +} + +static void +garrow_gpu_cuda_buffer_output_stream_class_init(GArrowGPUCUDABufferOutputStreamClass *klass) +{ +} + +/** + * garrow_gpu_cuda_buffer_output_stream_new: + * @buffer: A #GArrowGPUCUDABuffer. + * + * Returns: (transfer full): A newly created + * #GArrowGPUCUDABufferOutputStream. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new(GArrowGPUCUDABuffer *buffer) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_writer = + std::make_shared(arrow_buffer); + return garrow_gpu_cuda_buffer_output_stream_new_raw(&arrow_writer); +} + +/** + * garrow_gpu_cuda_buffer_output_stream_set_buffer_size: + * @stream: A #GArrowGPUCUDABufferOutputStream. + * @size: A size of CPU buffer in bytes. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Sets CPU buffer size. to limit `cudaMemcpy()` calls. If CPU buffer + * size is `0`, buffering is disabled. + * + * The default is `0`. + * + * Since: 0.8.0 + */ +gboolean +garrow_gpu_cuda_buffer_output_stream_set_buffer_size(GArrowGPUCUDABufferOutputStream *stream, + gint64 size, + GError **error) +{ + auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream); + auto status = arrow_stream->SetBufferSize(size); + return garrow_error_check(error, + status, + "[gpu][cuda][buffer-output-stream][set-buffer-size]"); +} + +/** + * garrow_gpu_cuda_buffer_output_stream_get_buffer_size: + * @stream: A #GArrowGPUCUDABufferOutputStream. + * + * Returns: The CPU buffer size in bytes. + * + * See garrow_gpu_cuda_buffer_output_stream_set_buffer_size() for CPU + * buffer size details. + * + * Since: 0.8.0 + */ +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffer_size(GArrowGPUCUDABufferOutputStream *stream) +{ + auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream); + return arrow_stream->buffer_size(); +} + +/** + * garrow_gpu_cuda_buffer_output_stream_get_buffered_size: + * @stream: A #GArrowGPUCUDABufferOutputStream. + * + * Returns: The size of buffered data in bytes. + * + * Since: 0.8.0 + */ +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffered_size(GArrowGPUCUDABufferOutputStream *stream) +{ + auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream); + return arrow_stream->num_bytes_buffered(); +} + + +G_END_DECLS + +GArrowGPUCUDAContext * +garrow_gpu_cuda_context_new_raw(std::shared_ptr *arrow_context) +{ + return GARROW_GPU_CUDA_CONTEXT(g_object_new(GARROW_GPU_TYPE_CUDA_CONTEXT, + "context", arrow_context, + NULL)); +} + +std::shared_ptr +garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context) +{ + if (!context) + return nullptr; + + auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(context); + return priv->context; +} + +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle) +{ + auto handle = g_object_new(GARROW_GPU_TYPE_CUDA_IPC_MEMORY_HANDLE, + "ipc-memory-handle", arrow_handle, + NULL); + return GARROW_GPU_CUDA_IPC_MEMORY_HANDLE(handle); +} + +arrow::gpu::CudaIpcMemHandle * +garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle) +{ + if (!handle) + return nullptr; + + auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(handle); + return priv->ipc_memory_handle; +} + +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_raw(std::shared_ptr *arrow_buffer) +{ + return GARROW_GPU_CUDA_BUFFER(g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER, + "buffer", arrow_buffer, + NULL)); +} + +std::shared_ptr +garrow_gpu_cuda_buffer_get_raw(GArrowGPUCUDABuffer *buffer) +{ + if (!buffer) + return nullptr; + + auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer)); + return std::static_pointer_cast(arrow_buffer); +} + +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new_raw(std::shared_ptr *arrow_buffer) +{ + auto buffer = g_object_new(GARROW_GPU_TYPE_CUDA_HOST_BUFFER, + "buffer", arrow_buffer, + NULL); + return GARROW_GPU_CUDA_HOST_BUFFER(buffer); +} + +std::shared_ptr +garrow_gpu_cuda_host_buffer_get_raw(GArrowGPUCUDAHostBuffer *buffer) +{ + if (!buffer) + return nullptr; + + auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer)); + return std::static_pointer_cast(arrow_buffer); +} + +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new_raw(std::shared_ptr *arrow_reader) +{ + auto input_stream = g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER_INPUT_STREAM, + "input-stream", arrow_reader, + NULL); + return GARROW_GPU_CUDA_BUFFER_INPUT_STREAM(input_stream); +} + +std::shared_ptr +garrow_gpu_cuda_buffer_input_stream_get_raw(GArrowGPUCUDABufferInputStream *input_stream) +{ + if (!input_stream) + return nullptr; + + auto arrow_reader = + garrow_input_stream_get_raw(GARROW_INPUT_STREAM(input_stream)); + return std::static_pointer_cast(arrow_reader); +} + +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new_raw(std::shared_ptr *arrow_writer) +{ + auto output_stream = g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER_OUTPUT_STREAM, + "output-stream", arrow_writer, + NULL); + return GARROW_GPU_CUDA_BUFFER_OUTPUT_STREAM(output_stream); +} + +std::shared_ptr +garrow_gpu_cuda_buffer_output_stream_get_raw(GArrowGPUCUDABufferOutputStream *output_stream) +{ + if (!output_stream) + return nullptr; + + auto arrow_writer = + garrow_output_stream_get_raw(GARROW_OUTPUT_STREAM(output_stream)); + return std::static_pointer_cast(arrow_writer); +} diff --git a/c_glib/arrow-gpu-glib/cuda.h b/c_glib/arrow-gpu-glib/cuda.h new file mode 100644 index 0000000000000..7c615a144e739 --- /dev/null +++ b/c_glib/arrow-gpu-glib/cuda.h @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +G_BEGIN_DECLS + +#define GARROW_GPU_TYPE_CUDA_DEVICE_MANAGER \ + (garrow_gpu_cuda_device_manager_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDADeviceManager, + garrow_gpu_cuda_device_manager, + GARROW_GPU, + CUDA_DEVICE_MANAGER, + GObject) +struct _GArrowGPUCUDADeviceManagerClass +{ + GObjectClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_CONTEXT (garrow_gpu_cuda_context_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAContext, + garrow_gpu_cuda_context, + GARROW_GPU, + CUDA_CONTEXT, + GObject) +struct _GArrowGPUCUDAContextClass +{ + GObjectClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_BUFFER (garrow_gpu_cuda_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABuffer, + garrow_gpu_cuda_buffer, + GARROW_GPU, + CUDA_BUFFER, + GArrowBuffer) +struct _GArrowGPUCUDABufferClass +{ + GArrowBufferClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_HOST_BUFFER (garrow_gpu_cuda_host_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAHostBuffer, + garrow_gpu_cuda_host_buffer, + GARROW_GPU, + CUDA_HOST_BUFFER, + GArrowMutableBuffer) +struct _GArrowGPUCUDAHostBufferClass +{ + GArrowMutableBufferClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_IPC_MEMORY_HANDLE \ + (garrow_gpu_cuda_ipc_memory_handle_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAIPCMemoryHandle, + garrow_gpu_cuda_ipc_memory_handle, + GARROW_GPU, + CUDA_IPC_MEMORY_HANDLE, + GObject) +struct _GArrowGPUCUDAIPCMemoryHandleClass +{ + GObjectClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_BUFFER_INPUT_STREAM \ + (garrow_gpu_cuda_buffer_input_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABufferInputStream, + garrow_gpu_cuda_buffer_input_stream, + GARROW_GPU, + CUDA_BUFFER_INPUT_STREAM, + GArrowBufferInputStream) +struct _GArrowGPUCUDABufferInputStreamClass +{ + GArrowBufferInputStreamClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_BUFFER_OUTPUT_STREAM \ + (garrow_gpu_cuda_buffer_output_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABufferOutputStream, + garrow_gpu_cuda_buffer_output_stream, + GARROW_GPU, + CUDA_BUFFER_OUTPUT_STREAM, + GArrowOutputStream) +struct _GArrowGPUCUDABufferOutputStreamClass +{ + GArrowOutputStreamClass parent_class; +}; + +GArrowGPUCUDADeviceManager * +garrow_gpu_cuda_device_manager_new(GError **error); + +GArrowGPUCUDAContext * +garrow_gpu_cuda_device_manager_get_context(GArrowGPUCUDADeviceManager *manager, + gint gpu_number, + GError **error); +gsize +garrow_gpu_cuda_device_manager_get_n_devices(GArrowGPUCUDADeviceManager *manager); + +gint64 +garrow_gpu_cuda_context_get_allocated_size(GArrowGPUCUDAContext *context); + + +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new(GArrowGPUCUDAContext *context, + gint64 size, + GError **error); +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_ipc(GArrowGPUCUDAContext *context, + GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error); +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_record_batch(GArrowGPUCUDAContext *context, + GArrowRecordBatch *record_batch, + GError **error); +GBytes * +garrow_gpu_cuda_buffer_copy_to_host(GArrowGPUCUDABuffer *buffer, + gint64 position, + gint64 size, + GError **error); +gboolean +garrow_gpu_cuda_buffer_copy_from_host(GArrowGPUCUDABuffer *buffer, + const guint8 *data, + gint64 size, + GError **error); +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_buffer_export(GArrowGPUCUDABuffer *buffer, + GError **error); +GArrowGPUCUDAContext * +garrow_gpu_cuda_buffer_get_context(GArrowGPUCUDABuffer *buffer); +GArrowRecordBatch * +garrow_gpu_cuda_buffer_read_record_batch(GArrowGPUCUDABuffer *buffer, + GArrowSchema *schema, + GError **error); + + +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new(gint64 size, GError **error); + +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new(const guint8 *data, + gsize size, + GError **error); + +GArrowBuffer * +garrow_gpu_cuda_ipc_memory_handle_serialize(GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error); + +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new(GArrowGPUCUDABuffer *buffer); + +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new(GArrowGPUCUDABuffer *buffer); + +gboolean +garrow_gpu_cuda_buffer_output_stream_set_buffer_size(GArrowGPUCUDABufferOutputStream *stream, + gint64 size, + GError **error); +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffer_size(GArrowGPUCUDABufferOutputStream *stream); +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffered_size(GArrowGPUCUDABufferOutputStream *stream); + +G_END_DECLS diff --git a/c_glib/arrow-gpu-glib/cuda.hpp b/c_glib/arrow-gpu-glib/cuda.hpp new file mode 100644 index 0000000000000..3eeff8b6f18ed --- /dev/null +++ b/c_glib/arrow-gpu-glib/cuda.hpp @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include + +GArrowGPUCUDAContext * +garrow_gpu_cuda_context_new_raw(std::shared_ptr *arrow_context); +std::shared_ptr +garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context); + +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle); +arrow::gpu::CudaIpcMemHandle * +garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle); + +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_raw(std::shared_ptr *arrow_buffer); +std::shared_ptr +garrow_gpu_cuda_buffer_get_raw(GArrowGPUCUDABuffer *buffer); + +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new_raw(std::shared_ptr *arrow_buffer); +std::shared_ptr +garrow_gpu_cuda_host_buffer_get_raw(GArrowGPUCUDAHostBuffer *buffer); + +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new_raw(std::shared_ptr *arrow_reader); +std::shared_ptr +garrow_gpu_cuda_buffer_input_stream_get_raw(GArrowGPUCUDABufferInputStream *input_stream); + +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new_raw(std::shared_ptr *arrow_writer); +std::shared_ptr +garrow_gpu_cuda_buffer_output_stream_get_raw(GArrowGPUCUDABufferOutputStream *output_stream); diff --git a/c_glib/arrow-gpu-glib/meson.build b/c_glib/arrow-gpu-glib/meson.build new file mode 100644 index 0000000000000..00c7f079d6485 --- /dev/null +++ b/c_glib/arrow-gpu-glib/meson.build @@ -0,0 +1,80 @@ +# -*- indent-tabs-mode: nil -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +sources = files( + 'cuda.cpp', +) + +c_headers = files( + 'arrow-gpu-glib.h', + 'cuda.h', +) + +cpp_headers = files( + 'arrow-gpu-glib.hpp', + 'cuda.hpp', +) + +headers = c_headers + cpp_headers +install_headers(headers, subdir: 'arrow-gpu-glib') + + +dependencies = [ + arrow_gpu_dependency, + libarrow_glib_dependency, +] +libarrow_gpu_glib = library('arrow-gpu-glib', + sources: sources, + install: true, + dependencies: dependencies, + include_directories: [ + root_inc, + ], + soversion: so_version, + version: library_version) +libarrow_gpu_glib_dependency = declare_dependency(link_with: libarrow_gpu_glib, + include_directories: [ + root_inc, + ], + dependencies: dependencies) + +pkgconfig.generate(filebase: 'arrow-gpu-glib', + name: 'Apache Arrow GPU GLib', + description: 'C API for Apache Arrow GPU based on GLib', + version: version, + requires: ['arrow-glib', 'arrow-gpu'], + libraries: [libarrow_gpu_glib], + subdirs: ['arrow-gpu-glib']) + +gnome.generate_gir(libarrow_gpu_glib, + dependencies: arrow_glib_gir_dependency, + sources: sources + c_headers, + namespace: 'ArrowGPU', + nsversion: api_version, + identifier_prefix: 'GArrowGPU', + symbol_prefix: 'garrow_gpu', + export_packages: 'arrow-gpu-glib', + includes: [ + 'Arrow-1.0', + ], + install: true, + extra_args: [ + '--warn-all', + '--include-uninstalled=./arrow-glib/Arrow-1.0.gir', + ]) diff --git a/c_glib/configure.ac b/c_glib/configure.ac index 5db435275a300..c6fa0192c944d 100644 --- a/c_glib/configure.ac +++ b/c_glib/configure.ac @@ -77,18 +77,34 @@ AC_ARG_WITH(arrow-cpp-build-dir, [GARROW_ARROW_CPP_BUILD_DIR=""]) if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then PKG_CHECK_MODULES([ARROW], [arrow arrow-compute]) + PKG_CHECK_MODULES([ARROW_GPU], + [arrow-gpu], + [HAVE_ARROW_GPU=yes], + [HAVE_ARROW_GPU=no]) else ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src" ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}" ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}" - ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow" - - AC_SUBST(ARROW_LIB_DIR) - AC_SUBST(ARROW_CFLAGS) AC_SUBST(ARROW_LIBS) + + ARROW_GPU_CFLAGS="" + if test -f "${GARROW_ARROW_CPP_BUILD_DIR}/src/arrow/gpu/arrow-gpu.pc"; then + HAVE_ARROW_GPU=yes + ARROW_GPU_LIBS="-larrow_gpu" + else + HAVE_ARROW_GPU=no + ARROW_GPU_LIBS="" + fi + AC_SUBST(ARROW_GPU_CFLAGS) + AC_SUBST(ARROW_GPU_LIBS) +fi + +AM_CONDITIONAL([HAVE_ARROW_GPU], [test "$HAVE_ARROW_GPU" = "yes"]) +if test "$HAVE_ARROW_GPU" = "yes"; then + AC_DEFINE(HAVE_ARROW_GPU, [1], [Define to 1 if Apache Arrow supports GPU.]) fi exampledir="\$(datadir)/arrow-glib/example" @@ -98,6 +114,8 @@ AC_CONFIG_FILES([ Makefile arrow-glib/Makefile arrow-glib/arrow-glib.pc + arrow-gpu-glib/Makefile + arrow-gpu-glib/arrow-gpu-glib.pc doc/Makefile doc/reference/Makefile doc/reference/xml/Makefile diff --git a/c_glib/doc/Makefile.am b/c_glib/doc/Makefile.am index 85c1d5126097c..1d491ab09110e 100644 --- a/c_glib/doc/Makefile.am +++ b/c_glib/doc/Makefile.am @@ -16,4 +16,4 @@ # under the License. SUBDIRS = \ - reference + reference diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am index 45b11f035183e..896aff544d454 100644 --- a/c_glib/doc/reference/Makefile.am +++ b/c_glib/doc/reference/Makefile.am @@ -51,6 +51,17 @@ AM_CFLAGS = \ GTKDOC_LIBS = \ $(top_builddir)/arrow-glib/libarrow-glib.la +if HAVE_ARROW_GPU +DOC_SOURCE_DIR += \ + $(top_srcdir)/arrow-gpu-glib +HFILE_GLOB += \ + $(top_srcdir)/arrow-gpu-glib/*.h +CFILE_GLOB += \ + $(top_srcdir)/arrow-gpu-glib/*.cpp +GTKDOC_LIBS += \ + $(top_builddir)/arrow-gpu-glib/libarrow-gpu-glib.la +endif + include $(srcdir)/gtk-doc.make CLEANFILES += \ diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml index a504ef1148383..e267ea2f9d356 100644 --- a/c_glib/doc/reference/arrow-glib-docs.sgml +++ b/c_glib/doc/reference/arrow-glib-docs.sgml @@ -125,6 +125,16 @@ + + Object Hierarchy diff --git a/c_glib/doc/reference/meson.build b/c_glib/doc/reference/meson.build index 08936daf87288..4c9552e83c303 100644 --- a/c_glib/doc/reference/meson.build +++ b/c_glib/doc/reference/meson.build @@ -32,13 +32,26 @@ glib_prefix = dependency('glib-2.0').get_pkgconfig_variable('prefix') glib_doc_path = join_paths(glib_prefix, 'share', 'gtk-doc', 'html') doc_path = join_paths(data_dir, meson.project_name(), 'gtk-doc', 'html') +source_directories = [ + join_paths(meson.source_root(), 'arrow-glib'), + join_paths(meson.build_root(), 'arrow-glib'), +] +dependencies = [ + libarrow_glib_dependency, +] +if arrow_gpu_dependency.found() + source_directories += [ + join_paths(meson.source_root(), 'arrow-gpu-glib'), + join_paths(meson.build_root(), 'arrow-gpu-glib'), + ] + dependencies += [ + libarrow_gpu_glib_dependency, + ] +endif gnome.gtkdoc(meson.project_name(), main_xml: meson.project_name() + '-docs.sgml', - src_dir: [ - join_paths(meson.source_root(), 'arrow-glib'), - join_paths(meson.build_root(), 'arrow-glib'), - ], - dependencies: libarrow_glib_dependency, + src_dir: source_directories, + dependencies: dependencies, gobject_typesfile: meson.project_name() + '.types', scan_args: [ '--rebuild-types', diff --git a/c_glib/meson.build b/c_glib/meson.build index 1fa64ba19c406..9fe1b8cbd7179 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -49,6 +49,10 @@ pkgconfig = import('pkgconfig') root_inc = include_directories('.') subdir('arrow-glib') +arrow_gpu_dependency = dependency('arrow-gpu', required: false) +if arrow_gpu_dependency.found() + subdir('arrow-gpu-glib') +endif subdir('example') if get_option('enable_gtk_doc') @@ -58,4 +62,7 @@ endif run_test = find_program('test/run-test.sh') test('unit test', run_test, - env: ['ARROW_GLIB_TYPELIB_DIR=@0@/arrow-glib'.format(meson.build_root())]) + env: [ + 'ARROW_GLIB_TYPELIB_DIR=@0@/arrow-glib'.format(meson.build_root()), + 'ARROW_GPU_GLIB_TYPELIB_DIR=@0@/arrow-gpu-glib'.format(meson.build_root()), + ]) diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb index 3451bd29fde1b..392c56f33ae51 100755 --- a/c_glib/test/run-test.rb +++ b/c_glib/test/run-test.rb @@ -37,6 +37,12 @@ def initialize(data) end end +begin + ArrowGPU = GI.load("ArrowGPU") +rescue GObjectIntrospection::RepositoryError::TypelibNotFound +end + +require "rbconfig" require "tempfile" require_relative "helper/buildable" require_relative "helper/omittable" diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh index 19ccf077833a5..d563e8586ce59 100755 --- a/c_glib/test/run-test.sh +++ b/c_glib/test/run-test.sh @@ -20,27 +20,34 @@ test_dir="$(cd $(dirname $0); pwd)" build_dir="$(cd .; pwd)" -arrow_glib_build_dir="${build_dir}/arrow-glib/" -libtool_dir="${arrow_glib_build_dir}/.libs" -if [ -d "${libtool_dir}" ]; then - LD_LIBRARY_PATH="${libtool_dir}:${LD_LIBRARY_PATH}" -else - if [ -d "${arrow_glib_build_dir}" ]; then - LD_LIBRARY_PATH="${arrow_glib_build_dir}:${LD_LIBRARY_PATH}" +modules="arrow-glib arrow-gpu-glib" + +for module in ${modules}; do + module_build_dir="${build_dir}/${module}" + libtool_dir="${module_build_dir}/.libs" + if [ -d "${libtool_dir}" ]; then + LD_LIBRARY_PATH="${libtool_dir}:${LD_LIBRARY_PATH}" + else + if [ -d "${module_build_dir}" ]; then + LD_LIBRARY_PATH="${module_build_dir}:${LD_LIBRARY_PATH}" + fi fi -fi +done if [ -f "Makefile" -a "${NO_MAKE}" != "yes" ]; then make -j8 > /dev/null || exit $? fi -arrow_glib_typelib_dir="${ARROW_GLIB_TYPELIB_DIR}" -if [ -z "${arrow_glib_typelib_dir}" ]; then - arrow_glib_typelib_dir="${build_dir}/arrow-glib" -fi +for module in ${modules}; do + MODULE_TYPELIB_DIR_VAR_NAME="$(echo ${module} | tr a-z- A-Z_)_TYPELIB_DIR" + module_typelib_dir=$(eval "echo \${${MODULE_TYPELIB_DIR_VAR_NAME}}") + if [ -z "${module_typelib_dir}" ]; then + module_typelib_dir="${build_dir}/${module}" + fi -if [ -d "${arrow_glib_typelib_dir}" ]; then - GI_TYPELIB_PATH="${arrow_glib_typelib_dir}:${GI_TYPELIB_PATH}" -fi + if [ -d "${module_typelib_dir}" ]; then + GI_TYPELIB_PATH="${module_typelib_dir}:${GI_TYPELIB_PATH}" + fi +done ${GDB} ruby ${test_dir}/run-test.rb "$@" diff --git a/c_glib/test/test-gpu-cuda.rb b/c_glib/test/test-gpu-cuda.rb new file mode 100644 index 0000000000000..c710ef2264976 --- /dev/null +++ b/c_glib/test/test-gpu-cuda.rb @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestGPUCUDA < Test::Unit::TestCase + include Helper::Buildable + + def setup + omit("Arrow GPU is required") unless defined?(::ArrowGPU) + @manager = ArrowGPU::CUDADeviceManager.new + omit("At least one GPU is required") if @manager.n_devices.zero? + @context = @manager.get_context(0) + end + + sub_test_case("Context") do + def test_allocated_size + allocated_size_before = @context.allocated_size + size = 128 + buffer = ArrowGPU::CUDABuffer.new(@context, size) + assert_equal(size, + @context.allocated_size - allocated_size_before) + end + end + + sub_test_case("Buffer") do + def setup + super + @buffer = ArrowGPU::CUDABuffer.new(@context, 128) + end + + def test_copy + @buffer.copy_from_host("Hello World") + assert_equal("llo W", @buffer.copy_to_host(2, 5).to_s) + end + + def test_export + @buffer.copy_from_host("Hello World") + handle = @buffer.export + serialized_handle = handle.serialize.data + Tempfile.open("arrow-gpu-cuda-export") do |output| + pid = spawn(RbConfig.ruby, "-e", <<-SCRIPT) +require "gi" + +Gio = GI.load("Gio") +Arrow = GI.load("Arrow") +ArrowGPU = GI.load("ArrowGPU") + +manager = ArrowGPU::CUDADeviceManager.new +context = manager.get_context(0) +serialized_handle = #{serialized_handle.to_s.dump} +handle = ArrowGPU::CUDAIPCMemoryHandle.new(serialized_handle) +buffer = ArrowGPU::CUDABuffer.new(context, handle) +File.open(#{output.path.dump}, "w") do |output| + output.print(buffer.copy_to_host(0, 6).to_s) +end + SCRIPT + Process.waitpid(pid) + assert_equal("Hello ", output.read) + end + end + + def test_context + assert_equal(@context.allocated_size, + @buffer.context.allocated_size) + end + + def test_record_batch + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + columns = [ + build_boolean_array([true]), + ] + cpu_record_batch = Arrow::RecordBatch.new(schema, 1, columns) + + buffer = ArrowGPU::CUDABuffer.new(@context, cpu_record_batch) + gpu_record_batch = buffer.read_record_batch(schema) + assert_equal(cpu_record_batch.n_rows, + gpu_record_batch.n_rows) + end + end + + sub_test_case("HostBuffer") do + def test_new + buffer = ArrowGPU::CUDAHostBuffer.new(128) + assert_equal(128, buffer.size) + end + end + + sub_test_case("BufferInputStream") do + def test_new + buffer = ArrowGPU::CUDABuffer.new(@context, 128) + buffer.copy_from_host("Hello World") + stream = ArrowGPU::CUDABufferInputStream.new(buffer) + begin + assert_equal("Hello Worl", stream.read(5).copy_to_host(0, 10).to_s) + ensure + stream.close + end + end + end + + sub_test_case("BufferOutputStream") do + def setup + super + @buffer = ArrowGPU::CUDABuffer.new(@context, 128) + @buffer.copy_from_host("\x00" * @buffer.size) + @stream = ArrowGPU::CUDABufferOutputStream.new(@buffer) + end + + def cleanup + super + @stream.close + end + + def test_new + @stream.write("Hello World") + assert_equal("Hello World", @buffer.copy_to_host(0, 11).to_s) + end + + def test_buffer + assert_equal(0, @stream.buffer_size) + @stream.buffer_size = 5 + assert_equal(5, @stream.buffer_size) + @stream.write("Hell") + assert_equal(4, @stream.buffered_size) + assert_equal("\x00" * 5, @buffer.copy_to_host(0, 5).to_s) + @stream.write("o") + assert_equal("Hello", @buffer.copy_to_host(0, 5).to_s) + end + end +end From b18bbeba0ed8b76f6575bd0501de3beb1e6e60f5 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Mon, 13 Nov 2017 22:56:13 -0500 Subject: [PATCH 042/177] ARROW-1371: [Website] Add "Powered By" page to the website Author: Korn, Uwe Author: Wes McKinney Author: Uwe L. Korn Closes #1280 from xhochy/ARROW-1371 and squashes the following commits: 84a76e94 [Uwe L. Korn] Add Red Data Tools 57338daa [Wes McKinney] Formatting 522a3592 [Wes McKinney] Add a few more projects ad511ed2 [Korn, Uwe] Add GeoMesa and Dremio 0e4f755c [Korn, Uwe] Add Ray and Spark ecabb167 [Korn, Uwe] ARROW-1371: [Website] Add "Powered By" page to the website --- site/_config.yml | 1 + site/_includes/header.html | 1 + site/powered_by.md | 126 +++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 site/powered_by.md diff --git a/site/_config.yml b/site/_config.yml index a6c5575d0680f..cbcf97dd3b0d1 100644 --- a/site/_config.yml +++ b/site/_config.yml @@ -30,6 +30,7 @@ exclude: - Gemfile - Gemfile.lock - _docs/format/* + - ruby - asf-site - scripts - README.md diff --git a/site/_includes/header.html b/site/_includes/header.html index 6c0ec30f39ca7..03b3c8750cdb8 100644 --- a/site/_includes/header.html +++ b/site/_includes/header.html @@ -27,6 +27,7 @@
  • Mailing List
  • Slack Channel
  • Committers
    • +
  • Powered By
  • diff --git a/site/powered_by.md b/site/powered_by.md new file mode 100644 index 0000000000000..38320c0ec91b0 --- /dev/null +++ b/site/powered_by.md @@ -0,0 +1,126 @@ +--- +layout: default +--- + + +## Powered By + +### Project and Product names using "Apache Arrow" + +Organizations creating products and projects for use with Apache Arrow, along +with associated marketing materials, should take care to respect the trademark +in "Apache Arrow" and its logo. Please refer to [ASF Trademarks Guidance][1] +and associated [FAQ][2] for comprehensive and authoritative guidance on proper +usage of ASF trademarks. + +Names that do not include "Apache Arrow" at all have no potential trademark +issue with the Apache Arrow project. This is recommended. + +Names like "Apache Arrow BigCoProduct" are not OK, as are names including +"Apache Arrow" in general. The above links, however, describe some exceptions, +like for names such as "BigCoProduct, powered by Apache Arrow" or +"BigCoProduct for Apache Arrow". + +It is common practice to create software identifiers (Maven coordinates, module +names, etc.) like "arrow-foo". These are permitted. Nominative use of trademarks +in descriptions is also always allowed, as in "BigCoProduct is a widget for +Apache Arrow". + +### Open Source Projects + +To add yourself to the list, please email dev@arrow.apache.org with your +organization name, URL, a list of which Arrow components you are using, and a +short description of your use case. + +* **[Apache Parquet][3]:** A columnar storage format available to any project + in the Hadoop ecosystem, regardless of the choice of data processing + framework, data model or programming language. The C++ and Java + implementation provide vectorized reads and write to/from Arrow data + structures. +* **[Apache Spark][7]:** Apache Spark™ is a fast and general engine for + large-scale data processing. Spark uses Apache Arrow to + 1. improve performance of conversion between Spark DataFrame and pandas DataFrame + 2. enable a set of vectorized user-defined functions (`pandas_udf`) in PySpark. +* **[Dask][15]:** Python library for parallel and distributed execution of + dynamic task graphs. Dask supports using pyarrow for accessing Parquet + files +* **[GeoMesa][8]:** A suite of tools that enables large-scale geospatial query + and analytics on distributed computing systems. GeoMesa supports query + results in the Arrow IPC format, which can then be used for in-browser + visualizations and/or further analytics. +* **[libgdf][14]:** A C library of CUDA-based analytics functions and GPU IPC + support for structured data. Uses the Arrow IPC format and targets the Arrow + memory layout in its analytic functions. This work is part of the [GPU Open + Analytics Initiative][11] +* **[MapD][10]:** in-memory columnar SQL engine designed to run on GPUs. MapD + supports Arrow for data ingest and data interchange via CUDA IPC + handles. This work is part of the [GPU Open Analytics Initiative][11] +* **[pandas][12]:** data analysis toolkit for Python programmers. pandas + supports reading and writing Parquet files using pyarrow. Several pandas + core developers are also contributors to Apache Arrow. +* **[Ray][5]:** A flexible, high-performance distributed execution framework + with a focus on machine learning and AI applications. Uses Arrow to + efficiently store Python data structures containing large arrays of numerical + data. Data can be accessed with zero-copy by multiple processes using the + [Plasma shared memory object store][6] which originated from Ray and is part + of Arrow now. +* **[Red Data Tools][16]:** A project that provides data processing + tools for Ruby. It provides [Red Arrow][17] that is a Ruby bindings + of Apache Arrow based on Apache Arrow GLib. Red Arrow is a core + library for it. It also provides many Ruby libraries to integrate + existing Ruby libraries with Apache Arrow. They use Red Arrow. +* **[Turbodbc][4]:** Python module to access relational databases via the Open + Database Connectivity (ODBC) interface. It provides the ability to return + Arrow Tables and RecordBatches in addition to the Python Database API + Specification 2.0. + +### Companies and Organizations + +To add yourself to the list, please email dev@arrow.apache.org with your +organization name, URL, a list of which Arrow components you are using, and a +short description of your use case. + +* **[Dremio][9]:** A self-service data platform. Dremio makes it easy for + users to discover, curate, accelerate, and share data from any source. + It includes a distributed SQL execution engine based on Apache Arrow. + Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow + buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI, + Python, R, and more (all backed by Apache Arrow). +* **[Quilt Data][13]:** Quilt is a data package manager, designed to make + managing data as easy as managing code. It supports Parquet format via + pyarrow for data access. + +[1]: https://www.apache.org/foundation/marks/ +[2]: https://www.apache.org/foundation/marks/faq/ +[3]: https://parquet.apache.org/ +[4]: https://github.com/blue-yonder/turbodbc +[5]: https://github.com/ray-project/ray +[6]: https://ray-project.github.io/2017/08/08/plasma-in-memory-object-store.html +[7]: https://spark.apache.org/ +[8]: https://github.com/locationtech/geomesa +[9]: https://www.dremio.com/ +[10]: https://github.com/mapd/mapd-core +[11]: https://gpuopenanalytics.com/ +[12]: https://pandas.pydata.org +[13]: https://quiltdata.com/ +[14]: https://github.com/gpuopenanalytics/libgdf +[15]: https://github.com/dask/dask +[16]: https://red-data-tools.github.io/ +[17]: https://github.com/red-data-tools/red-arrow/ From e3db5da648bb74b3d325042e595ae856f1ad8f7f Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Tue, 14 Nov 2017 11:17:59 -0500 Subject: [PATCH 043/177] ARROW-1806: [GLib] Add garrow_record_batch_writer_write_table() Author: Kouhei Sutou Closes #1315 from kou/glib-add-recored-batch-writer-write-table and squashes the following commits: 15743615 [Kouhei Sutou] [GLib] Add garrow_record_batch_writer_write_table() --- c_glib/arrow-glib/writer.cpp | 25 +++++++++++++++++++++++++ c_glib/arrow-glib/writer.h | 4 ++++ c_glib/test/test-file-writer.rb | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/c_glib/arrow-glib/writer.cpp b/c_glib/arrow-glib/writer.cpp index 7d3b59457fd33..9bcda2dbbfd2c 100644 --- a/c_glib/arrow-glib/writer.cpp +++ b/c_glib/arrow-glib/writer.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -165,6 +166,30 @@ garrow_record_batch_writer_write_record_batch(GArrowRecordBatchWriter *writer, "[record-batch-writer][write-record-batch]"); } +/** + * garrow_record_batch_writer_write_table: + * @writer: A #GArrowRecordBatchWriter. + * @table: The table to be written. + * @error: (nullable): Return locatipcn for a #GError or %NULL. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Since: 0.8.0 + */ +gboolean +garrow_record_batch_writer_write_table(GArrowRecordBatchWriter *writer, + GArrowTable *table, + GError **error) +{ + auto arrow_writer = garrow_record_batch_writer_get_raw(writer); + auto arrow_table = garrow_table_get_raw(table); + + auto status = arrow_writer->WriteTable(*arrow_table); + return garrow_error_check(error, + status, + "[record-batch-writer][write-table]"); +} + /** * garrow_record_batch_writer_close: * @writer: A #GArrowRecordBatchWriter. diff --git a/c_glib/arrow-glib/writer.h b/c_glib/arrow-glib/writer.h index 3853c2bd72c50..41b5f723d10eb 100644 --- a/c_glib/arrow-glib/writer.h +++ b/c_glib/arrow-glib/writer.h @@ -77,6 +77,10 @@ gboolean garrow_record_batch_writer_write_record_batch( GArrowRecordBatchWriter *writer, GArrowRecordBatch *record_batch, GError **error); +gboolean garrow_record_batch_writer_write_table( + GArrowRecordBatchWriter *writer, + GArrowTable *table, + GError **error); gboolean garrow_record_batch_writer_close( GArrowRecordBatchWriter *writer, GError **error); diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb index 6ba5c7aebca44..3de8e5cf34b51 100644 --- a/c_glib/test/test-file-writer.rb +++ b/c_glib/test/test-file-writer.rb @@ -16,6 +16,8 @@ # under the License. class TestFileWriter < Test::Unit::TestCase + include Helper::Buildable + def test_write_record_batch tempfile = Tempfile.open("arrow-ipc-file-writer") output = Arrow::FileOutputStream.new(tempfile.path, false) @@ -42,4 +44,35 @@ def test_write_record_batch input.close end end + + def test_write_table + tempfile = Tempfile.open("arrow-ipc-file-writer") + output = Arrow::FileOutputStream.new(tempfile.path, false) + + array = build_boolean_array([true, false, true]) + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + column = Arrow::Column.new(field, array) + + begin + file_writer = Arrow::RecordBatchFileWriter.new(output, schema) + begin + table = Arrow::Table.new(schema, [column]) + file_writer.write_table(table) + ensure + file_writer.close + end + ensure + output.close + end + + input = Arrow::MemoryMappedInputStream.new(tempfile.path) + begin + file_reader = Arrow::RecordBatchFileReader.new(input) + assert_equal(Arrow::RecordBatch.new(schema, array.length, [array]), + file_reader.read_record_batch(0)) + ensure + input.close + end + end end From 9fb806ce2ca0ccdee1b89c510dcfae16996cf243 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Wed, 15 Nov 2017 13:23:51 +0100 Subject: [PATCH 044/177] ARROW-1811: [C++/Python] Rename all Decimal based APIs to Decimal128 Author: Phillip Cloud Closes #1321 from cpcloud/ARROW-1811 and squashes the following commits: 1b0a3b8 [Phillip Cloud] ARROW-1811: [C++/Python] Rename all Decimal based APIs to Decimal128 --- cpp/src/arrow/array.cc | 2 +- cpp/src/arrow/array.h | 2 +- cpp/src/arrow/builder.cc | 9 +++++---- cpp/src/arrow/builder.h | 8 +++++--- cpp/src/arrow/compare.cc | 4 ++-- cpp/src/arrow/ipc/json-internal.cc | 8 ++++---- cpp/src/arrow/ipc/metadata-internal.cc | 2 +- cpp/src/arrow/python/arrow_to_pandas.cc | 4 ++-- cpp/src/arrow/python/builtin_convert.cc | 2 +- cpp/src/arrow/python/numpy_to_arrow.cc | 6 +++--- cpp/src/arrow/type.cc | 6 +++--- cpp/src/arrow/type.h | 6 ++++-- cpp/src/arrow/type_fwd.h | 4 ++-- cpp/src/arrow/type_traits.h | 4 ++-- cpp/src/arrow/visitor.h | 2 +- cpp/src/arrow/visitor_inline.h | 2 +- 16 files changed, 38 insertions(+), 33 deletions(-) diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index 651fa26ba9026..f2dd75335df8b 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -320,7 +320,7 @@ Decimal128Array::Decimal128Array(const std::shared_ptr& data) } std::string Decimal128Array::FormatValue(int64_t i) const { - const auto& type_ = static_cast(*type()); + const auto& type_ = static_cast(*type()); const Decimal128 value(GetValue(i)); return value.ToString(type_.scale()); } diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index 3337e4b158267..28756a6abdafe 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -562,7 +562,7 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray { // Decimal128Array class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray { public: - using TypeClass = DecimalType; + using TypeClass = Decimal128Type; using FixedSizeBinaryArray::FixedSizeBinaryArray; diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index c910170dd5c87..d2d3dbdf7d13e 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -1147,17 +1147,18 @@ template class DictionaryBuilder; template class DictionaryBuilder; // ---------------------------------------------------------------------- -// DecimalBuilder +// Decimal128Builder -DecimalBuilder::DecimalBuilder(const std::shared_ptr& type, MemoryPool* pool) +Decimal128Builder::Decimal128Builder(const std::shared_ptr& type, + MemoryPool* pool) : FixedSizeBinaryBuilder(type, pool) {} -Status DecimalBuilder::Append(const Decimal128& value) { +Status Decimal128Builder::Append(const Decimal128& value) { RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1)); return FixedSizeBinaryBuilder::Append(value.ToBytes()); } -Status DecimalBuilder::FinishInternal(std::shared_ptr* out) { +Status Decimal128Builder::FinishInternal(std::shared_ptr* out) { std::shared_ptr data; RETURN_NOT_OK(byte_builder_.Finish(&data)); diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index c580eeb3b35fb..bc25d0d211934 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -753,10 +753,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder { BufferBuilder byte_builder_; }; -class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder { +class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder { public: - explicit DecimalBuilder(const std::shared_ptr& type, - MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT); + explicit Decimal128Builder(const std::shared_ptr& type, + MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT); using FixedSizeBinaryBuilder::Append; @@ -765,6 +765,8 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder { Status FinishInternal(std::shared_ptr* out) override; }; +using DecimalBuilder = Decimal128Builder; + // ---------------------------------------------------------------------- // Struct diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index 253c2e1feb801..72ef122e3481e 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -615,8 +615,8 @@ class TypeEqualsVisitor { return Status::OK(); } - Status Visit(const DecimalType& left) { - const auto& right = static_cast(right_); + Status Visit(const Decimal128Type& left) { + const auto& right = static_cast(right_); result_ = left.precision() == right.precision() && left.scale() == right.scale(); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index 976f9660a1b78..bdf1ef52b40eb 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -253,7 +253,7 @@ class SchemaWriter { writer_->Int(type.byte_width()); } - void WriteTypeMetadata(const DecimalType& type) { + void WriteTypeMetadata(const Decimal128Type& type) { writer_->Key("precision"); writer_->Int(type.precision()); writer_->Key("scale"); @@ -347,7 +347,7 @@ class SchemaWriter { return WritePrimitive("fixedsizebinary", type); } - Status Visit(const DecimalType& type) { return WritePrimitive("decimal", type); } + Status Visit(const Decimal128Type& type) { return WritePrimitive("decimal", type); } Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); } Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); } @@ -1063,7 +1063,7 @@ class ArrayReader { template typename std::enable_if::value && - !std::is_base_of::value, + !std::is_base_of::value, Status>::type Visit(const T& type) { typename TypeTraits::BuilderType builder(type_, pool_); @@ -1105,7 +1105,7 @@ class ArrayReader { } template - typename std::enable_if::value, Status>::type Visit( + typename std::enable_if::value, Status>::type Visit( const T& type) { typename TypeTraits::BuilderType builder(type_, pool_); diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index 63ef8a549f236..87b4708bf6346 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -458,7 +458,7 @@ static Status TypeToFlatbuffer(FBB& fbb, const DataType& type, *offset = flatbuf::CreateTimestamp(fbb, fb_unit, fb_timezone).Union(); } break; case Type::DECIMAL: { - const auto& dec_type = static_cast(*value_type); + const auto& dec_type = static_cast(*value_type); *out_type = flatbuf::Type_Decimal; *offset = flatbuf::CreateDecimal(fbb, dec_type.precision(), dec_type.scale()).Union(); diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index f966b2987b93f..2b0f964391cb1 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -634,7 +634,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data, for (int c = 0; c < data.num_chunks(); c++) { auto* arr(static_cast(data.chunk(c).get())); - auto type(std::dynamic_pointer_cast(arr->type())); + auto type(std::dynamic_pointer_cast(arr->type())); const int scale = type->scale(); for (int64_t i = 0; i < arr->length(); ++i) { @@ -1600,7 +1600,7 @@ class ArrowDeserializer { return VisitObjects(ConvertFixedSizeBinary); } - Status Visit(const DecimalType& type) { return VisitObjects(ConvertDecimals); } + Status Visit(const Decimal128Type& type) { return VisitObjects(ConvertDecimals); } Status Visit(const Time32Type& type) { return VisitObjects(ConvertTimes); } diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc index 0e775a0fb0e99..fa0098bdf63e9 100644 --- a/cpp/src/arrow/python/builtin_convert.cc +++ b/cpp/src/arrow/python/builtin_convert.cc @@ -700,7 +700,7 @@ class ListConverter : public TypedConverterVisitor { }; class DecimalConverter - : public TypedConverterVisitor { + : public TypedConverterVisitor { public: inline Status AppendItem(const OwnedRef& item) { /// TODO(phillipc): Check for nan? diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index c5aff2e4f2e3a..a6c28af9b7443 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -305,7 +305,7 @@ class NumPyConverter { return TypeNotImplemented(type.ToString()); } - Status Visit(const DecimalType& type) { return TypeNotImplemented(type.ToString()); } + Status Visit(const Decimal128Type& type) { return TypeNotImplemented(type.ToString()); } Status Visit(const DictionaryType& type) { return TypeNotImplemented(type.ToString()); } @@ -669,9 +669,9 @@ Status NumPyConverter::ConvertDecimals() { RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(object, &precision, &scale)); - type_ = std::make_shared(precision, scale); + type_ = std::make_shared(precision, scale); - DecimalBuilder builder(type_, pool_); + Decimal128Builder builder(type_, pool_); RETURN_NOT_OK(builder.Resize(length_)); for (int64_t i = 0; i < length_; ++i) { diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index 0d1985fb2d914..ae17118323450 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -373,7 +373,7 @@ ACCEPT_VISITOR(FixedSizeBinaryType); ACCEPT_VISITOR(StringType); ACCEPT_VISITOR(ListType); ACCEPT_VISITOR(StructType); -ACCEPT_VISITOR(DecimalType); +ACCEPT_VISITOR(Decimal128Type); ACCEPT_VISITOR(UnionType); ACCEPT_VISITOR(Date32Type); ACCEPT_VISITOR(Date64Type); @@ -471,7 +471,7 @@ std::shared_ptr field(const std::string& name, } std::shared_ptr decimal(int32_t precision, int32_t scale) { - return std::make_shared(precision, scale); + return std::make_shared(precision, scale); } static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1); @@ -511,7 +511,7 @@ std::vector UnionType::GetBufferLayout() const { } } -std::string DecimalType::ToString() const { +std::string Decimal128Type::ToString() const { std::stringstream s; s << "decimal(" << precision_ << ", " << scale_ << ")"; return s.str(); diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 9e11a034420e5..d86e7ef57bec6 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -498,11 +498,11 @@ class ARROW_EXPORT StructType : public NestedType { std::vector GetBufferLayout() const override; }; -class ARROW_EXPORT DecimalType : public FixedSizeBinaryType { +class ARROW_EXPORT Decimal128Type : public FixedSizeBinaryType { public: static constexpr Type::type type_id = Type::DECIMAL; - explicit DecimalType(int32_t precision, int32_t scale) + explicit Decimal128Type(int32_t precision, int32_t scale) : FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {} Status Accept(TypeVisitor* visitor) const override; @@ -517,6 +517,8 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType { int32_t scale_; }; +using DecimalType = Decimal128Type; + struct UnionMode { enum type { SPARSE, DENSE }; }; diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h index 343487055c6fc..b8b3c5aa5adb7 100644 --- a/cpp/src/arrow/type_fwd.h +++ b/cpp/src/arrow/type_fwd.h @@ -68,9 +68,9 @@ class StructType; class StructArray; class StructBuilder; -class DecimalType; +class Decimal128Type; class Decimal128Array; -class DecimalBuilder; +class Decimal128Builder; class UnionType; class UnionArray; diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index bc600372eb01f..6707f3759581d 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -230,9 +230,9 @@ struct TypeTraits { }; template <> -struct TypeTraits { +struct TypeTraits { using ArrayType = Decimal128Array; - using BuilderType = DecimalBuilder; + using BuilderType = Decimal128Builder; constexpr static bool is_parameter_free = false; }; diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h index 34679eb950d4b..030ffc85744e4 100644 --- a/cpp/src/arrow/visitor.h +++ b/cpp/src/arrow/visitor.h @@ -83,7 +83,7 @@ class ARROW_EXPORT TypeVisitor { virtual Status Visit(const Time64Type& type); virtual Status Visit(const TimestampType& type); virtual Status Visit(const IntervalType& type); - virtual Status Visit(const DecimalType& type); + virtual Status Visit(const Decimal128Type& type); virtual Status Visit(const ListType& type); virtual Status Visit(const StructType& type); virtual Status Visit(const UnionType& type); diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h index 5ecabd2a4fbb1..72c82a3bb871b 100644 --- a/cpp/src/arrow/visitor_inline.h +++ b/cpp/src/arrow/visitor_inline.h @@ -55,7 +55,7 @@ inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) { TYPE_VISIT_INLINE(TimestampType); TYPE_VISIT_INLINE(Time32Type); TYPE_VISIT_INLINE(Time64Type); - TYPE_VISIT_INLINE(DecimalType); + TYPE_VISIT_INLINE(Decimal128Type); TYPE_VISIT_INLINE(ListType); TYPE_VISIT_INLINE(StructType); TYPE_VISIT_INLINE(UnionType); From 725546059134c1895742aca131f486e909b58ce8 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Wed, 15 Nov 2017 13:25:17 +0100 Subject: [PATCH 045/177] ARROW-1810: [Plasma] Remove unused Plasma test shell scripts These tests are run via CMake, so it is confusing to have the scripts in the repo. Author: Philipp Moritz Closes #1320 from pcmoritz/remove-test-sh and squashes the following commits: ed6fec4 [Philipp Moritz] removed unused test shell scripts --- cpp/src/plasma/test/run_tests.sh | 61 ----------------------------- cpp/src/plasma/test/run_valgrind.sh | 27 ------------- 2 files changed, 88 deletions(-) delete mode 100644 cpp/src/plasma/test/run_tests.sh delete mode 100644 cpp/src/plasma/test/run_valgrind.sh diff --git a/cpp/src/plasma/test/run_tests.sh b/cpp/src/plasma/test/run_tests.sh deleted file mode 100644 index 958bd08398e23..0000000000000 --- a/cpp/src/plasma/test/run_tests.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Cause the script to exit if a single command fails. -set -e - -./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & -sleep 1 -./src/plasma/manager_tests -killall plasma_store -./src/plasma/serialization_tests - -# Start the Redis shards. -./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 & -redis_pid1=$! -./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 & -redis_pid2=$! -sleep 1 - -# Flush the redis server -./src/common/thirdparty/redis/src/redis-cli flushall -# Register the shard location with the primary shard. -./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 -./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 -sleep 1 -./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 & -plasma1_pid=$! -./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 & -plasma2_pid=$! -./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 & -plasma3_pid=$! -./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 & -plasma4_pid=$! -sleep 1 - -./src/plasma/client_tests - -kill $plasma4_pid -kill $plasma3_pid -kill $plasma2_pid -kill $plasma1_pid -kill $redis_pid1 -wait $redis_pid1 -kill $redis_pid2 -wait $redis_pid2 diff --git a/cpp/src/plasma/test/run_valgrind.sh b/cpp/src/plasma/test/run_valgrind.sh deleted file mode 100644 index 0472194128679..0000000000000 --- a/cpp/src/plasma/test/run_valgrind.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Cause the script to exit if a single command fails. -set -e - -./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & -sleep 1 -valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests -killall plasma_store -valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests From 1d951b5d2799b08b9ce81902e525390054b05fe4 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Wed, 15 Nov 2017 13:26:28 +0100 Subject: [PATCH 046/177] ARROW-1809: [GLib] Use .xml instead of .sgml for GTK-Doc main file .sgml is old extension: https://developer.gnome.org/gtk-doc-manual/unstable/settingup_firstrun.html.en > `-docs.xml` (in the past `.sgml`) Author: Kouhei Sutou Closes #1318 from kou/glib-doc-use-xml and squashes the following commits: 138c199 [Kouhei Sutou] [GLib] Use .xml instead of .sgml for GTK-Doc main file --- c_glib/doc/reference/Makefile.am | 2 +- .../doc/reference/{arrow-glib-docs.sgml => arrow-glib-docs.xml} | 0 c_glib/doc/reference/meson.build | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename c_glib/doc/reference/{arrow-glib-docs.sgml => arrow-glib-docs.xml} (100%) diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am index 896aff544d454..4c005c237b300 100644 --- a/c_glib/doc/reference/Makefile.am +++ b/c_glib/doc/reference/Makefile.am @@ -20,7 +20,7 @@ SUBDIRS = \ DOC_MODULE = arrow-glib -DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.sgml +DOC_MAIN_SGML_FILE = $(DOC_MODULE)-docs.xml DOC_SOURCE_DIR = \ $(top_srcdir)/arrow-glib diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.xml similarity index 100% rename from c_glib/doc/reference/arrow-glib-docs.sgml rename to c_glib/doc/reference/arrow-glib-docs.xml diff --git a/c_glib/doc/reference/meson.build b/c_glib/doc/reference/meson.build index 4c9552e83c303..3374fbde5b9ed 100644 --- a/c_glib/doc/reference/meson.build +++ b/c_glib/doc/reference/meson.build @@ -49,7 +49,7 @@ if arrow_gpu_dependency.found() ] endif gnome.gtkdoc(meson.project_name(), - main_xml: meson.project_name() + '-docs.sgml', + main_xml: meson.project_name() + '-docs.xml', src_dir: source_directories, dependencies: dependencies, gobject_typesfile: meson.project_name() + '.types', From 9812aea13e810626f4f5b74f1ca4ecbb843578e9 Mon Sep 17 00:00:00 2001 From: Stephanie Date: Wed, 15 Nov 2017 10:51:45 -0500 Subject: [PATCH 047/177] ARROW-1812: [C++] Plasma store modifies hash table while iterating during client disconnect Author: Stephanie Closes #1322 from stephanie-wang/plasma-disconnect-bug and squashes the following commits: cf1872d8 [Stephanie] Fix disconnect client memory error 8929b6bc [Stephanie] Introduce more complicated test for multiple Plasma clients --- cpp/src/plasma/store.cc | 10 +++++- cpp/src/plasma/test/client_tests.cc | 53 +++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc index 31033ccbb3202..7094aed6f8e86 100644 --- a/cpp/src/plasma/store.cc +++ b/cpp/src/plasma/store.cc @@ -460,14 +460,22 @@ void PlasmaStore::disconnect_client(int client_fd) { ARROW_LOG(INFO) << "Disconnecting client on fd " << client_fd; // If this client was using any objects, remove it from the appropriate // lists. + // TODO(swang): Avoid iteration through the object table. auto client = it->second.get(); + std::vector unsealed_objects; for (const auto& entry : store_info_.objects) { if (entry.second->state == PLASMA_SEALED) { remove_client_from_object_clients(entry.second.get(), client); } else { - abort_object(entry.first, client); + // Add unsealed objects to a temporary list of object IDs. Do not perform + // the abort here, since it potentially modifies the object table. + unsealed_objects.push_back(entry.first); } } + // If the client was creating any objects, abort them. + for (const auto& entry : unsealed_objects) { + abort_object(entry, client); + } // Note, the store may still attempt to send a message to the disconnected // client (for example, when an object ID that the client was waiting for diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc index d4285f8988404..f44ed2510c582 100644 --- a/cpp/src/plasma/test/client_tests.cc +++ b/cpp/src/plasma/test/client_tests.cc @@ -209,6 +209,59 @@ TEST_F(TestPlasmaStore, MultipleClientTest) { ASSERT_EQ(has_object, true); } +TEST_F(TestPlasmaStore, ManyObjectTest) { + // Create many objects on the first client. Seal one third, abort one third, + // and leave the last third unsealed. + std::vector object_ids; + for (int i = 0; i < 100; i++) { + ObjectID object_id = ObjectID::from_random(); + object_ids.push_back(object_id); + + // Test for object non-existence on the first client. + bool has_object; + ARROW_CHECK_OK(client_.Contains(object_id, &has_object)); + ASSERT_EQ(has_object, false); + + // Test for the object being in local Plasma store. + // First create and seal object on the first client. + int64_t data_size = 100; + uint8_t metadata[] = {5}; + int64_t metadata_size = sizeof(metadata); + uint8_t* data; + ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); + + if (i % 3 == 0) { + // Seal one third of the objects. + ARROW_CHECK_OK(client_.Seal(object_id)); + // Test that the first client can get the object. + ARROW_CHECK_OK(client_.Contains(object_id, &has_object)); + ASSERT_EQ(has_object, true); + } else if (i % 3 == 1) { + // Abort one third of the objects. + ARROW_CHECK_OK(client_.Release(object_id)); + ARROW_CHECK_OK(client_.Abort(object_id)); + } + } + // Disconnect the first client. All unsealed objects should be aborted. + ARROW_CHECK_OK(client_.Disconnect()); + + // Check that the second client can query the object store for the first + // client's objects. + int i = 0; + for (auto const& object_id : object_ids) { + bool has_object; + ARROW_CHECK_OK(client2_.Contains(object_id, &has_object)); + if (i % 3 == 0) { + // The first third should be sealed. + ASSERT_EQ(has_object, true); + } else { + // The rest were aborted, so the object is not in the store. + ASSERT_EQ(has_object, false); + } + i++; + } +} + } // namespace plasma int main(int argc, char** argv) { From 42353ba993314e6bba963aae50861566150f96eb Mon Sep 17 00:00:00 2001 From: Siddharth Teotia Date: Sat, 14 Oct 2017 11:38:30 -0400 Subject: [PATCH 048/177] ARROW-1473: ValueVector new hierarchy prototype (implementation phase 1) Close #1164 Close #1198 Change-Id: If18e42d2edfdfef83e83621334a5b65a390e9db9 --- .../codegen/templates/ComplexReaders.java | 24 +- .../codegen/templates/ComplexWriters.java | 45 +- .../templates/NullableValueVectors.java | 542 ++++++++++-- .../main/codegen/templates/UnionVector.java | 18 +- .../vector/BaseNullableFixedWidthVector.java | 701 ++++++++++++++++ .../BaseNullableVariableWidthVector.java | 764 +++++++++++++++++ .../apache/arrow/vector/BaseValueVector.java | 20 + .../apache/arrow/vector/BitVectorHelper.java | 60 ++ .../arrow/vector/NullableIntVector.java | 299 +++++++ .../arrow/vector/NullableVarCharVector.java | 451 ++++++++++ .../org/apache/arrow/vector/ValueVector.java | 6 + .../apache/arrow/vector/VectorUnloader.java | 15 +- .../org/apache/arrow/vector/ZeroVector.java | 9 + .../vector/complex/FixedSizeListVector.java | 27 +- .../arrow/vector/complex/ListVector.java | 23 +- .../arrow/vector/complex/MapVector.java | 33 +- .../vector/file/json/JsonFileReader.java | 2 +- .../vector/TestBufferOwnershipTransfer.java | 6 +- .../arrow/vector/TestDictionaryVector.java | 59 +- .../arrow/vector/TestFixedSizeListVector.java | 13 +- .../arrow/vector/TestSplitAndTransfer.java | 15 +- .../apache/arrow/vector/TestValueVector.java | 788 +++++++++++++----- .../arrow/vector/TestVectorReAlloc.java | 11 +- .../arrow/vector/TestVectorUnloadLoad.java | 104 ++- .../arrow/vector/file/BaseFileTest.java | 70 +- .../arrow/vector/file/TestArrowFile.java | 2 +- 26 files changed, 3658 insertions(+), 449 deletions(-) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index 38cd1bfdeb3c5..791064905156d 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -70,7 +70,11 @@ public Field getField(){ public boolean isSet(){ <#if nullMode == "Nullable"> - return !vector.getAccessor().isNull(idx()); + <#if minor.class != "Int" && minor.class != "VarChar"> + return !vector.getAccessor().isNull(idx()); + <#else> + return !vector.isNull(idx()); + <#else> return true; @@ -93,11 +97,19 @@ public void read(${minor.class?cap_first}Holder h){ public void read(Nullable${minor.class?cap_first}Holder h){ - vector.getAccessor().get(idx(), h); + <#if minor.class != "Int" && minor.class != "VarChar"> + vector.getAccessor().get(idx(), h); + <#else> + vector.get(idx(), h); + } public ${friendlyType} read${safeType}(){ - return vector.getAccessor().getObject(idx()); + <#if minor.class == "Int" || minor.class == "VarChar"> + return vector.getObject(idx()); + <#else> + return vector.getAccessor().getObject(idx()); + } <#if minor.class == "TimeStampSec" || @@ -115,7 +127,11 @@ public void copyValue(FieldWriter w){ } public Object readObject(){ - return vector.getAccessor().getObject(idx()); + <#if minor.class == "Int" || minor.class == "VarChar"> + return (Object)vector.getObject(idx()); + <#else> + return vector.getAccessor().getObject(idx()); + } } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index fe099bede3568..77f6594a07804 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -39,11 +39,16 @@ @SuppressWarnings("unused") public class ${eName}WriterImpl extends AbstractFieldWriter { - private final Nullable${name}Vector.Mutator mutator; + <#if minor.class != "Int" && minor.class != "VarChar"> + private final Nullable${name}Vector.Mutator mutator; + + final Nullable${name}Vector vector; public ${eName}WriterImpl(Nullable${name}Vector vector) { - this.mutator = vector.getMutator(); + <#if minor.class != "Int" && minor.class != "VarChar"> + this.mutator = vector.getMutator(); + this.vector = vector; } @@ -103,18 +108,33 @@ public void setPosition(int idx) { <#else> public void write(${minor.class}Holder h) { - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setSafe(idx(), h); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); + } public void write(Nullable${minor.class}Holder h) { - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setSafe(idx(), h); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); + } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - mutator.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.setValueCount(idx()+1); + } <#if minor.class == "Decimal"> @@ -126,8 +146,13 @@ public void write(Nullable${minor.class}Holder h) { <#if mode == "Nullable"> public void writeNull() { - mutator.setNull(idx()); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setNull(idx()); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setNull(idx()); + vector.setValueCount(idx()+1); + } diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 122cd23648286..5d1f5a33b90f5 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -19,8 +19,14 @@ <#list vv.types as type> <#list type.minor as minor> +<#if minor.class == "Int" || minor.class == "VarChar"> +<#assign className = "LegacyNullable${minor.class}Vector" /> +<#assign valuesName = "Nullable${minor.class}Vector" /> +<#else> <#assign className = "Nullable${minor.class}Vector" /> <#assign valuesName = "${minor.class}Vector" /> + + <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> @@ -44,15 +50,24 @@ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @SuppressWarnings("unused") -public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, NullableVector, FieldVector { +<#if minor.class == "Int" || minor.class == "VarChar"> +@Deprecated + +public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, FieldVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); protected final static byte[] emptyByteArray = new byte[]{}; + + <#if minor.class != "Int" && minor.class != "VarChar"> private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this); + private final String bitsField = "$bits$"; private final String valuesField = "$values$"; + + <#if minor.class != "Int" && minor.class != "VarChar"> private final Field field; + final BitVector bits = new BitVector(bitsField, allocator); final ${valuesName} values; @@ -60,7 +75,9 @@ public final class ${className} extends BaseValueVector implements <#if type.maj private final Mutator mutator; private final Accessor accessor; + <#if minor.class != "Int" && minor.class != "VarChar"> private final List innerVectors; + <#if minor.typeParams??> <#assign typeParams = minor.typeParams?reverse> @@ -105,6 +122,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj this.mutator = new Mutator(); this.accessor = new Accessor(); + <#if minor.class != "Int" && minor.class != "VarChar"> this.field = new Field(name, fieldType, null); innerVectors = Collections.unmodifiableList(Arrays.asList( bits, @@ -113,16 +131,24 @@ public final class ${className} extends BaseValueVector implements <#if type.maj values )); + } - @Override + <#if minor.class != "Int" && minor.class != "VarChar"> + /* not needed for new vectors */ public BitVector getValidityVector() { return bits; } + @Override public List getFieldInnerVectors() { - return innerVectors; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getFieldInnerVectors(); + <#else> + return innerVectors; + } @Override @@ -139,6 +165,10 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.loadFieldBuffers(fieldNode, ownBuffers); + <#else> <#if type.major = "VarLen"> // variable width values: truncate offset vector buffer to size (#1) org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, @@ -151,34 +181,64 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); bits.valueCount = fieldNode.getLength(); + } public List getFieldBuffers() { - return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getFieldBuffers(); + <#else> + return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors()); + } @Override public Field getField() { - return field; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getField(); + <#else> + return field; + } @Override public MinorType getMinorType() { - return MinorType.${minor.class?upper_case}; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getMinorType(); + <#else> + return MinorType.${minor.class?upper_case}; + } @Override public FieldReader getReader(){ - return reader; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getReader(); + <#else> + return reader; + } @Override public int getValueCapacity(){ - return Math.min(bits.getValueCapacity(), values.getValueCapacity()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValueCapacity(); + <#else> + return Math.min(bits.getValueCapacity(), values.getValueCapacity()); + } @Override public ArrowBuf[] getBuffers(boolean clear) { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getBuffers(clear); + <#else> final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class); if (clear) { for (final ArrowBuf buffer:buffers) { @@ -187,25 +247,41 @@ public ArrowBuf[] getBuffers(boolean clear) { clear(); } return buffers; + } @Override public void close() { - bits.close(); - values.close(); - super.close(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.close(); + <#else> + bits.close(); + values.close(); + super.close(); + } @Override public void clear() { - bits.clear(); - values.clear(); - super.clear(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.clear(); + <#else> + bits.clear(); + values.clear(); + super.clear(); + } @Override public int getBufferSize(){ - return values.getBufferSize() + bits.getBufferSize(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSize(); + <#else> + return values.getBufferSize() + bits.getBufferSize(); + } @Override @@ -214,34 +290,52 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - return values.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSizeFor(valueCount); + <#else> + return values.getBufferSizeFor(valueCount) + + bits.getBufferSizeFor(valueCount); + } public ArrowBuf getBuffer() { return values.getDataBuffer(); } - @Override public ${valuesName} getValuesVector() { return values; } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); - values.setInitialCapacity(numRecords); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.setInitialCapacity(numRecords); + <#else> + bits.setInitialCapacity(numRecords); + values.setInitialCapacity(numRecords); + } @Override public void allocateNew() { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(); + <#else> if(!allocateNewSafe()){ throw new OutOfMemoryException("Failure while allocating buffer."); } + } @Override public boolean allocateNewSafe() { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.allocateNewSafe(); + <#else> /* Boolean to keep track if all the memory allocations were successful * Used in the case of composite vectors when we need to allocate multiple * buffers for multiple vectors. If one of the allocations failed we need to @@ -259,23 +353,38 @@ public boolean allocateNewSafe() { mutator.reset(); accessor.reset(); return success; + } @Override public void reAlloc() { - bits.reAlloc(); - values.reAlloc(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.reAlloc(); + <#else> + bits.reAlloc(); + values.reAlloc(); + } public void reset() { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.reset(); + <#else> bits.zeroVector(); mutator.reset(); accessor.reset(); + } <#if type.major == "VarLen"> @Override public void allocateNew(int totalBytes, int valueCount) { + <#if minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(totalBytes, valueCount); + <#else> try { values.allocateNew(totalBytes, valueCount); bits.allocateNew(valueCount); @@ -286,6 +395,7 @@ public void allocateNew(int totalBytes, int valueCount) { bits.zeroVector(); mutator.reset(); accessor.reset(); + } @Override @@ -301,6 +411,10 @@ public int getCurrentSizeInBytes(){ <#else> @Override public void allocateNew(int valueCount) { + <#if minor.class == "Int"> + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(valueCount); + <#else> try { values.allocateNew(valueCount); bits.allocateNew(valueCount); @@ -311,6 +425,7 @@ public void allocateNew(int valueCount) { bits.zeroVector(); mutator.reset(); accessor.reset(); + } /** @@ -318,32 +433,86 @@ public void allocateNew(int valueCount) { */ @Override public void zeroVector() { - bits.zeroVector(); - values.zeroVector(); + <#if minor.class == "Int"> + /* DELEGATE TO NEW VECTOR */ + values.zeroVector(); + <#else> + bits.zeroVector(); + values.zeroVector(); + } + + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator, callBack); + <#else> return getTransferPair(ref, allocator); + } + + @Override public TransferPair getTransferPair(BufferAllocator allocator){ - return new TransferImpl(name, allocator); - + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(allocator); + <#else> + return new TransferImpl(name, allocator); + } + + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator); + <#else> + return new TransferImpl(ref, allocator); + } + + @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${className}) to); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.makeTransferPair(to); + <#else> + return new TransferImpl((${className}) to); + + } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void transferTo(${valuesName} target) { + /* DELEGATE TO NEW VECTOR */ + <#if type.major == "VarLen"> + values.transferTo((BaseNullableVariableWidthVector) target); + <#else> + values.transferTo((BaseNullableFixedWidthVector) target); + + } + + public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) { + /* DELEGATE TO NEW VECTOR */ + <#if type.major == "VarLen"> + values.splitAndTransferTo(startIndex, length, (BaseNullableVariableWidthVector) target); + <#else> + values.splitAndTransferTo(startIndex, length, (BaseNullableFixedWidthVector) target); + } + <#else> public void transferTo(${className} target){ bits.transferTo(target.bits); values.transferTo(target.values); @@ -360,7 +529,11 @@ public void splitAndTransferTo(int startIndex, int length, ${className} target) target.mutator.lastSet = length - 1; } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> private class TransferImpl implements TransferPair { ${className} to; @@ -392,6 +565,9 @@ public void copyValueSafe(int fromIndex, int toIndex) { to.copyFromSafe(fromIndex, toIndex, ${className}.this); } } + + + @Override public Accessor getAccessor(){ @@ -403,7 +579,20 @@ public Mutator getMutator(){ return mutator; } - public void copyFrom(int fromIndex, int thisIndex, ${className} from){ + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) { + /* DELEGATE TO NEW VECTOR */ + values.copyFrom(fromIndex, thisIndex, from); + } + + public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) { + /* DELEGATE TO NEW VECTOR */ + values.copyFromSafe(fromIndex, thisIndex, from); + } + <#else> + public void copyFrom(int fromIndex, int thisIndex, ${className} from) { final Accessor fromAccessor = from.getAccessor(); if (!fromAccessor.isNull(fromIndex)) { mutator.set(thisIndex, fromAccessor.get(fromIndex)); @@ -428,17 +617,28 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ values.copyFromSafe(fromIndex, thisIndex, from.values); <#if type.major == "VarLen">mutator.lastSet = thisIndex; } + @Override public long getValidityBufferAddress() { /* address of the databuffer associated with the bitVector */ - return (bits.getDataBuffer().memoryAddress()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBufferAddress(); + <#else> + return (bits.getDataBuffer().memoryAddress()); + } @Override public long getDataBufferAddress() { /* address of the dataBuffer associated with the valueVector */ - return (values.getDataBuffer().memoryAddress()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getDataBufferAddress(); + <#else> + return (bits.getDataBuffer().memoryAddress()); + } @Override @@ -446,17 +646,26 @@ public long getOffsetBufferAddress() { /* address of the dataBuffer associated with the offsetVector * this operation is not supported for fixed-width vector types. */ - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBufferAddress(); <#else> - return (values.getOffsetAddr()); + <#if type.major != "VarLen"> + throw new UnsupportedOperationException(); + <#else> + return (values.getOffsetAddr()); + } @Override public ArrowBuf getValidityBuffer() { - /* dataBuffer associated with the bitVector */ - return (bits.getDataBuffer()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBuffer(); + <#else> + return (bits.getDataBuffer()); + } @Override @@ -468,10 +677,15 @@ public ArrowBuf getDataBuffer() { @Override public ArrowBuf getOffsetBuffer() { /* dataBuffer associated with the offsetVector of the valueVector */ - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBuffer(); <#else> - return (values.getOffsetBuffer()); + <#if type.major != "VarLen"> + throw new UnsupportedOperationException(); + <#else> + return (values.getOffsetBuffer()); + } @@ -485,38 +699,80 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m * @param index position of the value * @return value of the element, if not null */ - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - if (isNull(index)) { + <#if minor.class == "Int" || minor.class == "VarChar"> + public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { + /* DELEGATE TO NEW VECTOR */ + return values.get(index); + } + <#else> + + public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { + if (isNull(index)) { throw new IllegalStateException("Can't get a null value"); + } + return vAccessor.get(index); } - return vAccessor.get(index); - } + @Override public boolean isNull(int index) { - return isSet(index) == 0; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.isNull(index); + <#else> + return isSet(index) == 0; + } public int isSet(int index){ - return bAccessor.get(index); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.isSet(index); + <#else> + return bAccessor.get(index); + } <#if type.major == "VarLen"> public long getStartEnd(int index){ - return vAccessor.getStartEnd(index); + <#if minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getStartEnd(index); + <#else> + return vAccessor.getStartEnd(index); + } @Override public int getValueLength(int index) { - return values.getAccessor().getValueLength(index); + <#if minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValueLength(index); + <#else> + return values.getAccessor().getValueLength(index); + } + <#if minor.class == "Int" || minor.class == "VarChar"> + public void get(int index, Nullable${minor.class}Holder holder){ + /* DELEGATE TO NEW VECTOR */ + values.get(index, holder); + } + <#else> public void get(int index, Nullable${minor.class}Holder holder){ vAccessor.get(index, holder); holder.isSet = bAccessor.get(index); } + + <#if minor.class == "Int" || minor.class == "VarChar"> + @Override + public ${friendlyType} getObject(int index) { + /* DELEGATE TO NEW VECTOR */ + return values.getObject(index); + } + <#else> @Override public ${friendlyType} getObject(int index) { if (isNull(index)) { @@ -525,6 +781,7 @@ public void get(int index, Nullable${minor.class}Holder holder){ return vAccessor.getObject(index); } } + <#if minor.class == "IntervalYear" || minor.class == "IntervalDay"> public StringBuilder getAsStringBuilder(int index) { @@ -538,7 +795,12 @@ public StringBuilder getAsStringBuilder(int index) { @Override public int getValueCount(){ - return bits.getAccessor().getValueCount(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValueCount(); + <#else> + return bits.getAccessor().getValueCount(); + } public void reset(){} @@ -551,21 +813,35 @@ public final class Mutator extends BaseDataValueVector.BaseMutator implements Nu private Mutator(){ } - public ${valuesName} getVectorWithValues(){ + public ${valuesName} getVectorWithValues() { return values; } + @Override public void setIndexDefined(int index){ + <#if minor.class == "Int" || minor.class == "VarChar"> + values.setIndexDefined(index); + <#else> bits.getMutator().setToOne(index); + } + + /** * Set the variable length element at the specified index to the supplied byte array. * * @param index position of the bit to set * @param value array of bytes (or int if smaller than 4 bytes) to write */ + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, value); + } + <#else> public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { setCount++; final ${valuesName}.Mutator valuesMutator = values.getMutator(); @@ -579,9 +855,24 @@ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.widt valuesMutator.set(index, value); <#if type.major == "VarLen">lastSet = index; } + + + <#if type.major == "VarLen"> + <#if minor.class == "VarChar"> + public void fillEmpties(int index) { + /* DELEGATE TO NEW VECTOR */ + values.fillEmpties(index); + } + @Override + public void setValueLengthSafe(int index, int length) { + /* DELEGATE TO NEW VECTOR */ + values.setValueLengthSafe(index, length); + } + + <#else> public void fillEmpties(int index){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); for (int i = lastSet + 1; i < index; i++) { @@ -599,7 +890,16 @@ public void setValueLengthSafe(int index, int length) { lastSet = index; } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void setSafe(int index, byte[] value, int start, int length) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value, start, length); + } + <#else> public void setSafe(int index, byte[] value, int start, int length) { <#if type.major != "VarLen"> throw new UnsupportedOperationException(); @@ -612,7 +912,16 @@ public void setSafe(int index, byte[] value, int start, int length) { <#if type.major == "VarLen">lastSet = index; } + + + + <#if minor.class == "VarChar"> + public void setSafe(int index, ByteBuffer value, int start, int length) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value, start, length); + } + <#else> public void setSafe(int index, ByteBuffer value, int start, int length) { <#if type.major != "VarLen"> throw new UnsupportedOperationException(); @@ -625,11 +934,25 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { <#if type.major == "VarLen">lastSet = index; } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void setNull(int index) { + /* DELEGATE TO NEW VECTOR */ + values.setNull(index); + } + <#else> public void setNull(int index){ bits.getMutator().setSafe(index, 0); } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> + /* these methods are probably not needed */ public void setSkipNull(int index, ${minor.class}Holder holder){ values.getMutator().set(index, holder); } @@ -637,8 +960,17 @@ public void setSkipNull(int index, ${minor.class}Holder holder){ public void setSkipNull(int index, Nullable${minor.class}Holder holder){ values.getMutator().set(index, holder); } + + - public void set(int index, Nullable${minor.class}Holder holder){ + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void set(int index, Nullable${minor.class}Holder holder) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, holder); + } + <#else> + public void set(int index, Nullable${minor.class}Holder holder) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -649,8 +981,17 @@ public void set(int index, Nullable${minor.class}Holder holder){ valuesMutator.set(index, holder); <#if type.major == "VarLen">lastSet = index; } + + + - public void set(int index, ${minor.class}Holder holder){ + <#if minor.class == "Int" || minor.class == "VarChar"> + public void set(int index, ${minor.class}Holder holder) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, holder); + } + <#else> + public void set(int index, ${minor.class}Holder holder) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -661,11 +1002,45 @@ public void set(int index, ${minor.class}Holder holder){ valuesMutator.set(index, holder); <#if type.major == "VarLen">lastSet = index; } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public boolean isSafe(int outIndex) { + /* DELEGATE TO NEW VECTOR */ + return values.isSafe(outIndex); + } + <#else> public boolean isSafe(int outIndex) { return outIndex < ${className}.this.getValueCapacity(); } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + <#if minor.class == "Int"> + public void set(int index, int isSet, int valueField) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, isSet, valueField); + } + public void setSafe(int index, int isSet, int valueField) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, isSet, valueField); + } + + <#if minor.class == "VarChar"> + public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, isSet, startField, endField, bufferField); + } + public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, isSet, startField, endField, bufferField); + } + + <#else> <#assign fields = minor.fields!type.fields /> public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field ){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); @@ -688,8 +1063,21 @@ public void setSafe(int index, int isSet<#list fields as field><#if field.includ setCount++; <#if type.major == "VarLen">lastSet = index; } + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void setSafe(int index, Nullable${minor.class}Holder value) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); + } + + public void setSafe(int index, ${minor.class}Holder value) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); + } + <#else> public void setSafe(int index, Nullable${minor.class}Holder value) { <#if type.major == "VarLen"> fillEmpties(index); @@ -709,15 +1097,25 @@ public void setSafe(int index, ${minor.class}Holder value) { setCount++; <#if type.major == "VarLen">lastSet = index; } + + + <#if !(type.major == "VarLen" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { + <#if minor.class == "Int"> + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); + <#else> bits.getMutator().setSafeToOne(index); values.getMutator().setSafe(index, value); setCount++; + } - + + + <#if minor.class == "Decimal"> public void set(int index, ${friendlyType} value) { bits.getMutator().setToOne(index); @@ -729,8 +1127,17 @@ public void setSafe(int index, ${friendlyType} value) { values.getMutator().setSafe(index, value); setCount++; } - + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + @Override + public void setValueCount(int valueCount) { + /* DELEGATE TO NEW VECTOR */ + values.setValueCount(valueCount); + } + <#else> @Override public void setValueCount(int valueCount) { assert valueCount >= 0; @@ -740,7 +1147,12 @@ public void setValueCount(int valueCount) { values.getMutator().setValueCount(valueCount); bits.getMutator().setValueCount(valueCount); } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> + /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */ @Override public void generateTestData(int valueCount){ bits.getMutator().generateTestDataAlt(valueCount); @@ -748,13 +1160,27 @@ public void generateTestData(int valueCount){ <#if type.major = "VarLen">lastSet = valueCount; setValueCount(valueCount); } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> + /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */ @Override public void reset(){ setCount = 0; <#if type.major = "VarLen">lastSet = -1; } + + + + <#if minor.class == "VarChar"> + public void setLastSet(int value) { + /* DELEGATE TO NEW VECTOR */ + values.setLastSet(value); + } + <#else> public void setLastSet(int value) { <#if type.major = "VarLen"> lastSet = value; @@ -762,7 +1188,16 @@ public void setLastSet(int value) { throw new UnsupportedOperationException(); } + + + + <#if minor.class == "VarChar"> + public int getLastSet() { + /* DELEGATE TO NEW VECTOR */ + return values.getLastSet(); + } + <#else> public int getLastSet() { <#if type.major != "VarLen"> throw new UnsupportedOperationException(); @@ -770,6 +1205,7 @@ public int getLastSet() { return lastSet; } + } } diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index fe24a8674bdc5..3c7ed0190196a 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -436,7 +436,11 @@ public Object getObject(int index) { <#assign uncappedName = name?uncap_first/> <#if !minor.typeParams?? > case ${name?upper_case}: - return get${name}Vector().getAccessor().getObject(index); + <#if minor.class != "Int" && minor.class != "VarChar"> + return get${name}Vector().getAccessor().getObject(index); + <#else> + return get${name}Vector().getObject(index); + @@ -530,7 +534,11 @@ public void setSafe(int index, UnionHolder holder) { <#if !minor.typeParams?? > public void setSafe(int index, Nullable${name}Holder holder) { setType(index, MinorType.${name?upper_case}); - get${name}Vector().getMutator().setSafe(index, holder); + <#if minor.class != "Int" && minor.class != "VarChar"> + get${name}Vector().getMutator().setSafe(index, holder); + <#else> + get${name}Vector().setSafe(index, holder); + } @@ -547,4 +555,10 @@ public void reset() { } @Override public void generateTestData(int values) { } } + + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java new file mode 100644 index 0000000000000..c5f781000c9a9 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -0,0 +1,701 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +public abstract class BaseNullableFixedWidthVector extends BaseValueVector + implements FixedWidthVector, FieldVector { + private final byte typeWidth; + + private int valueAllocationSizeInBytes; + private int validityAllocationSizeInBytes; + + protected final Field field; + private int allocationMonitor; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected int valueCount; + + public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType, final byte typeWidth) { + super(name, allocator); + this.typeWidth = typeWidth; + valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; + validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); + field = new Field(name, fieldType, null); + valueCount = 0; + allocationMonitor = 0; + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + } + + /* TODO: + * Determine how writerIndex and readerIndex need to be used. Right now we + * are setting the writerIndex and readerIndex in the call to getFieldBuffers + * using the valueCount -- this assumes that the caller of getFieldBuffers + * on the vector has already invoked setValueCount. + * + * Do we need to set them during vector transfer and splitAndTransfer? + */ + + /* TODO: + * + * see if getNullCount() can be made faster -- O(1) + */ + + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy and those vectors have not + * yet been refactored so moving things to the top class as of now + * is not a good idea. + */ + + /* TODO: + * See if we need logger -- unnecessary object probably + */ + + protected abstract org.slf4j.Logger getLogger(); + + @Override + public Mutator getMutator() { + throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + } + + @Override + public Accessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + } + + @Override + public long getValidityBufferAddress() { + return (validityBuffer.memoryAddress()); + } + + @Override + public long getDataBufferAddress() { + return (valueBuffer.memoryAddress()); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long)valueCount * typeWidth; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int)size; + validityAllocationSizeInBytes = getSizeFromCount(valueCount); + } + + @Override + public int getValueCapacity(){ + return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); + } + + /* for test purposes */ + private int getValueBufferValueCapacity() { + return (int)((valueBuffer.capacity() * 1.0)/typeWidth); + } + + /* for test purposes */ + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + + /* number of bytes for the validity buffer for the given valueCount */ + protected int getSizeFromCount(int valueCount) { + return (int) Math.ceil(valueCount / 8.0); + } + + @Override + public void zeroVector() { + initValidityBuffer(); + initValueBuffer(); + } + + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + private void initValueBuffer() { + valueBuffer.setZero(0, valueBuffer.capacity()); + } + + public void reset() { + zeroVector(); + } + + @Override + public void close() { clear(); } + + @Override + public void clear() { + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + } + + /* used to step down the memory allocation */ + protected void incrementAllocationMonitor() { + if (allocationMonitor < 0) { + allocationMonitor = 0; + } + allocationMonitor++; + } + + /* used to step up the memory allocation */ + protected void decrementAllocationMonitor() { + if (allocationMonitor > 0) { + allocationMonitor = 0; + } + allocationMonitor--; + } + + @Override + public void allocateNew() { + if(!allocateNewSafe()){ + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try{ + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); + } catch (Exception e) { + getLogger().error("ERROR: Failure in allocateNewSafe"); + getLogger().error(e.getMessage()); + clear(); + return false; + } + + return true; + } + + public void allocateNew(int valueCount) { + long valueBufferSize = valueCount * typeWidth; + long validityBufferSize = getSizeFromCount(valueCount); + + if (allocationMonitor > 10) { + /* step down the default memory allocation since we have observed + * multiple times that provisioned value capacity was much larger than + * actually needed. see setValueCount for more details. + */ + valueBufferSize = Math.max(8, valueBufferSize / 2); + validityBufferSize = Math.max(8, validityBufferSize / 2); + allocationMonitor = 0; + } else if (allocationMonitor < -2) { + valueBufferSize = valueBufferSize * 2L; + validityBufferSize = validityBufferSize * 2L; + allocationMonitor = 0; + } + + if (valueBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(valueBufferSize, validityBufferSize); + } catch(Exception e) { + getLogger().error("ERROR: Failure in allocateNew"); + getLogger().error(e.getMessage()); + clear(); + throw e; + } + } + + /** + * Actual memory allocation is done by this function. All the calculations + * and knowledge about what size to allocate is upto the callers of this + * method. + * Callers appropriately handle errors if memory allocation fails here. + * Callers should also take care of determining that desired size is + * within the bounds of max allocation allowed and any other error + * conditions. + */ + private void allocateBytes(final long valueBufferSize, final long validityBufferSize) { + /* allocate data buffer */ + int curSize = (int)valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; + + /* allocate validity buffer */ + allocateValidityBuffer((int)validityBufferSize); + initValidityBuffer(); + } + + /* + * during splitAndTransfer, if we splitting from a random position within a byte, + * we can't just slice the source buffer so we have to explicitly allocate the + * validityBuffer of the target vector. This is unlike the databuffer which we can + * always slice for the target vector. + */ + private void allocateValidityBuffer(final int validityBufferSize) { + validityBuffer = allocator.buffer(validityBufferSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = validityBufferSize; + initValidityBuffer(); + } + + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { return 0; } + return (count * typeWidth) + getSizeFromCount(count); + } + + @Override + public int getBufferSize() { + if (valueCount == 0) { return 0; } + return (valueCount * typeWidth) + getSizeFromCount(valueCount); + } + + @Override + public Field getField() { + return field; + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers = new ArrowBuf[2]; + buffers[0] = validityBuffer; + buffers[1] = valueBuffer; + if (clear) { + for (final ArrowBuf buffer:buffers) { + buffer.retain(1); + } + clear(); + } + return buffers; + } + + @Override + public void reAlloc() { + valueBuffer = reallocBufferHelper(valueBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (dataBuffer ? valueAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", + name, (dataBuffer ? valueAllocationSizeInBytes : validityAllocationSizeInBytes), + newAllocationSize); + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (dataBuffer) { + valueAllocationSizeInBytes = (int)newAllocationSize; + } + else { + validityAllocationSizeInBytes = (int)newAllocationSize; + } + + return buffer; + } + + @Override + public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + if (ownBuffers.size() != 2) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf dataBuffer = ownBuffers.get(1); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + valueCount = fieldNode.getLength(); + + valueAllocationSizeInBytes = valueBuffer.capacity(); + validityAllocationSizeInBytes = validityBuffer.capacity(); + } + + public List getFieldBuffers() { + List result = new ArrayList<>(2); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getSizeFromCount(valueCount)); + valueBuffer.readerIndex(0); + valueBuffer.writerIndex(valueCount * typeWidth); + + result.add(validityBuffer); + result.add(valueBuffer); + + return result; + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator){ + return getTransferPair(name, allocator); + } + + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + public void transferTo(BaseNullableFixedWidthVector target){ + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.valueCount = valueCount; + clear(); + } + + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferValueBuffer(startIndex, length, target); + target.setValueCount(length); + } + + private void splitAndTransferValueBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + final int startPoint = startIndex * typeWidth; + final int sliceLength = length * typeWidth; + target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; + } + + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) & 0xFF) >>> offset); + } + + private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) << (8 - offset))); + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + int count = 0; + final int sizeInBytes = getSizeFromCount(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + + /** + * Get the value count of vector. This will always be zero unless + * {@link #setValueCount(int)} has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount(){ + return valueCount; + } + + + /** + * Set value count for the vector. + * + * @param valueCount value count to set + */ + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + final int currentValueCapacity = getValueCapacity(); + while (valueCount > getValueCapacity()) { + reAlloc(); + } + /* + * We are trying to understand the pattern of memory allocation. + * If initially, the user did vector.allocateNew(), we would have + * allocated memory of default size (4096 * type width). + * Later on user invokes setValueCount(count). + * + * If the existing value capacity is twice as large as the + * valueCount, we know that we over-provisioned memory in the + * first place when default memory allocation was done because user + * really needs a much less value count in the vector. + * + * We record this by bumping up the allocationMonitor. If this pattern + * happens for certain number of times and allocationMonitor + * reaches the threshold (internal hardcoded) value, subsequent + * call to allocateNew() will take care of stepping down the + * default memory allocation size. + * + * Another case would be under-provisioning the initial memory and + * thus going through a lot of realloc(). Here the goal is to + * see if we can minimize the number of reallocations. Again the + * state is recorded in allocationMonitor by decrementing it + * (negative value). If a threshold is hit, realloc will try to + * allocate more memory in order to possibly avoid a future realloc. + * This case is also applicable to setSafe() methods which can trigger + * a realloc() and thus we record the state there as well. + */ + if (valueCount > 0) { + if (currentValueCapacity >= (valueCount * 2)) { + incrementAllocationMonitor(); + } else if (currentValueCapacity <= (valueCount/2)) { + decrementAllocationMonitor(); + } + } + } + + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null, false otherwise + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + public void setIndexDefined(int index) { + handleSafe(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + public void set(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void set(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + + protected void handleSafe(int index) { + while (index >= getValueCapacity()) { + decrementAllocationMonitor(); + reAlloc(); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java new file mode 100644 index 0000000000000..a79709da4dd86 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -0,0 +1,764 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + + +import io.netty.buffer.ArrowBuf; + +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public abstract class BaseNullableVariableWidthVector extends BaseValueVector + implements VariableWidthVector, FieldVector { + private static final int DEFAULT_RECORD_BYTE_COUNT = 8; + private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; + + private int valueAllocationSizeInBytes; + private int validityAllocationSizeInBytes; + private int offsetAllocationSizeInBytes; + + /* protected members */ + protected static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ + protected static final byte[] emptyByteArray = new byte[]{}; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected ArrowBuf offsetBuffer; + protected int valueCount; + protected int lastSet; + protected final Field field; + private boolean cleared; + + public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType) { + super(name, allocator); + valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; + validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); + offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; + field = new Field(name, fieldType, null); + valueCount = 0; + lastSet = -1; + offsetBuffer = allocator.getEmpty(); + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + cleared = false; + } + + /* TODO: + * Determine how writerIndex and readerIndex need to be used. Right now we + * are setting the writerIndex and readerIndex in the call to getFieldBuffers + * using the valueCount -- this assumes that the caller of getFieldBuffers + * on the vector has already invoked setValueCount. + * + * Do we need to set them during vector transfer and splitAndTransfer? + */ + + /* TODO: + * + * see if getNullCount() can be made faster -- O(1) + */ + + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy and those vectors have not + * yet been refactored so moving things to the top class as of now + * is not a good idea. + */ + + /* TODO: + * See if we need logger -- unnecessary object probably + */ + + /* TODO: + * Implement getBufferSize(), getCurrentSizeInBytes(). + */ + + protected abstract org.slf4j.Logger getLogger(); + + public VariableWidthMutator getMutator() { + throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + } + + public VariableWidthAccessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + } + + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + @Override + public ArrowBuf getOffsetBuffer() { + return offsetBuffer; + } + + @Override + public long getOffsetBufferAddress() { + return offsetBuffer.memoryAddress(); + } + + @Override + public long getValidityBufferAddress() { + return validityBuffer.memoryAddress(); + } + + @Override + public long getDataBufferAddress() { + return valueBuffer.memoryAddress(); + } + + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int)size; + validityAllocationSizeInBytes = getSizeFromCount(valueCount); + /* to track the end offset of last data element in vector, we need + * an additional slot in offset buffer. + */ + offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; + } + + @Override + public int getValueCapacity(){ + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); + return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); + } + + /* for test purposes */ + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + + /* for test purposes */ + private int getOffsetBufferValueCapacity() { + return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); + } + + /* number of bytes for the validity buffer for a given valueCount */ + protected int getSizeFromCount(int valueCount) { + return (int) Math.ceil(valueCount / 8.0); + } + + public void zeroVector() { + initValidityBuffer(); + initOffsetBuffer(); + } + + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + private void initOffsetBuffer() { + offsetBuffer.setZero(0, offsetBuffer.capacity()); + } + + public void reset() { + zeroVector(); + lastSet = -1; + } + + @Override + public void close() { + clear(); + } + + @Override + public void clear() { + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + offsetBuffer = releaseBuffer(offsetBuffer); + cleared = true; + lastSet = -1; + valueCount = 0; + } + + @Override + public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf offBuffer = ownBuffers.get(1); + ArrowBuf dataBuffer = ownBuffers.get(2); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + offsetBuffer.release(); + offsetBuffer = offBuffer.retain(allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + lastSet = fieldNode.getLength() - 1; + valueCount = fieldNode.getLength(); + } + + public List getFieldBuffers() { + List result = new ArrayList<>(3); + final int lastDataOffset = getstartOffset(valueCount); + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getSizeFromCount(valueCount)); + offsetBuffer.readerIndex(0); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + valueBuffer.readerIndex(0); + valueBuffer.writerIndex(lastDataOffset); + + result.add(validityBuffer); + result.add(offsetBuffer); + result.add(valueBuffer); + + return result; + } + + @Override + public void allocateNew() { + if(!allocateNewSafe()){ + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + @Override + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + long curAllocationSizeOffset = offsetAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE || + curAllocationSizeOffset > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); + } catch (Exception e) { + getLogger().error("ERROR: Failure in allocateNewSafe"); + getLogger().error(e.getMessage()); + clear(); + return false; + } + + return true; + } + + @Override + public void allocateNew(int totalBytes, int valueCount) { + assert totalBytes >= 0; + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getSizeFromCount(valueCount); + + if (totalBytes > MAX_ALLOCATION_SIZE || + offsetBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); + } catch (Exception e) { + getLogger().error("ERROR: Failure in allocateNewSafe"); + getLogger().error(e.getMessage()); + clear(); + } + } + + private void allocateBytes(final long valueBufferSize, final long validityBufferSize, + final long offsetBufferSize) { + /* allocate data buffer */ + int curSize = (int)valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; + allocateValidityBuffer(validityBufferSize); + allocateOffsetBuffer(offsetBufferSize); + } + + private void allocateOffsetBuffer(final long size) { + final int curSize = (int)size; + offsetBuffer = allocator.buffer(curSize); + offsetBuffer.readerIndex(0); + offsetAllocationSizeInBytes = curSize; + initOffsetBuffer(); + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + initValidityBuffer(); + } + + public void reAlloc() { + reallocValueBuffer(); + reallocValidityAndOffsetBuffers(); + } + + protected void reallocValueBuffer() { + long baseSize = valueAllocationSizeInBytes; + final int currentBufferCapacity = valueBuffer.capacity(); + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity); + valueBuffer.release(); + valueBuffer = newBuf; + valueAllocationSizeInBytes = (int)newAllocationSize; + } + + protected void reallocValidityAndOffsetBuffers() { + offsetBuffer = reallocBufferHelper(offsetBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + /* need to refactor this to keep the logic in an single place and make callers + * more intelligent. see handleSafe() for more comments on realloc + */ + + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", + name, (offsetBuffer ? offsetAllocationSizeInBytes : validityAllocationSizeInBytes), + newAllocationSize); + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (offsetBuffer) { + offsetAllocationSizeInBytes = (int)newAllocationSize; + } + else { + validityAllocationSizeInBytes = (int)newAllocationSize; + } + + return buffer; + } + + @Override + public int getByteCapacity(){ + return valueBuffer.capacity(); + } + + @Override + public int getCurrentSizeInBytes(){ + /* TODO */ + return 0; + } + + @Override + public int getBufferSize() { + /* TODO */ + return 0; + } + + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } + + final int validityBufferSize = getSizeFromCount(valueCount); + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + /* get the end offset for this valueCount */ + final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + return validityBufferSize + offsetBufferSize + dataBufferSize; + } + + @Override + public Field getField() { + return field; + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers = new ArrowBuf[3]; + buffers[0] = validityBuffer; + buffers[1] = offsetBuffer; + buffers[1] = valueBuffer; + if (clear) { + for (final ArrowBuf buffer:buffers) { + buffer.retain(1); + } + clear(); + } + return buffers; + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator){ + return getTransferPair(name, allocator); + } + + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + public void transferTo(BaseNullableVariableWidthVector target){ + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; + target.valueCount = valueCount; + target.setLastSet(lastSet); + clear(); + } + + public void splitAndTransferTo(int startIndex, int length, + BaseNullableVariableWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferOffsetBuffer(startIndex, length, target); + target.setLastSet(length - 1); + target.setValueCount(length); + } + + /* + * transfer the offsets along with data + */ + private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { + final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); + final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH); + final int dataLength = end - start; + target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + for (int i = 0; i < length + 1; i++) { + final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start; + target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset); + } + target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer; + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableVariableWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) & 0xFF) >>> offset); + } + + private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) << (8 - offset))); + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + int count = 0; + final int sizeInBytes = getSizeFromCount(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the value count of vector. This will always be zero unless + * setValueCount(int) has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount(){ + return valueCount; + } + + /** + * Sets the value count for the vector + * + * @param valueCount value count + */ + public void setValueCount(int valueCount) { + assert valueCount >= 0; + this.valueCount = valueCount; + while (valueCount > getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + fillHoles(valueCount); + lastSet = valueCount - 1; + } + + public void fillEmpties(int index) { + handleSafe(index, emptyByteArray.length); + fillHoles(index); + lastSet = index - 1; + } + + public void setLastSet(int value) { + lastSet = value; + } + + public int getLastSet() { + return lastSet; + } + + public long getStartEnd(int index) { + return (long)offsetBuffer.getInt(index * OFFSET_WIDTH); + } + + public void setIndexDefined(int index) { + handleSafe(index, 0); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + protected final void fillHoles(int index) { + for (int i = lastSet + 1; i < index; i++) { + setBytes(i, emptyByteArray, 0, emptyByteArray.length); + } + lastSet = index - 1; + } + + protected final void setBytes(int index, byte[] value, int start, int length) { + /* end offset of current last element in the vector. this will + * be the start offset of new element we are trying to store. + */ + final int startOffset = getstartOffset(index); + /* set new end offset */ + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + /* store the var length data in value buffer */ + valueBuffer.setBytes(startOffset, value, start, length); + } + + protected final int getstartOffset(int index) { + return offsetBuffer.getInt(index * OFFSET_WIDTH); + } + + protected final void handleSafe(int index, int dataLength) { + /* + * IMPORTANT: + * value buffer for variable length vectors moves independent + * of the companion validity and offset buffers. This is in + * contrast to what we have for fixed width vectors. + * + * Here there is no concept of getValueCapacity() in the + * data stream. getValueCapacity() is applicable only to validity + * and offset buffers. + * + * So even though we may have setup an initial capacity of 1024 + * elements in the vector, it is quite possible + * that we need to reAlloc() the data buffer when we are setting + * the 5th element in the vector simply because previous + * variable length elements have exhausted the buffer capacity. + * However, we really don't need to reAlloc() validity and + * offset buffers until we try to set the 1025th element + * This is why we do a separate check for safe methods to + * determine which buffer needs reallocation. + */ + while (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + final int startOffset = getstartOffset(index); + while (valueBuffer.capacity() < (startOffset + dataLength)) { + reallocValueBuffer(); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 598e578e55a6d..1db96246485d3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -18,8 +18,10 @@ package org.apache.arrow.vector; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; +import java.util.List; import com.google.flatbuffers.FlatBufferBuilder; import org.apache.arrow.memory.BufferAllocator; @@ -122,5 +124,23 @@ public static boolean checkBufRefs(final ValueVector vv) { public BufferAllocator getAllocator() { return allocator; } + + protected void compareTypes(BaseValueVector target, String caller) { + if (this.getMinorType() != target.getMinorType()) { + throw new UnsupportedOperationException(caller + " should have vectors of exact same type"); + } + } + + protected ArrowBuf releaseBuffer(ArrowBuf buffer) { + buffer.release(); + buffer = allocator.getEmpty(); + return buffer; + } + + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java new file mode 100644 index 0000000000000..2439bd26eab56 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; + +class BitVectorHelper { + + /** + * Get the index of byte corresponding to bit index in validity buffer + */ + protected static int byteIndex(int absoluteBitIndex) { + return absoluteBitIndex >> 3; + } + + /** + * Get the relative index of bit within the byte in validity buffer + */ + private static int bitIndex(int absoluteBitIndex) { + return absoluteBitIndex & 7; + } + + protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + currentByte |= bitMask; + validityBuffer.setByte(byteIndex, currentByte); + } + + protected static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + if (value != 0) { + currentByte |= bitMask; + } else { + currentByte -= (bitMask & currentByte); + } + validityBuffer.setByte(byteIndex, currentByte); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java new file mode 100644 index 0000000000000..26b19fa7bd562 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -0,0 +1,299 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntHolder; +import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableIntVector implements a fixed width vector of values which could + * be null. A validity buffer (bit vector) is maintained to track which + * elements in the vector are null. + */ +public class NullableIntVector extends BaseNullableFixedWidthVector { + private static final org.slf4j.Logger logger = + org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + public NullableIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), + allocator); + } + + public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntReaderImpl(NullableIntVector.this); + } + + @Override + protected org.slf4j.Logger getLogger() { + return logger; + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int valueField ) { + if (isSet > 0) { + set(index, valueField); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, int valueField ) { + handleSafe(index); + set(index, isSet, valueField); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntVector to){ + this.to = to; + } + + @Override + public NullableIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java new file mode 100644 index 0000000000000..b9021544b0e10 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -0,0 +1,451 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.VarCharReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.VarCharHolder; +import org.apache.arrow.vector.holders.NullableVarCharHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.Text; +import org.apache.arrow.vector.util.TransferPair; + +import java.nio.ByteBuffer; + +public class NullableVarCharVector extends BaseNullableVariableWidthVector { + private static final org.slf4j.Logger logger = + org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); + private final FieldReader reader; + + public NullableVarCharVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); + } + + public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarCharReaderImpl(NullableVarCharVector.this); + } + + @Override + protected org.slf4j.Logger getLogger() { + return logger; + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARCHAR; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return Text object for non-null element, null otherwise + */ + public Text getObject(int index) { + Text result = new Text(); + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + result.set(b); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if(isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarCharHolder holder){ + assert index >= 0; + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + holder.isSet = 1; + holder.start = startOffset; + holder.end = dataLength; + holder.buffer = valueBuffer; + } + + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + + public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { + fillHoles(thisIndex); + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { + fillEmpties(thisIndex); + if (from.isSet(fromIndex) != 0) { + setSafe(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index, 0); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + endField); + final ArrowBuf bb = bufferField.slice(startField, endField); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + assert index >= 0; + handleSafe(index, endField); + set(index, isSet, startField, endField, bufferField); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarCharVector)to); + } + + private class TransferImpl implements TransferPair { + NullableVarCharVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableVarCharVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarCharVector to){ + this.to = to; + } + + @Override + public NullableVarCharVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index fb7286f852c06..e6048b4b517c9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -255,4 +255,10 @@ interface Mutator { * @return buffer */ public ArrowBuf getOffsetBuffer(); + + public int getValueCount(); + + public void setValueCount(int valueCount); + + public Object getObject(int index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index f8385a7262a21..96243eb5876b0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -27,6 +27,8 @@ import org.apache.arrow.vector.schema.ArrowRecordBatch; import org.apache.arrow.vector.schema.ArrowVectorType; +import javax.annotation.Nullable; + public class VectorUnloader { private final VectorSchemaRoot root; @@ -53,8 +55,17 @@ public ArrowRecordBatch getRecordBatch() { } private void appendNodes(FieldVector vector, List nodes, List buffers) { - Accessor accessor = vector.getAccessor(); - nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); + Accessor accessor = null; + if (vector instanceof NullableIntVector) { + nodes.add(new ArrowFieldNode(((NullableIntVector)vector).getValueCount(), + includeNullCount ? ((NullableIntVector)vector).getNullCount() : -1)); + } else if (vector instanceof NullableVarCharVector) { + nodes.add(new ArrowFieldNode(((NullableVarCharVector)vector).getValueCount(), + includeNullCount ? ((NullableVarCharVector)vector).getNullCount() : -1)); + } else { + accessor = vector.getAccessor(); + nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); + } List fieldBuffers = vector.getFieldBuffers(); List expectedBuffers = vector.getField().getTypeLayout().getVectorTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 5ac00375f8317..fcef02f4a1ba8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -259,4 +259,13 @@ public ArrowBuf getDataBuffer() { public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } + + @Override + public int getValueCount() { return 0; } + + @Override + public void setValueCount(int valueCount) { } + + @Override + public Object getObject(int index) { return null; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index b3be37541c172..e95442acf6d41 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -34,14 +34,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; -import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BaseDataValueVector; -import org.apache.arrow.vector.BaseValueVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; @@ -331,9 +324,15 @@ public Object getObject(int index) { return null; } final List vals = new JsonStringArrayList<>(listSize); - final ValueVector.Accessor valuesAccessor = vector.getAccessor(); - for (int i = 0; i < listSize; i++) { - vals.add(valuesAccessor.getObject(index * listSize + i)); + if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { + for (int i = 0; i < listSize; i++) { + vals.add(vector.getObject(index * listSize + i)); + } + } else { + final ValueVector.Accessor valuesAccessor = vector.getAccessor(); + for (int i = 0; i < listSize; i++) { + vals.add(valuesAccessor.getObject(index * listSize + i)); + } } return vals; } @@ -367,7 +366,11 @@ public void setNotNull(int index) { @Override public void setValueCount(int valueCount) { bits.getMutator().setValueCount(valueCount); - vector.getMutator().setValueCount(valueCount * listSize); + if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { + vector.setValueCount(valueCount * listSize); + } else { + vector.getMutator().setValueCount(valueCount * listSize); + } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index ea28a60619209..4b2c9139219f3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -40,6 +40,8 @@ import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.NullableIntVector; +import org.apache.arrow.vector.NullableVarCharVector; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; @@ -376,10 +378,18 @@ public Object getObject(int index) { final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); final int start = offsetsAccessor.get(index); final int end = offsetsAccessor.get(index + 1); - final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor(); - for (int i = start; i < end; i++) { - vals.add(valuesAccessor.getObject(i)); + final ValueVector vv = getDataVector(); + if (vv instanceof NullableIntVector || vv instanceof NullableVarCharVector) { + for (int i = start; i < end; i++) { + vals.add(vv.getObject(i)); + } + } else { + final ValueVector.Accessor valuesAccessor = vv.getAccessor(); + for (int i = start; i < end; i++) { + vals.add(valuesAccessor.getObject(i)); + } } + return vals; } @@ -432,7 +442,12 @@ public void setValueCount(int valueCount) { offsets.getMutator().setValueCount(valueCount + 1); } final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); + if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { + vector.setValueCount(childValueCount); + } else { + vector.getMutator().setValueCount(childValueCount); + } + bits.getMutator().setValueCount(valueCount); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index f46635ad76021..95efa60871806 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -35,9 +35,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BaseValueVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.ComplexHolder; @@ -272,10 +270,19 @@ public Object getObject(int index) { Map vv = new JsonStringHashMap<>(); for (String child : getChildFieldNames()) { ValueVector v = getChild(child); - if (v != null && index < v.getAccessor().getValueCount()) { - Object value = v.getAccessor().getObject(index); - if (value != null) { - vv.put(child, value); + if (v instanceof NullableVarCharVector || v instanceof NullableIntVector) { + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); + } + } + } else { + if (v != null && index < v.getAccessor().getValueCount()) { + Object value = v.getAccessor().getObject(index); + if (value != null) { + vv.put(child, value); + } } } } @@ -302,7 +309,11 @@ public class Mutator extends BaseValueVector.BaseMutator { @Override public void setValueCount(int valueCount) { for (final ValueVector v : getChildren()) { - v.getMutator().setValueCount(valueCount); + if (v instanceof NullableIntVector || v instanceof NullableVarCharVector) { + v.setValueCount(valueCount); + } else { + v.getMutator().setValueCount(valueCount); + } } MapVector.this.valueCount = valueCount; } @@ -362,4 +373,10 @@ public List getChildrenFromFields() { return getChildren(); } + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } + } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index e1c7c909f10be..0de804406f7b3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -265,7 +265,7 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1); break; case VARCHAR: - ((NullableVarCharVector) vector).getMutator().setLastSet(count - 1); + ((NullableVarCharVector) vector).setLastSet(count - 1); break; } vector.getMutator().setValueCount(count); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java index 1a801a63ec688..4fe2861450a34 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java @@ -40,7 +40,7 @@ public void testTransferFixedWidth() { NullableIntVector v1 = new NullableIntVector("v1", childAllocator1); v1.allocateNew(); - v1.getMutator().setValueCount(4095); + v1.setValueCount(4095); NullableIntVector v2 = new NullableIntVector("v2", childAllocator2); @@ -60,8 +60,8 @@ public void testTransferVariableidth() { NullableVarCharVector v1 = new NullableVarCharVector("v1", childAllocator1); v1.allocateNew(); - v1.getMutator().setSafe(4094, "hello world".getBytes(), 0, 11); - v1.getMutator().setValueCount(4001); + v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setValueCount(4001); NullableVarCharVector v2 = new NullableVarCharVector("v2", childAllocator2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index f8c16e7fc8176..1185246f88cde 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -56,24 +56,22 @@ public void testEncodeStrings() { // Create a new value vector try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator); final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(512, 5); // set some values - m.setSafe(0, zero, 0, zero.length); - m.setSafe(1, one, 0, one.length); - m.setSafe(2, one, 0, one.length); - m.setSafe(3, two, 0, two.length); - m.setSafe(4, zero, 0, zero.length); - m.setValueCount(5); + vector.setSafe(0, zero, 0, zero.length); + vector.setSafe(1, one, 0, one.length); + vector.setSafe(2, one, 0, one.length); + vector.setSafe(3, two, 0, two.length); + vector.setSafe(4, zero, 0, zero.length); + vector.setValueCount(5); // set some dictionary values - final NullableVarCharVector.Mutator m2 = dictionaryVector.getMutator(); dictionaryVector.allocateNew(512, 3); - m2.setSafe(0, zero, 0, zero.length); - m2.setSafe(1, one, 0, one.length); - m2.setSafe(2, two, 0, two.length); - m2.setValueCount(3); + dictionaryVector.setSafe(0, zero, 0, zero.length); + dictionaryVector.setSafe(1, one, 0, one.length); + dictionaryVector.setSafe(2, two, 0, two.length); + dictionaryVector.setValueCount(3); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); @@ -81,20 +79,20 @@ public void testEncodeStrings() { // verify indices assertEquals(NullableIntVector.class, encoded.getClass()); - NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor(); - assertEquals(5, indexAccessor.getValueCount()); - assertEquals(0, indexAccessor.get(0)); - assertEquals(1, indexAccessor.get(1)); - assertEquals(1, indexAccessor.get(2)); - assertEquals(2, indexAccessor.get(3)); - assertEquals(0, indexAccessor.get(4)); + NullableIntVector index = ((NullableIntVector)encoded); + assertEquals(5, index.getValueCount()); + assertEquals(0, index.get(0)); + assertEquals(1, index.get(1)); + assertEquals(1, index.get(2)); + assertEquals(2, index.get(3)); + assertEquals(0, index.get(4)); // now run through the decoder and verify we get the original back try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) { assertEquals(vector.getClass(), decoded.getClass()); - assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount()); + assertEquals(vector.getValueCount(), ((NullableVarCharVector)decoded).getValueCount()); for (int i = 0; i < 5; i++) { - assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i)); + assertEquals(vector.getObject(i), ((NullableVarCharVector)decoded).getObject(i)); } } } @@ -106,21 +104,20 @@ public void testEncodeLargeVector() { // Create a new value vector try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator); final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(); int count = 10000; for (int i = 0; i < 10000; ++i) { - vector.getMutator().setSafe(i, data[i % 3], 0, data[i % 3].length); + vector.setSafe(i, data[i % 3], 0, data[i % 3].length); } - vector.getMutator().setValueCount(count); + vector.setValueCount(count); dictionaryVector.allocateNew(512, 3); - dictionaryVector.getMutator().setSafe(0, zero, 0, zero.length); - dictionaryVector.getMutator().setSafe(1, one, 0, one.length); - dictionaryVector.getMutator().setSafe(2, two, 0, two.length); - dictionaryVector.getMutator().setValueCount(3); + dictionaryVector.setSafe(0, zero, 0, zero.length); + dictionaryVector.setSafe(1, one, 0, one.length); + dictionaryVector.setSafe(2, two, 0, two.length); + dictionaryVector.setValueCount(3); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); @@ -129,10 +126,10 @@ public void testEncodeLargeVector() { // verify indices assertEquals(NullableIntVector.class, encoded.getClass()); - NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor(); - assertEquals(count, indexAccessor.getValueCount()); + NullableIntVector index = ((NullableIntVector) encoded); + assertEquals(count, index.getValueCount()); for (int i = 0; i < count; ++i) { - assertEquals(i % 3, indexAccessor.get(i)); + assertEquals(i % 3, index.get(i)); } // now run through the decoder and verify we get the original back diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 43d9387b106a4..168deacd6a640 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -54,15 +54,14 @@ public void terminate() throws Exception { public void testIntType() { try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); - NullableIntVector.Mutator mutator = nested.getMutator(); vector.allocateNew(); for (int i = 0; i < 10; i++) { vector.getMutator().setNotNull(i); - mutator.set(i * 2, i); - mutator.set(i * 2 + 1, i + 10); + nested.set(i * 2, i); + nested.set(i * 2 + 1, i + 10); } - vector.getMutator().setValueCount(10); + vector.setValueCount(10); UnionFixedSizeListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -119,7 +118,7 @@ public void testNestedInList() { ListVector.Mutator mutator = vector.getMutator(); FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector(); FixedSizeListVector.Mutator tupleMutator = tuples.getMutator(); - NullableIntVector.Mutator innerMutator = (NullableIntVector.Mutator) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector().getMutator(); + NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { @@ -127,8 +126,8 @@ public void testNestedInList() { int position = mutator.startNewValue(i); for (int j = 0; j < i % 7; j++) { tupleMutator.setNotNull(position + j); - innerMutator.set((position + j) * 2, j); - innerMutator.set((position + j) * 2 + 1, j + 1); + innerVector.set((position + j) * 2, j); + innerVector.set((position + j) * 2 + 1, j + 1); } mutator.endValue(i, i % 7); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index 66e5375e3bd0d..7de3bcb33195f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -25,7 +25,6 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.NullableVarCharVector; -import org.apache.arrow.vector.NullableVarCharVector.Accessor; import org.apache.arrow.vector.util.TransferPair; import org.junit.After; @@ -54,32 +53,30 @@ public void test() throws Exception { final int valueCount = 500; final String[] compareArray = new String[valueCount]; - final NullableVarCharVector.Mutator mutator = varCharVector.getMutator(); for (int i = 0; i < valueCount; i += 3) { final String s = String.format("%010d", i); - mutator.set(i, s.getBytes()); + varCharVector.set(i, s.getBytes()); compareArray[i] = s; } - mutator.setValueCount(valueCount); + varCharVector.setValueCount(valueCount); final TransferPair tp = varCharVector.getTransferPair(allocator); final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo(); - final Accessor accessor = newVarCharVector.getAccessor(); final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}}; for (final int[] startLength : startLengths) { final int start = startLength[0]; final int length = startLength[1]; tp.splitAndTransfer(start, length); - newVarCharVector.getMutator().setValueCount(length); + newVarCharVector.setValueCount(length); for (int i = 0; i < length; i++) { final boolean expectedSet = ((start + i) % 3) == 0; if (expectedSet) { final byte[] expectedValue = compareArray[start + i].getBytes(); - assertFalse(accessor.isNull(i)); - assertArrayEquals(expectedValue, accessor.get(i)); + assertFalse(newVarCharVector.isNull(i)); + assertArrayEquals(expectedValue, newVarCharVector.get(i)); } else { - assertTrue(accessor.isNull(i)); + assertTrue(newVarCharVector.isNull(i)); } } newVarCharVector.clear(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index a239861d9b32f..b7f88c3babf1b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -694,8 +694,6 @@ public void testNullableFixedType2() { public void testNullableFixedType3() { // Create a new value vector for 1024 integers try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { - final NullableIntVector.Mutator mutator = vector.getMutator(); - final NullableIntVector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 1024; @@ -706,26 +704,26 @@ public void testNullableFixedType3() { /* underlying buffer should be able to store 16 values */ assertEquals(initialCapacity, vector.getValueCapacity()); - mutator.set(0, 1); - mutator.set(1, 2); - mutator.set(100, 3); - mutator.set(1022, 4); - mutator.set(1023, 5); + vector.set(0, 1); + vector.set(1, 2); + vector.set(100, 3); + vector.set(1022, 4); + vector.set(1023, 5); /* check vector contents */ int j = 1; for(int i = 0; i <= 1023; i++) { if((i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } else { - assertFalse("null data not expected at index: " + i, accessor.isNull(i)); - assertEquals("unexpected value at index: " + i, j, accessor.get(i)); + assertFalse("null data not expected at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, j, vector.get(i)); j++; } } - mutator.setValueCount(1024); + vector.setValueCount(1024); Field field = vector.getField(); TypeLayout typeLayout = field.getTypeLayout(); @@ -749,7 +747,7 @@ public void testNullableFixedType3() { assertEquals(-64, validityVectorBuf.getByte(127)); // 1022nd and 1023rd bit defined /* this should trigger a realloc() */ - mutator.setSafe(1024, 6); + vector.setSafe(1024, 6); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); @@ -758,11 +756,11 @@ public void testNullableFixedType3() { j = 1; for(int i = 0; i < (initialCapacity * 2); i++) { if((i > 1024) || (i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } else { - assertFalse("null data not expected at index: " + i, accessor.isNull(i)); - assertEquals("unexpected value at index: " + i, j, accessor.get(i)); + assertFalse("null data not expected at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, j, vector.get(i)); j++; } } @@ -775,13 +773,101 @@ public void testNullableFixedType3() { /* vector data should have been zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } vector.allocateNew(4096); // vector has been erased for(int i = 0; i < 4096; i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + } + } + } + + @Test /* NullableIntVector */ + public void testNullableFixedType4() { + try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { + + /* no memory allocation has happened yet */ + assertEquals(0, vector.getValueCapacity()); + + vector.allocateNew(); + int valueCapacity = vector.getValueCapacity(); + assertEquals(vector.INITIAL_VALUE_ALLOCATION, valueCapacity); + + int baseValue = 20000; + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + vector.set(i, baseValue + i); + } + } + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + vector.setSafe(valueCapacity, 20000000); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); + + for (int i = 0; i < vector.getValueCapacity(); i++) { + if (i == valueCapacity) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, 20000000, vector.get(i)); + } else if (i < valueCapacity) { + if ((i & 1) == 1) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + } + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + vector.zeroVector(); + + for (int i = 0; i < vector.getValueCapacity(); i+=2) { + vector.set(i, baseValue + i); + } + + for (int i = 0; i < vector.getValueCapacity(); i++) { + if (i%2 == 0) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + vector.setSafe((valueCapacity * 2) + 1000, 400000000); + assertEquals(valueCapacity * 4, vector.getValueCapacity()); + + for (int i = 0; i < vector.getValueCapacity(); i++) { + if (i == (valueCapacity*2 + 1000)) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, 400000000, vector.get(i)); + } else if (i < valueCapacity*2 && (i%2) == 0) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, baseValue + i, vector.get(i)); + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + /* reset the vector */ + vector.reset(); + + /* capacity shouldn't change after reset */ + assertEquals(valueCapacity * 4, vector.getValueCapacity()); + + /* vector data should be zeroed out */ + for(int i = 0; i < (valueCapacity * 4); i++) { + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -805,32 +891,30 @@ public void testNullableVarType1() { // Create a new value vector for 1024 integers. try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(1024 * 10, 1024); - m.set(0, STR1); - m.set(1, STR2); - m.set(2, STR3); - m.setSafe(3, STR3, 1, STR3.length - 1); - m.setSafe(4, STR3, 2, STR3.length - 2); + vector.set(0, STR1); + vector.set(1, STR2); + vector.set(2, STR3); + vector.setSafe(3, STR3, 1, STR3.length - 1); + vector.setSafe(4, STR3, 2, STR3.length - 2); ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3); - m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); - m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); + vector.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); + vector.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); // Check the sample strings. - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(3)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(4)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(5)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(6)); // Ensure null value throws. boolean b = false; try { - vector.getAccessor().get(7); + vector.get(7); } catch (IllegalStateException e) { b = true; } finally { @@ -1070,43 +1154,42 @@ public void testReallocAfterVectorTransfer2() { @Test /* NullableVarCharVector */ public void testReallocAfterVectorTransfer3() { try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableVarCharVector.Mutator mutator = vector.getMutator(); - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - /* 4096 values with 10 byte per record */ vector.allocateNew(4096 * 10, 4096); int valueCapacity = vector.getValueCapacity(); + assertEquals(4096, valueCapacity); /* populate the vector */ for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - mutator.set(i, STR1); + vector.set(i, STR1); } else { - mutator.set(i, STR2); + vector.set(i, STR2); } } /* Check the vector output */ for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertArrayEquals(STR1, accessor.get(i)); + assertArrayEquals(STR1, vector.get(i)); } else { - assertArrayEquals(STR2, accessor.get(i)); + assertArrayEquals(STR2, vector.get(i)); } } /* trigger first realloc */ - mutator.setSafe(valueCapacity, STR2, 0, STR2.length); + vector.setSafe(valueCapacity, STR2, 0, STR2.length); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); /* populate the remaining vector */ for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { if ((i & 1) == 1) { - mutator.set(i, STR1); + vector.set(i, STR1); } else { - mutator.set(i, STR2); + vector.set(i, STR2); } } @@ -1114,23 +1197,24 @@ public void testReallocAfterVectorTransfer3() { valueCapacity = vector.getValueCapacity(); for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertArrayEquals(STR1, accessor.get(i)); + assertArrayEquals(STR1, vector.get(i)); } else { - assertArrayEquals(STR2, accessor.get(i)); + assertArrayEquals(STR2, vector.get(i)); } } /* trigger second realloc */ - mutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length); + vector.setSafe(valueCapacity + 10, STR2, 0, STR2.length); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); /* populate the remaining vector */ for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { if ((i & 1) == 1) { - mutator.set(i, STR1); + vector.set(i, STR1); } else { - mutator.set(i, STR2); + vector.set(i, STR2); } } @@ -1138,10 +1222,10 @@ public void testReallocAfterVectorTransfer3() { valueCapacity = vector.getValueCapacity(); for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertArrayEquals(STR1, accessor.get(i)); + assertArrayEquals(STR1, vector.get(i)); } else { - assertArrayEquals(STR2, accessor.get(i)); + assertArrayEquals(STR2, vector.get(i)); } } @@ -1152,13 +1236,112 @@ public void testReallocAfterVectorTransfer3() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); NullableVarCharVector toVector = (NullableVarCharVector)transferPair.getTo(); - NullableVarCharVector.Mutator toMutator = toVector.getMutator(); - NullableVarCharVector.Accessor toAccessor = toVector.getAccessor(); - valueCapacity = toVector.getValueCapacity(); - /* trigger a realloc of this toVector */ - toMutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(STR1, toVector.get(i)); + } + else { + assertArrayEquals(STR2, toVector.get(i)); + } + } + + toVector.close(); + } + } + + @Test /* NullableIntVector */ + public void testReallocAfterVectorTransfer4() { + try (final NullableIntVector vector = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) { + + /* 4096 values */ + vector.allocateNew(4096); + int valueCapacity = vector.getValueCapacity(); + assertEquals(4096, valueCapacity); + + /* populate the vector */ + int baseValue = 1000; + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + vector.set(i, 1000 + i); + } + } + + /* Check the vector output */ + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, vector.get(i)); + } + else { + assertTrue(vector.isNull(i)); + } + } + + /* trigger first realloc */ + vector.setSafe(valueCapacity, 10000000); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); + + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 0) { + vector.set(i, 1000 + i); + } + } + + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, vector.get(i)); + } + else { + assertTrue(vector.isNull(i)); + } + } + + /* trigger second realloc */ + vector.setSafe(valueCapacity, 10000000); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); + + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 0) { + vector.set(i, 1000 + i); + } + } + + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, vector.get(i)); + } + else { + assertTrue(vector.isNull(i)); + } + } + + /* we are potentially working with 4x the size of vector buffer + * that we initially started with. Now let's transfer the vector. + */ + + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.transfer(); + NullableIntVector toVector = (NullableIntVector)transferPair.getTo(); + /* value capacity of source and target vectors should be same after + * the transfer. + */ + assertEquals(valueCapacity, toVector.getValueCapacity()); + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, toVector.get(i)); + } + else { + assertTrue(toVector.isNull(i)); + } + } toVector.close(); } @@ -1199,31 +1382,33 @@ public void testReAllocNullableFixedWidthVector() { @Test public void testReAllocNullableVariableWidthVector() { - // Create a new value vector for 1024 integers try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(); int initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); - // Put values in indexes that fall within the initial allocation - m.setSafe(0, STR1, 0, STR1.length); - m.setSafe(initialCapacity - 1, STR2, 0, STR2.length); + /* Put values in indexes that fall within the initial allocation */ + vector.setSafe(0, STR1, 0, STR1.length); + vector.setSafe(initialCapacity - 1, STR2, 0, STR2.length); - // Now try to put values in space that falls beyond the initial allocation - m.setSafe(initialCapacity + 200, STR3, 0, STR3.length); + /* the above set calls should NOT have triggered a realloc */ + initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); - // Check valueCapacity is more than initial allocation - assertEquals((initialCapacity + 1) * 2 - 1, vector.getValueCapacity()); + /* Now try to put values in space that falls beyond the initial allocation */ + vector.setSafe(initialCapacity + 200, STR3, 0, STR3.length); - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(initialCapacity - 1)); - assertArrayEquals(STR3, accessor.get(initialCapacity + 200)); + /* Check valueCapacity is more than initial allocation */ + assertEquals(((initialCapacity + 1) * 2) - 1, vector.getValueCapacity()); + + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(initialCapacity - 1)); + assertArrayEquals(STR3, vector.get(initialCapacity + 200)); // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors // as we don't call setSafe for null values, but we do call setValueCount when the current batch is processed. - m.setValueCount(vector.getValueCapacity() + 200); + vector.setValueCount(vector.getValueCapacity() + 200); } } @@ -1232,10 +1417,18 @@ public void testFillEmptiesNotOverfill() { try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { vector.allocateNew(); - vector.getMutator().setSafe(4094, "hello".getBytes(), 0, 5); - vector.getMutator().setValueCount(4095); + int initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); + + vector.setSafe(4094, "hello".getBytes(), 0, 5); + /* the above set method should NOT have trigerred a realloc */ + initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); - assertEquals(4096 * 4, vector.getFieldBuffers().get(1).capacity()); + vector.setValueCount(4095); + assertEquals(4096 * vector.OFFSET_WIDTH, vector.getFieldBuffers().get(1).capacity()); + initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); } } @@ -1243,42 +1436,129 @@ public void testFillEmptiesNotOverfill() { public void testCopyFromWithNulls() { try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); for (int i = 0; i < 4095; i++) { if (i % 3 == 0) { continue; } byte[] b = Integer.toString(i).getBytes(); - vector.getMutator().setSafe(i, b, 0, b.length); + vector.setSafe(i, b, 0, b.length); } - vector.getMutator().setValueCount(4095); + /* NO reAlloc() should have happened in setSafe() */ + capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + vector.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + } + } vector2.allocateNew(); + capacity = vector2.getValueCapacity(); + assertEquals(4095, capacity); for (int i = 0; i < 4095; i++) { vector2.copyFromSafe(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } } - vector2.getMutator().setValueCount(4095); + /* NO reAlloc() should have happened in copyFrom */ + capacity = vector2.getValueCapacity(); + assertEquals(4095, capacity); + + vector2.setValueCount(4095); for (int i = 0; i < 4095; i++) { if (i % 3 == 0) { - assertNull(vector2.getAccessor().getObject(i)); + assertNull(vector2.getObject(i)); } else { - assertEquals(Integer.toString(i), vector2.getAccessor().getObject(i).toString()); + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); } } } } @Test - public void testSetLastSetUsage() { - try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { + public void testCopyFromWithNulls1() { + try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); + final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { - final NullableVarCharVector.Mutator mutator = vector.getMutator(); + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + continue; + } + byte[] b = Integer.toString(i).getBytes(); + vector.setSafe(i, b, 0, b.length); + } + + /* NO reAlloc() should have happened in setSafe() */ + capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + vector.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024 * 10, 1024); + + capacity = vector2.getValueCapacity(); + assertEquals(1024, capacity); + + for (int i = 0; i < 4095; i++) { + vector2.copyFromSafe(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + + /* 2 reAllocs should have happened in copyFromSafe() */ + capacity = vector2.getValueCapacity(); + assertEquals(4096, capacity); + + vector2.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + } + } + + @Test + public void testSetLastSetUsage() { + try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { vector.allocateNew(1024 * 10, 1024); setBytes(0, STR1, vector); @@ -1289,32 +1569,102 @@ public void testSetLastSetUsage() { setBytes(5, STR6, vector); /* Check current lastSet */ - assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(-1), Integer.toString(vector.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); /* * If we don't do setLastSe(5) before setValueCount(), then the latter will corrupt * the value vector by filling in all positions [0,valuecount-1] will empty byte arrays. * Run the test by commenting out next line and we should see incorrect vector output. */ - mutator.setLastSet(5); - mutator.setValueCount(20); + vector.setLastSet(5); + vector.setValueCount(20); + + /* Check current lastSet */ + assertEquals(Integer.toString(19), Integer.toString(vector.getLastSet())); /* Check the vector output again */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(10))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(11))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(12))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(13))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(14))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(15))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(16))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(17))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(18))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(19))); + + /* Check offsets */ + assertEquals(Integer.toString(0), + Integer.toString(vector.offsetBuffer.getInt(0 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(6), + Integer.toString(vector.offsetBuffer.getInt(1 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(16), + Integer.toString(vector.offsetBuffer.getInt(2 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(21), + Integer.toString(vector.offsetBuffer.getInt(3 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(30), + Integer.toString(vector.offsetBuffer.getInt(4 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(34), + Integer.toString(vector.offsetBuffer.getInt(5 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(6 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(7 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(8 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(9 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(10 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(11 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(12 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(13 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(14 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(15 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(16 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(17 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(18 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(19 * vector.OFFSET_WIDTH))); + + vector.set(19, STR6); + assertArrayEquals(STR6, vector.get(19)); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(19 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(46), + Integer.toString(vector.offsetBuffer.getInt(20 * vector.OFFSET_WIDTH))); } } @@ -1322,29 +1672,25 @@ public void testSetLastSetUsage() { public void testVectorLoadUnload() { try (final NullableVarCharVector vector1 = new NullableVarCharVector("myvector", allocator)) { - - final NullableVarCharVector.Mutator mutator1 = vector1.getMutator(); - vector1.allocateNew(1024 * 10, 1024); - mutator1.set(0, STR1); - mutator1.set(1, STR2); - mutator1.set(2, STR3); - mutator1.set(3, STR4); - mutator1.set(4, STR5); - mutator1.set(5, STR6); - assertEquals(Integer.toString(5), Integer.toString(mutator1.getLastSet())); - mutator1.setValueCount(15); - assertEquals(Integer.toString(14), Integer.toString(mutator1.getLastSet())); + vector1.set(0, STR1); + vector1.set(1, STR2); + vector1.set(2, STR3); + vector1.set(3, STR4); + vector1.set(4, STR5); + vector1.set(5, STR6); + assertEquals(Integer.toString(5), Integer.toString(vector1.getLastSet())); + vector1.setValueCount(15); + assertEquals(Integer.toString(14), Integer.toString(vector1.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor(); - assertArrayEquals(STR1, accessor1.get(0)); - assertArrayEquals(STR2, accessor1.get(1)); - assertArrayEquals(STR3, accessor1.get(2)); - assertArrayEquals(STR4, accessor1.get(3)); - assertArrayEquals(STR5, accessor1.get(4)); - assertArrayEquals(STR6, accessor1.get(5)); + assertArrayEquals(STR1, vector1.get(0)); + assertArrayEquals(STR2, vector1.get(1)); + assertArrayEquals(STR3, vector1.get(2)); + assertArrayEquals(STR4, vector1.get(3)); + assertArrayEquals(STR5, vector1.get(4)); + assertArrayEquals(STR6, vector1.get(5)); Field field = vector1.getField(); String fieldName = field.getName(); @@ -1357,7 +1703,7 @@ public void testVectorLoadUnload() { Schema schema = new Schema(fields); - VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, accessor1.getValueCount()); + VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, vector1.getValueCount()); VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1); try ( @@ -1370,24 +1716,21 @@ public void testVectorLoadUnload() { vectorLoader.load(recordBatch); NullableVarCharVector vector2 = (NullableVarCharVector) schemaRoot2.getVector(fieldName); - NullableVarCharVector.Mutator mutator2 = vector2.getMutator(); - /* * lastSet would have internally been set by VectorLoader.load() when it invokes * loadFieldBuffers. */ - assertEquals(Integer.toString(14), Integer.toString(mutator2.getLastSet())); - mutator2.setValueCount(25); - assertEquals(Integer.toString(24), Integer.toString(mutator2.getLastSet())); + assertEquals(Integer.toString(14), Integer.toString(vector2.getLastSet())); + vector2.setValueCount(25); + assertEquals(Integer.toString(24), Integer.toString(vector2.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor(); - assertArrayEquals(STR1, accessor2.get(0)); - assertArrayEquals(STR2, accessor2.get(1)); - assertArrayEquals(STR3, accessor2.get(2)); - assertArrayEquals(STR4, accessor2.get(3)); - assertArrayEquals(STR5, accessor2.get(4)); - assertArrayEquals(STR6, accessor2.get(5)); + assertArrayEquals(STR1, vector2.get(0)); + assertArrayEquals(STR2, vector2.get(1)); + assertArrayEquals(STR3, vector2.get(2)); + assertArrayEquals(STR4, vector2.get(3)); + assertArrayEquals(STR5, vector2.get(4)); + assertArrayEquals(STR6, vector2.get(5)); } } } @@ -1396,8 +1739,6 @@ public void testVectorLoadUnload() { public void testFillEmptiesUsage() { try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { - final NullableVarCharVector.Mutator mutator = vector.getMutator(); - vector.allocateNew(1024 * 10, 1024); setBytes(0, STR1, vector); @@ -1408,84 +1749,98 @@ public void testFillEmptiesUsage() { setBytes(5, STR6, vector); /* Check current lastSet */ - assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(-1), Integer.toString(vector.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); - - mutator.setLastSet(5); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + + vector.setLastSet(5); /* fill empty byte arrays from index [6, 9] */ - mutator.fillEmpties(10); + vector.fillEmpties(10); /* Check current lastSet */ - assertEquals(Integer.toString(9), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(9), Integer.toString(vector.getLastSet())); /* Check the vector output */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9))); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9))); setBytes(10, STR1, vector); setBytes(11, STR2, vector); - mutator.setLastSet(11); + vector.setLastSet(11); /* fill empty byte arrays from index [12, 14] */ - mutator.setValueCount(15); + vector.setValueCount(15); /* Check current lastSet */ - assertEquals(Integer.toString(14), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(14), Integer.toString(vector.getLastSet())); /* Check the vector output */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9))); - assertArrayEquals(STR1, accessor.get(10)); - assertArrayEquals(STR2, accessor.get(11)); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(12))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(13))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(14))); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9))); + assertArrayEquals(STR1, vector.get(10)); + assertArrayEquals(STR2, vector.get(11)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(12))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(13))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(14))); /* Check offsets */ - final UInt4Vector.Accessor offsetAccessor = vector.values.offsetVector.getAccessor(); - assertEquals(Integer.toString(0), Integer.toString(offsetAccessor.get(0))); - assertEquals(Integer.toString(6), Integer.toString(offsetAccessor.get(1))); - assertEquals(Integer.toString(16), Integer.toString(offsetAccessor.get(2))); - assertEquals(Integer.toString(21), Integer.toString(offsetAccessor.get(3))); - assertEquals(Integer.toString(30), Integer.toString(offsetAccessor.get(4))); - assertEquals(Integer.toString(34), Integer.toString(offsetAccessor.get(5))); - - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(6))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(7))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(8))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(9))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(10))); - - assertEquals(Integer.toString(46), Integer.toString(offsetAccessor.get(11))); - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(12))); - - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(13))); - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(14))); - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(15))); + assertEquals(Integer.toString(0), + Integer.toString(vector.offsetBuffer.getInt(0 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(6), + Integer.toString(vector.offsetBuffer.getInt(1 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(16), + Integer.toString(vector.offsetBuffer.getInt(2 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(21), + Integer.toString(vector.offsetBuffer.getInt(3 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(30), + Integer.toString(vector.offsetBuffer.getInt(4 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(34), + Integer.toString(vector.offsetBuffer.getInt(5 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(6 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(7 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(8 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(9 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(10 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(46), + Integer.toString(vector.offsetBuffer.getInt(11 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(12 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(13 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(14 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(15 * vector.OFFSET_WIDTH))); } } @@ -1493,29 +1848,25 @@ public void testFillEmptiesUsage() { public void testGetBufferAddress1() { try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { - - final NullableVarCharVector.Mutator mutator = vector.getMutator(); - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - vector.allocateNew(1024 * 10, 1024); /* populate the vector */ - mutator.set(0, STR1); - mutator.set(1, STR2); - mutator.set(2, STR3); - mutator.set(3, STR4); - mutator.set(4, STR5); - mutator.set(5, STR6); + vector.set(0, STR1); + vector.set(1, STR2); + vector.set(2, STR3); + vector.set(3, STR4); + vector.set(4, STR5); + vector.set(5, STR6); - mutator.setValueCount(15); + vector.setValueCount(15); /* check the vector output */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); List buffers = vector.getFieldBuffers(); long bitAddress = vector.getValidityBufferAddress(); @@ -1531,23 +1882,18 @@ public void testGetBufferAddress1() { @Test /* NullableIntVector */ public void testGetBufferAddress2() { - try (final NullableIntVector vector = new NullableIntVector("myvector", allocator)) { - - final NullableIntVector.Mutator mutator = vector.getMutator(); - final NullableIntVector.Accessor accessor = vector.getAccessor(); boolean error = false; - vector.allocateNew(16); /* populate the vector */ for(int i = 0; i < 16; i += 2) { - mutator.set(i, i+10); + vector.set(i, i+10); } /* check the vector output */ for(int i = 0; i < 16; i += 2) { - assertEquals(i+10, accessor.get(i)); + assertEquals(i+10, vector.get(i)); } List buffers = vector.getFieldBuffers(); @@ -1580,11 +1926,15 @@ public void testMultipleClose() { vectorAllocator.close(); } + /* this method is used by the tests to bypass the vector set methods that manipulate + * lastSet. The method is to test the lastSet property and that's why we load the vector + * in a way that lastSet is not set automatically. + */ public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) { - final int currentOffset = vector.values.offsetVector.getAccessor().get(index); + final int currentOffset = vector.offsetBuffer.getInt(index * vector.OFFSET_WIDTH); - vector.bits.getMutator().setToOne(index); - vector.values.offsetVector.getMutator().set(index + 1, currentOffset + bytes.length); - vector.values.data.setBytes(currentOffset, bytes, 0, bytes.length); + BitVectorHelper.setValidityBitToOne(vector.validityBuffer, index); + vector.offsetBuffer.setInt((index + 1) * vector.OFFSET_WIDTH, currentOffset + bytes.length); + vector.valueBuffer.setBytes(currentOffset, bytes, 0, bytes.length); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 4ac7536c017db..531a46c2734ef 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -76,24 +76,23 @@ public void testFixedType() { @Test public void testNullableType() { try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.setInitialCapacity(512); vector.allocateNew(); assertEquals(512, vector.getValueCapacity()); try { - m.set(512, "foo".getBytes(StandardCharsets.UTF_8)); + vector.set(512, "foo".getBytes(StandardCharsets.UTF_8)); Assert.fail("Expected out of bounds exception"); } catch (Exception e) { // ok } vector.reAlloc(); - assertEquals(1023, vector.getValueCapacity()); + assertEquals(1024, vector.getValueCapacity()); - m.set(512, "foo".getBytes(StandardCharsets.UTF_8)); - assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8)); + vector.set(512, "foo".getBytes(StandardCharsets.UTF_8)); + assertEquals("foo", new String(vector.get(512), StandardCharsets.UTF_8)); } } @@ -105,7 +104,7 @@ public void testListType() { vector.setInitialCapacity(512); vector.allocateNew(); - assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason... + assertEquals(1023, vector.getValueCapacity()); try { vector.getOffsetVector().getAccessor().get(2014); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 7facf73f511da..0b7928dca2c39 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -45,13 +45,21 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Test; +import org.junit.*; public class TestVectorUnloadLoad { - static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + private BufferAllocator allocator; + + @Before + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void terminate() throws Exception { + allocator.close(); + } @Test public void testUnloadLoad() throws IOException { @@ -183,24 +191,40 @@ public void testUnloadLoadAddPadding() throws IOException { * @throws IOException */ @Test - public void testLoadEmptyValidityBuffer() throws IOException { + public void testLoadValidityBuffer() throws IOException { Schema schema = new Schema(asList( new Field("intDefined", FieldType.nullable(new ArrowType.Int(32, true)), Collections.emptyList()), new Field("intNull", FieldType.nullable(new ArrowType.Int(32, true)), Collections.emptyList()) )); int count = 10; - ArrowBuf validity = allocator.buffer(10).slice(0, 0); - ArrowBuf[] values = new ArrowBuf[2]; - for (int i = 0; i < values.length; i++) { - ArrowBuf arrowBuf = allocator.buffer(count * 4); // integers - values[i] = arrowBuf; + ArrowBuf[] values = new ArrowBuf[4]; + for (int i = 0; i < 4; i+=2) { + ArrowBuf buf1 = allocator.buffer((int)Math.ceil(count / 8.0)); + ArrowBuf buf2 = allocator.buffer(count * 4); // integers + values[i] = buf1; + values[i+1] = buf2; for (int j = 0; j < count; j++) { - arrowBuf.setInt(j * 4, j); + if (i == 2) { + BitVectorHelper.setValidityBit(buf1, j, 0); + } else { + BitVectorHelper.setValidityBitToOne(buf1, j); + } + + buf2.setInt(j * 4, j); } - arrowBuf.writerIndex(count * 4); + buf1.writerIndex((int)Math.ceil(count / 8)); + buf2.writerIndex(count * 4); } + + /* + * values[0] - validity buffer for first vector + * values[1] - data buffer for first vector + * values[2] - validity buffer for second vector + * values[3] - data buffer for second vector + */ + try ( - ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(validity, values[0], validity, values[1])); + ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(values[0], values[1], values[2], values[3])); BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE); VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator); ) { @@ -213,32 +237,31 @@ public void testLoadEmptyValidityBuffer() throws IOException { NullableIntVector intDefinedVector = (NullableIntVector) newRoot.getVector("intDefined"); NullableIntVector intNullVector = (NullableIntVector) newRoot.getVector("intNull"); for (int i = 0; i < count; i++) { - assertFalse("#" + i, intDefinedVector.getAccessor().isNull(i)); - assertEquals("#" + i, i, intDefinedVector.getAccessor().get(i)); - assertTrue("#" + i, intNullVector.getAccessor().isNull(i)); + assertFalse("#" + i, intDefinedVector.isNull(i)); + assertEquals("#" + i, i, intDefinedVector.get(i)); + assertTrue("#" + i, intNullVector.isNull(i)); } - intDefinedVector.getMutator().setSafe(count + 10, 1234); - assertTrue(intDefinedVector.getAccessor().isNull(count + 1)); + intDefinedVector.setSafe(count + 10, 1234); + assertTrue(intDefinedVector.isNull(count + 1)); // empty slots should still default to unset - intDefinedVector.getMutator().setSafe(count + 1, 789); - assertFalse(intDefinedVector.getAccessor().isNull(count + 1)); - assertEquals(789, intDefinedVector.getAccessor().get(count + 1)); - assertTrue(intDefinedVector.getAccessor().isNull(count)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 2)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 3)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 4)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 5)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 6)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 7)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 8)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 9)); - assertFalse(intDefinedVector.getAccessor().isNull(count + 10)); - assertEquals(1234, intDefinedVector.getAccessor().get(count + 10)); + intDefinedVector.setSafe(count + 1, 789); + assertFalse(intDefinedVector.isNull(count + 1)); + assertEquals(789, intDefinedVector.get(count + 1)); + assertTrue(intDefinedVector.isNull(count)); + assertTrue(intDefinedVector.isNull(count + 2)); + assertTrue(intDefinedVector.isNull(count + 3)); + assertTrue(intDefinedVector.isNull(count + 4)); + assertTrue(intDefinedVector.isNull(count + 5)); + assertTrue(intDefinedVector.isNull(count + 6)); + assertTrue(intDefinedVector.isNull(count + 7)); + assertTrue(intDefinedVector.isNull(count + 8)); + assertTrue(intDefinedVector.isNull(count + 9)); + assertFalse(intDefinedVector.isNull(count + 10)); + assertEquals(1234, intDefinedVector.get(count + 10)); } finally { for (ArrowBuf arrowBuf : values) { arrowBuf.release(); } - validity.release(); } } @@ -258,11 +281,11 @@ public void testUnloadLoadDuplicates() throws IOException { FieldVector vector = field.createVector(originalVectorsAllocator); vector.allocateNew(); sources.add(vector); - NullableIntVector.Mutator mutator = (NullableIntVector.Mutator) vector.getMutator(); + NullableIntVector intVector = (NullableIntVector)vector; for (int i = 0; i < count; i++) { - mutator.set(i, i); + intVector.set(i, i); } - mutator.setValueCount(count); + intVector.setValueCount(count); } try (VectorSchemaRoot root = new VectorSchemaRoot(schema.getFields(), sources, count)) { @@ -277,8 +300,8 @@ public void testUnloadLoadDuplicates() throws IOException { List targets = newRoot.getFieldVectors(); Assert.assertEquals(sources.size(), targets.size()); for (int k = 0; k < sources.size(); k++) { - NullableIntVector.Accessor src = (NullableIntVector.Accessor) sources.get(k).getAccessor(); - NullableIntVector.Accessor tgt = (NullableIntVector.Accessor) targets.get(k).getAccessor(); + NullableIntVector src = (NullableIntVector) sources.get(k); + NullableIntVector tgt = (NullableIntVector) targets.get(k); Assert.assertEquals(src.getValueCount(), tgt.getValueCount()); for (int i = 0; i < count; i++) { Assert.assertEquals(src.get(i), tgt.get(i)); @@ -296,9 +319,4 @@ public static VectorUnloader newVectorUnloader(FieldVector root) { VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount); return new VectorUnloader(vsr); } - - @AfterClass - public static void afterClass() { - allocator.close(); - } } \ No newline at end of file diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index ba62de0a6d93a..60009b0a46396 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -97,8 +97,14 @@ public void tearDown() { protected void validateContent(int count, VectorSchemaRoot root) { for (int i = 0; i < count; i++) { - Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); + FieldVector fv = root.getVector("int"); + if (fv instanceof NullableIntVector) { + Assert.assertEquals(i, fv.getObject(i)); + Assert.assertEquals(Integer.valueOf(i), fv.getObject(i)); + } else { + Assert.assertEquals(i, fv.getAccessor().getObject(i)); + Assert.assertEquals(Long.valueOf(i), fv.getAccessor().getObject(i)); + } } } @@ -152,6 +158,7 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { + Object intVal = root.getVector("int").getAccessor().getObject(i); if (i % 5 != 3) { Assert.assertEquals(i, intVal); @@ -220,22 +227,20 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat // Define dictionaries and add to provider NullableVarCharVector dictionary1Vector = newNullableVarCharVector("D1", bufferAllocator); dictionary1Vector.allocateNewSafe(); - NullableVarCharVector.Mutator mutator = dictionary1Vector.getMutator(); - mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8)); - mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - mutator.set(2, "baz".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(3); + dictionary1Vector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + dictionary1Vector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + dictionary1Vector.set(2, "baz".getBytes(StandardCharsets.UTF_8)); + dictionary1Vector.setValueCount(3); Dictionary dictionary1 = new Dictionary(dictionary1Vector, new DictionaryEncoding(1L, false, null)); provider.put(dictionary1); NullableVarCharVector dictionary2Vector = newNullableVarCharVector("D2", bufferAllocator); dictionary2Vector.allocateNewSafe(); - mutator = dictionary2Vector.getMutator(); - mutator.set(0, "micro".getBytes(StandardCharsets.UTF_8)); - mutator.set(1, "small".getBytes(StandardCharsets.UTF_8)); - mutator.set(2, "large".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(3); + dictionary2Vector.set(0, "micro".getBytes(StandardCharsets.UTF_8)); + dictionary2Vector.set(1, "small".getBytes(StandardCharsets.UTF_8)); + dictionary2Vector.set(2, "large".getBytes(StandardCharsets.UTF_8)); + dictionary2Vector.setValueCount(3); Dictionary dictionary2 = new Dictionary(dictionary2Vector, new DictionaryEncoding(2L, false, null)); provider.put(dictionary2); @@ -243,13 +248,12 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat // Populate the vectors NullableVarCharVector vector1A = newNullableVarCharVector("varcharA", bufferAllocator); vector1A.allocateNewSafe(); - mutator = vector1A.getMutator(); - mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8)); - mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - mutator.set(3, "baz".getBytes(StandardCharsets.UTF_8)); - mutator.set(4, "bar".getBytes(StandardCharsets.UTF_8)); - mutator.set(5, "baz".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(6); + vector1A.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + vector1A.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + vector1A.set(3, "baz".getBytes(StandardCharsets.UTF_8)); + vector1A.set(4, "bar".getBytes(StandardCharsets.UTF_8)); + vector1A.set(5, "baz".getBytes(StandardCharsets.UTF_8)); + vector1A.setValueCount(6); FieldVector encodedVector1A = (FieldVector) DictionaryEncoder.encode(vector1A, dictionary1); vector1A.close(); // Done with this vector after encoding @@ -257,22 +261,20 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat // Write this vector using indices instead of encoding NullableIntVector encodedVector1B = new NullableIntVector("varcharB", bufferAllocator); encodedVector1B.allocateNewSafe(); - NullableIntVector.Mutator mutator1B = encodedVector1B.getMutator(); - mutator1B.set(0, 2); // "baz" - mutator1B.set(1, 1); // "bar" - mutator1B.set(2, 2); // "baz" - mutator1B.set(4, 1); // "bar" - mutator1B.set(5, 0); // "foo" - mutator1B.setValueCount(6); + encodedVector1B.set(0, 2); // "baz" + encodedVector1B.set(1, 1); // "bar" + encodedVector1B.set(2, 2); // "baz" + encodedVector1B.set(4, 1); // "bar" + encodedVector1B.set(5, 0); // "foo" + encodedVector1B.setValueCount(6); NullableVarCharVector vector2 = newNullableVarCharVector("sizes", bufferAllocator); vector2.allocateNewSafe(); - mutator = vector2.getMutator(); - mutator.set(1, "large".getBytes(StandardCharsets.UTF_8)); - mutator.set(2, "small".getBytes(StandardCharsets.UTF_8)); - mutator.set(3, "small".getBytes(StandardCharsets.UTF_8)); - mutator.set(4, "large".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(6); + vector2.set(1, "large".getBytes(StandardCharsets.UTF_8)); + vector2.set(2, "small".getBytes(StandardCharsets.UTF_8)); + vector2.set(3, "small".getBytes(StandardCharsets.UTF_8)); + vector2.set(4, "large".getBytes(StandardCharsets.UTF_8)); + vector2.setValueCount(6); FieldVector encodedVector2 = (FieldVector) DictionaryEncoder.encode(vector2, dictionary2); vector2.close(); // Done with this vector after encoding @@ -355,8 +357,8 @@ protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAlloc // Define the dictionary and add to the provider NullableVarCharVector dictionaryVector = newNullableVarCharVector("D2", bufferAllocator); dictionaryVector.allocateNewSafe(); - dictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8)); - dictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8)); + dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); dictionaryVector.getMutator().setValueCount(2); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java index 81e58989fccc4..feae08ea223e1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java @@ -581,7 +581,7 @@ public void testWriteReadFixedSizeList() throws IOException { tuples.getMutator().setNotNull(i); floats.getMutator().set(i * 2, i + 0.1f); floats.getMutator().set(i * 2 + 1, i + 10.1f); - ints.getMutator().set(i, i); + ints.set(i, i); } parent.getMutator().setValueCount(10); From 9ee838ac9538666ae6d7b0b1e680e80410503f7e Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 16 Oct 2017 00:39:30 -0700 Subject: [PATCH 049/177] ARROW-1474:[JAVA] ValueVector hierarchy (Implementation Phase 2) Implementation of all scalar types and complex types with corresponding legacy versions. Closes #1203 --- .../arrow/tools/ArrowFileTestFixtures.java | 4 +- .../apache/arrow/tools/EchoServerTest.java | 84 +- .../codegen/templates/ComplexReaders.java | 24 +- .../codegen/templates/ComplexWriters.java | 51 +- .../codegen/templates/LegacyUnionVector.java | 356 ++++++++ .../main/codegen/templates/MapWriters.java | 4 +- .../templates/NullableValueVectors.java | 787 ++---------------- .../codegen/templates/UnionListWriter.java | 11 +- .../main/codegen/templates/UnionVector.java | 212 +++-- .../main/codegen/templates/UnionWriter.java | 26 +- .../vector/BaseNullableFixedWidthVector.java | 315 +++++-- .../BaseNullableVariableWidthVector.java | 412 +++++++-- .../apache/arrow/vector/BaseValueVector.java | 33 +- .../apache/arrow/vector/BitVectorHelper.java | 115 ++- .../arrow/vector/NullableBigIntVector.java | 403 +++++++++ .../arrow/vector/NullableBitVector.java | 502 +++++++++++ .../arrow/vector/NullableDateDayVector.java | 404 +++++++++ .../arrow/vector/NullableDateMilliVector.java | 408 +++++++++ .../arrow/vector/NullableDecimalVector.java | 454 ++++++++++ .../arrow/vector/NullableFloat4Vector.java | 404 +++++++++ .../arrow/vector/NullableFloat8Vector.java | 404 +++++++++ .../arrow/vector/NullableIntVector.java | 140 +++- .../vector/NullableIntervalDayVector.java | 429 ++++++++++ .../vector/NullableIntervalYearVector.java | 387 +++++++++ .../arrow/vector/NullableSmallIntVector.java | 431 ++++++++++ .../arrow/vector/NullableTimeMicroVector.java | 404 +++++++++ .../arrow/vector/NullableTimeMilliVector.java | 405 +++++++++ .../arrow/vector/NullableTimeNanoVector.java | 402 +++++++++ .../arrow/vector/NullableTimeSecVector.java | 403 +++++++++ .../NullableTimeStampMicroTZVector.java | 219 +++++ .../vector/NullableTimeStampMicroVector.java | 220 +++++ .../NullableTimeStampMilliTZVector.java | 218 +++++ .../vector/NullableTimeStampMilliVector.java | 218 +++++ .../vector/NullableTimeStampNanoTZVector.java | 219 +++++ .../vector/NullableTimeStampNanoVector.java | 219 +++++ .../vector/NullableTimeStampSecTZVector.java | 217 +++++ .../vector/NullableTimeStampSecVector.java | 219 +++++ .../arrow/vector/NullableTimeStampVector.java | 253 ++++++ .../arrow/vector/NullableTinyIntVector.java | 432 ++++++++++ .../arrow/vector/NullableUInt1Vector.java | 318 +++++++ .../arrow/vector/NullableUInt2Vector.java | 319 +++++++ .../arrow/vector/NullableUInt4Vector.java | 291 +++++++ .../arrow/vector/NullableUInt8Vector.java | 291 +++++++ .../arrow/vector/NullableVarBinaryVector.java | 514 ++++++++++++ .../arrow/vector/NullableVarCharVector.java | 100 ++- .../org/apache/arrow/vector/ValueVector.java | 8 + .../apache/arrow/vector/VectorSchemaRoot.java | 2 +- .../apache/arrow/vector/VectorUnloader.java | 12 +- .../org/apache/arrow/vector/ZeroVector.java | 6 + .../vector/complex/AbstractMapVector.java | 4 +- .../complex/BaseRepeatedValueVector.java | 174 ++-- .../vector/complex/FixedSizeListVector.java | 238 ++++-- .../LegacyBaseRepeatedValueVector.java | 240 ++++++ .../complex/LegacyFixedSizeListVector.java | 298 +++++++ .../vector/complex/LegacyListVector.java | 288 +++++++ .../arrow/vector/complex/LegacyMapVector.java | 229 +++++ .../arrow/vector/complex/ListVector.java | 424 ++++++---- .../arrow/vector/complex/MapVector.java | 96 +-- .../vector/complex/NullableMapVector.java | 299 +++++-- .../complex/impl/NullableMapReaderImpl.java | 2 +- .../vector/complex/impl/PromotableWriter.java | 2 +- .../complex/impl/SingleMapReaderImpl.java | 2 +- .../impl/UnionFixedSizeListReader.java | 4 +- .../vector/complex/impl/UnionListReader.java | 12 +- .../vector/dictionary/DictionaryEncoder.java | 28 +- .../apache/arrow/vector/file/ArrowWriter.java | 2 +- .../vector/file/json/JsonFileReader.java | 341 ++++---- .../vector/file/json/JsonFileWriter.java | 198 +++-- .../apache/arrow/vector/util/Validator.java | 10 +- .../apache/arrow/vector/TestBitVector.java | 35 +- .../arrow/vector/TestDecimalVector.java | 12 +- .../arrow/vector/TestDictionaryVector.java | 4 +- .../arrow/vector/TestFixedSizeListVector.java | 33 +- .../apache/arrow/vector/TestListVector.java | 196 ++--- .../apache/arrow/vector/TestUnionVector.java | 269 +++--- .../apache/arrow/vector/TestValueVector.java | 222 +++-- .../arrow/vector/TestVectorReAlloc.java | 5 +- .../arrow/vector/TestVectorUnloadLoad.java | 6 +- .../complex/impl/TestPromotableWriter.java | 21 +- .../complex/writer/TestComplexWriter.java | 4 +- .../arrow/vector/file/BaseFileTest.java | 139 ++-- .../arrow/vector/file/TestArrowFile.java | 28 +- .../arrow/vector/file/TestArrowStream.java | 6 +- .../vector/file/TestArrowStreamPipe.java | 14 +- .../arrow/vector/file/json/TestJSONFile.java | 2 +- 85 files changed, 14169 insertions(+), 2427 deletions(-) create mode 100644 java/vector/src/main/codegen/templates/LegacyUnionVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java index 6d9a6c1323e76..c56a5a3303b75 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java @@ -78,8 +78,8 @@ static void validateOutput(File testOutFile, BufferAllocator allocator) throws E static void validateContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < count; i++) { - Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); + Assert.assertEquals(i, root.getVector("int").getObject(i)); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); } } diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index ecac6d6e53067..89714e4479180 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -104,10 +104,10 @@ private void testEchoServer(int serverPort, for (int i = 0; i < batches; i++) { vector.allocateNew(16); for (int j = 0; j < 8; j++) { - vector.getMutator().set(j, j + i); - vector.getMutator().set(j + 8, 0, (byte) (j + i)); + vector.set(j, j + i); + vector.set(j + 8, 0, (byte) (j + i)); } - vector.getMutator().setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); writer.writeBatch(); } @@ -120,10 +120,10 @@ private void testEchoServer(int serverPort, for (int i = 0; i < batches; i++) { Assert.assertTrue(reader.loadNextBatch()); assertEquals(16, reader.getVectorSchemaRoot().getRowCount()); - assertEquals(16, readVector.getAccessor().getValueCount()); + assertEquals(16, readVector.getValueCount()); for (int j = 0; j < 8; j++) { - assertEquals(j + i, readVector.getAccessor().get(j)); - assertTrue(readVector.getAccessor().isNull(j + 8)); + assertEquals(j + i, readVector.get(j)); + assertTrue(readVector.isNull(j + 8)); } } Assert.assertFalse(reader.loadNextBatch()); @@ -169,20 +169,18 @@ public void testFlatDictionary() throws IOException { FieldType.nullable(VARCHAR.getType()), allocator)) { writeVector.allocateNewSafe(); - NullableIntVector.Mutator mutator = writeVector.getMutator(); - mutator.set(0, 0); - mutator.set(1, 1); - mutator.set(3, 2); - mutator.set(4, 1); - mutator.set(5, 2); - mutator.setValueCount(6); + writeVector.set(0, 0); + writeVector.set(1, 1); + writeVector.set(3, 2); + writeVector.set(4, 1); + writeVector.set(5, 2); + writeVector.setValueCount(6); writeDictionaryVector.allocateNewSafe(); - NullableVarCharVector.Mutator dictionaryMutator = writeDictionaryVector.getMutator(); - dictionaryMutator.set(0, "foo".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.set(2, "baz".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.setValueCount(3); + writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(2, "baz".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.setValueCount(3); List fields = ImmutableList.of(writeVector.getField()); List vectors = ImmutableList.of((FieldVector) writeVector); @@ -210,23 +208,21 @@ public void testFlatDictionary() throws IOException { Assert.assertNotNull(readEncoding); Assert.assertEquals(1L, readEncoding.getId()); - FieldVector.Accessor accessor = readVector.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(0, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(null, accessor.getObject(2)); - Assert.assertEquals(2, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(2, accessor.getObject(5)); + Assert.assertEquals(6, readVector.getValueCount()); + Assert.assertEquals(0, readVector.getObject(0)); + Assert.assertEquals(1, readVector.getObject(1)); + Assert.assertEquals(null, readVector.getObject(2)); + Assert.assertEquals(2, readVector.getObject(3)); + Assert.assertEquals(1, readVector.getObject(4)); + Assert.assertEquals(2, readVector.getObject(5)); Dictionary dictionary = reader.lookup(1L); Assert.assertNotNull(dictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary - .getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2)); } } } @@ -243,9 +239,9 @@ public void testNestedDictionary() throws IOException { // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]] writeDictionaryVector.allocateNew(); - writeDictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8)); - writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8)); - writeDictionaryVector.getMutator().setValueCount(2); + writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.setValueCount(2); writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null)); writeVector.allocateNew(); @@ -297,19 +293,17 @@ public void testNestedDictionary() throws IOException { Assert.assertEquals(2L, encoding.getId()); Assert.assertEquals(new Int(32, true), encoding.getIndexType()); - ListVector.Accessor accessor = readVector.getAccessor(); - Assert.assertEquals(3, accessor.getValueCount()); - Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0)); - Assert.assertEquals(Arrays.asList(0), accessor.getObject(1)); - Assert.assertEquals(Arrays.asList(1), accessor.getObject(2)); + Assert.assertEquals(3, readVector.getValueCount()); + Assert.assertEquals(Arrays.asList(0, 1), readVector.getObject(0)); + Assert.assertEquals(Arrays.asList(0), readVector.getObject(1)); + Assert.assertEquals(Arrays.asList(1), readVector.getObject(2)); Dictionary readDictionary = reader.lookup(2L); Assert.assertNotNull(readDictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) - readDictionary.getVector()).getAccessor(); - Assert.assertEquals(2, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) readDictionary.getVector()); + Assert.assertEquals(2, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } } } diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index 791064905156d..60347a2fcb8e2 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -70,11 +70,7 @@ public Field getField(){ public boolean isSet(){ <#if nullMode == "Nullable"> - <#if minor.class != "Int" && minor.class != "VarChar"> - return !vector.getAccessor().isNull(idx()); - <#else> return !vector.isNull(idx()); - <#else> return true; @@ -97,19 +93,11 @@ public void read(${minor.class?cap_first}Holder h){ public void read(Nullable${minor.class?cap_first}Holder h){ - <#if minor.class != "Int" && minor.class != "VarChar"> - vector.getAccessor().get(idx(), h); - <#else> - vector.get(idx(), h); - + vector.get(idx(), h); } public ${friendlyType} read${safeType}(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - return vector.getObject(idx()); - <#else> - return vector.getAccessor().getObject(idx()); - + return vector.getObject(idx()); } <#if minor.class == "TimeStampSec" || @@ -118,7 +106,7 @@ public void read(Nullable${minor.class?cap_first}Holder h){ minor.class == "TimeStampNano"> @Override public ${minor.boxedType} read${minor.boxedType}(){ - return vector.getAccessor().get(idx()); + return vector.get(idx()); } @@ -127,11 +115,7 @@ public void copyValue(FieldWriter w){ } public Object readObject(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - return (Object)vector.getObject(idx()); - <#else> - return vector.getAccessor().getObject(idx()); - + return (Object)vector.getObject(idx()); } } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 77f6594a07804..406bbb39c7f4a 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -39,16 +39,9 @@ @SuppressWarnings("unused") public class ${eName}WriterImpl extends AbstractFieldWriter { - <#if minor.class != "Int" && minor.class != "VarChar"> - private final Nullable${name}Vector.Mutator mutator; - - final Nullable${name}Vector vector; public ${eName}WriterImpl(Nullable${name}Vector vector) { - <#if minor.class != "Int" && minor.class != "VarChar"> - this.mutator = vector.getMutator(); - this.vector = vector; } @@ -108,51 +101,31 @@ public void setPosition(int idx) { <#else> public void write(${minor.class}Holder h) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx(), h); - vector.setValueCount(idx()+1); - + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); } public void write(Nullable${minor.class}Holder h) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx(), h); - vector.setValueCount(idx()+1); - + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.setValueCount(idx()+1); - + vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.setValueCount(idx()+1); } - <#if minor.class == "Decimal"> + <#if minor.class == "Decimal"> public void write${minor.class}(${friendlyType} value) { - mutator.setSafe(idx(), value); - vector.getMutator().setValueCount(idx()+1); + vector.setSafe(idx(), value); + vector.setValueCount(idx()+1); } - <#if mode == "Nullable"> + <#if mode == "Nullable"> public void writeNull() { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setNull(idx()); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setNull(idx()); - vector.setValueCount(idx()+1); - + vector.setNull(idx()); + vector.setValueCount(idx()+1); } diff --git a/java/vector/src/main/codegen/templates/LegacyUnionVector.java b/java/vector/src/main/codegen/templates/LegacyUnionVector.java new file mode 100644 index 0000000000000..b9cb6dfdc2482 --- /dev/null +++ b/java/vector/src/main/codegen/templates/LegacyUnionVector.java @@ -0,0 +1,356 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/LegacyUnionVector.java" /> + + +<#include "/@includes/license.ftl" /> + + package org.apache.arrow.vector.complex; + +<#include "/@includes/vv_imports.ftl" /> + import com.google.common.collect.ImmutableList; + import java.util.ArrayList; + import java.util.Collections; + import java.util.Iterator; + import org.apache.arrow.vector.BaseDataValueVector; + import org.apache.arrow.vector.complex.impl.ComplexCopier; + import org.apache.arrow.vector.util.CallBack; + import org.apache.arrow.vector.schema.ArrowFieldNode; + + import static org.apache.arrow.vector.types.UnionMode.Sparse; + + + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ +@SuppressWarnings("unused") + + +/** + * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each + * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, + * as well as the addOrGet method. + * + * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup + * each time the vector is accessed. + * Source code generated using FreeMarker template ${.template_name} + */ +public class LegacyUnionVector implements FieldVector { + + private Accessor accessor = new Accessor(); + private Mutator mutator = new Mutator(); + private final UnionVector unionVector; + + public LegacyUnionVector(String name, BufferAllocator allocator, CallBack callBack) { + unionVector = new UnionVector(name, allocator, callBack); + } + + public BufferAllocator getAllocator() { + return unionVector.getAllocator(); + } + + @Override + public MinorType getMinorType() { + return MinorType.UNION; + } + + @Override + public void initializeChildrenFromFields(List children) { + unionVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return unionVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + unionVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return unionVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return unionVector.getFieldInnerVectors(); + } + + @Override + public long getValidityBufferAddress() { + return unionVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return unionVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } + + @Override + public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } + + public NullableMapVector getMap() { + return unionVector.getMap(); + } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#assign lowerCaseName = name?lower_case/> + <#if !minor.typeParams?? > + + private Nullable${name}Vector ${uncappedName}Vector; + + public Nullable${name}Vector get${name}Vector() { + return unionVector.get${name}Vector(); + } + + + + + public ListVector getList() { + return unionVector.getList(); + } + + public int getTypeValue(int index) { + return unionVector.getTypeValue(index); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + unionVector.allocateNew(); + } + + @Override + public boolean allocateNewSafe() { + return unionVector.allocateNewSafe(); + } + + @Override + public void reAlloc() { + unionVector.reAlloc(); + } + + @Override + public void setInitialCapacity(int numRecords) { + } + + @Override + public int getValueCapacity() { + return unionVector.getValueCapacity(); + } + + @Override + public void close() { + unionVector.close(); + } + + @Override + public void clear() { + unionVector.clear(); + } + + @Override + public Field getField() { + return unionVector.getField(); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return unionVector.getTransferPair(allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return unionVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return unionVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return unionVector.makeTransferPair(((LegacyUnionVector)target).unionVector); + } + + public void copyFrom(int inIndex, int outIndex, UnionVector from) { + unionVector.copyFrom(inIndex, outIndex, from); + } + + public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { + unionVector.copyFromSafe(inIndex, outIndex, from); + } + + public FieldVector addVector(FieldVector v) { + return unionVector.addVector(v); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public FieldReader getReader() { + return unionVector.getReader(); + } + + public FieldWriter getWriter() { + return unionVector.getWriter(); + } + + @Override + public int getBufferSize() { + return unionVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(final int valueCount) { + return unionVector.getBufferSizeFor(valueCount); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return unionVector.getBuffers(clear); + } + + @Override + public Iterator iterator() { + return unionVector.iterator(); + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return unionVector.getObject(index); + } + + public byte[] get(int index) { + return unionVector.get(index); + } + + public void get(int index, ComplexHolder holder) { + } + + public void get(int index, UnionHolder holder) { + unionVector.get(index, holder); + } + + public int getNullCount() { + return unionVector.getNullCount(); + } + + @Override + public int getValueCount() { + return unionVector.getValueCount(); + } + + @Override + public boolean isNull(int index) { + return unionVector.isNull(index); + } + + public int isSet(int index) { + return unionVector.isSet(index); + } + } + + public class Mutator extends BaseValueVector.BaseMutator { + + UnionWriter writer; + + @Override + public void setValueCount(int valueCount) { + unionVector.setValueCount(valueCount); + } + + public void setSafe(int index, UnionHolder holder) { + unionVector.setSafe(index, holder); + } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.typeParams?? > + public void setSafe(int index, Nullable${name}Holder holder) { + unionVector.setSafe(index, holder); + } + + + + + + public void setType(int index, MinorType type) { + unionVector.setType(index, type); + } + + @Override + public void reset() { } + + @Override + public void generateTestData(int values) { } + } + + @Override + @Deprecated + public int getValueCount() { return getAccessor().getValueCount(); } + + @Override + @Deprecated + public void setValueCount(int valueCount) { getMutator().setValueCount(valueCount);} + + @Override + @Deprecated + public Object getObject(int index) { return getAccessor().getObject(index); } + + @Override + @Deprecated + public int getNullCount() { return getAccessor().getNullCount(); } + + @Override + @Deprecated + public boolean isNull(int index) { return getAccessor().isNull(index); } +} diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index b89f91457e8b2..6afd508e68837 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -184,7 +184,7 @@ public ListWriter list(String name) { } public void setValueCount(int count) { - container.getMutator().setValueCount(count); + container.setValueCount(count); } @Override @@ -199,7 +199,7 @@ public void setPosition(int index) { public void start() { <#if mode == "Single"> <#else> - container.getMutator().setIndexDefined(idx()); + container.setIndexDefined(idx()); } diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 5d1f5a33b90f5..8e5b10b649b13 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -19,14 +19,8 @@ <#list vv.types as type> <#list type.minor as minor> -<#if minor.class == "Int" || minor.class == "VarChar"> <#assign className = "LegacyNullable${minor.class}Vector" /> <#assign valuesName = "Nullable${minor.class}Vector" /> -<#else> -<#assign className = "Nullable${minor.class}Vector" /> -<#assign valuesName = "${minor.class}Vector" /> - - <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> @@ -50,35 +44,21 @@ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @SuppressWarnings("unused") -<#if minor.class == "Int" || minor.class == "VarChar"> @Deprecated - public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, FieldVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); protected final static byte[] emptyByteArray = new byte[]{}; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this); - - private final String bitsField = "$bits$"; private final String valuesField = "$values$"; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final Field field; - - final BitVector bits = new BitVector(bitsField, allocator); final ${valuesName} values; private final Mutator mutator; private final Accessor accessor; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final List innerVectors; - - <#if minor.typeParams??> <#assign typeParams = minor.typeParams?reverse> <#list typeParams as typeParam> @@ -122,33 +102,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj this.mutator = new Mutator(); this.accessor = new Accessor(); - <#if minor.class != "Int" && minor.class != "VarChar"> - this.field = new Field(name, fieldType, null); - innerVectors = Collections.unmodifiableList(Arrays.asList( - bits, - <#if type.major = "VarLen"> - values.offsetVector, - - values - )); - } - <#if minor.class != "Int" && minor.class != "VarChar"> - /* not needed for new vectors */ - public BitVector getValidityVector() { - return bits; - } - - @Override public List getFieldInnerVectors() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getFieldInnerVectors(); - <#else> - return innerVectors; - + /* DELEGATE TO NEW VECTOR */ + return values.getFieldInnerVectors(); } @Override @@ -165,123 +124,61 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.loadFieldBuffers(fieldNode, ownBuffers); - <#else> - <#if type.major = "VarLen"> - // variable width values: truncate offset vector buffer to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, - values.offsetVector.getBufferSizeFor( - fieldNode.getLength() == 0? 0 : fieldNode.getLength() + 1)); - mutator.lastSet = fieldNode.getLength() - 1; - <#else> - // fixed width values truncate value vector to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength())); - - org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); - bits.valueCount = fieldNode.getLength(); - + /* DELEGATE TO NEW VECTOR */ + values.loadFieldBuffers(fieldNode, ownBuffers); } public List getFieldBuffers() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getFieldBuffers(); - <#else> - return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors()); - + /* DELEGATE TO NEW VECTOR */ + return values.getFieldBuffers(); } @Override public Field getField() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getField(); - <#else> - return field; - + /* DELEGATE TO NEW VECTOR */ + return values.getField(); } @Override public MinorType getMinorType() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getMinorType(); - <#else> - return MinorType.${minor.class?upper_case}; - + /* DELEGATE TO NEW VECTOR */ + return values.getMinorType(); } @Override public FieldReader getReader(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getReader(); - <#else> - return reader; - + /* DELEGATE TO NEW VECTOR */ + return values.getReader(); } @Override public int getValueCapacity(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValueCapacity(); - <#else> - return Math.min(bits.getValueCapacity(), values.getValueCapacity()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValueCapacity(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBuffers(clear); - <#else> - final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class); - if (clear) { - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); - } - return buffers; - + /* DELEGATE TO NEW VECTOR */ + return values.getBuffers(clear); } @Override public void close() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.close(); - <#else> - bits.close(); - values.close(); - super.close(); - + /* DELEGATE TO NEW VECTOR */ + values.close(); } @Override public void clear() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.clear(); - <#else> - bits.clear(); - values.clear(); - super.clear(); - + /* DELEGATE TO NEW VECTOR */ + values.clear(); } @Override public int getBufferSize(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSize(); - <#else> - return values.getBufferSize() + bits.getBufferSize(); - + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSize(); } @Override @@ -289,14 +186,8 @@ public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSizeFor(valueCount); - <#else> - return values.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); - + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSizeFor(valueCount); } public ArrowBuf getBuffer() { @@ -309,93 +200,38 @@ public ArrowBuf getBuffer() { @Override public void setInitialCapacity(int numRecords) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.setInitialCapacity(numRecords); - <#else> - bits.setInitialCapacity(numRecords); - values.setInitialCapacity(numRecords); - + /* DELEGATE TO NEW VECTOR */ + values.setInitialCapacity(numRecords); } @Override public void allocateNew() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(); - <#else> - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating buffer."); - } - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(); } @Override public boolean allocateNewSafe() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.allocateNewSafe(); - <#else> - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - success = values.allocateNewSafe() && bits.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - return success; - + /* DELEGATE TO NEW VECTOR */ + return values.allocateNewSafe(); } @Override public void reAlloc() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.reAlloc(); - <#else> - bits.reAlloc(); - values.reAlloc(); - + /* DELEGATE TO NEW VECTOR */ + values.reAlloc(); } public void reset() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.reset(); - <#else> - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.reset(); } <#if type.major == "VarLen"> @Override public void allocateNew(int totalBytes, int valueCount) { - <#if minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(totalBytes, valueCount); - <#else> - try { - values.allocateNew(totalBytes, valueCount); - bits.allocateNew(valueCount); - } catch(RuntimeException e) { - clear(); - throw e; - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(totalBytes, valueCount); } @Override @@ -411,21 +247,8 @@ public int getCurrentSizeInBytes(){ <#else> @Override public void allocateNew(int valueCount) { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(valueCount); - <#else> - try { - values.allocateNew(valueCount); - bits.allocateNew(valueCount); - } catch(OutOfMemoryException e) { - clear(); - throw e; - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(valueCount); } /** @@ -433,13 +256,8 @@ public void allocateNew(int valueCount) { */ @Override public void zeroVector() { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.zeroVector(); - <#else> - bits.zeroVector(); - values.zeroVector(); - + /* DELEGATE TO NEW VECTOR */ + values.zeroVector(); } @@ -447,53 +265,36 @@ public void zeroVector() { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator, callBack); - <#else> - return getTransferPair(ref, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator, callBack); } @Override public TransferPair getTransferPair(BufferAllocator allocator){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(allocator); - <#else> - return new TransferImpl(name, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(allocator); } @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator); - <#else> - return new TransferImpl(ref, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator); } @Override public TransferPair makeTransferPair(ValueVector to) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.makeTransferPair(to); - <#else> - return new TransferImpl((${className}) to); - + /* DELEGATE TO NEW VECTOR */ + return values.makeTransferPair(to); } - <#if minor.class == "Int" || minor.class == "VarChar"> public void transferTo(${valuesName} target) { /* DELEGATE TO NEW VECTOR */ <#if type.major == "VarLen"> @@ -512,61 +313,6 @@ public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) } - <#else> - public void transferTo(${className} target){ - bits.transferTo(target.bits); - values.transferTo(target.values); - <#if type.major == "VarLen"> - target.mutator.lastSet = mutator.lastSet; - - clear(); - } - - public void splitAndTransferTo(int startIndex, int length, ${className} target) { - bits.splitAndTransferTo(startIndex, length, target.bits); - values.splitAndTransferTo(startIndex, length, target.values); - <#if type.major == "VarLen"> - target.mutator.lastSet = length - 1; - - } - - - - - <#if minor.class != "Int" && minor.class != "VarChar"> - private class TransferImpl implements TransferPair { - ${className} to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new ${className}(ref, field.getFieldType(), allocator); - } - - public TransferImpl(${className} to){ - this.to = to; - } - - @Override - public ${className} getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${className}.this); - } - } - - @Override @@ -580,8 +326,6 @@ public Mutator getMutator(){ } - - <#if minor.class == "Int" || minor.class == "VarChar"> public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) { /* DELEGATE TO NEW VECTOR */ values.copyFrom(fromIndex, thisIndex, from); @@ -591,107 +335,43 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) { /* DELEGATE TO NEW VECTOR */ values.copyFromSafe(fromIndex, thisIndex, from); } - <#else> - public void copyFrom(int fromIndex, int thisIndex, ${className} from) { - final Accessor fromAccessor = from.getAccessor(); - if (!fromAccessor.isNull(fromIndex)) { - mutator.set(thisIndex, fromAccessor.get(fromIndex)); - } - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){ - <#if type.major == "VarLen"> - mutator.fillEmpties(thisIndex); - - values.copyFromSafe(fromIndex, thisIndex, from); - bits.getMutator().setSafeToOne(thisIndex); - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ - <#if type.major == "VarLen"> - mutator.fillEmpties(thisIndex); - - bits.copyFromSafe(fromIndex, thisIndex, from.bits); - values.copyFromSafe(fromIndex, thisIndex, from.values); - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - @Override public long getValidityBufferAddress() { - /* address of the databuffer associated with the bitVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBufferAddress(); - <#else> - return (bits.getDataBuffer().memoryAddress()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBufferAddress(); } @Override public long getDataBufferAddress() { - /* address of the dataBuffer associated with the valueVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getDataBufferAddress(); - <#else> - return (bits.getDataBuffer().memoryAddress()); - + /* DELEGATE TO NEW VECTOR */ + return values.getDataBufferAddress(); } @Override public long getOffsetBufferAddress() { - /* address of the dataBuffer associated with the offsetVector - * this operation is not supported for fixed-width vector types. - */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBufferAddress(); - <#else> - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return (values.getOffsetAddr()); - - + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBufferAddress(); } @Override public ArrowBuf getValidityBuffer() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBuffer(); - <#else> - return (bits.getDataBuffer()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBuffer(); } @Override public ArrowBuf getDataBuffer() { - /* dataBuffer associated with the valueVector */ return (values.getDataBuffer()); } @Override public ArrowBuf getOffsetBuffer() { - /* dataBuffer associated with the offsetVector of the valueVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBuffer(); - <#else> - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return (values.getOffsetBuffer()); - - + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBuffer(); } public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor { - final BitVector.Accessor bAccessor = bits.getAccessor(); - final ${valuesName}.Accessor vAccessor = values.getAccessor(); /** * Get the element at the specified position. @@ -699,119 +379,67 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m * @param index position of the value * @return value of the element, if not null */ - <#if minor.class == "Int" || minor.class == "VarChar"> - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.get(index); - } - <#else> - - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - if (isNull(index)) { - throw new IllegalStateException("Can't get a null value"); - } - return vAccessor.get(index); - } - + public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { + /* DELEGATE TO NEW VECTOR */ + return values.get(index); + } @Override public boolean isNull(int index) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.isNull(index); - <#else> - return isSet(index) == 0; - + /* DELEGATE TO NEW VECTOR */ + return values.isNull(index); } public int isSet(int index){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.isSet(index); - <#else> - return bAccessor.get(index); - + /* DELEGATE TO NEW VECTOR */ + return values.isSet(index); } <#if type.major == "VarLen"> public long getStartEnd(int index){ - <#if minor.class == "VarChar"> /* DELEGATE TO NEW VECTOR */ return values.getStartEnd(index); - <#else> - return vAccessor.getStartEnd(index); - } @Override public int getValueLength(int index) { - <#if minor.class == "VarChar"> /* DELEGATE TO NEW VECTOR */ return values.getValueLength(index); - <#else> - return values.getAccessor().getValueLength(index); - } - <#if minor.class == "Int" || minor.class == "VarChar"> public void get(int index, Nullable${minor.class}Holder holder){ /* DELEGATE TO NEW VECTOR */ values.get(index, holder); } - <#else> - public void get(int index, Nullable${minor.class}Holder holder){ - vAccessor.get(index, holder); - holder.isSet = bAccessor.get(index); - } - - <#if minor.class == "Int" || minor.class == "VarChar"> @Override public ${friendlyType} getObject(int index) { /* DELEGATE TO NEW VECTOR */ return values.getObject(index); } - <#else> - @Override - public ${friendlyType} getObject(int index) { - if (isNull(index)) { - return null; - }else{ - return vAccessor.getObject(index); - } - } - <#if minor.class == "IntervalYear" || minor.class == "IntervalDay"> public StringBuilder getAsStringBuilder(int index) { - if (isNull(index)) { - return null; - }else{ - return vAccessor.getAsStringBuilder(index); - } + /* DELEGATE TO NEW VECTOR */ + return values.getAsStringBuilder(index); } @Override public int getValueCount(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValueCount(); - <#else> - return bits.getAccessor().getValueCount(); - + /* DELEGATE TO NEW VECTOR */ + return values.getValueCount(); } - public void reset(){} + public void reset() { } } public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator { private int setCount; <#if type.major = "VarLen"> private int lastSet = -1; - private Mutator(){ - } + private Mutator() { } public ${valuesName} getVectorWithValues() { return values; @@ -819,12 +447,9 @@ private Mutator(){ @Override - public void setIndexDefined(int index){ - <#if minor.class == "Int" || minor.class == "VarChar"> + public void setIndexDefined(int index) { + /* DELEGATE TO NEW VECTOR */ values.setIndexDefined(index); - <#else> - bits.getMutator().setToOne(index); - } @@ -835,32 +460,14 @@ public void setIndexDefined(int index){ * @param index position of the bit to set * @param value array of bytes (or int if smaller than 4 bytes) to write */ - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { /* DELEGATE TO NEW VECTOR */ values.set(index, value); } - <#else> - public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { - setCount++; - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - final BitVector.Mutator bitsMutator = bits.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bitsMutator.setToOne(index); - valuesMutator.set(index, value); - <#if type.major == "VarLen">lastSet = index; - } - <#if type.major == "VarLen"> - <#if minor.class == "VarChar"> public void fillEmpties(int index) { /* DELEGATE TO NEW VECTOR */ values.fillEmpties(index); @@ -871,203 +478,55 @@ public void setValueLengthSafe(int index, int length) { /* DELEGATE TO NEW VECTOR */ values.setValueLengthSafe(index, length); } - - <#else> - public void fillEmpties(int index){ - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.setSafe(i, emptyByteArray); - } - while(index > bits.getValueCapacity()) { - bits.reAlloc(); - } - lastSet = index - 1; - } - - @Override - public void setValueLengthSafe(int index, int length) { - values.getMutator().setValueLengthSafe(index, length); - lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setSafe(int index, byte[] value, int start, int length) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value, start, length); } - <#else> - public void setSafe(int index, byte[] value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value, start, length); - setCount++; - <#if type.major == "VarLen">lastSet = index; - - } - - - <#if minor.class == "VarChar"> public void setSafe(int index, ByteBuffer value, int start, int length) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value, start, length); } - <#else> - public void setSafe(int index, ByteBuffer value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value, start, length); - setCount++; - <#if type.major == "VarLen">lastSet = index; - - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setNull(int index) { /* DELEGATE TO NEW VECTOR */ values.setNull(index); } - <#else> - public void setNull(int index){ - bits.getMutator().setSafe(index, 0); - } - - - - - <#if minor.class != "Int" && minor.class != "VarChar"> - /* these methods are probably not needed */ - public void setSkipNull(int index, ${minor.class}Holder holder){ - values.getMutator().set(index, holder); - } - - public void setSkipNull(int index, Nullable${minor.class}Holder holder){ - values.getMutator().set(index, holder); - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, Nullable${minor.class}Holder holder) { /* DELEGATE TO NEW VECTOR */ values.set(index, holder); } - <#else> - public void set(int index, Nullable${minor.class}Holder holder) { - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().set(index, holder.isSet); - valuesMutator.set(index, holder); - <#if type.major == "VarLen">lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, ${minor.class}Holder holder) { /* DELEGATE TO NEW VECTOR */ values.set(index, holder); } - <#else> - public void set(int index, ${minor.class}Holder holder) { - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().setToOne(index); - valuesMutator.set(index, holder); - <#if type.major == "VarLen">lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public boolean isSafe(int outIndex) { /* DELEGATE TO NEW VECTOR */ return values.isSafe(outIndex); } - <#else> - public boolean isSafe(int outIndex) { - return outIndex < ${className}.this.getValueCapacity(); - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> - <#if minor.class == "Int"> - public void set(int index, int isSet, int valueField) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, isSet, valueField); - } - public void setSafe(int index, int isSet, int valueField) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, isSet, valueField); - } - - <#if minor.class == "VarChar"> - public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, isSet, startField, endField, bufferField); - } - public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, isSet, startField, endField, bufferField); - } - - <#else> <#assign fields = minor.fields!type.fields /> public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field ){ - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().set(index, isSet); - valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field); - <#if type.major == "VarLen">lastSet = index; + values.set(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); } public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field ) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafe(index, isSet); - values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field); - setCount++; - <#if type.major == "VarLen">lastSet = index; + values.setSafe(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setSafe(int index, Nullable${minor.class}Holder value) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value); @@ -1077,40 +536,12 @@ public void setSafe(int index, ${minor.class}Holder value) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value); } - <#else> - public void setSafe(int index, Nullable${minor.class}Holder value) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafe(index, value.isSet); - values.getMutator().setSafe(index, value); - setCount++; - <#if type.major == "VarLen">lastSet = index; - } - - public void setSafe(int index, ${minor.class}Holder value) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; - <#if type.major == "VarLen">lastSet = index; - } - - <#if !(type.major == "VarLen" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - <#else> - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; - + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); } @@ -1118,93 +549,45 @@ public void setSafe(int index, ${minor.javaType!type.javaType} value) { <#if minor.class == "Decimal"> public void set(int index, ${friendlyType} value) { - bits.getMutator().setToOne(index); - values.getMutator().set(index, value); + /* DELEGATE TO NEW VECTOR */ + values.set(index, value); } public void setSafe(int index, ${friendlyType} value) { - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); } - - <#if minor.class == "Int" || minor.class == "VarChar"> @Override public void setValueCount(int valueCount) { /* DELEGATE TO NEW VECTOR */ values.setValueCount(valueCount); } - <#else> - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - <#if type.major == "VarLen"> - fillEmpties(valueCount); - - values.getMutator().setValueCount(valueCount); - bits.getMutator().setValueCount(valueCount); - } - - - <#if minor.class != "Int" && minor.class != "VarChar"> /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */ @Override - public void generateTestData(int valueCount){ - bits.getMutator().generateTestDataAlt(valueCount); - values.getMutator().generateTestData(valueCount); - <#if type.major = "VarLen">lastSet = valueCount; - setValueCount(valueCount); - } - + public void generateTestData(int valueCount) { } - - <#if minor.class != "Int" && minor.class != "VarChar"> /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */ @Override - public void reset(){ - setCount = 0; - <#if type.major = "VarLen">lastSet = -1; - } - - + public void reset() { } - <#if minor.class == "VarChar"> + <#if type.major == "VarLen"> public void setLastSet(int value) { /* DELEGATE TO NEW VECTOR */ values.setLastSet(value); } - <#else> - public void setLastSet(int value) { - <#if type.major = "VarLen"> - lastSet = value; - <#else> - throw new UnsupportedOperationException(); - - } - - - <#if minor.class == "VarChar"> public int getLastSet() { /* DELEGATE TO NEW VECTOR */ return values.getLastSet(); } - <#else> - public int getLastSet() { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return lastSet; - - } } } diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index 9fe41d0d96335..8ac23fe46f7d2 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -36,11 +36,11 @@ public class UnionListWriter extends AbstractFieldWriter { private ListVector vector; - private UInt4Vector offsets; private PromotableWriter writer; private boolean inMap = false; private String mapName; private int lastIndex = 0; + private static final int OFFSET_WIDTH = 4; public UnionListWriter(ListVector vector) { this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance()); @@ -49,7 +49,6 @@ public UnionListWriter(ListVector vector) { public UnionListWriter(ListVector vector, NullableMapWriterFactory nullableMapWriterFactory) { this.vector = vector; this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableMapWriterFactory); - this.offsets = vector.getOffsetVector(); } public UnionListWriter(ListVector vector, AbstractFieldWriter parent) { @@ -72,7 +71,7 @@ public Field getField() { } public void setValueCount(int count) { - vector.getMutator().setValueCount(count); + vector.setValueCount(count); } @Override @@ -133,13 +132,13 @@ public MapWriter map(String name) { @Override public void startList() { - vector.getMutator().startNewValue(idx()); - writer.setPosition(offsets.getAccessor().get(idx() + 1)); + vector.startNewValue(idx()); + writer.setPosition(vector.getOffsetBuffer().getInt((idx() + 1) * OFFSET_WIDTH)); } @Override public void endList() { - offsets.getMutator().set(idx() + 1, writer.idx()); + vector.getOffsetBuffer().setInt((idx() + 1) * OFFSET_WIDTH, writer.idx()); setPosition(idx() + 1); } diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 3c7ed0190196a..3da383fe6f3c5 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -25,6 +25,7 @@ <#include "/@includes/vv_imports.ftl" /> import com.google.common.collect.ImmutableList; +import io.netty.buffer.ArrowBuf; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -32,6 +33,9 @@ import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.util.OversizedAllocationException; import static org.apache.arrow.vector.types.UnionMode.Sparse; @@ -56,12 +60,10 @@ public class UnionVector implements FieldVector { private String name; private BufferAllocator allocator; - private Accessor accessor = new Accessor(); - private Mutator mutator = new Mutator(); int valueCount; MapVector internalMap; - UInt1Vector typeVector; + ArrowBuf typeBuffer; private NullableMapVector mapVector; private ListVector listVector; @@ -71,16 +73,17 @@ public class UnionVector implements FieldVector { private int singleType = 0; private ValueVector singleVector; + private static final byte TYPE_WIDTH = 1; private final CallBack callBack; - private final List innerVectors; + private int typeBufferAllocationSizeInBytes; public UnionVector(String name, BufferAllocator allocator, CallBack callBack) { this.name = name; this.allocator = allocator; this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); - this.typeVector = new UInt1Vector("types", allocator); + this.typeBuffer = allocator.getEmpty(); this.callBack = callBack; - this.innerVectors = Collections.unmodifiableList(Arrays.asList(typeVector)); + this.typeBufferAllocationSizeInBytes = BaseValueVector.INITIAL_VALUE_ALLOCATION * TYPE_WIDTH; } public BufferAllocator getAllocator() { @@ -104,20 +107,33 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - // truncate types vector buffer to size (#0) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 0, typeVector.getBufferSizeFor(fieldNode.getLength())); - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf buffer = ownBuffers.get(0); + typeBuffer.release(); + typeBuffer = buffer.retain(allocator); + typeBufferAllocationSizeInBytes = typeBuffer.capacity(); this.valueCount = fieldNode.getLength(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(1); + + typeBuffer.readerIndex(0); + typeBuffer.writerIndex(valueCount * TYPE_WIDTH); + + result.add(typeBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return this.innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use geFieldBuffers"); } private String fieldName(MinorType type) { @@ -134,7 +150,7 @@ private T addOrGet(MinorType minorType, Class c) { @Override public long getValidityBufferAddress() { - return typeVector.getDataBuffer().memoryAddress(); + return typeBuffer.memoryAddress(); } @Override @@ -148,7 +164,7 @@ public long getOffsetBufferAddress() { } @Override - public ArrowBuf getValidityBuffer() { return typeVector.getDataBuffer(); } + public ArrowBuf getValidityBuffer() { return typeBuffer; } @Override public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } @@ -211,47 +227,76 @@ public ListVector getList() { } public int getTypeValue(int index) { - return typeVector.getAccessor().get(index); - } - - public UInt1Vector getTypeVector() { - return typeVector; + return typeBuffer.getByte(index * TYPE_WIDTH); } @Override public void allocateNew() throws OutOfMemoryException { internalMap.allocateNew(); - typeVector.allocateNew(); - if (typeVector != null) { - typeVector.zeroVector(); + try { + allocateTypeBuffer(); + } catch (Exception e) { + clear(); + throw e; } } @Override public boolean allocateNewSafe() { boolean safe = internalMap.allocateNewSafe(); - safe = safe && typeVector.allocateNewSafe(); - if (safe) { - if (typeVector != null) { - typeVector.zeroVector(); - } + if (!safe) { return false; } + try { + allocateTypeBuffer(); + } catch (Exception e) { + clear(); + return false; } - return safe; + + return true; + } + + private void allocateTypeBuffer() { + typeBuffer = allocator.buffer(typeBufferAllocationSizeInBytes); + typeBuffer.readerIndex(0); + typeBuffer.setZero(0, typeBuffer.capacity()); } @Override public void reAlloc() { internalMap.reAlloc(); - typeVector.reAlloc(); + reallocTypeBuffer(); } - @Override - public void setInitialCapacity(int numRecords) { + private void reallocTypeBuffer() { + final int currentBufferCapacity = typeBuffer.capacity(); + long baseSize = typeBufferAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, typeBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + typeBuffer.release(1); + typeBuffer = newBuf; + typeBufferAllocationSizeInBytes = (int)newAllocationSize; } + @Override + public void setInitialCapacity(int numRecords) { } + @Override public int getValueCapacity() { - return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity()); + return Math.min(getTypeBufferValueCapacity(), internalMap.getValueCapacity()); } @Override @@ -261,7 +306,9 @@ public void close() { @Override public void clear() { - typeVector.clear(); + valueCount = 0; + typeBuffer.release(); + typeBuffer = allocator.getEmpty(); internalMap.clear(); } @@ -300,7 +347,7 @@ public TransferPair makeTransferPair(ValueVector target) { public void copyFrom(int inIndex, int outIndex, UnionVector from) { from.getReader().setPosition(inIndex); getWriter().setPosition(outIndex); - ComplexCopier.copy(from.reader, mutator.writer); + ComplexCopier.copy(from.reader, writer); } public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { @@ -321,33 +368,34 @@ public FieldVector addVector(FieldVector v) { private class TransferImpl implements TransferPair { private final TransferPair internalMapVectorTransferPair; - private final TransferPair typeVectorTransferPair; private final UnionVector to; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { to = new UnionVector(name, allocator, callBack); internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap); - typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector); } public TransferImpl(UnionVector to) { this.to = to; internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap); - typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector); } @Override public void transfer() { + to.clear(); + to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer; internalMapVectorTransferPair.transfer(); - typeVectorTransferPair.transfer(); to.valueCount = valueCount; } @Override public void splitAndTransfer(int startIndex, int length) { + to.clear(); internalMapVectorTransferPair.splitAndTransfer(startIndex, length); - typeVectorTransferPair.splitAndTransfer(startIndex, length); - to.getMutator().setValueCount(length); + final int startPoint = startIndex * TYPE_WIDTH; + final int sliceLength = length * TYPE_WIDTH; + to.typeBuffer = typeBuffer.slice(startPoint, sliceLength).transferOwnership(to.allocator).buffer; + to.setValueCount(length); } @Override @@ -362,13 +410,15 @@ public void copyValueSafe(int from, int to) { } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from UNION"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to UNION"); } @Override @@ -380,15 +430,17 @@ public FieldReader getReader() { } public FieldWriter getWriter() { - if (mutator.writer == null) { - mutator.writer = new UnionWriter(this); + if (writer == null) { + writer = new UnionWriter(this); } - return mutator.writer; + return writer; } @Override public int getBufferSize() { - return typeVector.getBufferSize() + internalMap.getBufferSize(); + if (valueCount == 0) { return 0; } + + return (valueCount * TYPE_WIDTH) + internalMap.getBufferSize(); } @Override @@ -402,13 +454,13 @@ public int getBufferSizeFor(final int valueCount) { bufferSize += v.getBufferSizeFor(valueCount); } - return (int) bufferSize; + return (int) bufferSize + (valueCount * TYPE_WIDTH); } @Override public ArrowBuf[] getBuffers(boolean clear) { ImmutableList.Builder builder = ImmutableList.builder(); - builder.add(typeVector.getBuffers(clear)); + builder.add(typeBuffer); builder.add(internalMap.getBuffers(clear)); List list = builder.build(); return list.toArray(new ArrowBuf[list.size()]); @@ -417,15 +469,12 @@ public ArrowBuf[] getBuffers(boolean clear) { @Override public Iterator iterator() { List vectors = Lists.newArrayList(internalMap.iterator()); - vectors.add(typeVector); return vectors.iterator(); } - public class Accessor extends BaseValueVector.BaseAccessor { - @Override public Object getObject(int index) { - int type = typeVector.getAccessor().get(index); + int type = typeBuffer.getByte(index * TYPE_WIDTH); switch (MinorType.values()[type]) { case NULL: return null; @@ -436,18 +485,14 @@ public Object getObject(int index) { <#assign uncappedName = name?uncap_first/> <#if !minor.typeParams?? > case ${name?upper_case}: - <#if minor.class != "Int" && minor.class != "VarChar"> - return get${name}Vector().getAccessor().getObject(index); - <#else> return get${name}Vector().getObject(index); - case MAP: - return getMap().getAccessor().getObject(index); + return getMap().getObject(index); case LIST: - return getList().getAccessor().getObject(index); + return getList().getObject(index); default: throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]); } @@ -466,30 +511,37 @@ public void get(int index, UnionHolder holder) { holder.reader = reader; } - @Override public int getValueCount() { return valueCount; } - @Override public boolean isNull(int index) { - return typeVector.getAccessor().get(index) == 0; + return (typeBuffer.getByte(index * TYPE_WIDTH) == 0); + } + + @Override + public int getNullCount() { + int nullCount = 0; + for (int i = 0; i < getValueCount(); i++) { + if (isNull(i)) { + nullCount++; + } + } + return nullCount; } public int isSet(int index) { return isNull(index) ? 0 : 1; } - } - - public class Mutator extends BaseValueVector.BaseMutator { UnionWriter writer; - @Override public void setValueCount(int valueCount) { - UnionVector.this.valueCount = valueCount; - typeVector.getMutator().setValueCount(valueCount); - internalMap.getMutator().setValueCount(valueCount); + this.valueCount = valueCount; + while (valueCount > getTypeBufferValueCapacity()) { + reallocTypeBuffer(); + } + internalMap.setValueCount(valueCount); } public void setSafe(int index, UnionHolder holder) { @@ -534,11 +586,7 @@ public void setSafe(int index, UnionHolder holder) { <#if !minor.typeParams?? > public void setSafe(int index, Nullable${name}Holder holder) { setType(index, MinorType.${name?upper_case}); - <#if minor.class != "Int" && minor.class != "VarChar"> - get${name}Vector().getMutator().setSafe(index, holder); - <#else> - get${name}Vector().setSafe(index, holder); - + get${name}Vector().setSafe(index, holder); } @@ -546,19 +594,13 @@ public void setSafe(int index, Nullable${name}Holder holder) { public void setType(int index, MinorType type) { - typeVector.getMutator().setSafe(index, (byte) type.ordinal()); + while (index >= getTypeBufferValueCapacity()) { + reallocTypeBuffer(); + } + typeBuffer.setByte(index * TYPE_WIDTH , (byte) type.ordinal()); } - @Override - public void reset() { } - - @Override - public void generateTestData(int values) { } - } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } + private int getTypeBufferValueCapacity() { + return (int) ((typeBuffer.capacity() * 1.0) / TYPE_WIDTH); + } } diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index f892bac913392..526708a4c328b 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -62,7 +62,7 @@ public void setPosition(int index) { @Override public void start() { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().start(); } @@ -74,7 +74,7 @@ public void end() { @Override public void startList() { getListWriter().startList(); - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); } @Override @@ -92,7 +92,7 @@ private MapWriter getMapWriter() { } public MapWriter asMap() { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); return getMapWriter(); } @@ -106,7 +106,7 @@ private ListWriter getListWriter() { } public ListWriter asList() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); return getListWriter(); } @@ -150,19 +150,19 @@ BaseWriter getWriter(MinorType minorType) { } public ${name}Writer as${name}() { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); return get${name}Writer(); } @Override public void write(${name}Holder holder) { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); get${name}Writer().setPosition(idx()); get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); get${name}Writer().setPosition(idx()); get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); } @@ -175,28 +175,28 @@ public void writeNull() { @Override public MapWriter map() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().map(); } @Override public ListWriter list() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().list(); } @Override public ListWriter list(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().list(name); } @Override public MapWriter map(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().map(name); } @@ -209,14 +209,14 @@ public MapWriter map(String name) { <#if !minor.typeParams?? > @Override public ${capName}Writer ${lowerName}(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().${lowerName}(name); } @Override public ${capName}Writer ${lowerName}() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().${lowerName}(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index c5f781000c9a9..83dc409a74f14 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -35,12 +35,17 @@ import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; +/** + * BaseNullableFixedWidthVector provides an abstract interface for + * implementing vectors of fixed width values. The vectors are nullable + * implying that zero or more elements in the vector could be NULL. + */ public abstract class BaseNullableFixedWidthVector extends BaseValueVector implements FixedWidthVector, FieldVector { private final byte typeWidth; - private int valueAllocationSizeInBytes; - private int validityAllocationSizeInBytes; + protected int valueAllocationSizeInBytes; + protected int validityAllocationSizeInBytes; protected final Field field; private int allocationMonitor; @@ -52,23 +57,21 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all FieldType fieldType, final byte typeWidth) { super(name, allocator); this.typeWidth = typeWidth; - valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; - validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); field = new Field(name, fieldType, null); valueCount = 0; allocationMonitor = 0; validityBuffer = allocator.getEmpty(); valueBuffer = allocator.getEmpty(); + if (typeWidth > 0) { + valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + } else { + /* specialized handling for NullableBitVector */ + valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = valueAllocationSizeInBytes; + } } - /* TODO: - * Determine how writerIndex and readerIndex need to be used. Right now we - * are setting the writerIndex and readerIndex in the call to getFieldBuffers - * using the valueCount -- this assumes that the caller of getFieldBuffers - * on the vector has already invoked setValueCount. - * - * Do we need to set them during vector transfer and splitAndTransfer? - */ /* TODO: * @@ -84,57 +87,91 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all * abstracted out to top level base class. * * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy and those vectors have not - * yet been refactored so moving things to the top class as of now - * is not a good idea. + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. */ - /* TODO: - * See if we need logger -- unnecessary object probably - */ - - protected abstract org.slf4j.Logger getLogger(); @Override + @Deprecated public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); } @Override + @Deprecated public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ @Override public long getValidityBufferAddress() { return (validityBuffer.memoryAddress()); } + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getDataBufferAddress() { return (valueBuffer.memoryAddress()); } + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return starting address of the buffer + * @throws UnsupportedOperationException for fixed width vectors + */ @Override public long getOffsetBufferAddress() { throw new UnsupportedOperationException("not supported for fixed-width vectors"); } + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ @Override public ArrowBuf getValidityBuffer() { return validityBuffer; } + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ @Override public ArrowBuf getDataBuffer() { return valueBuffer; } + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + * @throws UnsupportedOperationException for fixed width vectors + */ @Override public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException("not supported for fixed-width vectors"); } + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ @Override public void setInitialCapacity(int valueCount) { final long size = (long)valueCount * typeWidth; @@ -142,52 +179,65 @@ public void setInitialCapacity(int valueCount) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getSizeFromCount(valueCount); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity(){ return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); } - /* for test purposes */ private int getValueBufferValueCapacity() { return (int)((valueBuffer.capacity() * 1.0)/typeWidth); } - /* for test purposes */ private int getValidityBufferValueCapacity() { return (int)(validityBuffer.capacity() * 8L); } - /* number of bytes for the validity buffer for the given valueCount */ - protected int getSizeFromCount(int valueCount) { - return (int) Math.ceil(valueCount / 8.0); - } - + /** + * zero out the vector and the data in associated buffers. + */ @Override public void zeroVector() { initValidityBuffer(); initValueBuffer(); } + /* zero out the validity buffer */ private void initValidityBuffer() { validityBuffer.setZero(0, validityBuffer.capacity()); } + /* zero out the data buffer */ private void initValueBuffer() { valueBuffer.setZero(0, valueBuffer.capacity()); } + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ public void reset() { zeroVector(); } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clear(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { + valueCount = 0; validityBuffer = releaseBuffer(validityBuffer); valueBuffer = releaseBuffer(valueBuffer); } @@ -208,6 +258,9 @@ protected void decrementAllocationMonitor() { allocationMonitor--; } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() { if(!allocateNewSafe()){ @@ -215,6 +268,15 @@ public void allocateNew() { } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override public boolean allocateNewSafe() { long curAllocationSizeValue = valueAllocationSizeInBytes; long curAllocationSizeValidity = validityAllocationSizeInBytes; @@ -229,8 +291,7 @@ public boolean allocateNewSafe() { try{ allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); return false; } @@ -238,9 +299,20 @@ public boolean allocateNewSafe() { return true; } + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ public void allocateNew(int valueCount) { long valueBufferSize = valueCount * typeWidth; - long validityBufferSize = getSizeFromCount(valueCount); + long validityBufferSize = getValidityBufferSizeFromCount(valueCount); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBufferSize = validityBufferSize; + } if (allocationMonitor > 10) { /* step down the default memory allocation since we have observed @@ -266,8 +338,7 @@ public void allocateNew(int valueCount) { try { allocateBytes(valueBufferSize, validityBufferSize); } catch(Exception e) { - getLogger().error("ERROR: Failure in allocateNew"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); throw e; } @@ -294,8 +365,8 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer initValidityBuffer(); } - /* - * during splitAndTransfer, if we splitting from a random position within a byte, + /** + * During splitAndTransfer, if we splitting from a random position within a byte, * we can't just slice the source buffer so we have to explicitly allocate the * validityBuffer of the target vector. This is unlike the databuffer which we can * always slice for the target vector. @@ -307,23 +378,49 @@ private void allocateValidityBuffer(final int validityBufferSize) { initValidityBuffer(); } + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int count) { if (count == 0) { return 0; } - return (count * typeWidth) + getSizeFromCount(count); + return (count * typeWidth) + getValidityBufferSizeFromCount(count); } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { if (valueCount == 0) { return 0; } - return (valueCount * typeWidth) + getSizeFromCount(valueCount); + return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); } + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ @Override public Field getField() { return field; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers = new ArrowBuf[2]; @@ -338,12 +435,20 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ @Override public void reAlloc() { valueBuffer = reallocBufferHelper(valueBuffer, true); validityBuffer = reallocBufferHelper(validityBuffer, false); } + /** + * Helper method for reallocating a particular internal buffer + * Returns the new buffer. + */ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { final int currentBufferCapacity = buffer.capacity(); long baseSize = (dataBuffer ? valueAllocationSizeInBytes @@ -360,10 +465,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) throw new OversizedAllocationException("Unable to expand the buffer"); } - getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", - name, (dataBuffer ? valueAllocationSizeInBytes : validityAllocationSizeInBytes), - newAllocationSize); - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); newBuf.setBytes(0, buffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; @@ -381,8 +482,17 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) } @Override - public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ @Override public void initializeChildrenFromFields(List children) { if (!children.isEmpty()) { @@ -390,11 +500,23 @@ public void initializeChildrenFromFields(List children) { } } + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ @Override public List getChildrenFromFields() { return Collections.emptyList(); } + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 2) { @@ -415,13 +537,22 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers validityAllocationSizeInBytes = validityBuffer.capacity(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ public List getFieldBuffers() { List result = new ArrayList<>(2); validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getSizeFromCount(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); valueBuffer.readerIndex(0); - valueBuffer.writerIndex(valueCount * typeWidth); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + } else { + valueBuffer.writerIndex(valueCount * typeWidth); + } result.add(validityBuffer); result.add(valueBuffer); @@ -429,18 +560,42 @@ public List getFieldBuffers() { return result; } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ @Override public TransferPair getTransferPair(BufferAllocator allocator){ return getTransferPair(name, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ public void transferTo(BaseNullableFixedWidthVector target){ compareTypes(target, "transferTo"); target.clear(); @@ -450,6 +605,13 @@ public void transferTo(BaseNullableFixedWidthVector target){ clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ public void splitAndTransferTo(int startIndex, int length, BaseNullableFixedWidthVector target) { compareTypes(target, "splitAndTransferTo"); @@ -459,6 +621,9 @@ public void splitAndTransferTo(int startIndex, int length, target.setValueCount(length); } + /** + * Data buffer can always be split and transferred using slicing. + */ private void splitAndTransferValueBuffer(int startIndex, int length, BaseNullableFixedWidthVector target) { final int startPoint = startIndex * typeWidth; @@ -466,17 +631,21 @@ private void splitAndTransferValueBuffer(int startIndex, int length, target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; } + /** + * Validity buffer has multiple cases of split and transfer depending on + * the starting position of the source index. + */ private void splitAndTransferValidityBuffer(int startIndex, int length, BaseNullableFixedWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getSizeFromCount(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { if (offset == 0) { - // slice + /* slice */ if (target.validityBuffer != null) { target.validityBuffer.release(); } @@ -493,8 +662,10 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + i + 1, offset); target.validityBuffer.setByte(i, (b1 + b2)); } @@ -509,15 +680,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * by shifting data from the current byte. */ if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); } else { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); } @@ -525,14 +696,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, } } - private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } - - private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } - /****************************************************************** * * @@ -547,27 +710,9 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { * @return the number of null elements. */ public int getNullCount() { - int count = 0; - final int sizeInBytes = getSizeFromCount(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. - */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; + return BitVectorHelper.getNullCount(validityBuffer, valueCount); } - /** * Get the value count of vector. This will always be zero unless * {@link #setValueCount(int)} has been called prior to calling this. @@ -578,7 +723,6 @@ public int getValueCount(){ return valueCount; } - /** * Set value count for the vector. * @@ -625,7 +769,6 @@ public void setValueCount(int valueCount) { } } - /** * Check if the given index is within the current value capacity * of the vector @@ -637,7 +780,6 @@ public boolean isSafe(int index) { return index < getValueCapacity(); } - /** * Check if element at given index is null. * @@ -648,7 +790,6 @@ public boolean isNull(int index) { return (isSet(index) == 0); } - /** * Same as {@link #isNull(int)}. * @@ -662,6 +803,11 @@ public int isSet(int index) { return Long.bitCount(b & (1L << bitIndex)); } + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ public void setIndexDefined(int index) { handleSafe(index); BitVectorHelper.setValidityBitToOne(validityBuffer, index); @@ -691,7 +837,6 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { ******************************************************************/ - protected void handleSafe(int index) { while (index >= getValueCapacity()) { decrementAllocationMonitor(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index a79709da4dd86..6587cde633ea9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -45,7 +45,7 @@ public abstract class BaseNullableVariableWidthVector extends BaseValueVector private int offsetAllocationSizeInBytes; /* protected members */ - protected static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ + public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ protected static final byte[] emptyByteArray = new byte[]{}; protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; @@ -59,7 +59,7 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator FieldType fieldType) { super(name, allocator); valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; - validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; field = new Field(name, fieldType, null); valueCount = 0; @@ -70,21 +70,12 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator cleared = false; } - /* TODO: - * Determine how writerIndex and readerIndex need to be used. Right now we - * are setting the writerIndex and readerIndex in the call to getFieldBuffers - * using the valueCount -- this assumes that the caller of getFieldBuffers - * on the vector has already invoked setValueCount. - * - * Do we need to set them during vector transfer and splitAndTransfer? - */ - /* TODO: * * see if getNullCount() can be made faster -- O(1) */ - /* TODO: + /* TODO: * Once the entire hierarchy has been refactored, move common functions * like getNullCount(), splitAndTransferValidityBuffer to top level * base class BaseValueVector. @@ -93,59 +84,89 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator * abstracted out to top level base class. * * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy and those vectors have not - * yet been refactored so moving things to the top class as of now - * is not a good idea. - */ - - /* TODO: - * See if we need logger -- unnecessary object probably + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. */ - /* TODO: - * Implement getBufferSize(), getCurrentSizeInBytes(). - */ - - protected abstract org.slf4j.Logger getLogger(); + @Override + @Deprecated public VariableWidthMutator getMutator() { - throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); } + @Override + @Deprecated public VariableWidthAccessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ @Override public ArrowBuf getValidityBuffer() { return validityBuffer; } + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ @Override public ArrowBuf getDataBuffer() { return valueBuffer; } + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + */ @Override public ArrowBuf getOffsetBuffer() { return offsetBuffer; } + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getOffsetBufferAddress() { return offsetBuffer.memoryAddress(); } + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ @Override public long getValidityBufferAddress() { return validityBuffer.memoryAddress(); } + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getDataBufferAddress() { return valueBuffer.memoryAddress(); } + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ @Override public void setInitialCapacity(int valueCount) { final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT; @@ -153,57 +174,69 @@ public void setInitialCapacity(int valueCount) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getSizeFromCount(valueCount); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); /* to track the end offset of last data element in vector, we need * an additional slot in offset buffer. */ offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity(){ final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); } - /* for test purposes */ private int getValidityBufferValueCapacity() { return (int)(validityBuffer.capacity() * 8L); } - /* for test purposes */ private int getOffsetBufferValueCapacity() { return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); } - /* number of bytes for the validity buffer for a given valueCount */ - protected int getSizeFromCount(int valueCount) { - return (int) Math.ceil(valueCount / 8.0); - } - + /** + * zero out the vector and the data in associated buffers. + */ public void zeroVector() { initValidityBuffer(); initOffsetBuffer(); } + /* zero out the validity buffer */ private void initValidityBuffer() { validityBuffer.setZero(0, validityBuffer.capacity()); } + /* zero out the offset buffer */ private void initOffsetBuffer() { offsetBuffer.setZero(0, offsetBuffer.capacity()); } + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ public void reset() { zeroVector(); lastSet = -1; } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clear(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { validityBuffer = releaseBuffer(validityBuffer); @@ -215,8 +248,17 @@ public void clear() { } @Override - public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ @Override public void initializeChildrenFromFields(List children) { if (!children.isEmpty()) { @@ -224,11 +266,24 @@ public void initializeChildrenFromFields(List children) { } } + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ @Override public List getChildrenFromFields() { return Collections.emptyList(); } + + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { ArrowBuf bitBuffer = ownBuffers.get(0); @@ -246,11 +301,15 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers valueCount = fieldNode.getLength(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ public List getFieldBuffers() { List result = new ArrayList<>(3); final int lastDataOffset = getstartOffset(valueCount); validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getSizeFromCount(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); offsetBuffer.readerIndex(0); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); valueBuffer.readerIndex(0); @@ -263,6 +322,9 @@ public List getFieldBuffers() { return result; } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() { if(!allocateNewSafe()){ @@ -270,6 +332,14 @@ public void allocateNew() { } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int, int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ @Override public boolean allocateNewSafe() { long curAllocationSizeValue = valueAllocationSizeInBytes; @@ -287,8 +357,7 @@ public boolean allocateNewSafe() { try { allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); return false; } @@ -296,11 +365,19 @@ public boolean allocateNewSafe() { return true; } + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param totalBytes desired total memory capacity + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ @Override public void allocateNew(int totalBytes, int valueCount) { assert totalBytes >= 0; final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getSizeFromCount(valueCount); + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); if (totalBytes > MAX_ALLOCATION_SIZE || offsetBufferSize > MAX_ALLOCATION_SIZE) { @@ -313,12 +390,12 @@ public void allocateNew(int totalBytes, int valueCount) { try { allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); } } + /* allocate the inner buffers */ private void allocateBytes(final long valueBufferSize, final long validityBufferSize, final long offsetBufferSize) { /* allocate data buffer */ @@ -330,6 +407,7 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer allocateOffsetBuffer(offsetBufferSize); } + /* allocate offset buffer */ private void allocateOffsetBuffer(final long size) { final int curSize = (int)size; offsetBuffer = allocator.buffer(curSize); @@ -338,6 +416,7 @@ private void allocateOffsetBuffer(final long size) { initOffsetBuffer(); } + /* allocate validity buffer */ private void allocateValidityBuffer(final long size) { final int curSize = (int)size; validityBuffer = allocator.buffer(curSize); @@ -346,6 +425,10 @@ private void allocateValidityBuffer(final long size) { initValidityBuffer(); } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ public void reAlloc() { reallocValueBuffer(); reallocValidityAndOffsetBuffers(); @@ -378,10 +461,7 @@ protected void reallocValidityAndOffsetBuffers() { validityBuffer = reallocBufferHelper(validityBuffer, false); } - /* need to refactor this to keep the logic in an single place and make callers - * more intelligent. see handleSafe() for more comments on realloc - */ - + /* helper method to realloc a particular buffer. returns the allocated buffer */ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { final int currentBufferCapacity = buffer.capacity(); long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes @@ -398,10 +478,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer throw new OversizedAllocationException("Unable to expand the buffer"); } - getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", - name, (offsetBuffer ? offsetAllocationSizeInBytes : validityAllocationSizeInBytes), - newAllocationSize); - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); newBuf.setBytes(0, buffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; @@ -418,6 +494,10 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer return buffer; } + /** + * Get the size (number of bytes) of underlying data buffer. + * @return + */ @Override public int getByteCapacity(){ return valueBuffer.capacity(); @@ -429,30 +509,55 @@ public int getCurrentSizeInBytes(){ return 0; } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { - /* TODO */ - return 0; + return getBufferSizeFor(this.valueCount); } + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getSizeFromCount(valueCount); + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); return validityBufferSize + offsetBufferSize + dataBufferSize; } + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ @Override public Field getField() { return field; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers = new ArrowBuf[3]; @@ -468,18 +573,42 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ @Override public TransferPair getTransferPair(BufferAllocator allocator){ return getTransferPair(name, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ public void transferTo(BaseNullableVariableWidthVector target){ compareTypes(target, "transferTo"); target.clear(); @@ -491,6 +620,13 @@ public void transferTo(BaseNullableVariableWidthVector target){ clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ public void splitAndTransferTo(int startIndex, int length, BaseNullableVariableWidthVector target) { compareTypes(target, "splitAndTransferTo"); @@ -502,7 +638,12 @@ public void splitAndTransferTo(int startIndex, int length, } /* - * transfer the offsets along with data + * Transfer the offsets along with data. Unlike the data buffer, we cannot simply + * slice the offset buffer for split and transfer. The reason is that offsets + * in the target vector have to be adjusted and made relative to the staring + * offset in source vector from the start index of split. This is why, we + * need to explicitly allocate the offset buffer and set the adjusted offsets + * in the target vector. */ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); @@ -517,14 +658,14 @@ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullab } /* - * transfer the validity. + * Transfer the validity. */ private void splitAndTransferValidityBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getSizeFromCount(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -546,8 +687,8 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); target.validityBuffer.setByte(i, (b1 + b2)); } @@ -562,15 +703,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * by shifting data from the current byte. */ if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); } else { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); } @@ -578,14 +719,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, } } - private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } - - private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } - /****************************************************************** * * @@ -600,24 +733,7 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { * @return the number of null elements. */ public int getNullCount() { - int count = 0; - final int sizeInBytes = getSizeFromCount(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. - */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; + return BitVectorHelper.getNullCount(validityBuffer, valueCount); } /** @@ -679,24 +795,55 @@ public void setValueCount(int valueCount) { lastSet = valueCount - 1; } + /** + * Create holes in the vector upto the given index (exclusive). + * Holes will be created from the current last set position in + * the vector. + * + * @param index target index + */ public void fillEmpties(int index) { handleSafe(index, emptyByteArray.length); fillHoles(index); lastSet = index - 1; } + /** + * Set the index of last non-null element in the vector. + * It is important to call this method with appropriate value + * before calling {@link #setValueCount(int)}. + * + * @param value desired index of last non-null element. + */ public void setLastSet(int value) { lastSet = value; } + /** + * Get the index of last non-null element in the vector. + * + * @return index of the last non-null element + */ public int getLastSet() { return lastSet; } + /** + * Get the starting position (offset) in the data stream for a given + * element in the vector. + * + * @param index position of the element in the vector + * @return starting offset for the element + */ public long getStartEnd(int index) { return (long)offsetBuffer.getInt(index * OFFSET_WIDTH); } + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ public void setIndexDefined(int index) { handleSafe(index, 0); BitVectorHelper.setValidityBitToOne(validityBuffer, index); @@ -761,4 +908,97 @@ protected final void handleSafe(int index, int dataLength) { reallocValueBuffer(); } } + + + /****************************************************************** + * * + * helper methods currently * + * used by JsonFileReader and * + * JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Method used by Json Reader to explicitly set the data of the variable + * width vector elements. The method takes care of allocating the memory + * for the vector if caller hasn't done so. + * + * This method should not be used externally. + * + * @param data ArrowBuf for storing variable width elements in the vector + * @param offset offset of the element + * @param allocator memory allocator + * @param index position of the element in the vector + * @param value array of bytes for the element + * @param valueCount number of elements in the vector + * @return buffer holding the variable width data. + */ + public static ArrowBuf set(ArrowBuf data, ArrowBuf offset, + BufferAllocator allocator, int index, byte[] value, + int valueCount) { + if (data == null) { + data = allocator.buffer(INITIAL_BYTE_COUNT); + } + final int currentBufferCapacity = data.capacity(); + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + while (currentBufferCapacity < currentStartOffset + value.length) { + final ArrowBuf newBuf = allocator.buffer(currentBufferCapacity * 2); + newBuf.setBytes(0, data, 0, currentBufferCapacity); + data.release(); + data = newBuf; + } + data.setBytes(currentStartOffset, value, 0, value.length); + if (index == (valueCount - 1)) { + data.writerIndex(offset.getInt(valueCount * OFFSET_WIDTH)); + } + return data; + } + + /** + * Method used by Json Writer to read a variable width element from + * the variable width vector and write to Json. + * + * This method should not be used externally. + * + * @param data buffer storing the variable width vector elements + * @param offset buffer storing the offsets of variable width vector elements + * @param index position of the element in the vector + * @return array of bytes + */ + public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) { + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + final int dataLength = + offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset; + final byte[] result = new byte[dataLength]; + data.getBytes(currentStartOffset, result, 0, dataLength); + return result; + } + + /** + * Method used by Json Reader to explicitly set the offsets of the variable + * width vector data. The method takes care of allocating the memory for + * offsets if the caller hasn't done so. + * + * This method should not be used externally. + * + * @param buffer ArrowBuf to store offsets for variable width elements + * @param allocator memory allocator + * @param valueCount number of elements + * @param index position of the element + * @param value offset of the element + * @return buffer holding the offsets + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * OFFSET_WIDTH); + } + buffer.setInt(index * OFFSET_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * OFFSET_WIDTH); + } + + return buffer; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 1db96246485d3..1b96782681f26 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -56,7 +56,6 @@ public String toString() { @Override public void clear() { - getMutator().reset(); } @Override @@ -137,10 +136,36 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) { return buffer; } - public int getValueCount() { return 0; } + @Override + public int getValueCount() { + return getAccessor().getValueCount(); + } + + @Override + public void setValueCount(int valueCount) { + getMutator().setValueCount(valueCount); + } + + @Override + + public Object getObject(int index) { + return getAccessor().getObject(index); + } + + @Override + + public int getNullCount() { + return getAccessor().getNullCount(); + } - public void setValueCount(int valueCount) { } + @Override + public boolean isNull(int index) { + return getAccessor().isNull(index); + } - public Object getObject(int index) { return null; } + /* number of bytes for the validity buffer for the given valueCount */ + protected static int getValidityBufferSizeFromCount(final int valueCount) { + return (int) Math.ceil(valueCount / 8.0); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 2439bd26eab56..69325b5ea06b6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -19,24 +19,35 @@ package org.apache.arrow.vector; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; -class BitVectorHelper { +/** + * Helper class for performing generic operations on a bit vector buffer. + * External use of this class is not recommended. + */ +public class BitVectorHelper { /** * Get the index of byte corresponding to bit index in validity buffer */ - protected static int byteIndex(int absoluteBitIndex) { + public static int byteIndex(int absoluteBitIndex) { return absoluteBitIndex >> 3; } /** * Get the relative index of bit within the byte in validity buffer */ - private static int bitIndex(int absoluteBitIndex) { + public static int bitIndex(int absoluteBitIndex) { return absoluteBitIndex & 7; } - protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { + /** + * Set the bit at provided index to 1. + * + * @param validityBuffer + * @param index + */ + public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); @@ -45,7 +56,14 @@ protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { validityBuffer.setByte(byteIndex, currentByte); } - protected static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { + /** + * Set the bit at a given index to provided value (1 or 0) + * + * @param validityBuffer + * @param index + * @param value + */ + public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); @@ -57,4 +75,91 @@ protected static void setValidityBit(ArrowBuf validityBuffer, int index, int val } validityBuffer.setByte(byteIndex, currentByte); } + + /** + * Set the bit at a given index to provided value (1 or 0). Internally + * takes care of allocating the buffer if the caller didn't do so. + * + * @param validityBuffer + * @param allocator + * @param valueCount + * @param index + * @param value + * @return ArrowBuf + */ + public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (validityBuffer == null) { + validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + } + setValidityBit(validityBuffer, index, value); + if (index == (valueCount - 1)) { + validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + } + + return validityBuffer; + } + + /** + * Check if a bit at a given index is set or not. + * + * @param buffer + * @param index + * @return 1 if bit is set, 0 otherwise. + */ + public static int get(final ArrowBuf buffer, int index) { + final int byteIndex = index >> 3; + final byte b = buffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Compute the size of validity buffer required to manage a given number + * of elements in a vector. + * + * @param valueCount + * @return buffer size + */ + public static int getValidityBufferSize(int valueCount) { + return ((int) Math.ceil(valueCount / 8.0)); + } + + /** + * Given a validity buffer, find the number of bits that are not set. + * This is used to compute the number of null elements in a nullable vector. + * + * @param validityBuffer + * @param valueCount + * @return number of bits not set. + */ + public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) { + if (valueCount == 0) { return 0; } + int count = 0; + final int sizeInBytes = getValidityBufferSize(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) { + return (byte)((data.getByte(index) & 0xFF) >>> offset); + } + + public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) << (8 - offset))); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java new file mode 100644 index 0000000000000..ee40d708cc199 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.BigIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.BigIntHolder; +import org.apache.arrow.vector.holders.NullableBigIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableBigIntVector implements a fixed width vector (8 bytes) of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableBigIntVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), + allocator); + } + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new BigIntReaderImpl(NullableBigIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIGINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBigIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BigIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BigIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBigIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableBigIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableBigIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBigIntVector to){ + this.to = to; + } + + @Override + public NullableBigIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java new file mode 100644 index 0000000000000..eceaa61b57f41 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java @@ -0,0 +1,502 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.BitReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.BitHolder; +import org.apache.arrow.vector.holders.NullableBitHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +import java.util.ArrayList; +import java.util.List; + +/** + * NullableBitVector implements a fixed width (1 bit) vector of + * boolean values which could be null. Each value in the vector corresponds + * to a single bit in the underlying data stream backing the vector. + */ +public class NullableBitVector extends BaseNullableFixedWidthVector { + private final FieldReader reader; + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIT.getType()), + allocator); + } + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, (byte)0); + reader = new BitReaderImpl(NullableBitVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIT; + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final int size = getValidityBufferSizeFromCount(valueCount); + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = size; + validityAllocationSizeInBytes = size; + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity(){ + return (int)(validityBuffer.capacity() * 8L); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { return 0; } + return 2 * getValidityBufferSizeFromCount(count); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + return getBufferSizeFor(valueCount); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, + validityBuffer, target.validityBuffer); + target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, + valueBuffer, target.valueBuffer); + + target.setValueCount(length); + } + + private ArrowBuf splitAndTransferBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target, + ArrowBuf sourceBuffer, ArrowBuf destBuffer) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + /* slice */ + if (destBuffer != null) { + destBuffer.release(); + } + destBuffer = destBuffer.slice(firstByteSource, byteSizeTarget); + destBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + destBuffer = allocator.buffer(byteSizeTarget); + destBuffer.readerIndex(0); + destBuffer.setZero(0, destBuffer.capacity()); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); + + destBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, + firstByteSource + byteSizeTarget, offset); + + destBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + destBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + + return destBuffer; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + private int getBit(int index) { + final int byteIndex = index >> 3; + final byte b = valueBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return getBit(index); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBitHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = getBit(index); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Boolean getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return new Boolean (getBit(index) != 0); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBitVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BitHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BitHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBitVector)to); + } + + private class TransferImpl implements TransferPair { + NullableBitVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableBitVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBitVector to){ + this.to = to; + } + + @Override + public NullableBitVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java new file mode 100644 index 0000000000000..949287ecafe45 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DateDayReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DateDayHolder; +import org.apache.arrow.vector.holders.NullableDateDayHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableDateDayVector implements a fixed width (4 bytes) vector of + * date values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDateDayVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateDayReaderImpl(NullableDateDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateDayHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateDayHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateDayHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateDayVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDateDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDateDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateDayVector to){ + this.to = to; + } + + @Override + public NullableDateDayVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java new file mode 100644 index 0000000000000..a0bdccedea405 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -0,0 +1,408 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DateMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DateMilliHolder; +import org.apache.arrow.vector.holders.NullableDateMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; +import org.joda.time.LocalDateTimes; +import org.slf4j.Logger; + +/** + * NullableDateMilliVector implements a fixed width vector (8 bytes) of + * date values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDateMilliVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateMilliReaderImpl(NullableDateMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = get(index); + final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateMilliVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDateMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateMilliVector to){ + this.to = to; + } + + @Override + public NullableDateMilliVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java new file mode 100644 index 0000000000000..8320f90830a5b --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -0,0 +1,454 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DecimalReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DecimalHolder; +import org.apache.arrow.vector.holders.NullableDecimalHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.DecimalUtility; +import org.apache.arrow.vector.util.TransferPair; + +import java.math.BigDecimal; + +/** + * NullableDecimalVector implements a fixed width vector (16 bytes) of + * decimal values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDecimalVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 16; + private final FieldReader reader; + + private final int precision; + private final int scale; + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, BufferAllocator allocator, + int precision, int scale) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), + allocator); + } + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal)fieldType.getType(); + reader = new DecimalReaderImpl(NullableDecimalVector.this); + this.precision = arrowType.getPrecision(); + this.scale = arrowType.getScale(); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DECIMAL; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDecimalHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.buffer = valueBuffer; + holder.precision = precision; + holder.scale = scale; + holder.start = index * TYPE_WIDTH; + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public BigDecimal getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { + if (from.isSet(fromIndex) != 0) { + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, int start, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void set(int index, BigDecimal value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + DecimalUtility.checkPrecisionAndScale(value, precision, scale); + DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DecimalHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, ArrowBuf buffer) { + handleSafe(index); + set(index, buffer); + } + + /** + * Same as {@link #set(int, int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, start, buffer); + } + + /** + * Same as {@link #set(int, BigDecimal)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void setSafe(int index, BigDecimal value){ + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DecimalHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void set(int index, int isSet, int start, ArrowBuf buffer) { + if (isSet > 0) { + set(index, start, buffer); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles + * the case when the position of new value is beyond the current value + * capacity of the vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, isSet, start, buffer); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value as array of bytes + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, byte[] value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + DecimalUtility.writeByteArrayToArrowBuf(value, buffer, index); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDecimalVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDecimalVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, + NullableDecimalVector.this.scale); + } + + public TransferImpl(NullableDecimalVector to){ + this.to = to; + } + + @Override + public NullableDecimalVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java new file mode 100644 index 0000000000000..5b28065c402c4 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.Float4ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.Float4Holder; +import org.apache.arrow.vector.holders.NullableFloat4Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableFloat4Vector implements a fixed width vector (4 bytes) of + * float values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableFloat4Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float4ReaderImpl(NullableFloat4Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public float get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat4Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Float getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, float value) { + valueBuffer.setFloat(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, float value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float4Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, float)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, float value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float4Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, float value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, float)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, float value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, float value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setFloat(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static float get(final ArrowBuf buffer, final int index) { + return buffer.getFloat(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat4Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableFloat4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat4Vector to){ + this.to = to; + } + + @Override + public NullableFloat4Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java new file mode 100644 index 0000000000000..624abf2f272cd --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.Float8ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.Float8Holder; +import org.apache.arrow.vector.holders.NullableFloat8Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableFloat8Vector implements a fixed width vector (8 bytes) of + * double values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableFloat8Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float8ReaderImpl(NullableFloat8Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public double get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat8Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Double getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, double value) { + valueBuffer.setDouble(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, double value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float8Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, double)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, double value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float8Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, double value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, double)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, double value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, double value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setDouble(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static double get(final ArrowBuf buffer, final int index) { + return buffer.getDouble(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat8Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableFloat8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat8Vector to){ + this.to = to; + } + + @Override + public NullableFloat8Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 26b19fa7bd562..6311daf4f5cc1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,10 +14,11 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.impl.IntReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -27,39 +27,53 @@ import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.slf4j.Logger; /** - * NullableIntVector implements a fixed width vector of values which could - * be null. A validity buffer (bit vector) is maintained to track which - * elements in the vector are null. + * NullableIntVector implements a fixed width (4 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. */ public class NullableIntVector extends BaseNullableFixedWidthVector { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); private static final byte TYPE_WIDTH = 4; private final FieldReader reader; + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ public NullableIntVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), allocator); } + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); reader = new IntReaderImpl(NullableIntVector.this); } - @Override - protected org.slf4j.Logger getLogger() { - return logger; - } - + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ @Override public FieldReader getReader(){ return reader; } + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ @Override public Types.MinorType getMinorType() { return Types.MinorType.INT; @@ -116,12 +130,27 @@ public Integer getObject(int index) { } } + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); } } + /** + * Same as {@link #copyFrom(int, int, NullableIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); @@ -234,17 +263,80 @@ public void setNull(int index){ BitVectorHelper.setValidityBit(validityBuffer, index, 0); } - public void set(int index, int isSet, int valueField ) { + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { if (isSet > 0) { - set(index, valueField); + set(index, value); } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); } } - public void setSafe(int index, int isSet, int valueField ) { + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { handleSafe(index); - set(index, isSet, valueField); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); } @@ -255,11 +347,23 @@ public void setSafe(int index, int isSet, int valueField ) { ******************************************************************/ + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ return new TransferImpl(ref, allocator); } + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((NullableIntVector)to); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java new file mode 100644 index 0000000000000..174b722b04316 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java @@ -0,0 +1,429 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntervalDayReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntervalDayHolder; +import org.apache.arrow.vector.holders.NullableIntervalDayHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.Period; + +/** + * NullableIntervalDayVector implements a fixed width vector (8 bytes) of + * interval (days and milliseconds) values which could be null. + * A validity buffer (bit vector) is maintained to track which elements in the + * vector are null. + */ +public class NullableIntervalDayVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private static final byte MILLISECOND_OFFSET = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + return null; + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalDayHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startIndex = index * TYPE_WIDTH; + holder.isSet = 1; + holder.days = valueBuffer.getInt(startIndex); + holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int startIndex = index * TYPE_WIDTH; + final int days = valueBuffer.getInt(startIndex); + final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + final Period p = new Period(); + return p.plusDays(days).plusMillis(milliseconds); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [days, hours, minutes, seconds, millis] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + }else{ + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + final int startIndex = index * TYPE_WIDTH; + + final int days = valueBuffer.getInt(startIndex); + int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + + final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + + final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + + final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + + final String dayString = (Math.abs(days) == 1) ? " day " : " days "; + + return(new StringBuilder(). + append(days).append(dayString). + append(hours).append(":"). + append(minutes).append(":"). + append(seconds).append("."). + append(millis)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + if (from.isSet(fromIndex) != 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, ArrowBuf value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void set(int index, int days, int milliseconds){ + final int offsetIndex = index * TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setInt(offsetIndex, days); + valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + set(index, holder.days, holder.milliseconds); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalDayHolder holder){ + set(index, holder.days, holder.milliseconds); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, ArrowBuf value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void setSafe(int index, int days, int milliseconds) { + handleSafe(index); + set(index, days, milliseconds); + } + + /** + * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalDayHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void set(int index, int isSet, int days, int milliseconds) { + if (isSet > 0) { + set(index, days, milliseconds); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void setSafe(int index, int isSet, int days, int milliseconds) { + handleSafe(index); + set(index, isSet, days, milliseconds); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalDayVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalDayVector to){ + this.to = to; + } + + @Override + public NullableIntervalDayVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java new file mode 100644 index 0000000000000..303286d0ed30c --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java @@ -0,0 +1,387 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntervalYearReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntervalYearHolder; +import org.apache.arrow.vector.holders.NullableIntervalYearHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.Period; + +/** + * NullableIntervalYearVector implements a fixed width (4 bytes) vector of + * interval (years and months) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableIntervalYearVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALYEAR; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalYearHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int interval = get(index); + final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final Period p = new Period(); + return p.plusYears(years).plusMonths(months); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [years, months] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + }else{ + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + int value = valueBuffer.getInt(index * TYPE_WIDTH); + + final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + + final String yearString = (Math.abs(years) == 1) ? " year " : " years "; + final String monthString = (Math.abs(months) == 1) ? " month " : " months "; + + return(new StringBuilder(). + append(years).append(yearString). + append(months).append(monthString)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalYearHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalYearHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalYearVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalYearVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalYearVector to){ + this.to = to; + } + + @Override + public NullableIntervalYearVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java new file mode 100644 index 0000000000000..c45a8d5f5bcba --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -0,0 +1,431 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.SmallIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.SmallIntHolder; +import org.apache.arrow.vector.holders.NullableSmallIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableSmallIntVector implements a fixed width (2 bytes) vector of + * short values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableSmallIntVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), + allocator); + } + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new SmallIntReaderImpl(NullableSmallIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.SMALLINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public short get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableSmallIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Short getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + private void setValue(int index, short value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, short value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, SmallIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, short)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, short value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, SmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, SmallIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, short value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, short)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, short value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, short value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setShort(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static short get(final ArrowBuf buffer, final int index) { + return buffer.getShort(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableSmallIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableSmallIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableSmallIntVector to){ + this.to = to; + } + + @Override + public NullableSmallIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java new file mode 100644 index 0000000000000..454a4ac41a6d9 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeMicroReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeMicroHolder; +import org.apache.arrow.vector.holders.NullableTimeMicroHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableTimeMicroVector implements a fixed width vector (8 bytes) of + * time (microsecond resolution) values which could be null. + * A validity buffer (bit vector) is maintained to track which elements in the + * vector are null. + */ +public class NullableTimeMicroVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMicroHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMicroHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMicroHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMicroVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMicroVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMicroVector to){ + this.to = to; + } + + @Override + public NullableTimeMicroVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java new file mode 100644 index 0000000000000..8540d16924210 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -0,0 +1,405 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeMilliHolder; +import org.apache.arrow.vector.holders.NullableTimeMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; +import org.slf4j.Logger; + +/** + * NullableTimeMilliVector implements a fixed width (4 bytes) vector of + * time (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeMilliVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } + org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), + org.joda.time.DateTimeZone.UTC); + return ldt; + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMilliVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMilliVector to){ + this.to = to; + } + + @Override + public NullableTimeMilliVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java new file mode 100644 index 0000000000000..015226da22318 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -0,0 +1,402 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeNanoReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeNanoHolder; +import org.apache.arrow.vector.holders.NullableTimeNanoHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeNanoVector implements a fixed width vector (8 bytes) of + * time (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeNanoVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMENANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeNanoHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeNanoHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeNanoHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer,final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeNanoVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeNanoVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeNanoVector to){ + this.to = to; + } + + @Override + public NullableTimeNanoVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java new file mode 100644 index 0000000000000..2b2375e92f1f4 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeSecReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeSecHolder; +import org.apache.arrow.vector.holders.NullableTimeSecHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeSecVector implements a fixed width (4 bytes) vector of + * time (seconds resolution) values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTimeSecVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeSecReaderImpl(NullableTimeSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeSecHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeSecHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeSecHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeSecVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeSecVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeSecVector to){ + this.to = to; + } + + @Override + public NullableTimeSecVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java new file mode 100644 index 0000000000000..0d25eff4e02b7 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMicroTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMicroTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampMicroTZVector implements a fixed width vector (8 bytes) of + * timestamp (microsecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMicroTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICROTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java new file mode 100644 index 0000000000000..41f7a1112f2df --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMicroReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMicroHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampMicroVector implements a fixed width vector (8 bytes) of + * timestamp (microsecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMicroVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */ + final long micros = get(index); + final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroVector )to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java new file mode 100644 index 0000000000000..6376b085e64e1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMilliTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampMilliTZVector implements a fixed width vector (8 bytes) of + * timestamp (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMilliTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLITZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java new file mode 100644 index 0000000000000..ff581d2ea2f03 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMilliHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampMilliVector implements a fixed width vector (8 bytes) of + * timestamp (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMilliVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = get(index); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java new file mode 100644 index 0000000000000..6c4445d500b39 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampNanoTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampNanoTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampNanoTZVector implements a fixed width vector (8 bytes) of + * timestamp (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampNanoTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANOTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoTZHolder holder) { + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java new file mode 100644 index 0000000000000..096913ebf1cf0 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampNanoReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampNanoHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampNanoVector implements a fixed width vector (8 bytes) of + * timestamp (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampNanoVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long nanos = get(index); + final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java new file mode 100644 index 0000000000000..8e5ba72ae1fe2 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java @@ -0,0 +1,217 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampSecTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampSecTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecTZHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampSecTZVector implements a fixed width vector (8 bytes) of + * timestamp (seconds resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampSecTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSECTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java new file mode 100644 index 0000000000000..bc800e0e4f04f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampSecReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampSecHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampSecVector implements a fixed width vector (8 bytes) of + * timestamp (seconds resolution) values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTimeStampSecVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long secs = get(index); + final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java new file mode 100644 index 0000000000000..b2a58bd4568f9 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampVector is an abstract interface for fixed width vector (8 bytes) + * of timestamp values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector { + protected static final byte TYPE_WIDTH = 8; + + /** + * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + protected void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and * + * JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + public class TransferImpl implements TransferPair { + NullableTimeStampVector to; + + public TransferImpl(NullableTimeStampVector to) { + this.to = to; + } + + @Override + public NullableTimeStampVector getTo(){ + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java new file mode 100644 index 0000000000000..15100530d0ec5 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -0,0 +1,432 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TinyIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TinyIntHolder; +import org.apache.arrow.vector.holders.NullableTinyIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTinyIntVector implements a fixed width (1 bytes) vector of + * byte values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTinyIntVector extends BaseNullableFixedWidthVector { + public static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), + allocator); + } + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TinyIntReaderImpl(NullableTinyIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TINYINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTinyIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TinyIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TinyIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, byte)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, byte value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setByte(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static byte get(final ArrowBuf buffer, final int index) { + return buffer.getByte(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTinyIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTinyIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTinyIntVector to){ + this.to = to; + } + + @Override + public NullableTinyIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java new file mode 100644 index 0000000000000..4d20f800ba1fd --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java @@ -0,0 +1,318 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt1ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt1Holder; +import org.apache.arrow.vector.holders.NullableUInt1Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableUInt1Vector implements a fixed width (1 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt1Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + public NullableUInt1Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), + allocator); + } + + public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt1ReaderImpl(NullableUInt1Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT1; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt1Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt1Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt1Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt1Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt1Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt1Vector to){ + this.to = to; + } + + @Override + public NullableUInt1Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java new file mode 100644 index 0000000000000..b140b9f5cbc67 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt2ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt2Holder; +import org.apache.arrow.vector.holders.NullableUInt2Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt2Vector implements a fixed width (2 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt2Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + public NullableUInt2Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), + allocator); + } + + public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt2ReaderImpl(NullableUInt2Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT2; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public char get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt2Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Character getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + private void setValue(int index, char value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, char value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt2Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, char)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, char value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt2Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, char value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, char value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt2Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt2Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt2Vector to){ + this.to = to; + } + + @Override + public NullableUInt2Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java new file mode 100644 index 0000000000000..fc8cb639581fb --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt4ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt4Holder; +import org.apache.arrow.vector.holders.NullableUInt4Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt4Vector implements a fixed width (4 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt4Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + public NullableUInt4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), + allocator); + } + + public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt4ReaderImpl(NullableUInt4Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt4Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt4Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt4Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt4Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt4Vector to){ + this.to = to; + } + + @Override + public NullableUInt4Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java new file mode 100644 index 0000000000000..2110cf03b9f05 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt8ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt8Holder; +import org.apache.arrow.vector.holders.NullableUInt8Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt8Vector implements a fixed width vector (8 bytes) of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt8Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + public NullableUInt8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), + allocator); + } + + public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt8ReaderImpl(NullableUInt8Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt8Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt8Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt8Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt8Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt8Vector to){ + this.to = to; + } + + @Override + public NullableUInt8Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java new file mode 100644 index 0000000000000..330e06710fcdd --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java @@ -0,0 +1,514 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.VarBinaryReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.VarBinaryHolder; +import org.apache.arrow.vector.holders.NullableVarBinaryHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +import java.nio.ByteBuffer; + +/** + * NullableVarBinaryVector implements a variable width vector of binary + * values which could be NULL. A validity buffer (bit vector) is maintained + * to track which elements in the vector are null. + */ +public class NullableVarBinaryVector extends BaseNullableVariableWidthVector { + private final FieldReader reader; + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); + } + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARBINARY; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return byte array for non-null element, null otherwise + */ + public byte[] getObject(int index) { + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + return b; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if(isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarBinaryHolder holder){ + assert index >= 0; + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + holder.isSet = 1; + holder.start = startOffset; + holder.end = dataLength; + holder.buffer = valueBuffer; + } + + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + fillHoles(thisIndex); + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + fillEmpties(thisIndex); + if (from.isSet(fromIndex) != 0) { + setSafe(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index, 0); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); + final ArrowBuf bb = buffer.slice(start, end); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + handleSafe(index, end); + set(index, isSet, start, end, buffer); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarBinaryVector)to); + } + + private class TransferImpl implements TransferPair { + NullableVarBinaryVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarBinaryVector to){ + this.to = to; + } + + @Override + public NullableVarBinaryVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java index b9021544b0e10..0b03164f0f385 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; @@ -32,30 +31,50 @@ import java.nio.ByteBuffer; +/** + * NullableVarCharVector implements a variable width vector of VARCHAR + * values which could be NULL. A validity buffer (bit vector) is maintained + * to track which elements in the vector are null. + */ public class NullableVarCharVector extends BaseNullableVariableWidthVector { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); private final FieldReader reader; + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ public NullableVarCharVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); } + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType); reader = new VarCharReaderImpl(NullableVarCharVector.this); } - @Override - protected org.slf4j.Logger getLogger() { - return logger; - } - + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ @Override public FieldReader getReader(){ return reader; } + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ @Override public Types.MinorType getMinorType() { return Types.MinorType.VARCHAR; @@ -154,7 +173,13 @@ public void get(int index, NullableVarCharHolder holder){ ******************************************************************/ - + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { fillHoles(thisIndex); if (from.isSet(fromIndex) != 0) { @@ -163,6 +188,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { } } + /** + * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { fillEmpties(thisIndex); if (from.isSet(fromIndex) != 0) { @@ -383,21 +416,42 @@ public void setNull(int index){ BitVectorHelper.setValidityBit(validityBuffer, index, 0); } - public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { assert index >= 0; fillHoles(index); BitVectorHelper.setValidityBit(validityBuffer, index, isSet); final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + endField); - final ArrowBuf bb = bufferField.slice(startField, endField); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); + final ArrowBuf bb = buffer.slice(start, end); valueBuffer.setBytes(startOffset, bb); lastSet = index; } - public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { assert index >= 0; - handleSafe(index, endField); - set(index, isSet, startField, endField, bufferField); + handleSafe(index, end); + set(index, isSet, start, end, buffer); } @@ -407,11 +461,23 @@ public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf * * ******************************************************************/ + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ return new TransferImpl(ref, allocator); } + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((NullableVarCharVector)to); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index e6048b4b517c9..dc74afaa2e49c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -256,9 +256,17 @@ interface Mutator { */ public ArrowBuf getOffsetBuffer(); + /* temporarily add these methods here until we remove other vectors + * (non-nullable) which are under ValueVector hierarchy and still + * use the mutator/accessor interfaces. + */ public int getValueCount(); public void setValueCount(int valueCount); public Object getObject(int index); + + public int getNullCount(); + + public boolean isNull(int index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java index 0c8868cad55b5..196393fb9584a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java @@ -38,7 +38,7 @@ public class VectorSchemaRoot implements AutoCloseable { private final Map fieldVectorsMap = new HashMap<>(); public VectorSchemaRoot(FieldVector parent) { - this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getAccessor().getValueCount()); + this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getValueCount()); } public VectorSchemaRoot(List fields, List fieldVectors, int rowCount) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 96243eb5876b0..288afc6bbec69 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -55,17 +55,7 @@ public ArrowRecordBatch getRecordBatch() { } private void appendNodes(FieldVector vector, List nodes, List buffers) { - Accessor accessor = null; - if (vector instanceof NullableIntVector) { - nodes.add(new ArrowFieldNode(((NullableIntVector)vector).getValueCount(), - includeNullCount ? ((NullableIntVector)vector).getNullCount() : -1)); - } else if (vector instanceof NullableVarCharVector) { - nodes.add(new ArrowFieldNode(((NullableVarCharVector)vector).getValueCount(), - includeNullCount ? ((NullableVarCharVector)vector).getNullCount() : -1)); - } else { - accessor = vector.getAccessor(); - nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); - } + nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1)); List fieldBuffers = vector.getFieldBuffers(); List expectedBuffers = vector.getField().getTypeLayout().getVectorTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index fcef02f4a1ba8..3cc93a2a34f43 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -268,4 +268,10 @@ public void setValueCount(int valueCount) { } @Override public Object getObject(int index) { return null; } + + @Override + public int getNullCount() { return 0; } + + @Override + public boolean isNull(int index) { return false; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java index 26b0f90581ffc..1b20b09d81b0b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java @@ -141,8 +141,8 @@ public T addOrGet(String childName, FieldType fieldType, } private boolean nullFilled(ValueVector vector) { - for (int r = 0; r < vector.getAccessor().getValueCount(); r++) { - if (!vector.getAccessor().isNull(r)) { + for (int r = 0; r < vector.getValueCount(); r++) { + if (!vector.isNull(r)) { return false; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 8e2877f892a64..3eab30a32d224 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.Iterator; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.AddOrGetResult; import org.apache.arrow.vector.BaseValueVector; @@ -31,6 +32,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.SchemaChangeRuntimeException; import com.google.common.base.Preconditions; @@ -41,12 +43,14 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; public final static String DATA_VECTOR_NAME = "$data$"; - protected final UInt4Vector offsets; + public final static byte OFFSET_WIDTH = 4; + protected ArrowBuf offsetBuffer; protected FieldVector vector; protected final CallBack callBack; + protected int valueCount; + protected int offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { this(name, allocator, DEFAULT_DATA_VECTOR, callBack); @@ -54,42 +58,73 @@ protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBa protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { super(name, allocator); - this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); + this.offsetBuffer = allocator.getEmpty(); this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); this.callBack = callBack; + this.valueCount = 0; } @Override public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; + boolean dataAlloc = false; try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - } finally { - if (!success) { + allocateOffsetBuffer(offsetAllocationSizeInBytes); + dataAlloc = vector.allocateNewSafe(); + } catch (Exception e) { + e.printStackTrace(); + clear(); + return false; + } + finally { + if (!dataAlloc) { clear(); } } - offsets.zeroVector(); - return success; + return dataAlloc; + } + + protected void allocateOffsetBuffer(final long size) { + final int curSize = (int)size; + offsetBuffer = allocator.buffer(curSize); + offsetBuffer.readerIndex(0); + offsetAllocationSizeInBytes = curSize; + offsetBuffer.setZero(0, offsetBuffer.capacity()); } @Override public void reAlloc() { - offsets.reAlloc(); + reallocOffsetBuffer(); vector.reAlloc(); } + protected void reallocOffsetBuffer() { + final int currentBufferCapacity = offsetBuffer.capacity(); + long baseSize = offsetAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + offsetBuffer.release(1); + offsetBuffer = newBuf; + offsetAllocationSizeInBytes = (int)newAllocationSize; + } + @Override + @Deprecated public UInt4Vector getOffsetVector() { - return offsets; + throw new UnsupportedOperationException("There is no inner offset vector"); } @Override @@ -99,25 +134,29 @@ public FieldVector getDataVector() { @Override public void setInitialCapacity(int numRecords) { - offsets.setInitialCapacity(numRecords + 1); + offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); } @Override public int getValueCapacity() { - final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); if (vector == DEFAULT_DATA_VECTOR) { return offsetValueCapacity; } return Math.min(vector.getValueCapacity(), offsetValueCapacity); } + private int getOffsetBufferValueCapacity() { + return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); + } + @Override public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return offsets.getBufferSize() + vector.getBufferSize(); + return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSize(); } @Override @@ -126,7 +165,7 @@ public int getBufferSizeFor(int valueCount) { return 0; } - return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); + return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSizeFor(valueCount); } @Override @@ -136,14 +175,16 @@ public Iterator iterator() { @Override public void clear() { - offsets.clear(); + offsetBuffer = releaseBuffer(offsetBuffer); vector.clear(); + valueCount = 0; super.clear(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, + vector.getBuffers(false), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -187,54 +228,61 @@ protected void replaceDataVector(FieldVector v) { vector = v; } - public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { - @Override - public int getValueCount() { - return Math.max(offsets.getAccessor().getValueCount() - 1, 0); - } + @Override + public int getValueCount() { + return valueCount; + } - @Override - public int getInnerValueCount() { - return vector.getAccessor().getValueCount(); - } + /* returns the value count for inner data vector for this list vector */ + public int getInnerValueCount() { + return vector.getValueCount(); + } - @Override - public int getInnerValueCountAt(int index) { - return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); - } - @Override - public boolean isNull(int index) { - return false; - } + /* returns the value count for inner data vector at a particular index */ + public int getInnerValueCountAt(int index) { + return offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - + offsetBuffer.getInt(index * OFFSET_WIDTH); + } - @Override - public boolean isEmpty(int index) { - return false; - } + public boolean isNull(int index) { + return false; } - public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { + public boolean isEmpty(int index) { + return false; + } - @Override - public int startNewValue(int index) { - while (offsets.getValueCapacity() <= index) { - offsets.reAlloc(); - } - int offset = offsets.getAccessor().get(index); - offsets.getMutator().setSafe(index + 1, offset); - setValueCount(index + 1); - return offset; + public int startNewValue(int index) { + while (index >= getOffsetBufferValueCapacity()) { + reallocOffsetBuffer(); } + int offset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, offset); + setValueCount(index + 1); + return offset; + } - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + while (valueCount > getOffsetBufferValueCapacity()) { + reallocOffsetBuffer(); } + final int childValueCount = valueCount == 0 ? 0 : + offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + vector.setValueCount(childValueCount); + } + + @Override + @Deprecated + public RepeatedAccessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from LIST."); } + @Override + @Deprecated + public RepeatedMutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing to LIST"); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index e95442acf6d41..bb091f68c1807 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -22,16 +22,14 @@ import static java.util.Collections.singletonList; import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; +import java.util.*; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.*; @@ -42,10 +40,7 @@ import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringArrayList; -import org.apache.arrow.vector.util.SchemaChangeRuntimeException; -import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.*; public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { @@ -55,15 +50,13 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a } private FieldVector vector; - private final BitVector bits; + private ArrowBuf validityBuffer; private final int listSize; private final FieldType fieldType; - private final List innerVectors; private UnionFixedSizeListReader reader; - - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); + private int valueCount; + private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @Deprecated @@ -80,13 +73,14 @@ public FixedSizeListVector(String name, FieldType fieldType, CallBack schemaChangeCallback) { super(name, allocator); - this.bits = new BitVector("$bits$", allocator); + this.validityBuffer = allocator.getEmpty(); this.vector = ZeroVector.INSTANCE; this.fieldType = fieldType; this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize(); Preconditions.checkArgument(listSize > 0, "list size must be positive"); - this.innerVectors = Collections.singletonList((BufferBacked) bits); this.reader = new UnionFixedSizeListReader(this); + this.valueCount = 0; + this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -124,27 +118,47 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - BaseDataValueVector.load(fieldNode, innerVectors, ownBuffers); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueCount = fieldNode.getLength(); + + validityAllocationSizeInBytes = validityBuffer.capacity(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(innerVectors); + List result = new ArrayList<>(1); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + + result.add(validityBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); } @Override @@ -154,7 +168,9 @@ public UnionFixedSizeListReader getReader() { @Override public void allocateNew() throws OutOfMemoryException { - allocateNewSafe(); + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } } @Override @@ -166,31 +182,63 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - success = bits.allocateNewSafe() && vector.allocateNewSafe(); + /* allocate validity buffer */ + allocateValidityBuffer(validityAllocationSizeInBytes); + success = vector.allocateNewSafe(); } finally { if (!success) { clear(); + return false; } } - if (success) { - bits.zeroVector(); - } - return success; + + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { - bits.reAlloc(); + reallocValidityBuffer(); vector.reAlloc(); } + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; + } + public FieldVector getDataVector() { return vector; } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); vector.setInitialCapacity(numRecords * listSize); } @@ -199,15 +247,15 @@ public int getValueCapacity() { if (vector == ZeroVector.INSTANCE) { return 0; } - return vector.getValueCapacity() / listSize; + return Math.min(vector.getValueCapacity() / listSize, getValidityBufferValueCapacity()); } @Override public int getBufferSize() { - if (accessor.getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return bits.getBufferSize() + vector.getBufferSize(); + return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize(); } @Override @@ -215,7 +263,8 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - return bits.getBufferSizeFor(valueCount) + vector.getBufferSizeFor(valueCount * listSize); + return getValidityBufferSizeFromCount(valueCount) + + vector.getBufferSizeFor(valueCount * listSize); } @Override @@ -225,14 +274,15 @@ public Iterator iterator() { @Override public void clear() { - bits.clear(); + validityBuffer = releaseBuffer(validityBuffer); vector.clear(); + valueCount = 0; super.clear(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -288,7 +338,7 @@ public UnionVector promoteToUnion() { @Override public long getValidityBufferAddress() { - return (bits.getDataBuffer().memoryAddress()); + return validityBuffer.memoryAddress(); } @Override @@ -303,7 +353,7 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getValidityBuffer() { - return (bits.getDataBuffer()); + return validityBuffer; } @Override @@ -316,62 +366,64 @@ public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } - final List vals = new JsonStringArrayList<>(listSize); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - for (int i = 0; i < listSize; i++) { - vals.add(vector.getObject(index * listSize + i)); - } - } else { - final ValueVector.Accessor valuesAccessor = vector.getAccessor(); - for (int i = 0; i < listSize; i++) { - vals.add(valuesAccessor.getObject(index * listSize + i)); - } - } - return vals; + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; } - - @Override - public boolean isNull(int index) { - return bits.getAccessor().get(index) == 0; + final List vals = new JsonStringArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + vals.add(vector.getObject(index * listSize + i)); } + return vals; + } - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); - } + public boolean isNull(int index) { + return (isSet(index) == 0); + } - @Override - public int getValueCount() { - return bits.getAccessor().getValueCount(); - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); } - public class Mutator extends BaseValueVector.BaseMutator { + @Override + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - public void setNull(int index) { - bits.getMutator().setSafe(index, 0); + @Override + public int getValueCount() { + return valueCount; + } + + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } - public void setNotNull(int index) { - bits.getMutator().setSafe(index, 1); + public void setNotNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } - @Override - public void setValueCount(int valueCount) { - bits.getMutator().setValueCount(valueCount); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - vector.setValueCount(valueCount * listSize); - } else { - vector.getMutator().setValueCount(valueCount * listSize); - } + @Override + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + while (valueCount > getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + vector.setValueCount(valueCount * listSize); } @Override @@ -392,7 +444,7 @@ public TransferPair makeTransferPair(ValueVector target) { private class TransferImpl implements TransferPair { FixedSizeListVector to; - TransferPair pairs[] = new TransferPair[2]; + TransferPair dataPair; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { this(new FixedSizeListVector(name, allocator, fieldType, callBack)); @@ -401,19 +453,20 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { public TransferImpl(FixedSizeListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); - pairs[0] = bits.makeTransferPair(to.bits); - pairs[1] = vector.makeTransferPair(to.vector); + dataPair = vector.makeTransferPair(to.vector); } @Override public void transfer() { - for (TransferPair pair : pairs) { - pair.transfer(); - } + to.clear(); + dataPair.transfer(); + to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; + to.setValueCount(valueCount); } @Override public void splitAndTransfer(int startIndex, int length) { + to.clear(); to.allocateNew(); for (int i = 0; i < length; i++) { copyValueSafe(startIndex + i, i); @@ -426,12 +479,15 @@ public ValueVector getTo() { } @Override - public void copyValueSafe(int from, int to) { - pairs[0].copyValueSafe(from, to); - int fromOffset = from * listSize; - int toOffset = to * listSize; + public void copyValueSafe(int fromIndex, int toIndex) { + while (toIndex >= to.getValueCapacity()) { + to.reAlloc(); + } + BitVectorHelper.setValidityBit(to.validityBuffer, toIndex, isSet(fromIndex)); + int fromOffset = fromIndex * listSize; + int toOffset = toIndex * listSize; for (int i = 0; i < listSize; i++) { - pairs[1].copyValueSafe(fromOffset + i, toOffset + i); + dataPair.copyValueSafe(fromOffset + i, toOffset + i); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java new file mode 100644 index 0000000000000..435d3d8ccf2e6 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import java.util.Collections; +import java.util.Iterator; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.AddOrGetResult; +import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.SchemaChangeRuntimeException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; + +public abstract class LegacyBaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { + + public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; + public final static String OFFSETS_VECTOR_NAME = "$offsets$"; + public final static String DATA_VECTOR_NAME = "$data$"; + + protected final UInt4Vector offsets; + protected FieldVector vector; + protected final CallBack callBack; + + protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { + this(name, allocator, DEFAULT_DATA_VECTOR, callBack); + } + + protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { + super(name, allocator); + this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); + this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); + this.callBack = callBack; + } + + @Override + public boolean allocateNewSafe() { + /* boolean to keep track if all the memory allocation were successful + * Used in the case of composite vectors when we need to allocate multiple + * buffers for multiple vectors. If one of the allocations failed we need to + * clear all the memory that we allocated + */ + boolean success = false; + try { + if (!offsets.allocateNewSafe()) { + return false; + } + success = vector.allocateNewSafe(); + } finally { + if (!success) { + clear(); + } + } + offsets.zeroVector(); + return success; + } + + @Override + public void reAlloc() { + offsets.reAlloc(); + vector.reAlloc(); + } + + @Override + public UInt4Vector getOffsetVector() { + return offsets; + } + + @Override + public FieldVector getDataVector() { + return vector; + } + + @Override + public void setInitialCapacity(int numRecords) { + offsets.setInitialCapacity(numRecords + 1); + vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); + } + + @Override + public int getValueCapacity() { + final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); + if (vector == DEFAULT_DATA_VECTOR) { + return offsetValueCapacity; + } + return Math.min(vector.getValueCapacity(), offsetValueCapacity); + } + + @Override + public int getBufferSize() { + if (getAccessor().getValueCount() == 0) { + return 0; + } + return offsets.getBufferSize() + vector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(int valueCount) { + if (valueCount == 0) { + return 0; + } + + return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); + } + + @Override + public Iterator iterator() { + return Collections.singleton(getDataVector()).iterator(); + } + + @Override + public void clear() { + offsets.clear(); + vector.clear(); + super.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + if (clear) { + for (ArrowBuf buffer : buffers) { + buffer.retain(); + } + clear(); + } + return buffers; + } + + /** + * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 + */ + public int size() { + return vector == DEFAULT_DATA_VECTOR ? 0 : 1; + } + + public AddOrGetResult addOrGetVector(FieldType fieldType) { + boolean created = false; + if (vector instanceof ZeroVector) { + vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack); + // returned vector must have the same field + created = true; + if (callBack != null && + // not a schema change if changing from ZeroVector to ZeroVector + (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { + callBack.doWork(); + } + } + + if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { + final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", + fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); + throw new SchemaChangeRuntimeException(msg); + } + + return new AddOrGetResult<>((T) vector, created); + } + + protected void replaceDataVector(FieldVector v) { + vector.clear(); + vector = v; + } + + public abstract class LegacyBaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { + + @Override + public int getValueCount() { + return Math.max(offsets.getAccessor().getValueCount() - 1, 0); + } + + @Override + public int getInnerValueCount() { + return vector.getAccessor().getValueCount(); + } + + @Override + public int getInnerValueCountAt(int index) { + return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); + } + + @Override + public boolean isNull(int index) { + return false; + } + + @Override + public boolean isEmpty(int index) { + return false; + } + } + + public abstract class LegacyBaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { + + @Override + public int startNewValue(int index) { + while (offsets.getValueCapacity() <= index) { + offsets.reAlloc(); + } + int offset = offsets.getAccessor().get(index); + offsets.getMutator().setSafe(index + 1, offset); + setValueCount(index + 1); + return offset; + } + + @Override + public void setValueCount(int valueCount) { + // TODO: populate offset end points + offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); + final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); + vector.getMutator().setValueCount(childValueCount); + } + } + +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java new file mode 100644 index 0000000000000..0843907d9df82 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java @@ -0,0 +1,298 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector.complex; + +import static java.util.Collections.singletonList; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.SchemaChangeRuntimeException; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyFixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { + + public static LegacyFixedSizeListVector empty(String name, int size, BufferAllocator allocator) { + FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size)); + return new LegacyFixedSizeListVector(name, allocator, fieldType, null); + } + + private Mutator mutator = new Mutator(); + private Accessor accessor = new Accessor(); + /* delegate */ + private final FixedSizeListVector fixedSizeListVector; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyFixedSizeListVector(String name, + BufferAllocator allocator, + int listSize, + DictionaryEncoding dictionary, + CallBack schemaChangeCallback) { + super(name, allocator); + fixedSizeListVector = new FixedSizeListVector(name, allocator, listSize, dictionary, schemaChangeCallback); + } + + public LegacyFixedSizeListVector(String name, + BufferAllocator allocator, + FieldType fieldType, + CallBack schemaChangeCallback) { + super(name, allocator); + fixedSizeListVector = new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback); + } + + @Override + public Field getField() { + return fixedSizeListVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.FIXED_SIZE_LIST; + } + + public int getListSize() { + return fixedSizeListVector.getListSize(); + } + + @Override + public void initializeChildrenFromFields(List children) { + fixedSizeListVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return fixedSizeListVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + fixedSizeListVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return fixedSizeListVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return fixedSizeListVector.getFieldInnerVectors(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public UnionFixedSizeListReader getReader() { + return fixedSizeListVector.getReader(); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + fixedSizeListVector.allocateNew(); + } + + @Override + public boolean allocateNewSafe() { + return fixedSizeListVector.allocateNewSafe(); + } + + @Override + public void reAlloc() { + fixedSizeListVector.reAlloc(); + } + + public FieldVector getDataVector() { + return fixedSizeListVector.getDataVector(); + } + + @Override + public void setInitialCapacity(int numRecords) { + fixedSizeListVector.setInitialCapacity(numRecords); + } + + @Override + public int getValueCapacity() { + return fixedSizeListVector.getValueCapacity(); + } + + @Override + public int getBufferSize() { + return fixedSizeListVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(int valueCount) { + return fixedSizeListVector.getBufferSizeFor(valueCount); + } + + @Override + public Iterator iterator() { + return fixedSizeListVector.iterator(); + } + + @Override + public void clear() { + fixedSizeListVector.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return fixedSizeListVector.getBuffers(clear); + } + + /** + * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 + */ + public int size() { + return fixedSizeListVector.size(); + } + + @Override + @SuppressWarnings("unchecked") + public AddOrGetResult addOrGetVector(FieldType type) { + return fixedSizeListVector.addOrGetVector(type); + } + + public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) { + fixedSizeListVector.copyFromSafe(inIndex, outIndex, from); + } + + public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) { + fixedSizeListVector.copyFrom(fromIndex, thisIndex, from); + } + + @Override + public UnionVector promoteToUnion() { + return fixedSizeListVector.promoteToUnion(); + } + + @Override + public long getValidityBufferAddress() { + return fixedSizeListVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return fixedSizeListVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return fixedSizeListVector.getObject(index); + } + + @Override + public boolean isNull(int index) { + return fixedSizeListVector.isNull(index); + } + + @Override + public int getNullCount() { + return fixedSizeListVector.getNullCount(); + } + + @Override + public int getValueCount() { + return fixedSizeListVector.getValueCount(); + } + } + + public class Mutator extends BaseValueVector.BaseMutator { + + public void setNull(int index) { + fixedSizeListVector.setNull(index); + } + + public void setNotNull(int index) { + fixedSizeListVector.setNotNull(index); + } + + @Override + public void setValueCount(int valueCount) { + fixedSizeListVector.setValueCount(valueCount); + } + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return fixedSizeListVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return fixedSizeListVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return fixedSizeListVector.makeTransferPair(((LegacyFixedSizeListVector)target).fixedSizeListVector); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java new file mode 100644 index 0000000000000..732699ada7a81 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java @@ -0,0 +1,288 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Collections.list; +import static java.util.Collections.singletonList; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.UnionListReader; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Null; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyListVector extends LegacyBaseRepeatedValueVector implements FieldVector, PromotableVector { + + public static LegacyListVector empty(String name, BufferAllocator allocator) { + return new LegacyListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); + } + + private Mutator mutator = new Mutator(); + private Accessor accessor = new Accessor(); + + private final ListVector listVector; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyListVector(String name, BufferAllocator allocator, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, callBack); + } + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, dictionary, callBack); + } + + public LegacyListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, fieldType, callBack); + } + + @Override + public void initializeChildrenFromFields(List children) { + listVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return listVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + listVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return listVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return listVector.getFieldInnerVectors(); + } + + public UnionListWriter getWriter() { + return listVector.getWriter(); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + listVector.allocateNew(); + } + + @Override + public void reAlloc() { + listVector.reAlloc(); + } + + public void copyFromSafe(int inIndex, int outIndex, ListVector from) { + listVector.copyFrom(inIndex, outIndex, from); + } + + public void copyFrom(int inIndex, int outIndex, ListVector from) { + listVector.copyFrom(inIndex, outIndex, from); + } + + @Override + public FieldVector getDataVector() { + return listVector.getDataVector(); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return listVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return listVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return listVector.makeTransferPair(((LegacyListVector)target).listVector); + } + + @Override + public long getValidityBufferAddress() { + return listVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + return listVector.getOffsetBufferAddress(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return listVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + return listVector.getOffsetBuffer(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public UnionListReader getReader() { + return listVector.getReader(); + } + + @Override + public boolean allocateNewSafe() { + return listVector.allocateNewSafe(); + } + + public AddOrGetResult addOrGetVector(FieldType fieldType) { + return listVector.addOrGetVector(fieldType); + } + + @Override + public int getBufferSize() { + return listVector.getBufferSize(); + } + + @Override + public Field getField() { + return listVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.LIST; + } + + @Override + public void clear() { + listVector.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return listVector.getBuffers(clear); + } + + @Override + public UnionVector promoteToUnion() { + return listVector.promoteToUnion(); + } + + private int lastSet = 0; + + public class Accessor extends LegacyBaseRepeatedAccessor { + + @Override + public Object getObject(int index) { + return listVector.getObject(index); + } + + @Override + public boolean isNull(int index) { + return listVector.isNull(index); + } + + @Override + public int getNullCount() { + return listVector.getNullCount(); + } + } + + public class Mutator extends LegacyBaseRepeatedMutator { + public void setNotNull(int index) { + listVector.setNotNull(index); + } + + @Override + public int startNewValue(int index) { + return listVector.startNewValue(index); + } + + /** + * End the current value + * + * @param index index of the value to end + * @param size number of elements in the list that was written + */ + public void endValue(int index, int size) { + listVector.endValue(index, size); + } + + @Override + public void setValueCount(int valueCount) { + listVector.setValueCount(valueCount); + } + + public void setLastSet(int value) { + listVector.setLastSet(value); + } + + public int getLastSet() { + return listVector.getLastSet(); + } + } + +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java new file mode 100644 index 0000000000000..77556ab774f02 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; + +import io.netty.buffer.ArrowBuf; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.ComplexHolder; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringHashMap; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyMapVector extends AbstractMapVector { + + /* delegate */ + private final MapVector mapVector; + + public static LegacyMapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); + return new LegacyMapVector(name, allocator, fieldType, null); + } + + private final Accessor accessor = new Accessor(); + private final Mutator mutator = new Mutator(); + + @Deprecated + public LegacyMapVector(String name, BufferAllocator allocator, CallBack callBack) { + super(name, allocator, callBack); + mapVector = new MapVector(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); + } + + public LegacyMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + mapVector = new MapVector(name, allocator, fieldType, callBack); + } + + @Override + public FieldReader getReader() { + return mapVector.getReader(); + } + + public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { + mapVector.copyFromSafe(fromIndex, thisIndex, from); + } + + @Override + protected boolean supportsDirectRead() { + return true; + } + + public Iterator fieldNameIterator() { + return mapVector.fieldNameIterator(); + } + + @Override + public void setInitialCapacity(int numRecords) { + mapVector.setInitialCapacity(numRecords); + } + + @Override + public int getBufferSize() { + return mapVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(final int valueCount) { + return (int) mapVector.getBufferSizeFor(valueCount); + } + + @Override + public ArrowBuf getValidityBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator, null); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return mapVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return mapVector.makeTransferPair(((LegacyMapVector)to).mapVector); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return mapVector.getTransferPair(ref, allocator); + } + + @Override + public int getValueCapacity() { + return mapVector.getValueCapacity(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return mapVector.getObject(index); + } + + public void get(int index, ComplexHolder holder) { + mapVector.get(index, holder); + } + + @Override + public int getValueCount() { + return mapVector.getValueCount(); + } + } + + public ValueVector getVectorById(int id) { + return mapVector.getVectorById(id); + } + + public class Mutator extends BaseValueVector.BaseMutator { + + @Override + public void setValueCount(int valueCount) { + mapVector.setValueCount(valueCount); + } + + @Override + public void reset() { } + + @Override + public void generateTestData(int values) { } + } + + @Override + public void clear() { + mapVector.clear(); + } + + @Override + public Field getField() { + return mapVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.MAP; + } + + @Override + public void close() { + mapVector.close(); + } + + public void initializeChildrenFromFields(List children) { + mapVector.initializeChildrenFromFields(children); + } + + public List getChildrenFromFields() { + return mapVector.getChildren(); + } + + public boolean isNull(int index) { return false; } + + public int getNullCount() { return 0; } + + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 4b2c9139219f3..ea577208cc238 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -22,26 +22,22 @@ import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Collections.singletonList; -import java.util.Arrays; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import com.google.common.collect.ImmutableList; import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BaseDataValueVector; -import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.BufferBacked; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; -import org.apache.arrow.vector.NullableIntVector; -import org.apache.arrow.vector.NullableVarCharVector; +import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; @@ -55,6 +51,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector { @@ -63,14 +60,12 @@ public static ListVector empty(String name, BufferAllocator allocator) { return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); } - final UInt4Vector offsets; - final BitVector bits; - private final List innerVectors; - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); + private ArrowBuf validityBuffer; private UnionListReader reader; private CallBack callBack; private final FieldType fieldType; + private int validityAllocationSizeInBytes; + private int lastSet; // deprecated, use FieldType or static constructor instead @Deprecated @@ -86,12 +81,12 @@ public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dic public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); - this.bits = new BitVector("$bits$", allocator); - this.offsets = getOffsetVector(); - this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits, offsets)); + this.validityBuffer = allocator.getEmpty(); this.reader = new UnionListReader(this); this.fieldType = checkNotNull(fieldType); this.callBack = callBack; + this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.lastSet = 0; } @Override @@ -115,20 +110,43 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - // variable width values: truncate offset vector buffer to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, offsets.getBufferSizeFor(fieldNode.getLength() + 1)); - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); + if (ownBuffers.size() != 2) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf offBuffer = ownBuffers.get(1); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + offsetBuffer.release(); + offsetBuffer = offBuffer.retain(allocator); + + validityAllocationSizeInBytes = validityBuffer.capacity(); + offsetAllocationSizeInBytes = offsetBuffer.capacity(); + lastSet = fieldNode.getLength(); + valueCount = fieldNode.getLength(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(2); + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + offsetBuffer.readerIndex(0); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + + result.add(validityBuffer); + result.add(offsetBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } public UnionListWriter getWriter() { @@ -137,14 +155,70 @@ public UnionListWriter getWriter() { @Override public void allocateNew() throws OutOfMemoryException { - super.allocateNewSafe(); - bits.allocateNewSafe(); + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } + } + + public boolean allocateNewSafe() { + boolean success = false; + try { + /* allocate validity buffer */ + allocateValidityBuffer(validityAllocationSizeInBytes); + /* allocate offset and data buffer */ + success = super.allocateNewSafe(); + } finally { + if (!success) { + clear(); + return false; + } + } + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { + /* reallocate the validity buffer */ + reallocValidityBuffer(); + /* reallocate the offset and data */ super.reAlloc(); - bits.reAlloc(); + } + + private void reallocValidityAndOffsetBuffers() { + reallocOffsetBuffer(); + reallocValidityBuffer(); + } + + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; } public void copyFromSafe(int inIndex, int outIndex, ListVector from) { @@ -181,7 +255,7 @@ public TransferPair makeTransferPair(ValueVector target) { @Override public long getValidityBufferAddress() { - return (bits.getDataBuffer().memoryAddress()); + return (validityBuffer.memoryAddress()); } @Override @@ -191,11 +265,11 @@ public long getDataBufferAddress() { @Override public long getOffsetBufferAddress() { - return (offsets.getDataBuffer().memoryAddress()); + return (offsetBuffer.memoryAddress()); } @Override - public ArrowBuf getValidityBuffer() { return bits.getDataBuffer(); } + public ArrowBuf getValidityBuffer() { return validityBuffer; } @Override public ArrowBuf getDataBuffer() { @@ -203,17 +277,13 @@ public ArrowBuf getDataBuffer() { } @Override - public ArrowBuf getOffsetBuffer() { return offsets.getDataBuffer(); } + public ArrowBuf getOffsetBuffer() { return offsetBuffer; } private class TransferImpl implements TransferPair { ListVector to; - TransferPair bitsTransferPair; - TransferPair offsetsTransferPair; TransferPair dataTransferPair; - TransferPair[] pairs; - public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { this(new ListVector(name, allocator, fieldType, callBack)); } @@ -221,39 +291,99 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { public TransferImpl(ListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); - offsetsTransferPair = offsets.makeTransferPair(to.offsets); - bitsTransferPair = bits.makeTransferPair(to.bits); if (to.getDataVector() instanceof ZeroVector) { to.addOrGetVector(vector.getField().getFieldType()); } dataTransferPair = getDataVector().makeTransferPair(to.getDataVector()); - pairs = new TransferPair[] {bitsTransferPair, offsetsTransferPair, dataTransferPair}; } @Override public void transfer() { - for (TransferPair pair : pairs) { - pair.transfer(); - } + dataTransferPair.transfer(); + to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; + to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer; to.lastSet = lastSet; + to.setValueCount(valueCount); } @Override public void splitAndTransfer(int startIndex, int length) { - UInt4Vector.Accessor offsetVectorAccessor = ListVector.this.offsets.getAccessor(); - final int startPoint = offsetVectorAccessor.get(startIndex); - final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint; + final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); + final int sliceLength = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH) - startPoint; to.clear(); - to.offsets.allocateNew(length + 1); - offsetVectorAccessor = ListVector.this.offsets.getAccessor(); - final UInt4Vector.Mutator targetOffsetVectorMutator = to.offsets.getMutator(); + to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + /* splitAndTransfer offset buffer */ for (int i = 0; i < length + 1; i++) { - targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint); + final int relativeOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - startPoint; + to.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeOffset); } - bitsTransferPair.splitAndTransfer(startIndex, length); + /* splitAndTransfer validity buffer */ + splitAndTransferValidityBuffer(startIndex, length, to); + /* splitAndTransfer data buffer */ dataTransferPair.splitAndTransfer(startPoint, sliceLength); to.lastSet = length; - to.mutator.setValueCount(length); + to.setValueCount(length); + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } } @Override @@ -267,47 +397,11 @@ public void copyValueSafe(int from, int to) { } } - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - @Override public UnionListReader getReader() { return reader; } - @Override - public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - success = success && bits.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - if (success) { - offsets.zeroVector(); - bits.zeroVector(); - } - return success; - } - public AddOrGetResult addOrGetVector(FieldType fieldType) { AddOrGetResult result = super.addOrGetVector(fieldType); reader = new UnionListReader(this); @@ -316,10 +410,12 @@ public AddOrGetResult addOrGetVector(FieldType fieldT @Override public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize(); + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + return offsetBufferSize + validityBufferSize + vector.getBufferSize(); } @Override @@ -334,16 +430,14 @@ public MinorType getMinorType() { @Override public void clear() { - offsets.clear(); - vector.clear(); - bits.clear(); - lastSet = 0; super.clear(); + validityBuffer = releaseBuffer(validityBuffer); + lastSet = 0; } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false), + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, ObjectArrays.concat(new ArrowBuf[] {validityBuffer}, vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { @@ -365,99 +459,97 @@ public UnionVector promoteToUnion() { return vector; } - private int lastSet = 0; + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } + final List vals = new JsonStringArrayList<>(); + final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); + final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + final ValueVector vv = getDataVector(); + for (int i = start; i < end; i++) { + vals.add(vv.getObject(i)); + } - public class Accessor extends BaseRepeatedAccessor { + return vals; + } - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } - final List vals = new JsonStringArrayList<>(); - final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); - final int start = offsetsAccessor.get(index); - final int end = offsetsAccessor.get(index + 1); - final ValueVector vv = getDataVector(); - if (vv instanceof NullableIntVector || vv instanceof NullableVarCharVector) { - for (int i = start; i < end; i++) { - vals.add(vv.getObject(i)); - } - } else { - final ValueVector.Accessor valuesAccessor = vv.getAccessor(); - for (int i = start; i < end; i++) { - vals.add(valuesAccessor.getObject(i)); - } - } + @Override + public boolean isNull(int index) { + return (isSet(index) == 0); + } - return vals; - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } - @Override - public boolean isNull(int index) { - return bits.getAccessor().get(index) == 0; - } + @Override + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); + public void setNotNull(int index) { + if (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + lastSet = index + 1; } - public class Mutator extends BaseRepeatedMutator { - public void setNotNull(int index) { - bits.getMutator().setSafe(index, 1); - lastSet = index + 1; + @Override + public int startNewValue(int index) { + if (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); } - - @Override - public int startNewValue(int index) { - for (int i = lastSet; i <= index; i++) { - offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); - } - setNotNull(index); - lastSet = index + 1; - return offsets.getAccessor().get(lastSet); + for (int i = lastSet; i <= index; i++) { + final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); + offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } + setNotNull(index); + lastSet = index + 1; + return offsetBuffer.getInt(lastSet * OFFSET_WIDTH); + } - /** - * End the current value - * - * @param index index of the value to end - * @param size number of elements in the list that was written - */ - public void endValue(int index, int size) { - offsets.getMutator().set(index + 1, offsets.getAccessor().get(index + 1) + size); - } + /** + * End the current value + * + * @param index index of the value to end + * @param size number of elements in the list that was written + */ + public void endValue(int index, int size) { + final int currentOffset = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size); + } - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - if (valueCount == 0) { - offsets.getMutator().setValueCount(0); - } else { - for (int i = lastSet; i < valueCount; i++) { - offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); - } - offsets.getMutator().setValueCount(valueCount + 1); + @Override + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + if (valueCount > 0) { + while (valueCount > getValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityAndOffsetBuffers(); } - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - vector.setValueCount(childValueCount); - } else { - vector.getMutator().setValueCount(childValueCount); + for (int i = lastSet; i < valueCount; i++) { + /* fill the holes with offsets */ + final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); + offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } - - bits.getMutator().setValueCount(valueCount); - } - - public void setLastSet(int value) { - lastSet = value; } + /* valueCount for the data vector is the current end offset */ + final int childValueCount = (valueCount == 0) ? 0 : + offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + vector.setValueCount(childValueCount); + } - public int getLastSet() { - return lastSet; - } + public void setLastSet(int value) { + lastSet = value; } + public int getLastSet() { + return lastSet; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 95efa60871806..6089a67924fe0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -56,8 +56,6 @@ public static MapVector empty(String name, BufferAllocator allocator) { } private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); - private final Accessor accessor = new Accessor(); - private final Mutator mutator = new Mutator(); protected final FieldType fieldType; public int valueCount; @@ -70,6 +68,7 @@ public MapVector(String name, BufferAllocator allocator, CallBack callBack) { public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); this.fieldType = checkNotNull(fieldType); + this.valueCount = 0; } @Override @@ -230,7 +229,7 @@ public void splitAndTransfer(int startIndex, int length) { for (TransferPair p : pairs) { p.splitAndTransfer(startIndex, length); } - to.getMutator().setValueCount(length); + to.setValueCount(length); } } @@ -255,76 +254,54 @@ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { @Override public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("accessor is not needed for MAP"); } @Override public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("mutator is not needed for MAP"); } - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - Map vv = new JsonStringHashMap<>(); - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v instanceof NullableVarCharVector || v instanceof NullableIntVector) { - if (v != null && index < v.getValueCount()) { - Object value = v.getObject(index); - if (value != null) { - vv.put(child, value); - } - } - } else { - if (v != null && index < v.getAccessor().getValueCount()) { - Object value = v.getAccessor().getObject(index); - if (value != null) { - vv.put(child, value); - } - } + @Override + public Object getObject(int index) { + Map vv = new JsonStringHashMap<>(); + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); } } - return vv; } + return vv; + } - public void get(int index, ComplexHolder holder) { - reader.setPosition(index); - holder.reader = reader; - } + @Override + public boolean isNull(int index) { return false; } + @Override + public int getNullCount() { return 0; } - @Override - public int getValueCount() { - return valueCount; - } + public void get(int index, ComplexHolder holder) { + reader.setPosition(index); + holder.reader = reader; } - public ValueVector getVectorById(int id) { - return getChildByOrdinal(id); + @Override + public int getValueCount() { + return valueCount; } - public class Mutator extends BaseValueVector.BaseMutator { - - @Override - public void setValueCount(int valueCount) { - for (final ValueVector v : getChildren()) { - if (v instanceof NullableIntVector || v instanceof NullableVarCharVector) { - v.setValueCount(valueCount); - } else { - v.getMutator().setValueCount(valueCount); - } - } - MapVector.this.valueCount = valueCount; - } - - @Override - public void reset() { - } + public ValueVector getVectorById(int id) { + return getChildByOrdinal(id); +} - @Override - public void generateTestData(int values) { + @Override + public void setValueCount(int valueCount) { + for (final ValueVector v : getChildren()) { + v.setValueCount(valueCount); } + MapVector.this.valueCount = valueCount; } @Override @@ -372,11 +349,4 @@ public void initializeChildrenFromFields(List children) { public List getChildrenFromFields() { return getChildren(); } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 1c9d5aa265963..a1478c5a7213d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -27,13 +28,9 @@ import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BaseDataValueVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.NullableVectorDefinitionSetter; -import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; import org.apache.arrow.vector.complex.impl.NullableMapWriter; import org.apache.arrow.vector.holders.ComplexHolder; @@ -44,6 +41,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; public class NullableMapVector extends MapVector implements FieldVector { @@ -56,12 +54,8 @@ public static NullableMapVector empty(String name, BufferAllocator allocator) { private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); private final NullableMapWriter writer = new NullableMapWriter(this); - protected final BitVector bits; - - private final List innerVectors; - - private final Accessor accessor; - private final Mutator mutator; + private ArrowBuf validityBuffer; + private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @Deprecated @@ -77,10 +71,8 @@ public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncod public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, checkNotNull(allocator), fieldType, callBack); - this.bits = new BitVector("$bits$", allocator); - this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits)); - this.accessor = new Accessor(); - this.mutator = new Mutator(); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); } @Override @@ -92,18 +84,33 @@ public Field getField() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); - this.valueCount = fieldNode.getLength(); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueCount = fieldNode.getLength(); + validityAllocationSizeInBytes = validityBuffer.capacity(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(1); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + result.add(validityBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } @Override @@ -146,49 +153,129 @@ protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, @Override public void transfer() { - bits.transferTo(target.bits); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; super.transfer(); } @Override public void copyValueSafe(int fromIndex, int toIndex) { - target.bits.copyFromSafe(fromIndex, toIndex, bits); + while (toIndex >= target.getValidityBufferValueCapacity()) { + target.reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); super.copyValueSafe(fromIndex, toIndex); } @Override public void splitAndTransfer(int startIndex, int length) { - bits.splitAndTransferTo(startIndex, length, target.bits); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); super.splitAndTransfer(startIndex, length); } } + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + @Override public int getValueCapacity() { - return Math.min(bits.getValueCapacity(), super.getValueCapacity()); + return Math.min(getValidityBufferValueCapacity(), + super.getValueCapacity()); } @Override public ArrowBuf[] getBuffers(boolean clear) { - return ObjectArrays.concat(bits.getBuffers(clear), super.getBuffers(clear), ArrowBuf.class); + if (clear) { + validityBuffer.retain(1); + } + return ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(clear), ArrowBuf.class); } @Override public void close() { - bits.close(); + clearValidityBuffer(); super.close(); } @Override public void clear() { - bits.clear(); + clearValidityBuffer(); super.clear(); } + private void clearValidityBuffer() { + validityBuffer.release(); + validityBuffer = allocator.getEmpty(); + } @Override public int getBufferSize() { - return super.getBufferSize() + bits.getBufferSize(); + if (valueCount == 0) { return 0; } + return super.getBufferSize() + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override @@ -197,12 +284,12 @@ public int getBufferSizeFor(final int valueCount) { return 0; } return super.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); super.setInitialCapacity(numRecords); } @@ -215,25 +302,59 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - success = super.allocateNewSafe() && bits.allocateNewSafe(); + clearValidityBuffer(); + allocateValidityBuffer(validityAllocationSizeInBytes); + success = super.allocateNewSafe(); } finally { if (!success) { clear(); + return false; } } - bits.zeroVector(); - return success; + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { - bits.reAlloc(); + /* reallocate the validity buffer */ + reallocValidityBuffer(); super.reAlloc(); } + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; + } + @Override public long getValidityBufferAddress() { - return bits.getBuffer().memoryAddress(); + return validityBuffer.memoryAddress(); } @Override @@ -248,7 +369,7 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getValidityBuffer() { - return bits.getDataBuffer(); + return validityBuffer; } @Override @@ -261,82 +382,76 @@ public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public final class Accessor extends MapVector.Accessor { - final BitVector.Accessor bAccessor = bits.getAccessor(); - - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } else { - return super.getObject(index); - } - } - - @Override - public void get(int index, ComplexHolder holder) { - holder.isSet = isSet(index); - super.get(index, holder); - } - - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); - } - - @Override - public boolean isNull(int index) { - return isSet(index) == 0; - } - - public int isSet(int index) { - return bAccessor.get(index); + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return super.getObject(index); } - } - public final class Mutator extends MapVector.Mutator implements NullableVectorDefinitionSetter { + @Override + public void get(int index, ComplexHolder holder) { + holder.isSet = isSet(index); + super.get(index, holder); + } - private Mutator() { - } + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - @Override - public void setIndexDefined(int index) { - bits.getMutator().setSafe(index, 1); - } + public boolean isNull(int index) { + return isSet(index) == 0; + } - public void setNull(int index) { - bits.getMutator().setSafe(index, 0); - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - super.setValueCount(valueCount); - bits.getMutator().setValueCount(valueCount); + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } - @Override - public void generateTestData(int valueCount) { - super.generateTestData(valueCount); - bits.getMutator().generateTestDataAlt(valueCount); + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } - @Override - public void reset() { - bits.getMutator().setValueCount(0); + @Override + public void setValueCount(int valueCount) { + assert valueCount >= 0; + while (valueCount > getValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + super.setValueCount(valueCount); + this.valueCount = valueCount; + } + public void reset() { + valueCount = 0; } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP"); } - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java index 614c266acf147..06b0f4d43272f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java @@ -52,6 +52,6 @@ public void copyAsField(String name, MapWriter writer) { @Override public boolean isSet() { - return !nullableMapVector.getAccessor().isNull(idx()); + return !nullableMapVector.isNull(idx()); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 9722196ed7cd2..5bd439cac4269 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -162,7 +162,7 @@ private FieldWriter promoteToUnion() { writer = new UnionWriter(unionVector, nullableMapWriterFactory); writer.setPosition(idx()); for (int i = 0; i <= idx(); i++) { - unionVector.getMutator().setType(i, vector.getMinorType()); + unionVector.setType(i, vector.getMinorType()); } vector = null; state = State.UNION; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java index 3ebd0cd7dd959..c77ca4e8f23ba 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java @@ -78,7 +78,7 @@ public void setPosition(int index) { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java index f3e9b8773f25e..4ad2f6f5fad7a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java @@ -46,7 +46,7 @@ public UnionFixedSizeListReader(FixedSizeListVector vector) { @Override public boolean isSet() { - return !vector.getAccessor().isNull(idx()); + return !vector.isNull(idx()); } @Override @@ -56,7 +56,7 @@ public FieldReader reader() { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java index b98c36d2bf721..6243a2833c055 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java @@ -19,6 +19,7 @@ package org.apache.arrow.vector.complex.impl; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.ListVector; @@ -33,12 +34,11 @@ public class UnionListReader extends AbstractFieldReader { private ListVector vector; private ValueVector data; - private UInt4Vector offsets; + private static final int OFFSET_WIDTH = 4; public UnionListReader(ListVector vector) { this.vector = vector; this.data = vector.getDataVector(); - this.offsets = vector.getOffsetVector(); } @Override @@ -48,7 +48,7 @@ public Field getField() { @Override public boolean isSet() { - return !vector.getAccessor().isNull(idx()); + return !vector.isNull(idx()); } private int currentOffset; @@ -57,8 +57,8 @@ public boolean isSet() { @Override public void setPosition(int index) { super.setPosition(index); - currentOffset = offsets.getAccessor().get(index) - 1; - maxOffset = offsets.getAccessor().get(index + 1); + currentOffset = vector.getOffsetBuffer().getInt(index * OFFSET_WIDTH) - 1; + maxOffset = vector.getOffsetBuffer().getInt((index + 1) * OFFSET_WIDTH); } @Override @@ -68,7 +68,7 @@ public FieldReader reader() { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java index 3b7dc4a56e1ac..762a442c983fe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java @@ -47,11 +47,10 @@ public class DictionaryEncoder { public static ValueVector encode(ValueVector vector, Dictionary dictionary) { validateType(vector.getMinorType()); // load dictionary values into a hashmap for lookup - ValueVector.Accessor dictionaryAccessor = dictionary.getVector().getAccessor(); - Map lookUps = new HashMap<>(dictionaryAccessor.getValueCount()); - for (int i = 0; i < dictionaryAccessor.getValueCount(); i++) { + Map lookUps = new HashMap<>(dictionary.getVector().getValueCount()); + for (int i = 0; i < dictionary.getVector().getValueCount(); i++) { // for primitive array types we need a wrapper that implements equals and hashcode appropriately - lookUps.put(dictionaryAccessor.getObject(i), i); + lookUps.put(dictionary.getVector().getObject(i), i); } Field valueField = vector.getField(); @@ -61,14 +60,13 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { // vector to hold our indices (dictionary encoded values) FieldVector indices = indexField.createVector(vector.getAllocator()); - ValueVector.Mutator mutator = indices.getMutator(); // use reflection to pull out the set method // TODO implement a common interface for int vectors Method setter = null; for (Class c : ImmutableList.of(int.class, long.class)) { try { - setter = mutator.getClass().getMethod("setSafe", int.class, c); + setter = indices.getClass().getMethod("setSafe", int.class, c); break; } catch (NoSuchMethodException e) { // ignore @@ -78,21 +76,20 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { throw new IllegalArgumentException("Dictionary encoding does not have a valid int type:" + indices.getClass()); } - ValueVector.Accessor accessor = vector.getAccessor(); - int count = accessor.getValueCount(); + int count = vector.getValueCount(); indices.allocateNew(); try { for (int i = 0; i < count; i++) { - Object value = accessor.getObject(i); + Object value = vector.getObject(i); if (value != null) { // if it's null leave it null // note: this may fail if value was not included in the dictionary Object encoded = lookUps.get(value); if (encoded == null) { throw new IllegalArgumentException("Dictionary encoding not defined for value:" + value); } - setter.invoke(mutator, i, encoded); + setter.invoke(indices, i, encoded); } } } catch (IllegalAccessException e) { @@ -101,7 +98,7 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { throw new RuntimeException("InvocationTargetException invoking vector mutator set():", e.getCause()); } - mutator.setValueCount(count); + indices.setValueCount(count); return indices; } @@ -114,15 +111,14 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { * @return vector with values restored from dictionary */ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { - ValueVector.Accessor accessor = indices.getAccessor(); - int count = accessor.getValueCount(); + int count = indices.getValueCount(); ValueVector dictionaryVector = dictionary.getVector(); - int dictionaryCount = dictionaryVector.getAccessor().getValueCount(); + int dictionaryCount = dictionaryVector.getValueCount(); // copy the dictionary values into the decoded vector TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator()); transfer.getTo().allocateNewSafe(); for (int i = 0; i < count; i++) { - Object index = accessor.getObject(i); + Object index = indices.getObject(i); if (index != null) { int indexAsInt = ((Number) index).intValue(); if (indexAsInt > dictionaryCount) { @@ -133,7 +129,7 @@ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { } // TODO do we need to worry about the field? ValueVector decoded = transfer.getTo(); - decoded.getMutator().setValueCount(count); + decoded.setValueCount(count); return decoded; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java index b35aba5426e4a..7dc10b5e6299f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java @@ -82,7 +82,7 @@ protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, Writab for (long id : dictionaryIdsUsed) { Dictionary dictionary = provider.lookup(id); FieldVector vector = dictionary.getVector(); - int count = vector.getAccessor().getValueCount(); + int count = vector.getValueCount(); VectorSchemaRoot dictRoot = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), count); VectorUnloader unloader = new VectorUnloader(dictRoot); ArrowRecordBatch batch = unloader.getRecordBatch(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 0de804406f7b3..b21fcaac391e8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -23,60 +23,28 @@ import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET; +import static org.apache.arrow.vector.schema.ArrowVectorType.*; import java.io.File; import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.*; import com.google.common.collect.ImmutableList; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.NullableVarBinaryVector; -import org.apache.arrow.vector.NullableVarCharVector; -import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TimeMicroVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeNanoVector; -import org.apache.arrow.vector.TimeSecVector; -import org.apache.arrow.vector.TimeStampMicroTZVector; -import org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.arrow.vector.TimeStampMilliTZVector; -import org.apache.arrow.vector.TimeStampMilliVector; -import org.apache.arrow.vector.TimeStampNanoTZVector; -import org.apache.arrow.vector.TimeStampNanoVector; -import org.apache.arrow.vector.TimeStampSecTZVector; -import org.apache.arrow.vector.TimeStampSecVector; -import org.apache.arrow.vector.TinyIntVector; -import org.apache.arrow.vector.UInt1Vector; -import org.apache.arrow.vector.UInt2Vector; -import org.apache.arrow.vector.UInt4Vector; -import org.apache.arrow.vector.UInt8Vector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.DictionaryUtility; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -153,7 +121,7 @@ private void readDictionaryBatches() throws JsonParseException, IOException { FieldVector vector = dict.getVector(); List fields = ImmutableList.of(vector.getField()); List vectors = ImmutableList.of(vector); - VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount()); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount()); read(root); readToken(END_OBJECT); @@ -177,7 +145,7 @@ public boolean read(VectorSchemaRoot root) throws IOException { { for (Field field : root.getSchema().getFields()) { FieldVector vector = root.getVector(field.getName()); - readVector(field, vector); + readFromJsonIntoVector(field, vector); } } readToken(END_ARRAY); @@ -204,7 +172,7 @@ public VectorSchemaRoot read() throws IOException { { for (Field field : schema.getFields()) { FieldVector vector = recordBatch.getVector(field.getName()); - readVector(field, vector); + readFromJsonIntoVector(field, vector); } } readToken(END_ARRAY); @@ -218,16 +186,24 @@ public VectorSchemaRoot read() throws IOException { } } - /** - * TODO: A better way of implementing this function is to use `loadFieldBuffers` methods in - * FieldVector to set the inner-vector data as done in `ArrowFileReader`. - */ - private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException { + private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); - List fieldInnerVectors = vector.getFieldInnerVectors(); - if (vectorTypes.size() != fieldInnerVectors.size()) { - throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size()); - } + ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()]; + /* + * The order of inner buffers is : + * Fixed width vector: + * -- validity buffer + * -- data buffer + * Variable width vector: + * -- validity buffer + * -- offset buffer + * -- data buffer + * + * This is similar to what getFieldInnerVectors() used to give but now that we don't have + * inner vectors anymore, we will work directly at the buffer level -- populate buffers + * locally as we read from Json parser and do loadFieldBuffers on the vector followed by + * releasing the local buffers. + */ readToken(START_OBJECT); { // If currently reading dictionaries, field name is not important so don't check @@ -236,39 +212,41 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name); } - // Initialize the vector with required capacity - int count = readNextField("count", Integer.class); - vector.setInitialCapacity(count); - vector.allocateNew(); + /* Initialize the vector with required capacity but don't allocate since we would + * be doing loadFieldBuffers. + */ + int valueCount = readNextField("count", Integer.class); + vector.setInitialCapacity(valueCount); - // Read inner vectors for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); - ValueVector valueVector = (ValueVector) fieldInnerVectors.get(v); nextFieldIs(vectorType.getName()); readToken(START_ARRAY); - int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count; - for (int i = 0; i < innerVectorCount; i++) { + int innerBufferValueCount = valueCount; + if (vectorType.equals(OFFSET)) { + /* offset buffer has 1 additional value capacity */ + innerBufferValueCount = valueCount + 1; + } + for (int i = 0; i < innerBufferValueCount; i++) { + /* write data to the buffer */ parser.nextToken(); - setValueFromParser(valueVector, i); + /* for variable width vectors, value count doesn't help pre-determining the capacity of + * the underlying data buffer. So we need to pass down the offset buffer (which was already + * populated in the previous iteration of this loop). + */ + if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR + || vector.getMinorType() == Types.MinorType.VARBINARY)) { + vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], + vectorBuffers[v-1], i, innerBufferValueCount); + } else { + vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], + null, i, innerBufferValueCount); + } } readToken(END_ARRAY); } - // Set lastSet before valueCount to prevent setValueCount from filling empty values - switch (vector.getMinorType()) { - case LIST: - // ListVector starts lastSet from index 0, so lastSet value is always last index written + 1 - ((ListVector) vector).getMutator().setLastSet(count); - break; - case VARBINARY: - ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1); - break; - case VARCHAR: - ((NullableVarCharVector) vector).setLastSet(count - 1); - break; - } - vector.getMutator().setValueCount(count); + vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers)); // read child vectors, if any List fields = field.getChildren(); @@ -282,12 +260,16 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti for (int i = 0; i < fields.size(); i++) { Field childField = fields.get(i); FieldVector childVector = vectorChildren.get(i); - readVector(childField, childVector); + readFromJsonIntoVector(childField, childVector); } readToken(END_ARRAY); } } readToken(END_OBJECT); + + for (ArrowBuf buffer: vectorBuffers) { + buffer.release(); + } } private byte[] decodeHexSafe(String hexString) throws IOException { @@ -298,99 +280,124 @@ private byte[] decodeHexSafe(String hexString) throws IOException { } } - private void setValueFromParser(ValueVector valueVector, int i) throws IOException { - switch (valueVector.getMinorType()) { - case BIT: - ((BitVector) valueVector).getMutator().set(i, parser.readValueAs(Boolean.class) ? 1 : 0); - break; - case TINYINT: - ((TinyIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case SMALLINT: - ((SmallIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case INT: - ((IntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case BIGINT: - ((BigIntVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case UINT1: - ((UInt1Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT2: - ((UInt2Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT4: - ((UInt4Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT8: - ((UInt8Vector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case FLOAT4: - ((Float4Vector) valueVector).getMutator().set(i, parser.readValueAs(Float.class)); - break; - case FLOAT8: - ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class)); - break; - case DECIMAL: { - DecimalVector decimalVector = (DecimalVector) valueVector; - // Here we assume the decimal value is the unscaled integer value as a string - BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); - DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, decimalVector.getBuffer(), i); - } - break; - case VARBINARY: - ((VarBinaryVector) valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class))); - break; - case VARCHAR: - ((VarCharVector) valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8)); - break; - case DATEDAY: - ((DateDayVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case DATEMILLI: - ((DateMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESEC: - ((TimeSecVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case TIMEMILLI: - ((TimeMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case TIMEMICRO: - ((TimeMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMENANO: - ((TimeNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSEC: - ((TimeStampSecVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLI: - ((TimeStampMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICRO: - ((TimeStampMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANO: - ((TimeStampNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSECTZ: - ((TimeStampSecTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLITZ: - ((TimeStampMilliTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICROTZ: - ((TimeStampMicroTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANOTZ: - ((TimeStampNanoTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - default: - throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType()); + private ArrowBuf setValueFromParser(ArrowVectorType bufferType, FieldVector vector, + ArrowBuf buffer, ArrowBuf offsetBuffer, int index, + int valueCount) throws IOException { + if (bufferType.equals(TYPE)) { + buffer = NullableTinyIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Byte.class)); + } else if (bufferType.equals(OFFSET)) { + buffer = BaseNullableVariableWidthVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + } else if (bufferType.equals(VALIDITY)) { + buffer = BitVectorHelper.setValidityBit(buffer, allocator, + valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); + } else if (bufferType.equals(DATA)) { + switch (vector.getMinorType()) { + case BIT: + buffer = BitVectorHelper.setValidityBit(buffer, allocator, + valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); + break; + case TINYINT: + buffer = NullableTinyIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Byte.class)); + break; + case SMALLINT: + buffer = NullableSmallIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Short.class)); + break; + case INT: + buffer = NullableIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case BIGINT: + buffer = NullableBigIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case FLOAT4: + buffer = NullableFloat4Vector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Float.class)); + break; + case FLOAT8: + buffer = NullableFloat8Vector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Double.class)); + break; + case DECIMAL: + buffer = NullableDecimalVector.set(buffer, allocator, + valueCount, index, decodeHexSafe(parser.readValueAs(String.class))); + break; + case VARBINARY: + assert (offsetBuffer != null); + buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, + decodeHexSafe(parser.readValueAs(String.class)), valueCount); + break; + case VARCHAR: + assert (offsetBuffer != null); + buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, + parser.readValueAs(String.class).getBytes(UTF_8), valueCount); + break; + case DATEDAY: + buffer = NullableDateDayVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case DATEMILLI: + buffer = NullableDateMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESEC: + buffer = NullableTimeSecVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case TIMEMILLI: + buffer = NullableTimeMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case TIMEMICRO: + buffer = NullableTimeMicroVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMENANO: + buffer = NullableTimeNanoVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPSEC: + buffer = NullableTimeStampSecVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLI: + buffer = NullableTimeStampMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICRO: + buffer = NullableTimeStampMicroVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANO: + buffer = NullableTimeStampNanoVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPSECTZ: + buffer = NullableTimeStampSecTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLITZ: + buffer = NullableTimeStampMilliTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICROTZ: + buffer = NullableTimeStampMicroTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANOTZ: + buffer = NullableTimeStampNanoTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + default: + throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); + } } + + return buffer; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java index 05341bec44ea3..7921cd34c773c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java @@ -18,9 +18,10 @@ package org.apache.arrow.vector.file.json; +import static org.apache.arrow.vector.schema.ArrowVectorType.*; + import java.io.File; import java.io.IOException; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -28,28 +29,11 @@ import com.google.common.collect.ImmutableList; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.TimeMicroVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeNanoVector; -import org.apache.arrow.vector.TimeSecVector; -import org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.arrow.vector.TimeStampMilliVector; -import org.apache.arrow.vector.TimeStampNanoVector; -import org.apache.arrow.vector.TimeStampSecVector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.ValueVector.Accessor; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.schema.ArrowVectorType; -import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -135,7 +119,7 @@ private void writeDictionaryBatches(JsonGenerator generator, Set dictionar FieldVector vector = dictionary.getVector(); List fields = ImmutableList.of(vector.getField()); List vectors = ImmutableList.of(vector); - VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount()); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount()); writeBatch(root); generator.writeEndObject(); @@ -157,31 +141,36 @@ private void writeBatch(VectorSchemaRoot recordBatch) throws IOException { generator.writeArrayFieldStart("columns"); for (Field field : recordBatch.getSchema().getFields()) { FieldVector vector = recordBatch.getVector(field.getName()); - writeVector(field, vector); + writeFromVectorIntoJson(field, vector); } generator.writeEndArray(); } generator.writeEndObject(); } - private void writeVector(Field field, FieldVector vector) throws IOException { + private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); - List fieldInnerVectors = vector.getFieldInnerVectors(); - if (vectorTypes.size() != fieldInnerVectors.size()) { - throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size()); + List vectorBuffers = vector.getFieldBuffers(); + if (vectorTypes.size() != vectorBuffers.size()) { + throw new IllegalArgumentException("vector types and inner vector buffers are not the same size: " + vectorTypes.size() + " != " + vectorBuffers.size()); } generator.writeStartObject(); { generator.writeObjectField("name", field.getName()); - int valueCount = vector.getAccessor().getValueCount(); + int valueCount = vector.getValueCount(); generator.writeObjectField("count", valueCount); for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); - BufferBacked innerVector = fieldInnerVectors.get(v); + ArrowBuf vectorBuffer = vectorBuffers.get(v); generator.writeArrayFieldStart(vectorType.getName()); - ValueVector valueVector = (ValueVector) innerVector; - for (int i = 0; i < valueVector.getAccessor().getValueCount(); i++) { - writeValueToGenerator(valueVector, i); + final int bufferValueCount = (vectorType.equals(OFFSET)) ? valueCount + 1 : valueCount; + for (int i = 0; i < bufferValueCount; i++) { + if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || + vector.getMinorType() == Types.MinorType.VARBINARY)) { + writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i); + } else { + writeValueToGenerator(vectorType, vectorBuffer, null, vector, i); + } } generator.writeEndArray(); } @@ -195,7 +184,7 @@ private void writeVector(Field field, FieldVector vector) throws IOException { for (int i = 0; i < fields.size(); i++) { Field childField = fields.get(i); FieldVector childVector = children.get(i); - writeVector(childField, childVector); + writeFromVectorIntoJson(childField, childVector); } generator.writeEndArray(); } @@ -203,62 +192,101 @@ private void writeVector(Field field, FieldVector vector) throws IOException { generator.writeEndObject(); } - private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException { - switch (valueVector.getMinorType()) { - case DATEDAY: - generator.writeNumber(((DateDayVector) valueVector).getAccessor().get(i)); - break; - case DATEMILLI: - generator.writeNumber(((DateMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMESEC: - generator.writeNumber(((TimeSecVector) valueVector).getAccessor().get(i)); - break; - case TIMEMILLI: - generator.writeNumber(((TimeMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMEMICRO: - generator.writeNumber(((TimeMicroVector) valueVector).getAccessor().get(i)); - break; - case TIMENANO: - generator.writeNumber(((TimeNanoVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPSEC: - generator.writeNumber(((TimeStampSecVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPMILLI: - generator.writeNumber(((TimeStampMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPMICRO: - generator.writeNumber(((TimeStampMicroVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPNANO: - generator.writeNumber(((TimeStampNanoVector) valueVector).getAccessor().get(i)); - break; - case BIT: - generator.writeNumber(((BitVector) valueVector).getAccessor().get(i)); - break; - case VARBINARY: { - String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i)); - generator.writeString(hexString); + private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, + ArrowBuf offsetBuffer, FieldVector vector, int index) throws IOException { + if (bufferType.equals(TYPE)) { + generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH)); + } else if (bufferType.equals(OFFSET)) { + generator.writeNumber(buffer.getInt(index * BaseNullableVariableWidthVector.OFFSET_WIDTH)); + } else if(bufferType.equals(VALIDITY)) { + generator.writeNumber(vector.isNull(index) ? 0 : 1); + } else if (bufferType.equals(DATA)) { + switch (vector.getMinorType()) { + case TINYINT: + generator.writeNumber(NullableTinyIntVector.get(buffer, index)); + break; + case SMALLINT: + generator.writeNumber(NullableSmallIntVector.get(buffer, index)); + break; + case INT: + generator.writeNumber(NullableIntVector.get(buffer, index)); + break; + case BIGINT: + generator.writeNumber(NullableBigIntVector.get(buffer, index)); + break; + case FLOAT4: + generator.writeNumber(NullableFloat4Vector.get(buffer, index)); + break; + case FLOAT8: + generator.writeNumber(NullableFloat8Vector.get(buffer, index)); + break; + case DATEDAY: + generator.writeNumber(NullableDateDayVector.get(buffer, index)); + break; + case DATEMILLI: + generator.writeNumber(NullableDateMilliVector.get(buffer, index)); + break; + case TIMESEC: + generator.writeNumber(NullableTimeSecVector.get(buffer, index)); + break; + case TIMEMILLI: + generator.writeNumber(NullableTimeMilliVector.get(buffer, index)); + break; + case TIMEMICRO: + generator.writeNumber(NullableTimeMicroVector.get(buffer, index)); + break; + case TIMENANO: + generator.writeNumber(NullableTimeNanoVector.get(buffer, index)); + break; + case TIMESTAMPSEC: + generator.writeNumber(NullableTimeStampSecVector.get(buffer, index)); + break; + case TIMESTAMPMILLI: + generator.writeNumber(NullableTimeStampMilliVector.get(buffer, index)); + break; + case TIMESTAMPMICRO: + generator.writeNumber(NullableTimeStampMicroVector.get(buffer, index)); + break; + case TIMESTAMPNANO: + generator.writeNumber(NullableTimeStampNanoVector.get(buffer, index)); + break; + case TIMESTAMPSECTZ: + generator.writeNumber(NullableTimeStampSecTZVector.get(buffer, index)); + break; + case TIMESTAMPMILLITZ: + generator.writeNumber(NullableTimeStampMilliTZVector.get(buffer, index)); + break; + case TIMESTAMPMICROTZ: + generator.writeNumber(NullableTimeStampMicroTZVector.get(buffer, index)); + break; + case TIMESTAMPNANOTZ: + generator.writeNumber(NullableTimeStampNanoTZVector.get(buffer, index)); + break; + case BIT: + generator.writeNumber(BitVectorHelper.get(buffer, index)); + break; + case VARBINARY: { + assert offsetBuffer != null; + String hexString = Hex.encodeHexString(BaseNullableVariableWidthVector.get(buffer, + offsetBuffer, index)); + generator.writeObject(hexString); + break; } - break; - case DECIMAL: { - BigDecimal decimalValue = ((DecimalVector) valueVector).getAccessor().getObject(i); - // We write the unscaled value, because the scale is stored in the type metadata. - generator.writeString(decimalValue.unscaledValue().toString()); + case VARCHAR: { + assert offsetBuffer != null; + byte[] b = (BaseNullableVariableWidthVector.get(buffer, offsetBuffer, index)); + generator.writeString(new String(b, "UTF-8")); + break; } - break; - default: - // TODO: each type - Accessor accessor = valueVector.getAccessor(); - Object value = accessor.getObject(i); - if (value instanceof Number || value instanceof Boolean) { - generator.writeObject(value); - } else { - generator.writeObject(value.toString()); + case DECIMAL: { + String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(buffer, + index)); + generator.writeString(hexString); + break; } - break; + default: + throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); + } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java index 5851bd5fa5d97..c27e5e5c85c30 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java @@ -114,13 +114,13 @@ public static void compareFieldVectors(FieldVector vector1, FieldVector vector2) if (!field1.equals(vector2.getField())) { throw new IllegalArgumentException("Different Fields:\n" + field1 + "\n!=\n" + vector2.getField()); } - int valueCount = vector1.getAccessor().getValueCount(); - if (valueCount != vector2.getAccessor().getValueCount()) { - throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getAccessor().getValueCount()); + int valueCount = vector1.getValueCount(); + if (valueCount != vector2.getValueCount()) { + throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getValueCount()); } for (int j = 0; j < valueCount; j++) { - Object obj1 = vector1.getAccessor().getObject(j); - Object obj2 = vector2.getAccessor().getObject(j); + Object obj1 = vector1.getObject(j); + Object obj2 = vector2.getObject(j); if (!equals(field1.getType(), obj1, obj2)) { throw new IllegalArgumentException( "Different values in column:\n" + field1 + " at index " + j + ": " + obj1 + " != " + obj2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java index 17fcf05fcd9bd..ada341497878e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java @@ -340,59 +340,56 @@ public void testReallocAfterVectorTransfer2() { int valueCapacity = vector.getValueCapacity(); assertEquals(4096, valueCapacity); - final NullableBitVector.Mutator mutator = vector.getMutator(); - final NullableBitVector.Accessor accessor = vector.getAccessor(); - for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger first realloc */ - mutator.setSafe(valueCapacity, 1, 1); + vector.setSafe(valueCapacity, 1, 1); assertEquals(valueCapacity * 2, vector.getValueCapacity()); for (int i = valueCapacity; i < valueCapacity*2; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*2; i++) { if (((i & 1) == 1) || (i == valueCapacity)) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger second realloc */ - mutator.setSafe(valueCapacity*2, 1, 1); + vector.setSafe(valueCapacity*2, 1, 1); assertEquals(valueCapacity * 4, vector.getValueCapacity()); for (int i = valueCapacity*2; i < valueCapacity*4; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*4; i++) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } @@ -400,26 +397,24 @@ public void testReallocAfterVectorTransfer2() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); final NullableBitVector toVector = (NullableBitVector)transferPair.getTo(); - final NullableBitVector.Accessor toAccessor = toVector.getAccessor(); - final NullableBitVector.Mutator toMutator = toVector.getMutator(); assertEquals(valueCapacity * 4, toVector.getValueCapacity()); /* realloc the toVector */ - toMutator.setSafe(valueCapacity * 4, 1, 1); + toVector.setSafe(valueCapacity * 4, 1, 1); for (int i = 0; i < toVector.getValueCapacity(); i++) { if (i <= valueCapacity * 4) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2) || (i == valueCapacity*4)) { - assertFalse("unexpected cleared bit at index: " + i, toAccessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, toVector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } else { - assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index 56d2293276404..4d844d6d3ca0f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -71,14 +71,14 @@ public void testValuesWriteRead() { for (int i = 0; i < intValues.length; i++) { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale); values[i] = decimal; - decimalVector.getMutator().setSafe(i, decimal); + decimalVector.setSafe(i, decimal); } - decimalVector.getMutator().setValueCount(intValues.length); + decimalVector.setValueCount(intValues.length); for (int i = 0; i < intValues.length; i++) { - BigDecimal value = decimalVector.getAccessor().getObject(i); - assertEquals(values[i], value); + BigDecimal value = decimalVector.getObject(i); + assertEquals("unexpected data at index: " + i, values[i], value); } } } @@ -92,7 +92,7 @@ public void testBigDecimalDifferentScaleAndPrecision() { boolean hasError = false; try { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(0), 3); - decimalVector.getMutator().setSafe(0, decimal); + decimalVector.setSafe(0, decimal); } catch (UnsupportedOperationException ue) { hasError = true; } finally { @@ -103,7 +103,7 @@ public void testBigDecimalDifferentScaleAndPrecision() { hasError = false; try { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(12345), 2); - decimalVector.getMutator().setSafe(0, decimal); + decimalVector.setSafe(0, decimal); } catch (UnsupportedOperationException ue) { hasError = true; } finally { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 1185246f88cde..46a2bafa00208 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -135,9 +135,9 @@ public void testEncodeLargeVector() { // now run through the decoder and verify we get the original back try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) { assertEquals(vector.getClass(), decoded.getClass()); - assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount()); + assertEquals(vector.getValueCount(), decoded.getValueCount()); for (int i = 0; i < count; ++i) { - assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i)); + assertEquals(vector.getObject(i), decoded.getObject(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 168deacd6a640..2af6cd53aa0c1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -57,7 +57,7 @@ public void testIntType() { vector.allocateNew(); for (int i = 0; i < 10; i++) { - vector.getMutator().setNotNull(i); + vector.setNotNull(i); nested.set(i * 2, i); nested.set(i * 2 + 1, i + 10); } @@ -81,17 +81,16 @@ public void testIntType() { public void testFloatTypeNullable() { try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); - NullableFloat4Vector.Mutator mutator = nested.getMutator(); vector.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - vector.getMutator().setNotNull(i); - mutator.set(i * 2, i + 0.1f); - mutator.set(i * 2 + 1, i + 10.1f); + vector.setNotNull(i); + nested.set(i * 2, i + 0.1f); + nested.set(i * 2 + 1, i + 10.1f); } } - vector.getMutator().setValueCount(10); + vector.setValueCount(10); UnionFixedSizeListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -115,24 +114,22 @@ public void testFloatTypeNullable() { @Test public void testNestedInList() { try (ListVector vector = ListVector.empty("list", allocator)) { - ListVector.Mutator mutator = vector.getMutator(); FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector(); - FixedSizeListVector.Mutator tupleMutator = tuples.getMutator(); NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - int position = mutator.startNewValue(i); + int position = vector.startNewValue(i); for (int j = 0; j < i % 7; j++) { - tupleMutator.setNotNull(position + j); + tuples.setNotNull(position + j); innerVector.set((position + j) * 2, j); innerVector.set((position + j) * 2 + 1, j + 1); } - mutator.endValue(i, i % 7); + vector.endValue(i, i % 7); } } - mutator.setValueCount(10); + vector.setValueCount(10); UnionListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -161,24 +158,24 @@ public void testTransferPair() { try (FixedSizeListVector from = new FixedSizeListVector("from", allocator, 2, null, null); FixedSizeListVector to = new FixedSizeListVector("to", allocator, 2, null, null)) { NullableFloat4Vector nested = (NullableFloat4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); - NullableFloat4Vector.Mutator mutator = nested.getMutator(); from.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - from.getMutator().setNotNull(i); - mutator.set(i * 2, i + 0.1f); - mutator.set(i * 2 + 1, i + 10.1f); + from.setNotNull(i); + nested.set(i * 2, i + 0.1f); + nested.set(i * 2 + 1, i + 10.1f); } } - from.getMutator().setValueCount(10); + from.setValueCount(10); TransferPair pair = from.makeTransferPair(to); pair.copyValueSafe(0, 1); pair.copyValueSafe(2, 2); to.copyFromSafe(4, 3, from); - to.getMutator().setValueCount(10); + + to.setValueCount(10); UnionFixedSizeListReader reader = to.getReader(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 59e1646e86e56..f6aa86a300876 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -85,7 +85,7 @@ public void testCopyFrom() throws Exception { for (int i = 0; i < 3; i++) { outVector.copyFrom(i, i, inVector); } - outVector.getMutator().setValueCount(3); + outVector.setValueCount(3); // assert the output vector is correct FieldReader reader = outVector.getReader(); @@ -95,11 +95,9 @@ public void testCopyFrom() throws Exception { reader.setPosition(2); Assert.assertTrue("shouldn't be null", reader.isSet()); - /* check the exact contents of vector */ - final ListVector.Accessor accessor = outVector.getAccessor(); /* index 0 */ - Object result = accessor.getObject(0); + Object result = outVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); assertEquals(new Long(1), (Long) resultSet.get(0)); @@ -107,11 +105,11 @@ public void testCopyFrom() throws Exception { assertEquals(new Long(3), (Long) resultSet.get(2)); /* index 1 */ - result = accessor.getObject(1); + result = outVector.getObject(1); assertNull(result); /* index 2 */ - result = accessor.getObject(2); + result = outVector.getObject(2); resultSet = (ArrayList) result; assertEquals(0, resultSet.size()); } @@ -128,46 +126,46 @@ public void testSetLastSetUsage() throws Exception { /* allocate memory */ listVector.allocateNew(); - /* get inner vectors; bitVector and offsetVector */ - List innerVectors = listVector.getFieldInnerVectors(); - BitVector bitVector = (BitVector) innerVectors.get(0); - UInt4Vector offsetVector = (UInt4Vector) innerVectors.get(1); + /* get inner buffers; validityBuffer and offsetBuffer */ + + ArrowBuf validityBuffer = listVector.getValidityBuffer(); + ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* get the underlying data vector -- NullableBigIntVector */ NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); /* check current lastSet */ - assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet())); + assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet())); int index = 0; int offset = 0; - /* write [10, 11, 12] to the list vector at index */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(0, 1, 10); - dataVector.getMutator().setSafe(1, 1, 11); - dataVector.getMutator().setSafe(2, 1, 12); - offsetVector.getMutator().setSafe(index + 1, 3); + /* write [10, 11, 12] to the list vector at index 0 */ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(0, 1, 10); + dataVector.setSafe(1, 1, 11); + dataVector.setSafe(2, 1, 12); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 3); index += 1; /* write [13, 14] to the list vector at index 1 */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(3, 1, 13); - dataVector.getMutator().setSafe(4, 1, 14); - offsetVector.getMutator().setSafe(index + 1, 5); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(3, 1, 13); + dataVector.setSafe(4, 1, 14); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 5); index += 1; /* write [15, 16, 17] to the list vector at index 2 */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(5, 1, 15); - dataVector.getMutator().setSafe(6, 1, 16); - dataVector.getMutator().setSafe(7, 1, 17); - offsetVector.getMutator().setSafe(index + 1, 8); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(5, 1, 15); + dataVector.setSafe(6, 1, 16); + dataVector.setSafe(7, 1, 17); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 8); /* check current lastSet */ - assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet())); + assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet())); /* set lastset and arbitrary valuecount for list vector. * @@ -208,54 +206,50 @@ public void testSetLastSetUsage() throws Exception { * [15, 16, 17] * } */ - listVector.getMutator().setLastSet(3); - listVector.getMutator().setValueCount(10); - - /* check the vector output */ - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); - final ValueVector.Accessor valueAccessor = dataVector.getAccessor(); + listVector.setLastSet(3); + listVector.setValueCount(10); index = 0; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(0), Integer.toString(offset)); - Object actual = valueAccessor.getObject(offset); + Object actual = dataVector.getObject(offset); assertEquals(new Long(10), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(11), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(12), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(13), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(14), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); - actual = valueAccessor.getObject(offsetAccessor.get(index)); + actual = dataVector.getObject(offset); assertEquals(new Long(15), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(16), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(17), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(8), Integer.toString(offset)); - actual = valueAccessor.getObject(offsetAccessor.get(index)); + actual = dataVector.getObject(offset); assertNull(actual); } } @@ -308,99 +302,97 @@ public void testSplitAndTransfer() throws Exception { listWriter.bigInt().writeBigInt(23); listWriter.endList(); - listVector.getMutator().setValueCount(5); + listVector.setValueCount(5); - assertEquals(5, listVector.getMutator().getLastSet()); + assertEquals(5, listVector.getLastSet()); - /* get offsetVector */ - UInt4Vector offsetVector = (UInt4Vector) listVector.getOffsetVector(); + /* get offset buffer */ + final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* get dataVector */ NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); /* check the vector output */ - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); - final ValueVector.Accessor valueAccessor = dataVector.getAccessor(); int index = 0; int offset = 0; Object actual = null; /* index 0 */ - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(0), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(10), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(11), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(12), (Long) actual); /* index 1 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(13), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(14), (Long) actual); /* index 2 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(15), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(16), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(17), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(18), (Long) actual); /* index 3 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(9), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(19), (Long) actual); /* index 4 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(10), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(20), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(21), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(22), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(23), (Long) actual); /* index 5 */ index++; - assertTrue(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertTrue(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(14), Integer.toString(offset)); /* do split and transfer */ @@ -423,26 +415,26 @@ public void testSplitAndTransfer() throws Exception { transferPair.splitAndTransfer(start, splitLength); /* get offsetVector of toVector */ - UInt4Vector offsetVector1 = (UInt4Vector) toVector.getOffsetVector(); - UInt4Vector.Accessor offsetAccessor1 = offsetVector1.getAccessor(); + final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer(); /* get dataVector of toVector */ NullableBigIntVector dataVector1 = (NullableBigIntVector) toVector.getDataVector(); - NullableBigIntVector.Accessor valueAccessor1 = dataVector1.getAccessor(); for (int i = 0; i < splitLength; i++) { - dataLength1 = offsetAccessor.get(start + i + 1) - offsetAccessor.get(start + i); - dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i); + dataLength1 = offsetBuffer.getInt((start + i + 1) * ListVector.OFFSET_WIDTH) - + offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH); + dataLength2 = toOffsetBuffer.getInt((i + 1) * ListVector.OFFSET_WIDTH) + - toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); assertEquals("Different data lengths at index: " + i + " and start: " + start, dataLength1, dataLength2); - offset1 = offsetAccessor.get(start + i); - offset2 = offsetAccessor1.get(i); + offset1 = offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH); + offset2 = toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); for (int j = 0; j < dataLength1; j++) { assertEquals("Different data at indexes: " + offset1 + " and " + offset2, - valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2)); + dataVector.getObject(offset1), dataVector1.getObject(offset2)); offset1++; offset2++; @@ -506,15 +498,14 @@ public void testNestedListVector() throws Exception { listWriter.endList(); - assertEquals(2, listVector.getMutator().getLastSet()); + assertEquals(2, listVector.getLastSet()); - listVector.getMutator().setValueCount(2); + listVector.setValueCount(2); - final ListVector.Accessor accessor = listVector.getAccessor(); - assertEquals(2, accessor.getValueCount()); + assertEquals(2, listVector.getValueCount()); /* get listVector value at index 0 -- the value itself is a listvector */ - Object result = accessor.getObject(0); + Object result = listVector.getObject(0); ArrayList> resultSet = (ArrayList>) result; ArrayList list; @@ -534,7 +525,7 @@ public void testNestedListVector() throws Exception { assertEquals(new Long(175), list.get(3)); /* get listVector value at index 1 -- the value itself is a listvector */ - result = accessor.getObject(1); + result = listVector.getObject(1); resultSet = (ArrayList>) result; assertEquals(3, resultSet.size()); /* 3 inner lists at index 1 */ @@ -555,17 +546,16 @@ public void testNestedListVector() throws Exception { assertEquals(new Long(35), list.get(2)); /* check underlying bitVector */ - assertFalse(accessor.isNull(0)); - assertFalse(accessor.isNull(1)); + assertFalse(listVector.isNull(0)); + assertFalse(listVector.isNull(1)); - /* check underlying offsetVector */ - UInt4Vector offsetVector = listVector.getOffsetVector(); - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); + /* check underlying offsets */ + final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* listVector has 2 lists at index 0 and 3 lists at index 1 */ - assertEquals(0, offsetAccessor.get(0)); - assertEquals(2, offsetAccessor.get(1)); - assertEquals(5, offsetAccessor.get(2)); + assertEquals(0, offsetBuffer.getInt(0 * ListVector.OFFSET_WIDTH)); + assertEquals(2, offsetBuffer.getInt(1 * ListVector.OFFSET_WIDTH)); + assertEquals(5, offsetBuffer.getInt(2 * ListVector.OFFSET_WIDTH)); } } @@ -591,17 +581,15 @@ public void testGetBufferAddress() throws Exception { listWriter.bigInt().writeBigInt(300); listWriter.endList(); - final ListVector.Accessor accessor = listVector.getAccessor(); - /* check listVector contents */ - Object result = accessor.getObject(0); + Object result = listVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); assertEquals(new Long(50), resultSet.get(0)); assertEquals(new Long(100), resultSet.get(1)); assertEquals(new Long(200), resultSet.get(2)); - result = accessor.getObject(1); + result = listVector.getObject(1); resultSet = (ArrayList) result; assertEquals(2, resultSet.size()); assertEquals(new Long(250), resultSet.get(0)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index 86f0bf337f9d4..ede4f992291b4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -64,26 +64,24 @@ public void testUnionVector() throws Exception { unionVector.allocateNew(); // write some data - final UnionVector.Mutator mutator = unionVector.getMutator(); - mutator.setType(0, Types.MinorType.UINT4); - mutator.setSafe(0, uInt4Holder); - mutator.setType(2, Types.MinorType.UINT4); - mutator.setSafe(2, uInt4Holder); - mutator.setValueCount(4); + unionVector.setType(0, Types.MinorType.UINT4); + unionVector.setSafe(0, uInt4Holder); + unionVector.setType(2, Types.MinorType.UINT4); + unionVector.setSafe(2, uInt4Holder); + unionVector.setValueCount(4); // check that what we wrote is correct - final UnionVector.Accessor accessor = unionVector.getAccessor(); - assertEquals(4, accessor.getValueCount()); + assertEquals(4, unionVector.getValueCount()); - assertEquals(false, accessor.isNull(0)); - assertEquals(100, accessor.getObject(0)); + assertEquals(false, unionVector.isNull(0)); + assertEquals(100, unionVector.getObject(0)); - assertEquals(true, accessor.isNull(1)); + assertEquals(true, unionVector.isNull(1)); - assertEquals(false, accessor.isNull(2)); - assertEquals(100, accessor.getObject(2)); + assertEquals(false, unionVector.isNull(2)); + assertEquals(100, unionVector.getObject(2)); - assertEquals(true, accessor.isNull(3)); + assertEquals(true, unionVector.isNull(3)); } } @@ -93,16 +91,15 @@ public void testTransfer() throws Exception { srcVector.allocateNew(); // write some data - final UnionVector.Mutator mutator = srcVector.getMutator(); - mutator.setType(0, MinorType.INT); - mutator.setSafe(0, newIntHolder(5)); - mutator.setType(1, MinorType.BIT); - mutator.setSafe(1, newBitHolder(false)); - mutator.setType(3, MinorType.INT); - mutator.setSafe(3, newIntHolder(10)); - mutator.setType(5, MinorType.BIT); - mutator.setSafe(5, newBitHolder(false)); - mutator.setValueCount(6); + srcVector.setType(0, MinorType.INT); + srcVector.setSafe(0, newIntHolder(5)); + srcVector.setType(1, MinorType.BIT); + srcVector.setSafe(1, newBitHolder(false)); + srcVector.setType(3, MinorType.INT); + srcVector.setSafe(3, newIntHolder(10)); + srcVector.setType(5, MinorType.BIT); + srcVector.setSafe(5, newBitHolder(false)); + srcVector.setValueCount(6); try (UnionVector destVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { TransferPair pair = srcVector.makeTransferPair(destVector); @@ -116,9 +113,9 @@ public void testTransfer() throws Exception { assertEquals(srcVector.getField(), destVector.getField()); // now check the values are transferred - assertEquals(srcVector.getAccessor().getValueCount(), destVector.getAccessor().getValueCount()); - for (int i = 0; i < srcVector.getAccessor().getValueCount(); i++) { - assertEquals("Different values at index " + i, srcVector.getAccessor().get(i), destVector.getAccessor().get(i)); + assertEquals(srcVector.getValueCount(), destVector.getValueCount()); + for (int i = 0; i < srcVector.getValueCount(); i++) { + assertEquals("Different values at index " + i, srcVector.get(i), destVector.get(i)); } } } @@ -127,61 +124,58 @@ public void testTransfer() throws Exception { @Test public void testSplitAndTransfer() throws Exception { try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator sourceMutator = sourceVector.getMutator(); - final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(); /* populate the UnionVector */ - sourceMutator.setType(0, MinorType.INT); - sourceMutator.setSafe(0, newIntHolder(5)); - sourceMutator.setType(1, MinorType.INT); - sourceMutator.setSafe(1, newIntHolder(10)); - sourceMutator.setType(2, MinorType.INT); - sourceMutator.setSafe(2, newIntHolder(15)); - sourceMutator.setType(3, MinorType.INT); - sourceMutator.setSafe(3, newIntHolder(20)); - sourceMutator.setType(4, MinorType.INT); - sourceMutator.setSafe(4, newIntHolder(25)); - sourceMutator.setType(5, MinorType.INT); - sourceMutator.setSafe(5, newIntHolder(30)); - sourceMutator.setType(6, MinorType.INT); - sourceMutator.setSafe(6, newIntHolder(35)); - sourceMutator.setType(7, MinorType.INT); - sourceMutator.setSafe(7, newIntHolder(40)); - sourceMutator.setType(8, MinorType.INT); - sourceMutator.setSafe(8, newIntHolder(45)); - sourceMutator.setType(9, MinorType.INT); - sourceMutator.setSafe(9, newIntHolder(50)); - sourceMutator.setValueCount(10); + sourceVector.setType(0, MinorType.INT); + sourceVector.setSafe(0, newIntHolder(5)); + sourceVector.setType(1, MinorType.INT); + sourceVector.setSafe(1, newIntHolder(10)); + sourceVector.setType(2, MinorType.INT); + sourceVector.setSafe(2, newIntHolder(15)); + sourceVector.setType(3, MinorType.INT); + sourceVector.setSafe(3, newIntHolder(20)); + sourceVector.setType(4, MinorType.INT); + sourceVector.setSafe(4, newIntHolder(25)); + sourceVector.setType(5, MinorType.INT); + sourceVector.setSafe(5, newIntHolder(30)); + sourceVector.setType(6, MinorType.INT); + sourceVector.setSafe(6, newIntHolder(35)); + sourceVector.setType(7, MinorType.INT); + sourceVector.setSafe(7, newIntHolder(40)); + sourceVector.setType(8, MinorType.INT); + sourceVector.setSafe(8, newIntHolder(45)); + sourceVector.setType(9, MinorType.INT); + sourceVector.setSafe(9, newIntHolder(50)); + sourceVector.setValueCount(10); /* check the vector output */ - assertEquals(10, sourceAccessor.getValueCount()); - assertEquals(false, sourceAccessor.isNull(0)); - assertEquals(5, sourceAccessor.getObject(0)); - assertEquals(false, sourceAccessor.isNull(1)); - assertEquals(10, sourceAccessor.getObject(1)); - assertEquals(false, sourceAccessor.isNull(2)); - assertEquals(15, sourceAccessor.getObject(2)); - assertEquals(false, sourceAccessor.isNull(3)); - assertEquals(20, sourceAccessor.getObject(3)); - assertEquals(false, sourceAccessor.isNull(4)); - assertEquals(25, sourceAccessor.getObject(4)); - assertEquals(false, sourceAccessor.isNull(5)); - assertEquals(30, sourceAccessor.getObject(5)); - assertEquals(false, sourceAccessor.isNull(6)); - assertEquals(35, sourceAccessor.getObject(6)); - assertEquals(false, sourceAccessor.isNull(7)); - assertEquals(40, sourceAccessor.getObject(7)); - assertEquals(false, sourceAccessor.isNull(8)); - assertEquals(45, sourceAccessor.getObject(8)); - assertEquals(false, sourceAccessor.isNull(9)); - assertEquals(50, sourceAccessor.getObject(9)); + assertEquals(10, sourceVector.getValueCount()); + assertEquals(false, sourceVector.isNull(0)); + assertEquals(5, sourceVector.getObject(0)); + assertEquals(false, sourceVector.isNull(1)); + assertEquals(10, sourceVector.getObject(1)); + assertEquals(false, sourceVector.isNull(2)); + assertEquals(15, sourceVector.getObject(2)); + assertEquals(false, sourceVector.isNull(3)); + assertEquals(20, sourceVector.getObject(3)); + assertEquals(false, sourceVector.isNull(4)); + assertEquals(25, sourceVector.getObject(4)); + assertEquals(false, sourceVector.isNull(5)); + assertEquals(30, sourceVector.getObject(5)); + assertEquals(false, sourceVector.isNull(6)); + assertEquals(35, sourceVector.getObject(6)); + assertEquals(false, sourceVector.isNull(7)); + assertEquals(40, sourceVector.getObject(7)); + assertEquals(false, sourceVector.isNull(8)); + assertEquals(45, sourceVector.getObject(8)); + assertEquals(false, sourceVector.isNull(9)); + assertEquals(50, sourceVector.getObject(9)); try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final UnionVector.Accessor toAccessor = toVector.getAccessor(); final int[][] transferLengths = {{0, 3}, {3, 1}, @@ -199,8 +193,8 @@ public void testSplitAndTransfer() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceAccessor.getObject(start + i), - toAccessor.getObject(i)); + assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceVector.getObject(start + i), + toVector.getObject(i)); } } } @@ -210,70 +204,67 @@ public void testSplitAndTransfer() throws Exception { @Test public void testSplitAndTransferWithMixedVectors() throws Exception { try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator sourceMutator = sourceVector.getMutator(); - final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(); /* populate the UnionVector */ - sourceMutator.setType(0, MinorType.INT); - sourceMutator.setSafe(0, newIntHolder(5)); + sourceVector.setType(0, MinorType.INT); + sourceVector.setSafe(0, newIntHolder(5)); - sourceMutator.setType(1, MinorType.FLOAT4); - sourceMutator.setSafe(1, newFloat4Holder(5.5f)); + sourceVector.setType(1, MinorType.FLOAT4); + sourceVector.setSafe(1, newFloat4Holder(5.5f)); - sourceMutator.setType(2, MinorType.INT); - sourceMutator.setSafe(2, newIntHolder(10)); + sourceVector.setType(2, MinorType.INT); + sourceVector.setSafe(2, newIntHolder(10)); - sourceMutator.setType(3, MinorType.FLOAT4); - sourceMutator.setSafe(3, newFloat4Holder(10.5f)); + sourceVector.setType(3, MinorType.FLOAT4); + sourceVector.setSafe(3, newFloat4Holder(10.5f)); - sourceMutator.setType(4, MinorType.INT); - sourceMutator.setSafe(4, newIntHolder(15)); + sourceVector.setType(4, MinorType.INT); + sourceVector.setSafe(4, newIntHolder(15)); - sourceMutator.setType(5, MinorType.FLOAT4); - sourceMutator.setSafe(5, newFloat4Holder(15.5f)); + sourceVector.setType(5, MinorType.FLOAT4); + sourceVector.setSafe(5, newFloat4Holder(15.5f)); - sourceMutator.setType(6, MinorType.INT); - sourceMutator.setSafe(6, newIntHolder(20)); + sourceVector.setType(6, MinorType.INT); + sourceVector.setSafe(6, newIntHolder(20)); - sourceMutator.setType(7, MinorType.FLOAT4); - sourceMutator.setSafe(7, newFloat4Holder(20.5f)); + sourceVector.setType(7, MinorType.FLOAT4); + sourceVector.setSafe(7, newFloat4Holder(20.5f)); - sourceMutator.setType(8, MinorType.INT); - sourceMutator.setSafe(8, newIntHolder(30)); + sourceVector.setType(8, MinorType.INT); + sourceVector.setSafe(8, newIntHolder(30)); - sourceMutator.setType(9, MinorType.FLOAT4); - sourceMutator.setSafe(9, newFloat4Holder(30.5f)); - sourceMutator.setValueCount(10); + sourceVector.setType(9, MinorType.FLOAT4); + sourceVector.setSafe(9, newFloat4Holder(30.5f)); + sourceVector.setValueCount(10); /* check the vector output */ - assertEquals(10, sourceAccessor.getValueCount()); - assertEquals(false, sourceAccessor.isNull(0)); - assertEquals(5, sourceAccessor.getObject(0)); - assertEquals(false, sourceAccessor.isNull(1)); - assertEquals(5.5f, sourceAccessor.getObject(1)); - assertEquals(false, sourceAccessor.isNull(2)); - assertEquals(10, sourceAccessor.getObject(2)); - assertEquals(false, sourceAccessor.isNull(3)); - assertEquals(10.5f, sourceAccessor.getObject(3)); - assertEquals(false, sourceAccessor.isNull(4)); - assertEquals(15, sourceAccessor.getObject(4)); - assertEquals(false, sourceAccessor.isNull(5)); - assertEquals(15.5f, sourceAccessor.getObject(5)); - assertEquals(false, sourceAccessor.isNull(6)); - assertEquals(20, sourceAccessor.getObject(6)); - assertEquals(false, sourceAccessor.isNull(7)); - assertEquals(20.5f, sourceAccessor.getObject(7)); - assertEquals(false, sourceAccessor.isNull(8)); - assertEquals(30, sourceAccessor.getObject(8)); - assertEquals(false, sourceAccessor.isNull(9)); - assertEquals(30.5f, sourceAccessor.getObject(9)); + assertEquals(10, sourceVector.getValueCount()); + assertEquals(false, sourceVector.isNull(0)); + assertEquals(5, sourceVector.getObject(0)); + assertEquals(false, sourceVector.isNull(1)); + assertEquals(5.5f, sourceVector.getObject(1)); + assertEquals(false, sourceVector.isNull(2)); + assertEquals(10, sourceVector.getObject(2)); + assertEquals(false, sourceVector.isNull(3)); + assertEquals(10.5f, sourceVector.getObject(3)); + assertEquals(false, sourceVector.isNull(4)); + assertEquals(15, sourceVector.getObject(4)); + assertEquals(false, sourceVector.isNull(5)); + assertEquals(15.5f, sourceVector.getObject(5)); + assertEquals(false, sourceVector.isNull(6)); + assertEquals(20, sourceVector.getObject(6)); + assertEquals(false, sourceVector.isNull(7)); + assertEquals(20.5f, sourceVector.getObject(7)); + assertEquals(false, sourceVector.isNull(8)); + assertEquals(30, sourceVector.getObject(8)); + assertEquals(false, sourceVector.isNull(9)); + assertEquals(30.5f, sourceVector.getObject(9)); try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final UnionVector.Accessor toAccessor = toVector.getAccessor(); final int[][] transferLengths = {{0, 2}, {2, 1}, @@ -290,7 +281,7 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different values at index: " + i, sourceAccessor.getObject(start + i), toAccessor.getObject(i)); + assertEquals("Different values at index: " + i, sourceVector.getObject(start + i), toVector.getObject(i)); } } } @@ -300,37 +291,35 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { @Test public void testGetBufferAddress() throws Exception { try (UnionVector vector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator mutator = vector.getMutator(); - final UnionVector.Accessor accessor = vector.getAccessor(); boolean error = false; vector.allocateNew(); /* populate the UnionVector */ - mutator.setType(0, MinorType.INT); - mutator.setSafe(0, newIntHolder(5)); + vector.setType(0, MinorType.INT); + vector.setSafe(0, newIntHolder(5)); - mutator.setType(1, MinorType.FLOAT4); - mutator.setSafe(1, newFloat4Holder(5.5f)); + vector.setType(1, MinorType.FLOAT4); + vector.setSafe(1, newFloat4Holder(5.5f)); - mutator.setType(2, MinorType.INT); - mutator.setSafe(2, newIntHolder(10)); + vector.setType(2, MinorType.INT); + vector.setSafe(2, newIntHolder(10)); - mutator.setType(3, MinorType.FLOAT4); - mutator.setSafe(3, newFloat4Holder(10.5f)); + vector.setType(3, MinorType.FLOAT4); + vector.setSafe(3, newFloat4Holder(10.5f)); - mutator.setValueCount(10); + vector.setValueCount(10); /* check the vector output */ - assertEquals(10, accessor.getValueCount()); - assertEquals(false, accessor.isNull(0)); - assertEquals(5, accessor.getObject(0)); - assertEquals(false, accessor.isNull(1)); - assertEquals(5.5f, accessor.getObject(1)); - assertEquals(false, accessor.isNull(2)); - assertEquals(10, accessor.getObject(2)); - assertEquals(false, accessor.isNull(3)); - assertEquals(10.5f, accessor.getObject(3)); + assertEquals(10, vector.getValueCount()); + assertEquals(false, vector.isNull(0)); + assertEquals(5, vector.getObject(0)); + assertEquals(false, vector.isNull(1)); + assertEquals(5.5f, vector.getObject(1)); + assertEquals(false, vector.isNull(2)); + assertEquals(10, vector.getObject(2)); + assertEquals(false, vector.isNull(3)); + assertEquals(10.5f, vector.getObject(3)); List buffers = vector.getFieldBuffers(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index b7f88c3babf1b..c7ee202f94661 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -487,8 +487,6 @@ public void testNullableFixedType1() { // Create a new value vector for 1024 integers. try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) { - final NullableUInt4Vector.Mutator mutator = vector.getMutator(); - final NullableUInt4Vector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 1024; @@ -500,33 +498,33 @@ public void testNullableFixedType1() { assertEquals(initialCapacity, vector.getValueCapacity()); // Put and set a few values - mutator.set(0, 100); - mutator.set(1, 101); - mutator.set(100, 102); - mutator.set(1022, 103); - mutator.set(1023, 104); + vector.set(0, 100); + vector.set(1, 101); + vector.set(100, 102); + vector.set(1022, 103); + vector.set(1023, 104); /* check vector contents */ - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); int val = 0; /* check unset bits/null values */ for (int i = 2, j = 101; i <= 99 || j <= 1021; i++, j++) { if (i <= 99) { - assertTrue(accessor.isNull(i)); + assertTrue(vector.isNull(i)); } if(j <= 1021) { - assertTrue(accessor.isNull(j)); + assertTrue(vector.isNull(j)); } } try { - mutator.set(1024, 10000); + vector.set(1024, 10000); } catch (IndexOutOfBoundsException ie) { error = true; @@ -537,7 +535,7 @@ public void testNullableFixedType1() { } try { - accessor.get(1024); + vector.get(1024); } catch (IndexOutOfBoundsException ie) { error = true; @@ -548,28 +546,28 @@ public void testNullableFixedType1() { } /* should trigger a realloc of the underlying bitvector and valuevector */ - mutator.setSafe(1024, 10000); + vector.setSafe(1024, 10000); /* check new capacity */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); /* vector contents should still be intact after realloc */ - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); - assertEquals(10000, accessor.get(1024)); + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); + assertEquals(10000, vector.get(1024)); val = 0; /* check unset bits/null values */ for (int i = 2, j = 101; i < 99 || j < 1021; i++, j++) { if (i <= 99) { - assertTrue(accessor.isNull(i)); + assertTrue(vector.isNull(i)); } if(j <= 1021) { - assertTrue(accessor.isNull(j)); + assertTrue(vector.isNull(j)); } } @@ -581,7 +579,7 @@ public void testNullableFixedType1() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -590,8 +588,6 @@ public void testNullableFixedType1() { public void testNullableFixedType2() { // Create a new value vector for 1024 integers try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) { - final NullableFloat4Vector.Mutator mutator = vector.getMutator(); - final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 16; @@ -603,17 +599,17 @@ public void testNullableFixedType2() { assertEquals(initialCapacity, vector.getValueCapacity()); /* populate the vector */ - mutator.set(0, 100.5f); - mutator.set(2, 201.5f); - mutator.set(4, 300.3f); - mutator.set(6, 423.8f); - mutator.set(8, 555.6f); - mutator.set(10, 66.6f); - mutator.set(12, 78.8f); - mutator.set(14, 89.5f); + vector.set(0, 100.5f); + vector.set(2, 201.5f); + vector.set(4, 300.3f); + vector.set(6, 423.8f); + vector.set(8, 555.6f); + vector.set(10, 66.6f); + vector.set(12, 78.8f); + vector.set(14, 89.5f); try { - mutator.set(16, 90.5f); + vector.set(16, 90.5f); } catch (IndexOutOfBoundsException ie) { error = true; @@ -624,25 +620,25 @@ public void testNullableFixedType2() { } /* check vector contents */ - assertEquals(100.5f, accessor.get(0), 0); - assertTrue(accessor.isNull(1)); - assertEquals(201.5f, accessor.get(2), 0); - assertTrue(accessor.isNull(3)); - assertEquals(300.3f, accessor.get(4), 0); - assertTrue(accessor.isNull(5)); - assertEquals(423.8f, accessor.get(6), 0); - assertTrue(accessor.isNull(7)); - assertEquals(555.6f, accessor.get(8), 0); - assertTrue(accessor.isNull(9)); - assertEquals(66.6f, accessor.get(10), 0); - assertTrue(accessor.isNull(11)); - assertEquals(78.8f, accessor.get(12), 0); - assertTrue(accessor.isNull(13)); - assertEquals(89.5f, accessor.get(14), 0); - assertTrue(accessor.isNull(15)); + assertEquals(100.5f, vector.get(0), 0); + assertTrue(vector.isNull(1)); + assertEquals(201.5f, vector.get(2), 0); + assertTrue(vector.isNull(3)); + assertEquals(300.3f, vector.get(4), 0); + assertTrue(vector.isNull(5)); + assertEquals(423.8f, vector.get(6), 0); + assertTrue(vector.isNull(7)); + assertEquals(555.6f, vector.get(8), 0); + assertTrue(vector.isNull(9)); + assertEquals(66.6f, vector.get(10), 0); + assertTrue(vector.isNull(11)); + assertEquals(78.8f, vector.get(12), 0); + assertTrue(vector.isNull(13)); + assertEquals(89.5f, vector.get(14), 0); + assertTrue(vector.isNull(15)); try { - accessor.get(16); + vector.get(16); } catch (IndexOutOfBoundsException ie) { error = true; @@ -653,29 +649,28 @@ public void testNullableFixedType2() { } /* this should trigger a realloc() */ - mutator.setSafe(16, 90.5f); + vector.setSafe(16, 90.5f); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); /* vector data should still be intact after realloc */ - assertEquals(100.5f, accessor.get(0), 0); - assertTrue(accessor.isNull(1)); - assertEquals(201.5f, accessor.get(2), 0); - assertTrue(accessor.isNull(3)); - assertEquals(300.3f, accessor.get(4), 0); - assertTrue(accessor.isNull(5)); - assertEquals(423.8f, accessor.get(6), 0); - assertTrue(accessor.isNull(7)); - assertEquals(555.6f, accessor.get(8), 0); - assertTrue(accessor.isNull(9)); - assertEquals(66.6f, accessor.get(10), 0); - assertTrue(accessor.isNull(11)); - assertEquals(78.8f, accessor.get(12), 0); - assertTrue(accessor.isNull(13)); - assertEquals(89.5f, accessor.get(14), 0); - assertTrue(accessor.isNull(15)); - assertEquals(90.5f, accessor.get(16), 0); + assertEquals(100.5f, vector.get(0), 0); + assertTrue(vector.isNull(1)); + assertEquals(201.5f, vector.get(2), 0); + assertTrue(vector.isNull(3)); + assertEquals(300.3f, vector.get(4), 0); + assertTrue(vector.isNull(5)); + assertEquals(423.8f, vector.get(6), 0); + assertTrue(vector.isNull(7)); + assertEquals(555.6f, vector.get(8), 0); + assertTrue(vector.isNull(9)); + assertEquals(66.6f, vector.get(10), 0); + assertTrue(vector.isNull(11)); + assertEquals(78.8f, vector.get(12), 0); + assertTrue(vector.isNull(13)); + assertEquals(89.5f, vector.get(14), 0); + assertTrue(vector.isNull(15)); /* reset the vector */ vector.reset(); @@ -685,7 +680,7 @@ public void testNullableFixedType2() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -928,32 +923,30 @@ public void testNullableVarType2() { // Create a new value vector for 1024 integers. try (final NullableVarBinaryVector vector = newNullableVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableVarBinaryVector.Mutator m = vector.getMutator(); vector.allocateNew(1024 * 10, 1024); - m.set(0, STR1); - m.set(1, STR2); - m.set(2, STR3); - m.setSafe(3, STR3, 1, STR3.length - 1); - m.setSafe(4, STR3, 2, STR3.length - 2); + vector.set(0, STR1); + vector.set(1, STR2); + vector.set(2, STR3); + vector.setSafe(3, STR3, 1, STR3.length - 1); + vector.setSafe(4, STR3, 2, STR3.length - 2); ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3); - m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); - m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); + vector.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); + vector.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); // Check the sample strings. - final NullableVarBinaryVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(3)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(4)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(5)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(6)); // Ensure null value throws. boolean b = false; try { - vector.getAccessor().get(7); + vector.get(7); } catch (IllegalStateException e) { b = true; } finally { @@ -1064,8 +1057,6 @@ public void testReallocAfterVectorTransfer1() { @Test /* NullableFloat8Vector */ public void testReallocAfterVectorTransfer2() { try (final NullableFloat8Vector vector = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableFloat8Vector.Mutator mutator = vector.getMutator(); - final NullableFloat8Vector.Accessor accessor = vector.getAccessor(); final int initialDefaultCapacity = 4096; boolean error = false; @@ -1076,7 +1067,7 @@ public void testReallocAfterVectorTransfer2() { double baseValue = 100.375; for (int i = 0; i < initialDefaultCapacity; i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } /* the above setSafe calls should not have triggered a realloc as @@ -1085,33 +1076,33 @@ public void testReallocAfterVectorTransfer2() { assertEquals(initialDefaultCapacity, vector.getValueCapacity()); for (int i = 0; i < initialDefaultCapacity; i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); + vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity()); for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 2); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); + vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity()); for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 4); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } @@ -1123,12 +1114,11 @@ public void testReallocAfterVectorTransfer2() { transferPair.transfer(); NullableFloat8Vector toVector = (NullableFloat8Vector)transferPair.getTo(); - final NullableFloat8Vector.Accessor toAccessor = toVector.getAccessor(); /* check toVector contents before realloc */ for (int i = 0; i < (initialDefaultCapacity * 4); i++) { - assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i)); - double value = toAccessor.get(i); + assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + double value = toVector.get(i); assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0); } @@ -1138,12 +1128,12 @@ public void testReallocAfterVectorTransfer2() { for (int i = 0; i < (initialDefaultCapacity * 8); i++) { if (i < (initialDefaultCapacity * 4)) { - assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i)); - double value = toAccessor.get(i); + assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + double value = toVector.get(i); assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0); } else { - assertTrue("unexpected non-null value at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected non-null value at index: " + i, toVector.isNull(i)); } } @@ -1351,32 +1341,30 @@ public void testReallocAfterVectorTransfer4() { public void testReAllocNullableFixedWidthVector() { // Create a new value vector for 1024 integers try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) { - final NullableFloat4Vector.Mutator m = vector.getMutator(); vector.allocateNew(1024); assertEquals(1024, vector.getValueCapacity()); // Put values in indexes that fall within the initial allocation - m.setSafe(0, 100.1f); - m.setSafe(100, 102.3f); - m.setSafe(1023, 104.5f); + vector.setSafe(0, 100.1f); + vector.setSafe(100, 102.3f); + vector.setSafe(1023, 104.5f); // Now try to put values in space that falls beyond the initial allocation - m.setSafe(2000, 105.5f); + vector.setSafe(2000, 105.5f); // Check valueCapacity is more than initial allocation assertEquals(1024 * 2, vector.getValueCapacity()); - final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); - assertEquals(100.1f, accessor.get(0), 0); - assertEquals(102.3f, accessor.get(100), 0); - assertEquals(104.5f, accessor.get(1023), 0); - assertEquals(105.5f, accessor.get(2000), 0); + assertEquals(100.1f, vector.get(0), 0); + assertEquals(102.3f, vector.get(100), 0); + assertEquals(104.5f, vector.get(1023), 0); + assertEquals(105.5f, vector.get(2000), 0); // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the // vector - m.setValueCount(vector.getValueCapacity() + 200); + vector.setValueCount(vector.getValueCapacity() + 200); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 531a46c2734ef..293ffbfe19233 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.After; import org.junit.Assert; @@ -115,7 +116,7 @@ public void testListType() { vector.reAlloc(); assertEquals(2047, vector.getValueCapacity()); // note: size - 1 - assertEquals(0, vector.getOffsetVector().getAccessor().get(2014)); + assertEquals(0, vector.getOffsetBuffer().getInt(2014 * ListVector.OFFSET_WIDTH)); } } @@ -138,7 +139,7 @@ public void testMapType() { vector.reAlloc(); assertEquals(1024, vector.getValueCapacity()); - assertNull(vector.getAccessor().getObject(513)); + assertNull(vector.getObject(513)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 0b7928dca2c39..3853eecac0552 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -199,8 +199,10 @@ public void testLoadValidityBuffer() throws IOException { int count = 10; ArrowBuf[] values = new ArrowBuf[4]; for (int i = 0; i < 4; i+=2) { - ArrowBuf buf1 = allocator.buffer((int)Math.ceil(count / 8.0)); + ArrowBuf buf1 = allocator.buffer(BitVectorHelper.getValidityBufferSize(count)); ArrowBuf buf2 = allocator.buffer(count * 4); // integers + buf1.setZero(0, buf1.capacity()); + buf2.setZero(0, buf2.capacity()); values[i] = buf1; values[i+1] = buf2; for (int j = 0; j < count; j++) { @@ -314,7 +316,7 @@ public void testUnloadLoadDuplicates() throws IOException { public static VectorUnloader newVectorUnloader(FieldVector root) { Schema schema = new Schema(root.getField().getChildren()); - int valueCount = root.getAccessor().getValueCount(); + int valueCount = root.getValueCount(); List fields = root.getChildrenFromFields(); VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount); return new VectorUnloader(vsr); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 97efb7d5a6d30..b0d6cf555e98d 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -80,24 +80,23 @@ public void testPromoteToUnion() throws Exception { writer.end(); - container.getMutator().setValueCount(5); + container.setValueCount(5); final UnionVector uv = v.getChild("A", UnionVector.class); - final UnionVector.Accessor accessor = uv.getAccessor(); - assertFalse("0 shouldn't be null", accessor.isNull(0)); - assertEquals(false, accessor.getObject(0)); + assertFalse("0 shouldn't be null", uv.isNull(0)); + assertEquals(false, uv.getObject(0)); - assertFalse("1 shouldn't be null", accessor.isNull(1)); - assertEquals(true, accessor.getObject(1)); + assertFalse("1 shouldn't be null", uv.isNull(1)); + assertEquals(true, uv.getObject(1)); - assertFalse("2 shouldn't be null", accessor.isNull(2)); - assertEquals(10, accessor.getObject(2)); + assertFalse("2 shouldn't be null", uv.isNull(2)); + assertEquals(10, uv.getObject(2)); - assertTrue("3 should be null", accessor.isNull(3)); + assertTrue("3 should be null", uv.isNull(3)); - assertFalse("4 shouldn't be null", accessor.isNull(4)); - assertEquals(100, accessor.getObject(4)); + assertFalse("4 shouldn't be null", uv.isNull(4)); + assertEquals(100, uv.getObject(4)); container.clear(); container.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 856d60724b085..52defdc2e1b00 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -456,7 +456,7 @@ public void simpleUnion() { unionWriter.writeFloat4((float) i); } } - vector.getMutator().setValueCount(COUNT); + vector.setValueCount(COUNT); UnionReader unionReader = new UnionReader(vector); for (int i = 0; i < COUNT; i++) { unionReader.setPosition(i); @@ -834,7 +834,7 @@ public void complexCopierWithList() { TransferPair tp = mapVector.getTransferPair(allocator); tp.splitAndTransfer(0, 1); MapVector toMapVector = (MapVector) tp.getTo(); - JsonStringHashMap toMapValue = (JsonStringHashMap) toMapVector.getAccessor().getObject(0); + JsonStringHashMap toMapValue = (JsonStringHashMap) toMapVector.getObject(0); JsonStringArrayList object = (JsonStringArrayList) toMapValue.get("list"); assertEquals(1, object.get(0)); assertEquals(2, object.get(1)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 60009b0a46396..874ba99e20f29 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -97,14 +97,8 @@ public void tearDown() { protected void validateContent(int count, VectorSchemaRoot root) { for (int i = 0; i < count; i++) { - FieldVector fv = root.getVector("int"); - if (fv instanceof NullableIntVector) { - Assert.assertEquals(i, fv.getObject(i)); - Assert.assertEquals(Integer.valueOf(i), fv.getObject(i)); - } else { - Assert.assertEquals(i, fv.getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), fv.getAccessor().getObject(i)); - } + Assert.assertEquals(i, root.getVector("int").getObject(i)); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); } } @@ -146,10 +140,9 @@ protected void writeComplexData(int count, MapVector parent) { public void printVectors(List vectors) { for (FieldVector vector : vectors) { LOGGER.debug(vector.getField().getName()); - Accessor accessor = vector.getAccessor(); - int valueCount = accessor.getValueCount(); + int valueCount = vector.getValueCount(); for (int i = 0; i < valueCount; i++) { - LOGGER.debug(String.valueOf(accessor.getObject(i))); + LOGGER.debug(String.valueOf(vector.getObject(i))); } } } @@ -159,14 +152,14 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) { printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { - Object intVal = root.getVector("int").getAccessor().getObject(i); + Object intVal = root.getVector("int").getObject(i); if (i % 5 != 3) { Assert.assertEquals(i, intVal); } else { Assert.assertNull(intVal); } - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); - Assert.assertEquals(i % 3, ((List) root.getVector("list").getAccessor().getObject(i)).size()); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); + Assert.assertEquals(i % 3, ((List) root.getVector("list").getObject(i)).size()); NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder(); FieldReader mapReader = root.getVector("map").getReader(); mapReader.setPosition(i); @@ -209,15 +202,15 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { - long dateVal = ((NullableDateMilliVector) root.getVector("date")).getAccessor().get(i); + long dateVal = ((NullableDateMilliVector) root.getVector("date")).get(i); LocalDateTime dt = makeDateTimeFromCount(i); LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal); - long timeVal = ((NullableTimeMilliVector) root.getVector("time")).getAccessor().get(i); + long timeVal = ((NullableTimeMilliVector) root.getVector("time")).get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); - Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i); + Object timestampMilliVal = root.getVector("timestamp-milli").getObject(i); Assert.assertEquals(dt, timestampMilliVal); - Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getAccessor().getObject(i); + Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getObject(i); Assert.assertEquals(DateUtility.toMillis(dt), timestampMilliTZVal); } } @@ -282,7 +275,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat List fields = ImmutableList.of(encodedVector1A.getField(), encodedVector1B.getField(), encodedVector2.getField()); List vectors = ImmutableList.of(encodedVector1A, encodedVector1B, encodedVector2); - return new VectorSchemaRoot(fields, vectors, encodedVector1A.getAccessor().getValueCount()); + return new VectorSchemaRoot(fields, vectors, encodedVector1A.getValueCount()); } protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider provider) { @@ -293,14 +286,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertNotNull(encoding1A); Assert.assertEquals(1L, encoding1A.getId()); - FieldVector.Accessor accessor = vector1A.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(0, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(null, accessor.getObject(2)); - Assert.assertEquals(2, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(2, accessor.getObject(5)); + Assert.assertEquals(6, vector1A.getValueCount()); + Assert.assertEquals(0, vector1A.getObject(0)); + Assert.assertEquals(1, vector1A.getObject(1)); + Assert.assertEquals(null, vector1A.getObject(2)); + Assert.assertEquals(2, vector1A.getObject(3)); + Assert.assertEquals(1, vector1A.getObject(4)); + Assert.assertEquals(2, vector1A.getObject(5)); FieldVector vector1B = root.getVector("varcharB"); Assert.assertNotNull(vector1B); @@ -310,14 +302,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertTrue(encoding1A.equals(encoding1B)); Assert.assertEquals(1L, encoding1B.getId()); - accessor = vector1B.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(2, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(2, accessor.getObject(2)); - Assert.assertEquals(null, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(0, accessor.getObject(5)); + Assert.assertEquals(6, vector1B.getValueCount()); + Assert.assertEquals(2, vector1B.getObject(0)); + Assert.assertEquals(1, vector1B.getObject(1)); + Assert.assertEquals(2, vector1B.getObject(2)); + Assert.assertEquals(null, vector1B.getObject(3)); + Assert.assertEquals(1, vector1B.getObject(4)); + Assert.assertEquals(0, vector1B.getObject(5)); FieldVector vector2 = root.getVector("sizes"); Assert.assertNotNull(vector2); @@ -326,30 +317,29 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertNotNull(encoding2); Assert.assertEquals(2L, encoding2.getId()); - accessor = vector2.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(null, accessor.getObject(0)); - Assert.assertEquals(2, accessor.getObject(1)); - Assert.assertEquals(1, accessor.getObject(2)); - Assert.assertEquals(1, accessor.getObject(3)); - Assert.assertEquals(2, accessor.getObject(4)); - Assert.assertEquals(null, accessor.getObject(5)); + Assert.assertEquals(6, vector2.getValueCount()); + Assert.assertEquals(null, vector2.getObject(0)); + Assert.assertEquals(2, vector2.getObject(1)); + Assert.assertEquals(1, vector2.getObject(2)); + Assert.assertEquals(1, vector2.getObject(3)); + Assert.assertEquals(2, vector2.getObject(4)); + Assert.assertEquals(null, vector2.getObject(5)); Dictionary dictionary1 = provider.lookup(1L); Assert.assertNotNull(dictionary1); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary1.getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary1.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2)); Dictionary dictionary2 = provider.lookup(2L); Assert.assertNotNull(dictionary2); - dictionaryAccessor = ((NullableVarCharVector) dictionary2.getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("micro"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("small"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("large"), dictionaryAccessor.getObject(2)); + dictionaryVector = ((NullableVarCharVector) dictionary2.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("micro"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("small"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("large"), dictionaryVector.getObject(2)); } protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) { @@ -359,7 +349,7 @@ protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAlloc dictionaryVector.allocateNewSafe(); dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - dictionaryVector.getMutator().setValueCount(2); + dictionaryVector.setValueCount(2); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null)); provider.put(dictionary); @@ -398,18 +388,17 @@ protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvide Assert.assertEquals(2L, encoding.getId()); Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType()); - FieldVector.Accessor accessor = vector.getAccessor(); - Assert.assertEquals(3, accessor.getValueCount()); - Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0)); - Assert.assertEquals(Arrays.asList(0), accessor.getObject(1)); - Assert.assertEquals(Arrays.asList(1), accessor.getObject(2)); + Assert.assertEquals(3, vector.getValueCount()); + Assert.assertEquals(Arrays.asList(0, 1), vector.getObject(0)); + Assert.assertEquals(Arrays.asList(0), vector.getObject(1)); + Assert.assertEquals(Arrays.asList(1), vector.getObject(2)); Dictionary dictionary = provider.lookup(2L); Assert.assertNotNull(dictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor(); - Assert.assertEquals(2, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + Assert.assertEquals(2, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { @@ -423,14 +412,14 @@ protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { decimalVector3.allocateNew(count); for (int i = 0; i < count; i++) { - decimalVector1.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3)); - decimalVector2.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2)); - decimalVector3.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8)); + decimalVector1.setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3)); + decimalVector2.setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2)); + decimalVector3.setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8)); } - decimalVector1.getMutator().setValueCount(count); - decimalVector2.getMutator().setValueCount(count); - decimalVector3.getMutator().setValueCount(count); + decimalVector1.setValueCount(count); + decimalVector2.setValueCount(count); + decimalVector3.setValueCount(count); List fields = ImmutableList.of(decimalVector1.getField(), decimalVector2.getField(), decimalVector3.getField()); List vectors = ImmutableList.of(decimalVector1, decimalVector2, decimalVector3); @@ -446,19 +435,19 @@ protected void validateDecimalData(VectorSchemaRoot root) { for (int i = 0; i < count; i++) { // Verify decimal 1 vector - BigDecimal readValue = decimalVector1.getAccessor().getObject(i); + BigDecimal readValue = decimalVector1.getObject(i); ArrowType.Decimal type = (ArrowType.Decimal) decimalVector1.getField().getType(); BigDecimal genValue = new BigDecimal(BigInteger.valueOf(i), type.getScale()); Assert.assertEquals(genValue, readValue); // Verify decimal 2 vector - readValue = decimalVector2.getAccessor().getObject(i); + readValue = decimalVector2.getObject(i); type = (ArrowType.Decimal) decimalVector2.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * (1 << 10)), type.getScale()); Assert.assertEquals(genValue, readValue); // Verify decimal 3 vector - readValue = decimalVector3.getAccessor().getObject(i); + readValue = decimalVector3.getObject(i); type = (ArrowType.Decimal) decimalVector3.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), type.getScale()); Assert.assertEquals(genValue, readValue); @@ -571,7 +560,7 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { int numVarBinaryValues = 0; for (int i = 0; i < count; i++) { expectedArray[i] = (byte) i; - Object obj = listVector.getAccessor().getObject(i); + Object obj = listVector.getObject(i); List objList = (List) obj; if (i % 3 == 0) { Assert.assertTrue(objList.isEmpty()); @@ -586,10 +575,10 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { } // ListVector lastSet should be the index of last value + 1 - Assert.assertEquals(listVector.getMutator().getLastSet(), count); + Assert.assertEquals(listVector.getLastSet(), count); // NullableVarBinaryVector lastSet should be the index of last value NullableVarBinaryVector binaryVector = (NullableVarBinaryVector) listVector.getChildrenFromFields().get(0); - Assert.assertEquals(binaryVector.getMutator().getLastSet(), numVarBinaryValues - 1); + Assert.assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java index feae08ea223e1..8559969a20424 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java @@ -317,11 +317,11 @@ public void testWriteReadTiny() throws IOException { try (VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); // write file @@ -368,9 +368,9 @@ private void validateTinyData(VectorSchemaRoot root) { NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i)); + Assert.assertEquals((byte) (i + 1), vector.get(i)); } else { - Assert.assertTrue(vector.getAccessor().isNull(i)); + Assert.assertTrue(vector.isNull(i)); } } } @@ -397,7 +397,7 @@ public void testWriteReadMetadata() throws IOException { try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) { vector.allocateNewSafe(); - vector.getMutator().setValueCount(0); + vector.setValueCount(0); List vectors = ImmutableList.of(vector); VectorSchemaRoot root = new VectorSchemaRoot(originalSchema, vectors, 0); @@ -578,13 +578,13 @@ public void testWriteReadFixedSizeList() throws IOException { parent.allocateNew(); for (int i = 0; i < 10; i++) { - tuples.getMutator().setNotNull(i); - floats.getMutator().set(i * 2, i + 0.1f); - floats.getMutator().set(i * 2 + 1, i + 10.1f); + tuples.setNotNull(i); + floats.set(i * 2, i + 0.1f); + floats.set(i * 2 + 1, i + 10.1f); ints.set(i, i); } - parent.getMutator().setValueCount(10); + parent.setValueCount(10); write(parent, file, stream); } @@ -600,8 +600,8 @@ public void testWriteReadFixedSizeList() throws IOException { arrowReader.loadRecordBatch(rbBlock); Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < 10; i++) { - Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i)); - Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i)); + Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i)); + Assert.assertEquals(i, root.getVector("ints").getObject(i)); } } } @@ -616,8 +616,8 @@ public void testWriteReadFixedSizeList() throws IOException { arrowReader.loadNextBatch(); Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < 10; i++) { - Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i)); - Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i)); + Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i)); + Assert.assertEquals(i, root.getVector("ints").getObject(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java index e2efabef0095b..c7e34191a3f1a 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java @@ -70,11 +70,11 @@ public void testReadWrite() throws IOException { int numBatches = 1; root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector)root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); ByteArrayOutputStream out = new ByteArrayOutputStream(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java index 40716942f02a2..f393733cc9021 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java @@ -62,13 +62,13 @@ public void run() { writer.start(); for (int j = 0; j < numBatches; j++) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); // Send a changing batch id first - mutator.set(0, j); + vector.set(0, j); for (int i = 1; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); writer.writeBatch(); @@ -117,12 +117,12 @@ public boolean loadNextBatch() throws IOException { VectorSchemaRoot root = getVectorSchemaRoot(); Assert.assertEquals(16, root.getRowCount()); NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); - Assert.assertEquals((byte) (batchesRead - 1), vector.getAccessor().get(0)); + Assert.assertEquals((byte) (batchesRead - 1), vector.get(0)); for (int i = 1; i < 16; i++) { if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i)); + Assert.assertEquals((byte) (i + 1), vector.get(i)); } else { - Assert.assertTrue(vector.getAccessor().isNull(i)); + Assert.assertTrue(vector.isNull(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java index ee90d340d7cc8..5c4c48cd26b60 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java @@ -280,7 +280,7 @@ public void testSetStructLength() throws IOException { // initialize vectors try (VectorSchemaRoot root = reader.read();) { FieldVector vector = root.getVector("struct_nullable"); - Assert.assertEquals(7, vector.getAccessor().getValueCount()); + Assert.assertEquals(7, vector.getValueCount()); } } } From 5bea9836ffa044e07cf376a5c0bfd88afc3560d4 Mon Sep 17 00:00:00 2001 From: Li Jin Date: Tue, 7 Nov 2017 11:12:57 -0500 Subject: [PATCH 050/177] ARROW-1717: [Java] Refactor JsonReader for new class hierarchy and fix integration tests --- .../BaseNullableVariableWidthVector.java | 46 -- .../arrow/vector/NullableBigIntVector.java | 37 +- .../arrow/vector/NullableDateDayVector.java | 34 -- .../arrow/vector/NullableDateMilliVector.java | 34 -- .../arrow/vector/NullableDecimalVector.java | 37 +- .../arrow/vector/NullableFloat4Vector.java | 37 +- .../arrow/vector/NullableFloat8Vector.java | 37 +- .../arrow/vector/NullableIntVector.java | 36 +- .../arrow/vector/NullableSmallIntVector.java | 37 +- .../arrow/vector/NullableTimeMicroVector.java | 34 -- .../arrow/vector/NullableTimeMilliVector.java | 34 -- .../arrow/vector/NullableTimeNanoVector.java | 34 -- .../arrow/vector/NullableTimeSecVector.java | 34 -- .../arrow/vector/NullableTimeStampVector.java | 36 -- .../arrow/vector/NullableTinyIntVector.java | 36 -- .../vector/file/json/JsonFileReader.java | 424 ++++++++++++------ 16 files changed, 290 insertions(+), 677 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index 6587cde633ea9..47daeee7f202c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -909,52 +909,6 @@ protected final void handleSafe(int index, int dataLength) { } } - - /****************************************************************** - * * - * helper methods currently * - * used by JsonFileReader and * - * JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Method used by Json Reader to explicitly set the data of the variable - * width vector elements. The method takes care of allocating the memory - * for the vector if caller hasn't done so. - * - * This method should not be used externally. - * - * @param data ArrowBuf for storing variable width elements in the vector - * @param offset offset of the element - * @param allocator memory allocator - * @param index position of the element in the vector - * @param value array of bytes for the element - * @param valueCount number of elements in the vector - * @return buffer holding the variable width data. - */ - public static ArrowBuf set(ArrowBuf data, ArrowBuf offset, - BufferAllocator allocator, int index, byte[] value, - int valueCount) { - if (data == null) { - data = allocator.buffer(INITIAL_BYTE_COUNT); - } - final int currentBufferCapacity = data.capacity(); - final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); - while (currentBufferCapacity < currentStartOffset + value.length) { - final ArrowBuf newBuf = allocator.buffer(currentBufferCapacity * 2); - newBuf.setBytes(0, data, 0, currentBufferCapacity); - data.release(); - data = newBuf; - } - data.setBytes(currentStartOffset, value, 0, value.length); - if (index == (valueCount - 1)) { - data.writerIndex(offset.getInt(valueCount * OFFSET_WIDTH)); - } - return data; - } - /** * Method used by Json Writer to read a variable width element from * the variable width vector and write to Json. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java index ee40d708cc199..253427333a2e0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableBigIntVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; + public static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** @@ -290,41 +290,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java index 949287ecafe45..e6b5b590f3f7e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -292,40 +292,6 @@ public void setSafe(int index, int isSet, int value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java index a0bdccedea405..8e15100f70818 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -296,40 +296,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index 8320f90830a5b..2a611c6a2d5ba 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -37,7 +37,7 @@ * maintained to track which elements in the vector are null. */ public class NullableDecimalVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 16; + public static final byte TYPE_WIDTH = 16; private final FieldReader reader; private final int precision; @@ -355,41 +355,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value as array of bytes - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, byte[] value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - DecimalUtility.writeByteArrayToArrowBuf(value, buffer, index); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - - /****************************************************************** * * * vector transfer * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java index 5b28065c402c4..3ba5cfcea5251 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat4Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; + public static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** @@ -291,41 +291,6 @@ public void setSafe(int index, int isSet, float value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, float value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setFloat(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java index 624abf2f272cd..2fb96a44fe78b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat8Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; + public static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** @@ -291,41 +291,6 @@ public void setSafe(int index, int isSet, double value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, double value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setDouble(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 6311daf4f5cc1..93deacbdfdc74 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableIntVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; + public static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** @@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, int value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java index c45a8d5f5bcba..ed337188b5b0b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableSmallIntVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 2; + public static final byte TYPE_WIDTH = 2; private final FieldReader reader; /** @@ -319,41 +319,6 @@ public void setSafe(int index, int isSet, short value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, short value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setShort(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java index 454a4ac41a6d9..d7ea322239741 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -292,40 +292,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java index 8540d16924210..07d8abba4e57b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -294,40 +294,6 @@ public void setSafe(int index, int isSet, int value) { } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java index 015226da22318..947b2392f7a15 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java index 2b2375e92f1f4..0a3cfaac047eb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, int value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java index b2a58bd4568f9..1bf2abc384d92 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -165,42 +165,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and * - * JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java index 15100530d0ec5..ccbfa32b85f0c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -318,42 +318,6 @@ public void setSafe(int index, int isSet, byte value) { set(index, isSet, value); } - - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, byte value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setByte(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index b21fcaac391e8..d3c8b990394d8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -35,16 +35,19 @@ import java.util.*; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.file.InvalidArrowFileException; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.schema.ArrowVectorType; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.DictionaryUtility; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -186,6 +189,282 @@ public VectorSchemaRoot read() throws IOException { } } + private abstract class BufferReader { + abstract protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException; + + final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException { + readToken(START_ARRAY); + ArrowBuf buf = read(allocator, count); + readToken(END_ARRAY); + return buf; + } + } + + private class BufferHelper { + BufferReader BIT = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + final int bufferSize = BitVectorHelper.getValidityBufferSize(count); + ArrowBuf buf = allocator.buffer(bufferSize); + + // C++ integration test fails without this. + buf.setZero(0, bufferSize); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + BitVectorHelper.setValidityBit(buf, i, parser.readValueAs(Boolean.class) ? 1 : 0); + } + + return buf; + } + }; + + BufferReader INT1 = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableTinyIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeByte(parser.getByteValue()); + } + + return buf; + } + }; + + BufferReader INT2 = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableSmallIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeShort(parser.getShortValue()); + } + + return buf; + } + }; + + BufferReader INT4 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeInt(parser.getIntValue()); + } + + return buf; + } + }; + + BufferReader INT8 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableBigIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeLong(parser.getLongValue()); + } + + return buf; + } + }; + + BufferReader FLOAT4 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableFloat4Vector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeFloat(parser.getFloatValue()); + } + + return buf; + } + }; + + BufferReader FLOAT8 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableFloat8Vector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeDouble(parser.getDoubleValue()); + } + + return buf; + } + }; + + BufferReader DECIMAL = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableDecimalVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + final byte[] value = decodeHexSafe(parser.getValueAsString()); + DecimalUtility.writeByteArrayToArrowBuf(value, buf, i); + } + + return buf; + } + }; + + BufferReader VARCHAR = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrayList values = Lists.newArrayList(); + int bufferSize = 0; + for (int i = 0; i < count; i++) { + parser.nextToken(); + final byte[] value = parser.getValueAsString().getBytes(UTF_8); + values.add(value); + bufferSize += value.length; + + } + + ArrowBuf buf = allocator.buffer(bufferSize); + + for (byte[] value : values) { + buf.writeBytes(value); + } + + return buf; + } + }; + + BufferReader VARBINARY = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrayList values = Lists.newArrayList(); + int bufferSize = 0; + for (int i = 0; i < count; i++) { + parser.nextToken(); + final byte[] value = decodeHexSafe(parser.readValueAs(String.class)); + values.add(value); + bufferSize += value.length; + + } + + ArrowBuf buf = allocator.buffer(bufferSize); + + for (byte[] value : values) { + buf.writeBytes(value); + } + + return buf; + } + }; + + } + + private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferType, Types.MinorType type, int count) throws IOException { + ArrowBuf buf; + + BufferHelper helper = new BufferHelper(); + + BufferReader reader = null; + + if (bufferType.equals(VALIDITY)) { + reader = helper.BIT; + } else if (bufferType.equals(OFFSET)) { + reader = helper.INT4; + } else if (bufferType.equals(TYPE)) { + reader = helper.INT1; + } else if (bufferType.equals(DATA)) { + switch (type) { + case BIT: + reader = helper.BIT; + break; + case TINYINT: + reader = helper.INT1; + break; + case SMALLINT: + reader = helper.INT2; + break; + case INT: + reader = helper.INT4; + break; + case BIGINT: + reader = helper.INT8; + break; + case UINT1: + reader = helper.INT1; + break; + case UINT2: + reader = helper.INT2; + break; + case UINT4: + reader = helper.INT4; + break; + case UINT8: + reader = helper.INT8; + break; + case FLOAT4: + reader = helper.FLOAT4; + break; + case FLOAT8: + reader = helper.FLOAT8; + break; + case DECIMAL: + reader = helper.DECIMAL; + break; + case VARCHAR: + reader = helper.VARCHAR; + break; + case VARBINARY: + reader = helper.VARBINARY; + break; + case DATEDAY: + reader = helper.INT4; + break; + case DATEMILLI: + reader = helper.INT8; + break; + case TIMESEC: + case TIMEMILLI: + reader = helper.INT4; + break; + case TIMEMICRO: + case TIMENANO: + reader = helper.INT8; + break; + case TIMESTAMPNANO: + case TIMESTAMPMICRO: + case TIMESTAMPMILLI: + case TIMESTAMPSEC: + case TIMESTAMPNANOTZ: + case TIMESTAMPMICROTZ: + case TIMESTAMPMILLITZ: + case TIMESTAMPSECTZ: + reader = helper.INT8; + break; + default: + throw new UnsupportedOperationException("Cannot read array of type " + type); + } + } else { + throw new InvalidArrowFileException("Unrecognized buffer type " + bufferType); + } + + buf = reader.readBuffer(allocator, count); + assert buf != null; + return buf; + } + private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()]; @@ -212,7 +491,7 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name); } - /* Initialize the vector with required capacity but don't allocate since we would + /* Initialize the vector with required capacity but don't allocateNew since we would * be doing loadFieldBuffers. */ int valueCount = readNextField("count", Integer.class); @@ -221,29 +500,13 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); nextFieldIs(vectorType.getName()); - readToken(START_ARRAY); int innerBufferValueCount = valueCount; if (vectorType.equals(OFFSET)) { /* offset buffer has 1 additional value capacity */ innerBufferValueCount = valueCount + 1; } - for (int i = 0; i < innerBufferValueCount; i++) { - /* write data to the buffer */ - parser.nextToken(); - /* for variable width vectors, value count doesn't help pre-determining the capacity of - * the underlying data buffer. So we need to pass down the offset buffer (which was already - * populated in the previous iteration of this loop). - */ - if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR - || vector.getMinorType() == Types.MinorType.VARBINARY)) { - vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], - vectorBuffers[v-1], i, innerBufferValueCount); - } else { - vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], - null, i, innerBufferValueCount); - } - } - readToken(END_ARRAY); + + vectorBuffers[v] = readBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); } vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers)); @@ -253,7 +516,8 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json if (!fields.isEmpty()) { List vectorChildren = vector.getChildrenFromFields(); if (fields.size() != vectorChildren.size()) { - throw new IllegalArgumentException("fields and children are not the same size: " + fields.size() + " != " + vectorChildren.size()); + throw new IllegalArgumentException( + "fields and children are not the same size: " + fields.size() + " != " + vectorChildren.size()); } nextFieldIs("children"); readToken(START_ARRAY); @@ -280,126 +544,6 @@ private byte[] decodeHexSafe(String hexString) throws IOException { } } - private ArrowBuf setValueFromParser(ArrowVectorType bufferType, FieldVector vector, - ArrowBuf buffer, ArrowBuf offsetBuffer, int index, - int valueCount) throws IOException { - if (bufferType.equals(TYPE)) { - buffer = NullableTinyIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Byte.class)); - } else if (bufferType.equals(OFFSET)) { - buffer = BaseNullableVariableWidthVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - } else if (bufferType.equals(VALIDITY)) { - buffer = BitVectorHelper.setValidityBit(buffer, allocator, - valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); - } else if (bufferType.equals(DATA)) { - switch (vector.getMinorType()) { - case BIT: - buffer = BitVectorHelper.setValidityBit(buffer, allocator, - valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); - break; - case TINYINT: - buffer = NullableTinyIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Byte.class)); - break; - case SMALLINT: - buffer = NullableSmallIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Short.class)); - break; - case INT: - buffer = NullableIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case BIGINT: - buffer = NullableBigIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case FLOAT4: - buffer = NullableFloat4Vector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Float.class)); - break; - case FLOAT8: - buffer = NullableFloat8Vector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Double.class)); - break; - case DECIMAL: - buffer = NullableDecimalVector.set(buffer, allocator, - valueCount, index, decodeHexSafe(parser.readValueAs(String.class))); - break; - case VARBINARY: - assert (offsetBuffer != null); - buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, - decodeHexSafe(parser.readValueAs(String.class)), valueCount); - break; - case VARCHAR: - assert (offsetBuffer != null); - buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, - parser.readValueAs(String.class).getBytes(UTF_8), valueCount); - break; - case DATEDAY: - buffer = NullableDateDayVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case DATEMILLI: - buffer = NullableDateMilliVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESEC: - buffer = NullableTimeSecVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case TIMEMILLI: - buffer = NullableTimeMilliVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case TIMEMICRO: - buffer = NullableTimeMicroVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMENANO: - buffer = NullableTimeNanoVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSEC: - buffer = NullableTimeStampSecVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLI: - buffer = NullableTimeStampMilliVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICRO: - buffer = NullableTimeStampMicroVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANO: - buffer = NullableTimeStampNanoVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSECTZ: - buffer = NullableTimeStampSecTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLITZ: - buffer = NullableTimeStampMilliTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICROTZ: - buffer = NullableTimeStampMicroTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANOTZ: - buffer = NullableTimeStampNanoTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - default: - throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); - } - } - - return buffer; - } - @Override public void close() throws IOException { parser.close(); From 837150e245823c6f0cd9e16dba89b6d1a0396aa7 Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 13 Nov 2017 17:20:56 -0800 Subject: [PATCH 051/177] ARROW-1476: [JAVA] Implement Final ValueVector Updates indentation and line length rebase on master, fix tests address review comments integration test fix, some checkstyle fixes review comments indentation and javadocs --- .../codegen/templates/LegacyUnionVector.java | 356 --- .../templates/NullableValueVectors.java | 595 ----- .../main/codegen/templates/UnionReader.java | 2 +- .../main/codegen/templates/UnionVector.java | 30 +- .../arrow/vector/BaseDataValueVector.java | 6 +- .../vector/BaseNullableFixedWidthVector.java | 1582 ++++++------- .../BaseNullableVariableWidthVector.java | 2100 ++++++++++------- .../apache/arrow/vector/BaseValueVector.java | 4 +- .../org/apache/arrow/vector/BitVector.java | 4 +- .../apache/arrow/vector/BitVectorHelper.java | 298 ++- .../arrow/vector/GenerateSampleData.java | 336 +++ .../arrow/vector/NullableBigIntVector.java | 652 ++--- .../arrow/vector/NullableBitVector.java | 888 +++---- .../arrow/vector/NullableDateDayVector.java | 654 ++--- .../arrow/vector/NullableDateMilliVector.java | 661 +++--- .../arrow/vector/NullableDecimalVector.java | 758 +++--- .../arrow/vector/NullableFloat4Vector.java | 654 ++--- .../arrow/vector/NullableFloat8Vector.java | 654 ++--- .../arrow/vector/NullableIntVector.java | 664 +++--- .../vector/NullableIntervalDayVector.java | 772 +++--- .../vector/NullableIntervalYearVector.java | 692 +++--- .../arrow/vector/NullableSmallIntVector.java | 708 +++--- .../arrow/vector/NullableTimeMicroVector.java | 652 ++--- .../arrow/vector/NullableTimeMilliVector.java | 656 ++--- .../arrow/vector/NullableTimeNanoVector.java | 652 ++--- .../arrow/vector/NullableTimeSecVector.java | 654 ++--- .../NullableTimeStampMicroTZVector.java | 362 ++- .../vector/NullableTimeStampMicroVector.java | 363 ++- .../NullableTimeStampMilliTZVector.java | 360 ++- .../vector/NullableTimeStampMilliVector.java | 361 ++- .../vector/NullableTimeStampNanoTZVector.java | 362 ++- .../vector/NullableTimeStampNanoVector.java | 363 ++- .../vector/NullableTimeStampSecTZVector.java | 360 ++- .../vector/NullableTimeStampSecVector.java | 363 ++- .../arrow/vector/NullableTimeStampVector.java | 368 +-- .../arrow/vector/NullableTinyIntVector.java | 708 +++--- .../arrow/vector/NullableUInt1Vector.java | 554 ++--- .../arrow/vector/NullableUInt2Vector.java | 554 ++--- .../arrow/vector/NullableUInt4Vector.java | 498 ++-- .../arrow/vector/NullableUInt8Vector.java | 498 ++-- .../arrow/vector/NullableVarBinaryVector.java | 768 +++--- .../arrow/vector/NullableVarCharVector.java | 772 +++--- .../org/apache/arrow/vector/ValueVector.java | 4 +- .../apache/arrow/vector/VectorUnloader.java | 3 - .../complex/BaseRepeatedValueVector.java | 36 +- .../vector/complex/FixedSizeListVector.java | 47 +- .../LegacyBaseRepeatedValueVector.java | 240 -- .../complex/LegacyFixedSizeListVector.java | 298 --- .../vector/complex/LegacyListVector.java | 288 --- .../arrow/vector/complex/LegacyMapVector.java | 229 -- .../arrow/vector/complex/ListVector.java | 219 +- .../vector/complex/NullableMapVector.java | 103 +- .../vector/file/json/JsonFileReader.java | 53 +- .../vector/file/json/JsonFileWriter.java | 16 +- .../apache/arrow/vector/TestUnionVector.java | 22 +- 55 files changed, 11713 insertions(+), 13143 deletions(-) delete mode 100644 java/vector/src/main/codegen/templates/LegacyUnionVector.java delete mode 100644 java/vector/src/main/codegen/templates/NullableValueVectors.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java diff --git a/java/vector/src/main/codegen/templates/LegacyUnionVector.java b/java/vector/src/main/codegen/templates/LegacyUnionVector.java deleted file mode 100644 index b9cb6dfdc2482..0000000000000 --- a/java/vector/src/main/codegen/templates/LegacyUnionVector.java +++ /dev/null @@ -1,356 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -<@pp.dropOutputFile /> -<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/LegacyUnionVector.java" /> - - -<#include "/@includes/license.ftl" /> - - package org.apache.arrow.vector.complex; - -<#include "/@includes/vv_imports.ftl" /> - import com.google.common.collect.ImmutableList; - import java.util.ArrayList; - import java.util.Collections; - import java.util.Iterator; - import org.apache.arrow.vector.BaseDataValueVector; - import org.apache.arrow.vector.complex.impl.ComplexCopier; - import org.apache.arrow.vector.util.CallBack; - import org.apache.arrow.vector.schema.ArrowFieldNode; - - import static org.apache.arrow.vector.types.UnionMode.Sparse; - - - -/* - * This class is generated using freemarker and the ${.template_name} template. - */ -@SuppressWarnings("unused") - - -/** - * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each - * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, - * as well as the addOrGet method. - * - * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup - * each time the vector is accessed. - * Source code generated using FreeMarker template ${.template_name} - */ -public class LegacyUnionVector implements FieldVector { - - private Accessor accessor = new Accessor(); - private Mutator mutator = new Mutator(); - private final UnionVector unionVector; - - public LegacyUnionVector(String name, BufferAllocator allocator, CallBack callBack) { - unionVector = new UnionVector(name, allocator, callBack); - } - - public BufferAllocator getAllocator() { - return unionVector.getAllocator(); - } - - @Override - public MinorType getMinorType() { - return MinorType.UNION; - } - - @Override - public void initializeChildrenFromFields(List children) { - unionVector.initializeChildrenFromFields(children); - } - - @Override - public List getChildrenFromFields() { - return unionVector.getChildrenFromFields(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - unionVector.loadFieldBuffers(fieldNode, ownBuffers); - } - - @Override - public List getFieldBuffers() { - return unionVector.getFieldBuffers(); - } - - @Override - public List getFieldInnerVectors() { - return unionVector.getFieldInnerVectors(); - } - - @Override - public long getValidityBufferAddress() { - return unionVector.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getValidityBuffer() { - return unionVector.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } - - @Override - public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - - public NullableMapVector getMap() { - return unionVector.getMap(); - } - <#list vv.types as type> - <#list type.minor as minor> - <#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#assign lowerCaseName = name?lower_case/> - <#if !minor.typeParams?? > - - private Nullable${name}Vector ${uncappedName}Vector; - - public Nullable${name}Vector get${name}Vector() { - return unionVector.get${name}Vector(); - } - - - - - public ListVector getList() { - return unionVector.getList(); - } - - public int getTypeValue(int index) { - return unionVector.getTypeValue(index); - } - - @Override - public void allocateNew() throws OutOfMemoryException { - unionVector.allocateNew(); - } - - @Override - public boolean allocateNewSafe() { - return unionVector.allocateNewSafe(); - } - - @Override - public void reAlloc() { - unionVector.reAlloc(); - } - - @Override - public void setInitialCapacity(int numRecords) { - } - - @Override - public int getValueCapacity() { - return unionVector.getValueCapacity(); - } - - @Override - public void close() { - unionVector.close(); - } - - @Override - public void clear() { - unionVector.clear(); - } - - @Override - public Field getField() { - return unionVector.getField(); - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return unionVector.getTransferPair(allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return unionVector.getTransferPair(ref, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return unionVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector target) { - return unionVector.makeTransferPair(((LegacyUnionVector)target).unionVector); - } - - public void copyFrom(int inIndex, int outIndex, UnionVector from) { - unionVector.copyFrom(inIndex, outIndex, from); - } - - public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { - unionVector.copyFromSafe(inIndex, outIndex, from); - } - - public FieldVector addVector(FieldVector v) { - return unionVector.addVector(v); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - @Override - public FieldReader getReader() { - return unionVector.getReader(); - } - - public FieldWriter getWriter() { - return unionVector.getWriter(); - } - - @Override - public int getBufferSize() { - return unionVector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - return unionVector.getBufferSizeFor(valueCount); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - return unionVector.getBuffers(clear); - } - - @Override - public Iterator iterator() { - return unionVector.iterator(); - } - - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - return unionVector.getObject(index); - } - - public byte[] get(int index) { - return unionVector.get(index); - } - - public void get(int index, ComplexHolder holder) { - } - - public void get(int index, UnionHolder holder) { - unionVector.get(index, holder); - } - - public int getNullCount() { - return unionVector.getNullCount(); - } - - @Override - public int getValueCount() { - return unionVector.getValueCount(); - } - - @Override - public boolean isNull(int index) { - return unionVector.isNull(index); - } - - public int isSet(int index) { - return unionVector.isSet(index); - } - } - - public class Mutator extends BaseValueVector.BaseMutator { - - UnionWriter writer; - - @Override - public void setValueCount(int valueCount) { - unionVector.setValueCount(valueCount); - } - - public void setSafe(int index, UnionHolder holder) { - unionVector.setSafe(index, holder); - } - <#list vv.types as type> - <#list type.minor as minor> - <#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? > - public void setSafe(int index, Nullable${name}Holder holder) { - unionVector.setSafe(index, holder); - } - - - - - - public void setType(int index, MinorType type) { - unionVector.setType(index, type); - } - - @Override - public void reset() { } - - @Override - public void generateTestData(int values) { } - } - - @Override - @Deprecated - public int getValueCount() { return getAccessor().getValueCount(); } - - @Override - @Deprecated - public void setValueCount(int valueCount) { getMutator().setValueCount(valueCount);} - - @Override - @Deprecated - public Object getObject(int index) { return getAccessor().getObject(index); } - - @Override - @Deprecated - public int getNullCount() { return getAccessor().getNullCount(); } - - @Override - @Deprecated - public boolean isNull(int index) { return getAccessor().isNull(index); } -} diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java deleted file mode 100644 index 8e5b10b649b13..0000000000000 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ /dev/null @@ -1,595 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -<@pp.dropOutputFile /> -<#list vv.types as type> -<#list type.minor as minor> - -<#assign className = "LegacyNullable${minor.class}Vector" /> -<#assign valuesName = "Nullable${minor.class}Vector" /> -<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> - -<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> - -<#include "/@includes/license.ftl" /> - -package org.apache.arrow.vector; - -import org.apache.arrow.vector.schema.ArrowFieldNode; -import java.util.Collections; - -<#include "/@includes/vv_imports.ftl" /> - -import org.apache.arrow.flatbuf.Precision; - -/** - * ${className} implements a vector of values which could be null. Elements in the vector - * are first checked against a fixed length vector of boolean values. Then the element is retrieved - * from the base class (if not null). - * - * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. - */ -@SuppressWarnings("unused") -@Deprecated -public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, FieldVector { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); - -protected final static byte[] emptyByteArray = new byte[]{}; - - private final String bitsField = "$bits$"; - private final String valuesField = "$values$"; - - final BitVector bits = new BitVector(bitsField, allocator); - final ${valuesName} values; - - private final Mutator mutator; - private final Accessor accessor; - - <#if minor.typeParams??> - <#assign typeParams = minor.typeParams?reverse> - <#list typeParams as typeParam> - private final ${typeParam.type} ${typeParam.name}; - - - /** - * Assumes the type is nullable and not dictionary encoded - * @param name name of the field - * @param allocator allocator to use to resize the vector<#list typeParams as typeParam> - * @param ${typeParam.name} type parameter ${typeParam.name} - */ - public ${className}(String name, BufferAllocator allocator<#list typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { - <#if minor.arrowTypeConstructorParams??> - <#assign constructorParams = minor.arrowTypeConstructorParams /> - <#else> - <#assign constructorParams = [] /> - <#list typeParams as typeParam> - <#assign constructorParams = constructorParams + [ typeParam.name ] /> - - - this(name, FieldType.nullable(new ${minor.arrowType}(${constructorParams?join(", ")})), allocator); - } - <#else> - public ${className}(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator); - } - - - public ${className}(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator); - <#if minor.typeParams??> - <#assign typeParams = minor.typeParams?reverse> - ${minor.arrowType} arrowType = (${minor.arrowType})fieldType.getType(); - <#list typeParams as typeParam> - this.${typeParam.name} = arrowType.get${typeParam.name?cap_first}(); - - this.values = new ${valuesName}(valuesField, allocator<#list typeParams as typeParam>, ${typeParam.name}); - <#else> - this.values = new ${valuesName}(valuesField, allocator); - - this.mutator = new Mutator(); - this.accessor = new Accessor(); - } - - @Override - public List getFieldInnerVectors() { - /* DELEGATE TO NEW VECTOR */ - return values.getFieldInnerVectors(); - } - - @Override - public void initializeChildrenFromFields(List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("primitive type vector ${className} can not have children: " + children); - } - } - - @Override - public List getChildrenFromFields() { - return Collections.emptyList(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - /* DELEGATE TO NEW VECTOR */ - values.loadFieldBuffers(fieldNode, ownBuffers); - } - - public List getFieldBuffers() { - /* DELEGATE TO NEW VECTOR */ - return values.getFieldBuffers(); - } - - @Override - public Field getField() { - /* DELEGATE TO NEW VECTOR */ - return values.getField(); - } - - @Override - public MinorType getMinorType() { - /* DELEGATE TO NEW VECTOR */ - return values.getMinorType(); - } - - @Override - public FieldReader getReader(){ - /* DELEGATE TO NEW VECTOR */ - return values.getReader(); - } - - @Override - public int getValueCapacity(){ - /* DELEGATE TO NEW VECTOR */ - return values.getValueCapacity(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - /* DELEGATE TO NEW VECTOR */ - return values.getBuffers(clear); - } - - @Override - public void close() { - /* DELEGATE TO NEW VECTOR */ - values.close(); - } - - @Override - public void clear() { - /* DELEGATE TO NEW VECTOR */ - values.clear(); - } - - @Override - public int getBufferSize(){ - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSize(); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSizeFor(valueCount); - } - - public ArrowBuf getBuffer() { - return values.getDataBuffer(); - } - - public ${valuesName} getValuesVector() { - return values; - } - - @Override - public void setInitialCapacity(int numRecords) { - /* DELEGATE TO NEW VECTOR */ - values.setInitialCapacity(numRecords); - } - - @Override - public void allocateNew() { - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(); - } - - @Override - public boolean allocateNewSafe() { - /* DELEGATE TO NEW VECTOR */ - return values.allocateNewSafe(); - } - - @Override - public void reAlloc() { - /* DELEGATE TO NEW VECTOR */ - values.reAlloc(); - } - - public void reset() { - /* DELEGATE TO NEW VECTOR */ - values.reset(); - } - - <#if type.major == "VarLen"> - @Override - public void allocateNew(int totalBytes, int valueCount) { - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(totalBytes, valueCount); - } - - @Override - public int getByteCapacity(){ - return values.getByteCapacity(); - } - - @Override - public int getCurrentSizeInBytes(){ - return values.getCurrentSizeInBytes(); - } - - <#else> - @Override - public void allocateNew(int valueCount) { - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(valueCount); - } - - /** - * {@inheritDoc} - */ - @Override - public void zeroVector() { - /* DELEGATE TO NEW VECTOR */ - values.zeroVector(); - } - - - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator, callBack); - } - - - - @Override - public TransferPair getTransferPair(BufferAllocator allocator){ - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(allocator); - } - - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator); - } - - - - @Override - public TransferPair makeTransferPair(ValueVector to) { - /* DELEGATE TO NEW VECTOR */ - return values.makeTransferPair(to); - } - - - - public void transferTo(${valuesName} target) { - /* DELEGATE TO NEW VECTOR */ - <#if type.major == "VarLen"> - values.transferTo((BaseNullableVariableWidthVector) target); - <#else> - values.transferTo((BaseNullableFixedWidthVector) target); - - } - - public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) { - /* DELEGATE TO NEW VECTOR */ - <#if type.major == "VarLen"> - values.splitAndTransferTo(startIndex, length, (BaseNullableVariableWidthVector) target); - <#else> - values.splitAndTransferTo(startIndex, length, (BaseNullableFixedWidthVector) target); - - } - - - - @Override - public Accessor getAccessor(){ - return accessor; - } - - @Override - public Mutator getMutator(){ - return mutator; - } - - - public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) { - /* DELEGATE TO NEW VECTOR */ - values.copyFrom(fromIndex, thisIndex, from); - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) { - /* DELEGATE TO NEW VECTOR */ - values.copyFromSafe(fromIndex, thisIndex, from); - } - - @Override - public long getValidityBufferAddress() { - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - /* DELEGATE TO NEW VECTOR */ - return values.getDataBufferAddress(); - } - - @Override - public long getOffsetBufferAddress() { - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBufferAddress(); - } - - @Override - public ArrowBuf getValidityBuffer() { - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { - return (values.getDataBuffer()); - } - - @Override - public ArrowBuf getOffsetBuffer() { - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBuffer(); - } - - public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor { - - /** - * Get the element at the specified position. - * - * @param index position of the value - * @return value of the element, if not null - */ - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.get(index); - } - - @Override - public boolean isNull(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.isNull(index); - } - - public int isSet(int index){ - /* DELEGATE TO NEW VECTOR */ - return values.isSet(index); - } - - <#if type.major == "VarLen"> - public long getStartEnd(int index){ - /* DELEGATE TO NEW VECTOR */ - return values.getStartEnd(index); - } - - @Override - public int getValueLength(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.getValueLength(index); - } - - - public void get(int index, Nullable${minor.class}Holder holder){ - /* DELEGATE TO NEW VECTOR */ - values.get(index, holder); - } - - @Override - public ${friendlyType} getObject(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.getObject(index); - } - - <#if minor.class == "IntervalYear" || minor.class == "IntervalDay"> - public StringBuilder getAsStringBuilder(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.getAsStringBuilder(index); - } - - - @Override - public int getValueCount(){ - /* DELEGATE TO NEW VECTOR */ - return values.getValueCount(); - } - - public void reset() { } - } - - public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator { - private int setCount; - <#if type.major = "VarLen"> private int lastSet = -1; - - private Mutator() { } - - public ${valuesName} getVectorWithValues() { - return values; - } - - - @Override - public void setIndexDefined(int index) { - /* DELEGATE TO NEW VECTOR */ - values.setIndexDefined(index); - } - - - - /** - * Set the variable length element at the specified index to the supplied byte array. - * - * @param index position of the bit to set - * @param value array of bytes (or int if smaller than 4 bytes) to write - */ - public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, value); - } - - - - <#if type.major == "VarLen"> - public void fillEmpties(int index) { - /* DELEGATE TO NEW VECTOR */ - values.fillEmpties(index); - } - - @Override - public void setValueLengthSafe(int index, int length) { - /* DELEGATE TO NEW VECTOR */ - values.setValueLengthSafe(index, length); - } - - - - public void setSafe(int index, byte[] value, int start, int length) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value, start, length); - } - - - public void setSafe(int index, ByteBuffer value, int start, int length) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value, start, length); - } - - - public void setNull(int index) { - /* DELEGATE TO NEW VECTOR */ - values.setNull(index); - } - - - public void set(int index, Nullable${minor.class}Holder holder) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, holder); - } - - - public void set(int index, ${minor.class}Holder holder) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, holder); - } - - - public boolean isSafe(int outIndex) { - /* DELEGATE TO NEW VECTOR */ - return values.isSafe(outIndex); - } - - - <#assign fields = minor.fields!type.fields /> - public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field ){ - values.set(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); - } - - public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field ) { - values.setSafe(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); - } - - - public void setSafe(int index, Nullable${minor.class}Holder value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - public void setSafe(int index, ${minor.class}Holder value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - - <#if !(type.major == "VarLen" || minor.class == "IntervalDay")> - public void setSafe(int index, ${minor.javaType!type.javaType} value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - - - - <#if minor.class == "Decimal"> - public void set(int index, ${friendlyType} value) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, value); - } - - public void setSafe(int index, ${friendlyType} value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - - - @Override - public void setValueCount(int valueCount) { - /* DELEGATE TO NEW VECTOR */ - values.setValueCount(valueCount); - } - - - /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */ - @Override - public void generateTestData(int valueCount) { } - - - /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */ - @Override - public void reset() { } - - - <#if type.major == "VarLen"> - public void setLastSet(int value) { - /* DELEGATE TO NEW VECTOR */ - values.setLastSet(value); - } - - - public int getLastSet() { - /* DELEGATE TO NEW VECTOR */ - return values.getLastSet(); - } - - } -} - - diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java index fd3a766fc2cb4..98bb7c1f53d84 100644 --- a/java/vector/src/main/codegen/templates/UnionReader.java +++ b/java/vector/src/main/codegen/templates/UnionReader.java @@ -60,7 +60,7 @@ public Field getField() { } public boolean isSet(){ - return !data.getAccessor().isNull(idx()); + return !data.isNull(idx()); } public void read(UnionHolder holder) { diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 3da383fe6f3c5..e44edbd47b67a 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -63,7 +63,7 @@ public class UnionVector implements FieldVector { int valueCount; MapVector internalMap; - ArrowBuf typeBuffer; + protected ArrowBuf typeBuffer; private NullableMapVector mapVector; private ListVector listVector; @@ -121,15 +121,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers @Override public List getFieldBuffers() { List result = new ArrayList<>(1); - - typeBuffer.readerIndex(0); - typeBuffer.writerIndex(valueCount * TYPE_WIDTH); - + setReaderAndWriterIndex(); result.add(typeBuffer); return result; } + private void setReaderAndWriterIndex() { + typeBuffer.readerIndex(0); + typeBuffer.writerIndex(valueCount * TYPE_WIDTH); + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -232,6 +234,8 @@ public int getTypeValue(int index) { @Override public void allocateNew() throws OutOfMemoryException { + /* new allocation -- clear the current buffers */ + clear(); internalMap.allocateNew(); try { allocateTypeBuffer(); @@ -243,6 +247,8 @@ public void allocateNew() throws OutOfMemoryException { @Override public boolean allocateNewSafe() { + /* new allocation -- clear the current buffers */ + clear(); boolean safe = internalMap.allocateNewSafe(); if (!safe) { return false; } try { @@ -386,6 +392,7 @@ public void transfer() { to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer; internalMapVectorTransferPair.transfer(); to.valueCount = valueCount; + clear(); } @Override @@ -460,8 +467,17 @@ public int getBufferSizeFor(final int valueCount) { @Override public ArrowBuf[] getBuffers(boolean clear) { ImmutableList.Builder builder = ImmutableList.builder(); - builder.add(typeBuffer); - builder.add(internalMap.getBuffers(clear)); + setReaderAndWriterIndex(); + if (getBufferSize() != 0) { + builder.add(typeBuffer); + builder.add(internalMap.getBuffers(clear)); + } + if (clear) { + valueCount = 0; + typeBuffer.retain(); + typeBuffer.release(); + typeBuffer = allocator.getEmpty(); + } List list = builder.build(); return list.toArray(new ArrowBuf[list.size()]); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java index 01340f66c4095..38524ff8adf82 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -85,7 +85,7 @@ public ArrowBuf[] getBuffers(boolean clear) { if (getBufferSize() == 0) { out = new ArrowBuf[0]; } else { - out = new ArrowBuf[] {data}; + out = new ArrowBuf[]{data}; data.readerIndex(0); if (clear) { data.retain(1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index 83dc409a74f14..209758e4ece9e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,806 +41,810 @@ * implying that zero or more elements in the vector could be NULL. */ public abstract class BaseNullableFixedWidthVector extends BaseValueVector - implements FixedWidthVector, FieldVector { - private final byte typeWidth; - - protected int valueAllocationSizeInBytes; - protected int validityAllocationSizeInBytes; - - protected final Field field; - private int allocationMonitor; - protected ArrowBuf validityBuffer; - protected ArrowBuf valueBuffer; - protected int valueCount; - - public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator, - FieldType fieldType, final byte typeWidth) { - super(name, allocator); - this.typeWidth = typeWidth; - field = new Field(name, fieldType, null); - valueCount = 0; + implements FixedWidthVector, FieldVector, NullableVectorDefinitionSetter { + private final byte typeWidth; + + protected int valueAllocationSizeInBytes; + protected int validityAllocationSizeInBytes; + + protected final Field field; + private int allocationMonitor; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected int valueCount; + + public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType, final byte typeWidth) { + super(name, allocator); + this.typeWidth = typeWidth; + field = new Field(name, fieldType, null); + valueCount = 0; + allocationMonitor = 0; + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + if (typeWidth > 0) { + valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + } else { + /* specialized handling for NullableBitVector */ + valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = valueAllocationSizeInBytes; + } + } + + + /* TODO: + * see if getNullCount() can be made faster -- O(1) + */ + + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. + */ + + + @Override + @Deprecated + public Mutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); + } + + @Override + @Deprecated + public Accessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); + } + + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ + @Override + public long getValidityBufferAddress() { + return (validityBuffer.memoryAddress()); + } + + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ + @Override + public long getDataBufferAddress() { + return (valueBuffer.memoryAddress()); + } + + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return starting address of the buffer + * @throws UnsupportedOperationException for fixed width vectors + */ + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + * @throws UnsupportedOperationException for fixed width vectors + */ + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long) valueCount * typeWidth; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int) size; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); + } + + private int getValueBufferValueCapacity() { + return (int) ((valueBuffer.capacity() * 1.0) / typeWidth); + } + + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + /** + * zero out the vector and the data in associated buffers. + */ + @Override + public void zeroVector() { + initValidityBuffer(); + initValueBuffer(); + } + + /* zero out the validity buffer */ + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + /* zero out the data buffer */ + private void initValueBuffer() { + valueBuffer.setZero(0, valueBuffer.capacity()); + } + + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ + public void reset() { + zeroVector(); + } + + /** + * Close the vector and release the associated buffers. + */ + @Override + public void close() { + clear(); + } + + /** + * Same as {@link #close()} + */ + @Override + public void clear() { + valueCount = 0; + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + } + + /* used to step down the memory allocation */ + protected void incrementAllocationMonitor() { + if (allocationMonitor < 0) { allocationMonitor = 0; - validityBuffer = allocator.getEmpty(); - valueBuffer = allocator.getEmpty(); - if (typeWidth > 0) { - valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); - } else { - /* specialized handling for NullableBitVector */ - valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); - validityAllocationSizeInBytes = valueAllocationSizeInBytes; - } - } - - - /* TODO: - * - * see if getNullCount() can be made faster -- O(1) - */ - - /* TODO: - * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level - * base class BaseValueVector. - * - * Along with this, some class members (validityBuffer) can also be - * abstracted out to top level base class. - * - * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy (non-nullable) and those - * vectors have not yet been refactored/removed so moving things to - * the top class as of now is not a good idea. - */ - - - @Override - @Deprecated - public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); - } - - @Override - @Deprecated - public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); - } - - /** - * Get the memory address of buffer that manages the validity - * (NULL or NON-NULL nature) of elements in the vector. - * @return starting address of the buffer - */ - @Override - public long getValidityBufferAddress() { - return (validityBuffer.memoryAddress()); - } - - /** - * Get the memory address of buffer that stores the data for elements - * in the vector. - * @return starting address of the buffer - */ - @Override - public long getDataBufferAddress() { - return (valueBuffer.memoryAddress()); - } - - /** - * Get the memory address of buffer that stores the offsets for elements - * in the vector. This operation is not supported for fixed-width vectors. - * @return starting address of the buffer - * @throws UnsupportedOperationException for fixed width vectors - */ - @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException("not supported for fixed-width vectors"); - } - - /** - * Get buffer that manages the validity (NULL or NON-NULL nature) of - * elements in the vector. Consider it as a buffer for internal bit vector - * data structure. - * @return buffer - */ - @Override - public ArrowBuf getValidityBuffer() { - return validityBuffer; - } - - /** - * Get the buffer that stores the data for elements in the vector. - * @return buffer - */ - @Override - public ArrowBuf getDataBuffer() { - return valueBuffer; - } - - /** - * buffer that stores the offsets for elements - * in the vector. This operation is not supported for fixed-width vectors. - * @return buffer - * @throws UnsupportedOperationException for fixed width vectors - */ - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException("not supported for fixed-width vectors"); - } - - /** - * Sets the desired value capacity for the vector. This function doesn't - * allocate any memory for the vector. - * @param valueCount desired number of elements in the vector - */ - @Override - public void setInitialCapacity(int valueCount) { - final long size = (long)valueCount * typeWidth; - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity(){ - return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); - } - - private int getValueBufferValueCapacity() { - return (int)((valueBuffer.capacity() * 1.0)/typeWidth); - } - - private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); - } - - /** - * zero out the vector and the data in associated buffers. - */ - @Override - public void zeroVector() { - initValidityBuffer(); - initValueBuffer(); - } - - /* zero out the validity buffer */ - private void initValidityBuffer() { - validityBuffer.setZero(0, validityBuffer.capacity()); - } - - /* zero out the data buffer */ - private void initValueBuffer() { - valueBuffer.setZero(0, valueBuffer.capacity()); - } - - /** - * Reset the vector to initial state. Same as {@link #zeroVector()}. - * Note that this method doesn't release any memory. - */ - public void reset() { - zeroVector(); - } - - /** - * Close the vector and release the associated buffers. - */ - @Override - public void close() { clear(); } - - /** - * Same as {@link #close()} - */ - @Override - public void clear() { - valueCount = 0; - validityBuffer = releaseBuffer(validityBuffer); - valueBuffer = releaseBuffer(valueBuffer); - } - - /* used to step down the memory allocation */ - protected void incrementAllocationMonitor() { - if (allocationMonitor < 0) { - allocationMonitor = 0; - } - allocationMonitor++; - } + } + allocationMonitor++; + } - /* used to step up the memory allocation */ - protected void decrementAllocationMonitor() { - if (allocationMonitor > 0) { - allocationMonitor = 0; - } - allocationMonitor--; - } - - /** - * Same as {@link #allocateNewSafe()}. - */ - @Override - public void allocateNew() { - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating memory."); - } - } - - /** - * Allocate memory for the vector. We internally use a default value count - * of 4096 to allocate memory for at least these many elements in the - * vector. See {@link #allocateNew(int)} for allocating memory for specific - * number of elements in the vector. - * - * @return false if memory allocation fails, true otherwise. - */ - @Override - public boolean allocateNewSafe() { - long curAllocationSizeValue = valueAllocationSizeInBytes; - long curAllocationSizeValidity = validityAllocationSizeInBytes; - - if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory exceeds limit"); - } + /* used to step up the memory allocation */ + protected void decrementAllocationMonitor() { + if (allocationMonitor > 0) { + allocationMonitor = 0; + } + allocationMonitor--; + } + + /** + * Same as {@link #allocateNewSafe()}. + */ + @Override + public void allocateNew() { + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } /* we are doing a new allocation -- release the current buffers */ - clear(); - - try{ - allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); - } catch (Exception e) { - e.printStackTrace(); - clear(); - return false; - } + clear(); - return true; - } - - /** - * Allocate memory for the vector to support storing at least the provided number of - * elements in the vector. This method must be called prior to using the ValueVector. - * - * @param valueCount the desired number of elements in the vector - * @throws org.apache.arrow.memory.OutOfMemoryException - */ - public void allocateNew(int valueCount) { - long valueBufferSize = valueCount * typeWidth; - long validityBufferSize = getValidityBufferSizeFromCount(valueCount); - if (typeWidth == 0) { - /* specialized handling for NullableBitVector */ - valueBufferSize = validityBufferSize; - } - - if (allocationMonitor > 10) { - /* step down the default memory allocation since we have observed - * multiple times that provisioned value capacity was much larger than - * actually needed. see setValueCount for more details. - */ - valueBufferSize = Math.max(8, valueBufferSize / 2); - validityBufferSize = Math.max(8, validityBufferSize / 2); - allocationMonitor = 0; - } else if (allocationMonitor < -2) { - valueBufferSize = valueBufferSize * 2L; - validityBufferSize = validityBufferSize * 2L; - allocationMonitor = 0; - } - - if (valueBufferSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - - /* we are doing a new allocation -- release the current buffers */ + try { + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); + } catch (Exception e) { + e.printStackTrace(); clear(); - - try { - allocateBytes(valueBufferSize, validityBufferSize); - } catch(Exception e) { - e.printStackTrace(); - clear(); - throw e; - } - } - - /** - * Actual memory allocation is done by this function. All the calculations - * and knowledge about what size to allocate is upto the callers of this - * method. - * Callers appropriately handle errors if memory allocation fails here. - * Callers should also take care of determining that desired size is - * within the bounds of max allocation allowed and any other error - * conditions. - */ - private void allocateBytes(final long valueBufferSize, final long validityBufferSize) { - /* allocate data buffer */ - int curSize = (int)valueBufferSize; - valueBuffer = allocator.buffer(curSize); - valueBuffer.readerIndex(0); - valueAllocationSizeInBytes = curSize; - - /* allocate validity buffer */ - allocateValidityBuffer((int)validityBufferSize); - initValidityBuffer(); - } - - /** - * During splitAndTransfer, if we splitting from a random position within a byte, - * we can't just slice the source buffer so we have to explicitly allocate the - * validityBuffer of the target vector. This is unlike the databuffer which we can - * always slice for the target vector. - */ - private void allocateValidityBuffer(final int validityBufferSize) { - validityBuffer = allocator.buffer(validityBufferSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = validityBufferSize; - initValidityBuffer(); - } - - /** - * Get the potential buffer size for a particular number of records. - * @param count desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int count) { - if (count == 0) { return 0; } - return (count * typeWidth) + getValidityBufferSizeFromCount(count); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - if (valueCount == 0) { return 0; } - return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); - } - - /** - * Get information about how this field is materialized. - * @return the field corresponding to this vector - */ - @Override - public Field getField() { - return field; - } - - /** - * Return the underlying buffers associated with this vector. Note that this doesn't - * impact the reference counts for this buffer so it only should be used for in-context - * access. Also note that this buffer changes regularly thus - * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear Whether to clear vector before returning; the buffers will still be refcounted - * but the returned array will be the only reference to them - * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this - * vector instance. - */ - @Override - public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = new ArrowBuf[2]; + return false; + } + + return true; + } + + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ + public void allocateNew(int valueCount) { + long valueBufferSize = valueCount * typeWidth; + long validityBufferSize = getValidityBufferSizeFromCount(valueCount); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBufferSize = validityBufferSize; + } + + if (valueBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(valueBufferSize, validityBufferSize); + } catch (Exception e) { + e.printStackTrace(); + clear(); + throw e; + } + } + + /** + * Actual memory allocation is done by this function. All the calculations + * and knowledge about what size to allocate is upto the callers of this + * method. + * Callers appropriately handle errors if memory allocation fails here. + * Callers should also take care of determining that desired size is + * within the bounds of max allocation allowed and any other error + * conditions. + */ + private void allocateBytes(final long valueBufferSize, final long validityBufferSize) { + /* allocate data buffer */ + int curSize = (int) valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; + /* allocate validity buffer */ + allocateValidityBuffer((int) validityBufferSize); + zeroVector(); + } + + /** + * During splitAndTransfer, if we splitting from a random position within a byte, + * we can't just slice the source buffer so we have to explicitly allocate the + * validityBuffer of the target vector. This is unlike the databuffer which we can + * always slice for the target vector. + */ + private void allocateValidityBuffer(final int validityBufferSize) { + validityBuffer = allocator.buffer(validityBufferSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = validityBufferSize; + } + + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { + return 0; + } + return (count * typeWidth) + getValidityBufferSizeFromCount(count); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + if (valueCount == 0) { + return 0; + } + return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); + } + + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ + @Override + public Field getField() { + return field; + } + + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers; + setReaderAndWriterIndex(); + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = new ArrowBuf[2]; buffers[0] = validityBuffer; buffers[1] = valueBuffer; - if (clear) { - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); + } + if (clear) { + for (final ArrowBuf buffer : buffers) { + buffer.retain(1); } - return buffers; - } - - /** - * Resize the vector to increase the capacity. The internal behavior is to - * double the current value capacity. - */ - @Override - public void reAlloc() { - valueBuffer = reallocBufferHelper(valueBuffer, true); - validityBuffer = reallocBufferHelper(validityBuffer, false); - } - - /** - * Helper method for reallocating a particular internal buffer - * Returns the new buffer. - */ - private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { - final int currentBufferCapacity = buffer.capacity(); - long baseSize = (dataBuffer ? valueAllocationSizeInBytes - : validityAllocationSizeInBytes); - - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, buffer, 0, currentBufferCapacity); - final int halfNewCapacity = newBuf.capacity() / 2; - newBuf.setZero(halfNewCapacity, halfNewCapacity); - buffer.release(1); - buffer = newBuf; - if (dataBuffer) { - valueAllocationSizeInBytes = (int)newAllocationSize; - } - else { - validityAllocationSizeInBytes = (int)newAllocationSize; - } - - return buffer; - } - - @Override - @Deprecated - public List getFieldInnerVectors() { - throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); - } - - /** - * Initialize the children in schema for this Field. This operation is a - * NO-OP for scalar types since they don't have any children. - * @param children the schema - * @throws IllegalArgumentException if children is a non-empty list for scalar types. - */ - @Override - public void initializeChildrenFromFields(List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("primitive type vector can not have children"); - } - } - - /** - * Get the inner child vectors. - * @return list of child vectors for complex types, empty list for scalar vector - * types - */ - @Override - public List getChildrenFromFields() { - return Collections.emptyList(); - } - - /** - * Load the buffers of this vector with provided source buffers. - * The caller manages the source buffers and populates them before invoking - * this method. - * @param fieldNode the fieldNode indicating the value count - * @param ownBuffers the buffers for this Field (own buffers only, children not included) - */ - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - if (ownBuffers.size() != 2) { - throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); - } - - ArrowBuf bitBuffer = ownBuffers.get(0); - ArrowBuf dataBuffer = ownBuffers.get(1); - - validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); - valueBuffer.release(); - valueBuffer = dataBuffer.retain(allocator); - - valueCount = fieldNode.getLength(); - - valueAllocationSizeInBytes = valueBuffer.capacity(); - validityAllocationSizeInBytes = validityBuffer.capacity(); - } - - /** - * Get the buffers belonging to this vector - * @return the inner buffers. - */ - public List getFieldBuffers() { - List result = new ArrayList<>(2); - - validityBuffer.readerIndex(0); + clear(); + } + return buffers; + } + + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ + @Override + public void reAlloc() { + valueBuffer = reallocBufferHelper(valueBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + /** + * Helper method for reallocating a particular internal buffer + * Returns the new buffer. + */ + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (dataBuffer ? valueAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (dataBuffer) { + valueAllocationSizeInBytes = (int) newAllocationSize; + } else { + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + return buffer; + } + + @Override + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + if (ownBuffers.size() != 2) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf dataBuffer = ownBuffers.get(1); + + validityBuffer.release(); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + valueCount = fieldNode.getLength(); + + valueAllocationSizeInBytes = valueBuffer.capacity(); + validityAllocationSizeInBytes = validityBuffer.capacity(); + } + + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ + public List getFieldBuffers() { + List result = new ArrayList<>(2); + setReaderAndWriterIndex(); + result.add(validityBuffer); + result.add(valueBuffer); + + return result; + } + + /** + * Set the reader and writer indexes for the inner buffers. + */ + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + valueBuffer.readerIndex(0); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + valueBuffer.writerIndex(0); + } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - valueBuffer.readerIndex(0); if (typeWidth == 0) { - /* specialized handling for NullableBitVector */ - valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + /* specialized handling for NullableBitVector */ + valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); } else { - valueBuffer.writerIndex(valueCount * typeWidth); + valueBuffer.writerIndex(valueCount * typeWidth); } - - result.add(validityBuffer); - result.add(valueBuffer); - - return result; - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @param callBack - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return getTransferPair(ref, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param allocator allocator for the target vector - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(BufferAllocator allocator){ - return getTransferPair(name, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return TransferPair - */ - public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); - - /** - * Transfer this vector'data to another vector. The memory associated - * with this vector is transferred to the allocator of target vector - * for accounting and management purposes. - * @param target destination vector for transfer - */ - public void transferTo(BaseNullableFixedWidthVector target){ - compareTypes(target, "transferTo"); - target.clear(); - target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; - target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; - target.valueCount = valueCount; - clear(); - } - - /** - * Slice this vector at desired index and length and transfer the - * corresponding data to the target vector. - * @param startIndex start position of the split in source vector. - * @param length length of the split. - * @param target destination vector - */ - public void splitAndTransferTo(int startIndex, int length, - BaseNullableFixedWidthVector target) { - compareTypes(target, "splitAndTransferTo"); - target.clear(); - splitAndTransferValidityBuffer(startIndex, length, target); - splitAndTransferValueBuffer(startIndex, length, target); - target.setValueCount(length); - } - - /** - * Data buffer can always be split and transferred using slicing. - */ - private void splitAndTransferValueBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target) { - final int startPoint = startIndex * typeWidth; - final int sliceLength = length * typeWidth; - target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; - } - - /** - * Validity buffer has multiple cases of split and transfer depending on - * the starting position of the source index. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - /* slice */ - if (target.validityBuffer != null) { - target.validityBuffer.release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.retain(1); - } - else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, - firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, - firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - - - /****************************************************************** - * * - * common getters and setters * - * * - ******************************************************************/ - - - /** - * Get the number of elements that are null in the vector - * - * @return the number of null elements. - */ - public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); - } - - /** - * Get the value count of vector. This will always be zero unless - * {@link #setValueCount(int)} has been called prior to calling this. - * - * @return valueCount for the vector - */ - public int getValueCount(){ - return valueCount; - } - - /** - * Set value count for the vector. - * - * @param valueCount value count to set - */ - public void setValueCount(int valueCount) { - this.valueCount = valueCount; - final int currentValueCapacity = getValueCapacity(); - while (valueCount > getValueCapacity()) { - reAlloc(); - } - /* - * We are trying to understand the pattern of memory allocation. - * If initially, the user did vector.allocateNew(), we would have - * allocated memory of default size (4096 * type width). - * Later on user invokes setValueCount(count). - * - * If the existing value capacity is twice as large as the - * valueCount, we know that we over-provisioned memory in the - * first place when default memory allocation was done because user - * really needs a much less value count in the vector. - * - * We record this by bumping up the allocationMonitor. If this pattern - * happens for certain number of times and allocationMonitor - * reaches the threshold (internal hardcoded) value, subsequent - * call to allocateNew() will take care of stepping down the - * default memory allocation size. - * - * Another case would be under-provisioning the initial memory and - * thus going through a lot of realloc(). Here the goal is to - * see if we can minimize the number of reallocations. Again the - * state is recorded in allocationMonitor by decrementing it - * (negative value). If a threshold is hit, realloc will try to - * allocate more memory in order to possibly avoid a future realloc. - * This case is also applicable to setSafe() methods which can trigger - * a realloc() and thus we record the state there as well. - */ - if (valueCount > 0) { - if (currentValueCapacity >= (valueCount * 2)) { - incrementAllocationMonitor(); - } else if (currentValueCapacity <= (valueCount/2)) { - decrementAllocationMonitor(); - } + } + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ + public void transferTo(BaseNullableFixedWidthVector target) { + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.valueCount = valueCount; + clear(); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferValueBuffer(startIndex, length, target); + target.setValueCount(length); + } + + /** + * Data buffer can always be split and transferred using slicing. + */ + private void splitAndTransferValueBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + final int startPoint = startIndex * typeWidth; + final int sliceLength = length * typeWidth; + target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; + } + + /** + * Validity buffer has multiple cases of split and transfer depending on + * the starting position of the source index. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + /* slice */ + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } } - } - - /** - * Check if the given index is within the current value capacity - * of the vector - * - * @param index position to check - * @return true if index is within the current value capacity - */ - public boolean isSafe(int index) { - return index < getValueCapacity(); - } - - /** - * Check if element at given index is null. - * - * @param index position of element - * @return true if element at given index is null, false otherwise - */ - public boolean isNull(int index) { - return (isSet(index) == 0); - } - - /** - * Same as {@link #isNull(int)}. - * - * @param index position of element - * @return 1 if element at given index is not null, 0 otherwise - */ - public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Mark the particular position in the vector as non-null. - * - * @param index position of the element. - */ - public void setIndexDefined(int index) { - handleSafe(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - } - - public void set(int index, byte[] value, int start, int length) { - throw new UnsupportedOperationException(); - } - - public void setSafe(int index, byte[] value, int start, int length) { - throw new UnsupportedOperationException(); - } - - public void set(int index, ByteBuffer value, int start, int length) { - throw new UnsupportedOperationException(); - } - - public void setSafe(int index, ByteBuffer value, int start, int length) { - throw new UnsupportedOperationException(); - } - - - /****************************************************************** - * * - * helper methods for setters * - * * - ******************************************************************/ - - - protected void handleSafe(int index) { - while (index >= getValueCapacity()) { - decrementAllocationMonitor(); - reAlloc(); + } + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + + /** + * Get the value count of vector. This will always be zero unless + * {@link #setValueCount(int)} has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount() { + return valueCount; + } + + /** + * Set value count for the vector. + * + * @param valueCount value count to set + */ + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + final int currentValueCapacity = getValueCapacity(); + while (valueCount > getValueCapacity()) { + reAlloc(); + } + /* + * We are trying to understand the pattern of memory allocation. + * If initially, the user did vector.allocateNew(), we would have + * allocated memory of default size (4096 * type width). + * Later on user invokes setValueCount(count). + * + * If the existing value capacity is twice as large as the + * valueCount, we know that we over-provisioned memory in the + * first place when default memory allocation was done because user + * really needs a much less value count in the vector. + * + * We record this by bumping up the allocationMonitor. If this pattern + * happens for certain number of times and allocationMonitor + * reaches the threshold (internal hardcoded) value, subsequent + * call to allocateNew() will take care of stepping down the + * default memory allocation size. + * + * Another case would be under-provisioning the initial memory and + * thus going through a lot of realloc(). Here the goal is to + * see if we can minimize the number of reallocations. Again the + * state is recorded in allocationMonitor by decrementing it + * (negative value). If a threshold is hit, realloc will try to + * allocate more memory in order to possibly avoid a future realloc. + * This case is also applicable to setSafe() methods which can trigger + * a realloc() and thus we record the state there as well. + */ + if (valueCount > 0) { + if (currentValueCapacity >= (valueCount * 2)) { + incrementAllocationMonitor(); + } else if (currentValueCapacity <= (valueCount / 2)) { + decrementAllocationMonitor(); } - } + } + setReaderAndWriterIndex(); + } + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null, false otherwise + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ + @Override + public void setIndexDefined(int index) { + handleSafe(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + public void set(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void set(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + protected void handleSafe(int index) { + while (index >= getValueCapacity()) { + decrementAllocationMonitor(); + reAlloc(); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index 47daeee7f202c..edf4987de5756 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,6 +24,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; @@ -31,928 +32,1219 @@ import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; public abstract class BaseNullableVariableWidthVector extends BaseValueVector - implements VariableWidthVector, FieldVector { - private static final int DEFAULT_RECORD_BYTE_COUNT = 8; - private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; - - private int valueAllocationSizeInBytes; - private int validityAllocationSizeInBytes; - private int offsetAllocationSizeInBytes; - - /* protected members */ - public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ - protected static final byte[] emptyByteArray = new byte[]{}; - protected ArrowBuf validityBuffer; - protected ArrowBuf valueBuffer; - protected ArrowBuf offsetBuffer; - protected int valueCount; - protected int lastSet; - protected final Field field; - private boolean cleared; - - public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator, - FieldType fieldType) { - super(name, allocator); - valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); - offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; - field = new Field(name, fieldType, null); - valueCount = 0; - lastSet = -1; - offsetBuffer = allocator.getEmpty(); - validityBuffer = allocator.getEmpty(); - valueBuffer = allocator.getEmpty(); - cleared = false; - } - - /* TODO: - * - * see if getNullCount() can be made faster -- O(1) - */ + implements VariableWidthVector, FieldVector, NullableVectorDefinitionSetter { + private static final int DEFAULT_RECORD_BYTE_COUNT = 8; + private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; + + private int valueAllocationSizeInBytes; + private int validityAllocationSizeInBytes; + private int offsetAllocationSizeInBytes; + + /* protected members */ + public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ + protected static final byte[] emptyByteArray = new byte[]{}; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected ArrowBuf offsetBuffer; + protected int valueCount; + protected int lastSet; + protected final Field field; + private boolean cleared; + + public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType) { + super(name, allocator); + valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; + field = new Field(name, fieldType, null); + valueCount = 0; + lastSet = -1; + offsetBuffer = allocator.getEmpty(); + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + cleared = false; + } /* TODO: - * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level - * base class BaseValueVector. - * - * Along with this, some class members (validityBuffer) can also be - * abstracted out to top level base class. - * - * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy (non-nullable) and those - * vectors have not yet been refactored/removed so moving things to - * the top class as of now is not a good idea. - */ - - - @Override - @Deprecated - public VariableWidthMutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); - } - - @Override - @Deprecated - public VariableWidthAccessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); - } - - /** - * Get buffer that manages the validity (NULL or NON-NULL nature) of - * elements in the vector. Consider it as a buffer for internal bit vector - * data structure. - * @return buffer - */ - @Override - public ArrowBuf getValidityBuffer() { - return validityBuffer; - } - - /** - * Get the buffer that stores the data for elements in the vector. - * @return buffer - */ - @Override - public ArrowBuf getDataBuffer() { - return valueBuffer; - } - - /** - * buffer that stores the offsets for elements - * in the vector. This operation is not supported for fixed-width vectors. - * @return buffer - */ - @Override - public ArrowBuf getOffsetBuffer() { - return offsetBuffer; - } - - /** - * Get the memory address of buffer that stores the offsets for elements - * in the vector. - * @return starting address of the buffer - */ - @Override - public long getOffsetBufferAddress() { - return offsetBuffer.memoryAddress(); - } - - /** - * Get the memory address of buffer that manages the validity - * (NULL or NON-NULL nature) of elements in the vector. - * @return starting address of the buffer - */ - @Override - public long getValidityBufferAddress() { - return validityBuffer.memoryAddress(); - } - - /** - * Get the memory address of buffer that stores the data for elements - * in the vector. - * @return starting address of the buffer - */ - @Override - public long getDataBufferAddress() { - return valueBuffer.memoryAddress(); - } - - /** - * Sets the desired value capacity for the vector. This function doesn't - * allocate any memory for the vector. - * @param valueCount desired number of elements in the vector - */ - @Override - public void setInitialCapacity(int valueCount) { - final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT; - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); - /* to track the end offset of last data element in vector, we need - * an additional slot in offset buffer. - */ - offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity(){ - final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); - return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); - } - - private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); - } - - private int getOffsetBufferValueCapacity() { - return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); - } - - /** - * zero out the vector and the data in associated buffers. - */ - public void zeroVector() { - initValidityBuffer(); - initOffsetBuffer(); - } - - /* zero out the validity buffer */ - private void initValidityBuffer() { - validityBuffer.setZero(0, validityBuffer.capacity()); - } - - /* zero out the offset buffer */ - private void initOffsetBuffer() { - offsetBuffer.setZero(0, offsetBuffer.capacity()); - } - - /** - * Reset the vector to initial state. Same as {@link #zeroVector()}. - * Note that this method doesn't release any memory. - */ - public void reset() { - zeroVector(); - lastSet = -1; - } - - /** - * Close the vector and release the associated buffers. - */ - @Override - public void close() { - clear(); - } - - /** - * Same as {@link #close()} - */ - @Override - public void clear() { - validityBuffer = releaseBuffer(validityBuffer); - valueBuffer = releaseBuffer(valueBuffer); - offsetBuffer = releaseBuffer(offsetBuffer); - cleared = true; - lastSet = -1; - valueCount = 0; - } - - @Override - @Deprecated - public List getFieldInnerVectors() { - throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); - } - - /** - * Initialize the children in schema for this Field. This operation is a - * NO-OP for scalar types since they don't have any children. - * @param children the schema - * @throws IllegalArgumentException if children is a non-empty list for scalar types. - */ - @Override - public void initializeChildrenFromFields(List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("primitive type vector can not have children"); - } - } - - /** - * Get the inner child vectors. - * @return list of child vectors for complex types, empty list for scalar vector - * types - */ - @Override - public List getChildrenFromFields() { - return Collections.emptyList(); - } - - - /** - * Load the buffers of this vector with provided source buffers. - * The caller manages the source buffers and populates them before invoking - * this method. - * @param fieldNode the fieldNode indicating the value count - * @param ownBuffers the buffers for this Field (own buffers only, children not included) - */ - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - ArrowBuf bitBuffer = ownBuffers.get(0); - ArrowBuf offBuffer = ownBuffers.get(1); - ArrowBuf dataBuffer = ownBuffers.get(2); - - validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); - offsetBuffer.release(); - offsetBuffer = offBuffer.retain(allocator); - valueBuffer.release(); - valueBuffer = dataBuffer.retain(allocator); - - lastSet = fieldNode.getLength() - 1; - valueCount = fieldNode.getLength(); - } - - /** - * Get the buffers belonging to this vector - * @return the inner buffers. - */ - public List getFieldBuffers() { - List result = new ArrayList<>(3); + * see if getNullCount() can be made faster -- O(1) + */ + + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. + */ + + @Override + @Deprecated + public VariableWidthMutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); + } + + @Override + @Deprecated + public VariableWidthAccessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); + } + + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + */ + @Override + public ArrowBuf getOffsetBuffer() { + return offsetBuffer; + } + + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. + * @return starting address of the buffer + */ + @Override + public long getOffsetBufferAddress() { + return offsetBuffer.memoryAddress(); + } + + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ + @Override + public long getValidityBufferAddress() { + return validityBuffer.memoryAddress(); + } + + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ + @Override + public long getDataBufferAddress() { + return valueBuffer.memoryAddress(); + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long) valueCount * DEFAULT_RECORD_BYTE_COUNT; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int) size; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); + /* to track the end offset of last data element in vector, we need + * an additional slot in offset buffer. + */ + offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); + return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); + } + + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + private int getOffsetBufferValueCapacity() { + return (int) ((offsetBuffer.capacity() * 1.0) / OFFSET_WIDTH); + } + + /** + * zero out the vector and the data in associated buffers. + */ + public void zeroVector() { + initValidityBuffer(); + initOffsetBuffer(); + } + + /* zero out the validity buffer */ + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + /* zero out the offset buffer */ + private void initOffsetBuffer() { + offsetBuffer.setZero(0, offsetBuffer.capacity()); + } + + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ + public void reset() { + zeroVector(); + lastSet = -1; + } + + /** + * Close the vector and release the associated buffers. + */ + @Override + public void close() { + clear(); + } + + /** + * Same as {@link #close()} + */ + @Override + public void clear() { + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + offsetBuffer = releaseBuffer(offsetBuffer); + cleared = true; + lastSet = -1; + valueCount = 0; + } + + @Override + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf offBuffer = ownBuffers.get(1); + ArrowBuf dataBuffer = ownBuffers.get(2); + + validityBuffer.release(); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); + offsetBuffer.release(); + offsetBuffer = offBuffer.retain(allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + lastSet = fieldNode.getLength() - 1; + valueCount = fieldNode.getLength(); + } + + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ + public List getFieldBuffers() { + List result = new ArrayList<>(3); + setReaderAndWriterIndex(); + result.add(validityBuffer); + result.add(offsetBuffer); + result.add(valueBuffer); + + return result; + } + + /** + * Set the reader and writer indexes for the inner buffers. + */ + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + offsetBuffer.readerIndex(0); + valueBuffer.readerIndex(0); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + offsetBuffer.writerIndex(0); + valueBuffer.writerIndex(0); + } else { final int lastDataOffset = getstartOffset(valueCount); - validityBuffer.readerIndex(0); validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.readerIndex(0); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); - valueBuffer.readerIndex(0); valueBuffer.writerIndex(lastDataOffset); - - result.add(validityBuffer); - result.add(offsetBuffer); - result.add(valueBuffer); - - return result; - } - - /** - * Same as {@link #allocateNewSafe()}. - */ - @Override - public void allocateNew() { - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating memory."); - } - } - - /** - * Allocate memory for the vector. We internally use a default value count - * of 4096 to allocate memory for at least these many elements in the - * vector. See {@link #allocateNew(int, int)} for allocating memory for specific - * number of elements in the vector. - * - * @return false if memory allocation fails, true otherwise. - */ - @Override - public boolean allocateNewSafe() { - long curAllocationSizeValue = valueAllocationSizeInBytes; - long curAllocationSizeValidity = validityAllocationSizeInBytes; - long curAllocationSizeOffset = offsetAllocationSizeInBytes; - - if (curAllocationSizeValue > MAX_ALLOCATION_SIZE || - curAllocationSizeOffset > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory exceeds limit"); - } - - /* we are doing a new allocation -- release the current buffers */ + } + } + + /** + * Same as {@link #allocateNewSafe()}. + */ + @Override + public void allocateNew() { + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int, int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + long curAllocationSizeOffset = offsetAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE || + curAllocationSizeOffset > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); + } catch (Exception e) { + e.printStackTrace(); clear(); - - try { - allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); - } catch (Exception e) { - e.printStackTrace(); - clear(); - return false; - } - - return true; - } - - /** - * Allocate memory for the vector to support storing at least the provided number of - * elements in the vector. This method must be called prior to using the ValueVector. - * - * @param totalBytes desired total memory capacity - * @param valueCount the desired number of elements in the vector - * @throws org.apache.arrow.memory.OutOfMemoryException - */ - @Override - public void allocateNew(int totalBytes, int valueCount) { - assert totalBytes >= 0; - final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); - - if (totalBytes > MAX_ALLOCATION_SIZE || - offsetBufferSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory exceeds limit"); - } - - /* we are doing a new allocation -- release the current buffers */ + return false; + } + + return true; + } + + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param totalBytes desired total memory capacity + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ + @Override + public void allocateNew(int totalBytes, int valueCount) { + assert totalBytes >= 0; + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + + if (totalBytes > MAX_ALLOCATION_SIZE || + offsetBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); + } catch (Exception e) { + e.printStackTrace(); clear(); - - try { - allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); - } catch (Exception e) { - e.printStackTrace(); - clear(); - } - } - - /* allocate the inner buffers */ - private void allocateBytes(final long valueBufferSize, final long validityBufferSize, - final long offsetBufferSize) { - /* allocate data buffer */ - int curSize = (int)valueBufferSize; - valueBuffer = allocator.buffer(curSize); - valueBuffer.readerIndex(0); - valueAllocationSizeInBytes = curSize; - allocateValidityBuffer(validityBufferSize); - allocateOffsetBuffer(offsetBufferSize); - } - - /* allocate offset buffer */ - private void allocateOffsetBuffer(final long size) { - final int curSize = (int)size; - offsetBuffer = allocator.buffer(curSize); - offsetBuffer.readerIndex(0); - offsetAllocationSizeInBytes = curSize; - initOffsetBuffer(); - } - - /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - initValidityBuffer(); - } - - /** - * Resize the vector to increase the capacity. The internal behavior is to - * double the current value capacity. - */ - public void reAlloc() { - reallocValueBuffer(); - reallocValidityAndOffsetBuffers(); - } - - protected void reallocValueBuffer() { - long baseSize = valueAllocationSizeInBytes; - final int currentBufferCapacity = valueBuffer.capacity(); - - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity); - valueBuffer.release(); - valueBuffer = newBuf; - valueAllocationSizeInBytes = (int)newAllocationSize; - } - - protected void reallocValidityAndOffsetBuffers() { - offsetBuffer = reallocBufferHelper(offsetBuffer, true); - validityBuffer = reallocBufferHelper(validityBuffer, false); - } - - /* helper method to realloc a particular buffer. returns the allocated buffer */ - private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { - final int currentBufferCapacity = buffer.capacity(); - long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes - : validityAllocationSizeInBytes); - - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, buffer, 0, currentBufferCapacity); - final int halfNewCapacity = newBuf.capacity() / 2; - newBuf.setZero(halfNewCapacity, halfNewCapacity); - buffer.release(1); - buffer = newBuf; - if (offsetBuffer) { - offsetAllocationSizeInBytes = (int)newAllocationSize; - } - else { - validityAllocationSizeInBytes = (int)newAllocationSize; - } - - return buffer; - } - - /** - * Get the size (number of bytes) of underlying data buffer. - * @return - */ - @Override - public int getByteCapacity(){ - return valueBuffer.capacity(); - } - - @Override - public int getCurrentSizeInBytes(){ + } + } + + /* allocate the inner buffers */ + private void allocateBytes(final long valueBufferSize, final long validityBufferSize, + final long offsetBufferSize) { + /* allocate data buffer */ + int curSize = (int) valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; + allocateValidityBuffer(validityBufferSize); + allocateOffsetBuffer(offsetBufferSize); + } + + /* allocate offset buffer */ + private void allocateOffsetBuffer(final long size) { + final int curSize = (int) size; + offsetBuffer = allocator.buffer(curSize); + offsetBuffer.readerIndex(0); + offsetAllocationSizeInBytes = curSize; + initOffsetBuffer(); + } + + /* allocate validity buffer */ + private void allocateValidityBuffer(final long size) { + final int curSize = (int) size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + initValidityBuffer(); + } + + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ + public void reAlloc() { + reallocDataBuffer(); + reallocValidityAndOffsetBuffers(); + } + + /** + * Reallocate the data buffer. Data Buffer stores the actual data for + * VARCHAR or VARBINARY elements in the vector. The behavior is to double + * the size of buffer. + * @throws OversizedAllocationException if the desired new size is more than + * max allowed + * @throws OutOfMemoryException if the internal memory allocation fails + */ + public void reallocDataBuffer() { + long baseSize = valueAllocationSizeInBytes; + final int currentBufferCapacity = valueBuffer.capacity(); + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity); + valueBuffer.release(); + valueBuffer = newBuf; + valueAllocationSizeInBytes = (int) newAllocationSize; + } + + /** + * Reallocate the validity and offset buffers for this vector. Validity + * buffer is used to track the NULL or NON-NULL nature of elements in + * the vector and offset buffer is used to store the lengths of variable + * width elements in the vector. + * + * Note that data buffer for variable length vectors moves independent + * of the companion validity and offset buffers. This is in + * contrast to what we have for fixed width vectors. + * + * So even though we may have setup an initial capacity of 1024 + * elements in the vector, it is quite possible + * that we need to reAlloc() the data buffer when we are setting + * the 5th element in the vector simply because previous + * variable length elements have exhausted the buffer capacity. + * However, we really don't need to reAlloc() validity and + * offset buffers until we try to set the 1025th element + * This is why we do a separate check for safe methods to + * determine which buffer needs reallocation. + * @throws OversizedAllocationException if the desired new size is more than + * max allowed + * @throws OutOfMemoryException if the internal memory allocation fails + */ + public void reallocValidityAndOffsetBuffers() { + offsetBuffer = reallocBufferHelper(offsetBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + /* helper method to realloc a particular buffer. returns the allocated buffer */ + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (offsetBuffer) { + offsetAllocationSizeInBytes = (int) newAllocationSize; + } else { + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + return buffer; + } + + /** + * Get the size (number of bytes) of underlying data buffer. + * @return + */ + @Override + public int getByteCapacity() { + return valueBuffer.capacity(); + } + + @Override + public int getCurrentSizeInBytes() { /* TODO */ + return 0; + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + return getBufferSizeFor(this.valueCount); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { return 0; - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - return getBufferSizeFor(this.valueCount); - } - - /** - * Get the potential buffer size for a particular number of records. - * @param valueCount desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } + } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); - final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ - final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); - return validityBufferSize + offsetBufferSize + dataBufferSize; - } - - /** - * Get information about how this field is materialized. - * @return the field corresponding to this vector - */ - @Override - public Field getField() { - return field; - } - - /** - * Return the underlying buffers associated with this vector. Note that this doesn't - * impact the reference counts for this buffer so it only should be used for in-context - * access. Also note that this buffer changes regularly thus - * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear Whether to clear vector before returning; the buffers will still be refcounted - * but the returned array will be the only reference to them - * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this - * vector instance. - */ - @Override - public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = new ArrowBuf[3]; + final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + return validityBufferSize + offsetBufferSize + dataBufferSize; + } + + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ + @Override + public Field getField() { + return field; + } + + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers; + setReaderAndWriterIndex(); + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = new ArrowBuf[3]; buffers[0] = validityBuffer; buffers[1] = offsetBuffer; - buffers[1] = valueBuffer; - if (clear) { - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); + buffers[2] = valueBuffer; + } + if (clear) { + for (final ArrowBuf buffer : buffers) { + buffer.retain(1); } - return buffers; - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @param callBack - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return getTransferPair(ref, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param allocator allocator for the target vector - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(BufferAllocator allocator){ - return getTransferPair(name, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return TransferPair - */ - public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); - - /** - * Transfer this vector'data to another vector. The memory associated - * with this vector is transferred to the allocator of target vector - * for accounting and management purposes. - * @param target destination vector for transfer - */ - public void transferTo(BaseNullableVariableWidthVector target){ - compareTypes(target, "transferTo"); - target.clear(); - target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; - target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; - target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; - target.valueCount = valueCount; - target.setLastSet(lastSet); clear(); - } - - /** - * Slice this vector at desired index and length and transfer the - * corresponding data to the target vector. - * @param startIndex start position of the split in source vector. - * @param length length of the split. - * @param target destination vector - */ - public void splitAndTransferTo(int startIndex, int length, - BaseNullableVariableWidthVector target) { - compareTypes(target, "splitAndTransferTo"); - target.clear(); - splitAndTransferValidityBuffer(startIndex, length, target); - splitAndTransferOffsetBuffer(startIndex, length, target); - target.setLastSet(length - 1); - target.setValueCount(length); - } - - /* - * Transfer the offsets along with data. Unlike the data buffer, we cannot simply - * slice the offset buffer for split and transfer. The reason is that offsets - * in the target vector have to be adjusted and made relative to the staring - * offset in source vector from the start index of split. This is why, we - * need to explicitly allocate the offset buffer and set the adjusted offsets - * in the target vector. - */ - private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { - final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); - final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH); - final int dataLength = end - start; - target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); - for (int i = 0; i < length + 1; i++) { - final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start; - target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset); + } + return buffers; + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ + public void transferTo(BaseNullableVariableWidthVector target) { + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; + target.setLastSet(this.lastSet); + if (this.valueCount > 0) { + target.setValueCount(this.valueCount); + } + clear(); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableVariableWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferOffsetBuffer(startIndex, length, target); + target.setLastSet(length - 1); + if (this.valueCount > 0) { + target.setValueCount(this.valueCount); + } + } + + /* + * Transfer the offsets along with data. Unlike the data buffer, we cannot simply + * slice the offset buffer for split and transfer. The reason is that offsets + * in the target vector have to be adjusted and made relative to the staring + * offset in source vector from the start index of split. This is why, we + * need to explicitly allocate the offset buffer and set the adjusted offsets + * in the target vector. + */ + private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { + final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); + final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH); + final int dataLength = end - start; + target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + for (int i = 0; i < length + 1; i++) { + final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start; + target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset); + } + target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer; + } + + /* + * Transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableVariableWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } } - target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer; - } - - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, - BaseNullableVariableWidthVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.retain(1); - } - else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, - firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - - - /****************************************************************** - * * - * common getters and setters * - * * - ******************************************************************/ - - - /** - * Get the number of elements that are null in the vector - * - * @return the number of null elements. - */ - public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); - } - - /** - * Check if the given index is within the current value capacity - * of the vector - * - * @param index position to check - * @return true if index is within the current value capacity - */ - public boolean isSafe(int index) { - return index < getValueCapacity(); - } - - /** - * Check if element at given index is null. - * - * @param index position of element - * @return true if element at given index is null - */ - public boolean isNull(int index) { - return (isSet(index) == 0); - } - - /** - * Same as {@link #isNull(int)}. - * - * @param index position of element - * @return 1 if element at given index is not null, 0 otherwise - */ - public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Get the value count of vector. This will always be zero unless - * setValueCount(int) has been called prior to calling this. - * - * @return valueCount for the vector - */ - public int getValueCount(){ - return valueCount; - } - - /** - * Sets the value count for the vector - * - * @param valueCount value count - */ - public void setValueCount(int valueCount) { - assert valueCount >= 0; - this.valueCount = valueCount; - while (valueCount > getValueCapacity()) { - reallocValidityAndOffsetBuffers(); - } - fillHoles(valueCount); - lastSet = valueCount - 1; - } - - /** - * Create holes in the vector upto the given index (exclusive). - * Holes will be created from the current last set position in - * the vector. - * - * @param index target index - */ - public void fillEmpties(int index) { - handleSafe(index, emptyByteArray.length); - fillHoles(index); - lastSet = index - 1; - } - - /** - * Set the index of last non-null element in the vector. - * It is important to call this method with appropriate value - * before calling {@link #setValueCount(int)}. - * - * @param value desired index of last non-null element. - */ - public void setLastSet(int value) { - lastSet = value; - } - - /** - * Get the index of last non-null element in the vector. - * - * @return index of the last non-null element - */ - public int getLastSet() { - return lastSet; - } - - /** - * Get the starting position (offset) in the data stream for a given - * element in the vector. - * - * @param index position of the element in the vector - * @return starting offset for the element - */ - public long getStartEnd(int index) { - return (long)offsetBuffer.getInt(index * OFFSET_WIDTH); - } - - /** - * Mark the particular position in the vector as non-null. - * - * @param index position of the element. - */ - public void setIndexDefined(int index) { - handleSafe(index, 0); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - } - - - /****************************************************************** - * * - * helper methods for setters * - * * - ******************************************************************/ - - - protected final void fillHoles(int index) { - for (int i = lastSet + 1; i < index; i++) { - setBytes(i, emptyByteArray, 0, emptyByteArray.length); - } - lastSet = index - 1; - } - - protected final void setBytes(int index, byte[] value, int start, int length) { - /* end offset of current last element in the vector. this will - * be the start offset of new element we are trying to store. - */ - final int startOffset = getstartOffset(index); - /* set new end offset */ - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - /* store the var length data in value buffer */ - valueBuffer.setBytes(startOffset, value, start, length); - } - - protected final int getstartOffset(int index) { - return offsetBuffer.getInt(index * OFFSET_WIDTH); - } - - protected final void handleSafe(int index, int dataLength) { - /* - * IMPORTANT: - * value buffer for variable length vectors moves independent - * of the companion validity and offset buffers. This is in - * contrast to what we have for fixed width vectors. - * - * Here there is no concept of getValueCapacity() in the - * data stream. getValueCapacity() is applicable only to validity - * and offset buffers. - * - * So even though we may have setup an initial capacity of 1024 - * elements in the vector, it is quite possible - * that we need to reAlloc() the data buffer when we are setting - * the 5th element in the vector simply because previous - * variable length elements have exhausted the buffer capacity. - * However, we really don't need to reAlloc() validity and - * offset buffers until we try to set the 1025th element - * This is why we do a separate check for safe methods to - * determine which buffer needs reallocation. - */ - while (index >= getValueCapacity()) { - reallocValidityAndOffsetBuffers(); - } - final int startOffset = getstartOffset(index); - while (valueBuffer.capacity() < (startOffset + dataLength)) { - reallocValueBuffer(); - } - } - - /** - * Method used by Json Writer to read a variable width element from - * the variable width vector and write to Json. - * - * This method should not be used externally. - * - * @param data buffer storing the variable width vector elements - * @param offset buffer storing the offsets of variable width vector elements - * @param index position of the element in the vector - * @return array of bytes - */ - public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) { - final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); - final int dataLength = - offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset; - final byte[] result = new byte[dataLength]; - data.getBytes(currentStartOffset, result, 0, dataLength); - return result; - } - - /** - * Method used by Json Reader to explicitly set the offsets of the variable - * width vector data. The method takes care of allocating the memory for - * offsets if the caller hasn't done so. - * - * This method should not be used externally. - * - * @param buffer ArrowBuf to store offsets for variable width elements - * @param allocator memory allocator - * @param valueCount number of elements - * @param index position of the element - * @param value offset of the element - * @return buffer holding the offsets - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * OFFSET_WIDTH); - } - buffer.setInt(index * OFFSET_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * OFFSET_WIDTH); - } - - return buffer; - } + } + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the value count of vector. This will always be zero unless + * setValueCount(int) has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount() { + return valueCount; + } + + /** + * Sets the value count for the vector + * + * @param valueCount value count + */ + public void setValueCount(int valueCount) { + assert valueCount >= 0; + this.valueCount = valueCount; + while (valueCount > getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + fillHoles(valueCount); + lastSet = valueCount - 1; + setReaderAndWriterIndex(); + } + + /** + * Create holes in the vector upto the given index (exclusive). + * Holes will be created from the current last set position in + * the vector. + * + * @param index target index + */ + public void fillEmpties(int index) { + handleSafe(index, emptyByteArray.length); + fillHoles(index); + lastSet = index - 1; + } + + /** + * Set the index of last non-null element in the vector. + * It is important to call this method with appropriate value + * before calling {@link #setValueCount(int)}. + * + * @param value desired index of last non-null element. + */ + public void setLastSet(int value) { + lastSet = value; + } + + /** + * Get the index of last non-null element in the vector. + * + * @return index of the last non-null element + */ + public int getLastSet() { + return lastSet; + } + + /** + * Get the starting position (offset) in the data stream for a given + * element in the vector. + * + * @param index position of the element in the vector + * @return starting offset for the element + */ + public long getStartEnd(int index) { + return offsetBuffer.getLong(index * OFFSET_WIDTH); + } + + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ + @Override + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + fillHoles(index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + lastSet = index; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if (isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + final int dataLength = end - start; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, buffer, start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + final int dataLength = end - start; + fillEmpties(index); + handleSafe(index, end); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, buffer, start, dataLength); + lastSet = index; + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param start start position of data in buffer + * @param length length of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int start, int length, ArrowBuf buffer) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + final ArrowBuf bb = buffer.slice(start, length); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param start start position of data in buffer + * @param length length of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int start, int length, ArrowBuf buffer) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + final ArrowBuf bb = buffer.slice(start, length); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + protected final void fillHoles(int index) { + for (int i = lastSet + 1; i < index; i++) { + setBytes(i, emptyByteArray, 0, emptyByteArray.length); + } + lastSet = index - 1; + } + + protected final void setBytes(int index, byte[] value, int start, int length) { + /* end offset of current last element in the vector. this will + * be the start offset of new element we are trying to store. + */ + final int startOffset = getstartOffset(index); + /* set new end offset */ + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + /* store the var length data in value buffer */ + valueBuffer.setBytes(startOffset, value, start, length); + } + + protected final int getstartOffset(int index) { + return offsetBuffer.getInt(index * OFFSET_WIDTH); + } + + protected final void handleSafe(int index, int dataLength) { + /* + * IMPORTANT: + * value buffer for variable length vectors moves independent + * of the companion validity and offset buffers. This is in + * contrast to what we have for fixed width vectors. + * + * Here there is no concept of getValueCapacity() in the + * data stream. getValueCapacity() is applicable only to validity + * and offset buffers. + * + * So even though we may have setup an initial capacity of 1024 + * elements in the vector, it is quite possible + * that we need to reAlloc() the data buffer when we are setting + * the 5th element in the vector simply because previous + * variable length elements have exhausted the buffer capacity. + * However, we really don't need to reAlloc() validity and + * offset buffers until we try to set the 1025th element + * This is why we do a separate check for safe methods to + * determine which buffer needs reallocation. + */ + while (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + final int startOffset = getstartOffset(index); + while (valueBuffer.capacity() < (startOffset + dataLength)) { + reallocDataBuffer(); + } + } + + /** + * Method used by Json Writer to read a variable width element from + * the variable width vector and write to Json. + * + * This method should not be used externally. + * + * @param data buffer storing the variable width vector elements + * @param offset buffer storing the offsets of variable width vector elements + * @param index position of the element in the vector + * @return array of bytes + */ + public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) { + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + final int dataLength = + offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset; + final byte[] result = new byte[dataLength]; + data.getBytes(currentStartOffset, result, 0, dataLength); + return result; + } + + /** + * Method used by Json Reader to explicitly set the offsets of the variable + * width vector data. The method takes care of allocating the memory for + * offsets if the caller hasn't done so. + * + * This method should not be used externally. + * + * @param buffer ArrowBuf to store offsets for variable width elements + * @param allocator memory allocator + * @param valueCount number of elements + * @param index position of the element + * @param value offset of the element + * @return buffer holding the offsets + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * OFFSET_WIDTH); + } + buffer.setInt(index * OFFSET_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * OFFSET_WIDTH); + } + + return buffer; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 1b96782681f26..fc0ab3e5915cc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 591d13c4bd5b4..c6d404e15430c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -251,12 +251,12 @@ public void copyFromSafe(int inIndex, int outIndex, BitVector from) { @Override public Mutator getMutator() { - return new Mutator(); + return mutator; } @Override public Accessor getAccessor() { - return new Accessor(); + return accessor; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 69325b5ea06b6..23252ca697b8d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,6 +20,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.schema.ArrowFieldNode; /** * Helper class for performing generic operations on a bit vector buffer. @@ -27,139 +28,174 @@ */ public class BitVectorHelper { - /** - * Get the index of byte corresponding to bit index in validity buffer - */ - public static int byteIndex(int absoluteBitIndex) { - return absoluteBitIndex >> 3; - } - - /** - * Get the relative index of bit within the byte in validity buffer - */ - public static int bitIndex(int absoluteBitIndex) { - return absoluteBitIndex & 7; - } - - /** - * Set the bit at provided index to 1. - * - * @param validityBuffer - * @param index - */ - public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { - final int byteIndex = byteIndex(index); - final int bitIndex = bitIndex(index); - byte currentByte = validityBuffer.getByte(byteIndex); - final byte bitMask = (byte) (1L << bitIndex); + /** + * Get the index of byte corresponding to bit index in validity buffer + */ + public static int byteIndex(int absoluteBitIndex) { + return absoluteBitIndex >> 3; + } + + /** + * Get the relative index of bit within the byte in validity buffer + */ + public static int bitIndex(int absoluteBitIndex) { + return absoluteBitIndex & 7; + } + + /** + * Set the bit at provided index to 1. + * + * @param validityBuffer + * @param index + */ + public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + currentByte |= bitMask; + validityBuffer.setByte(byteIndex, currentByte); + } + + /** + * Set the bit at a given index to provided value (1 or 0) + * + * @param validityBuffer + * @param index + * @param value + */ + public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + if (value != 0) { currentByte |= bitMask; - validityBuffer.setByte(byteIndex, currentByte); - } - - /** - * Set the bit at a given index to provided value (1 or 0) - * - * @param validityBuffer - * @param index - * @param value - */ - public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { - final int byteIndex = byteIndex(index); - final int bitIndex = bitIndex(index); - byte currentByte = validityBuffer.getByte(byteIndex); - final byte bitMask = (byte) (1L << bitIndex); - if (value != 0) { - currentByte |= bitMask; - } else { - currentByte -= (bitMask & currentByte); - } - validityBuffer.setByte(byteIndex, currentByte); - } - - /** - * Set the bit at a given index to provided value (1 or 0). Internally - * takes care of allocating the buffer if the caller didn't do so. - * - * @param validityBuffer - * @param allocator - * @param valueCount - * @param index - * @param value - * @return ArrowBuf - */ - public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (validityBuffer == null) { - validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + } else { + currentByte -= (bitMask & currentByte); + } + validityBuffer.setByte(byteIndex, currentByte); + } + + /** + * Set the bit at a given index to provided value (1 or 0). Internally + * takes care of allocating the buffer if the caller didn't do so. + * + * @param validityBuffer + * @param allocator + * @param valueCount + * @param index + * @param value + * @return ArrowBuf + */ + public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (validityBuffer == null) { + validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + } + setValidityBit(validityBuffer, index, value); + if (index == (valueCount - 1)) { + validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + } + + return validityBuffer; + } + + /** + * Check if a bit at a given index is set or not. + * + * @param buffer + * @param index + * @return 1 if bit is set, 0 otherwise. + */ + public static int get(final ArrowBuf buffer, int index) { + final int byteIndex = index >> 3; + final byte b = buffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Compute the size of validity buffer required to manage a given number + * of elements in a vector. + * + * @param valueCount + * @return buffer size + */ + public static int getValidityBufferSize(int valueCount) { + return ((int) Math.ceil(valueCount / 8.0)); + } + + /** + * Given a validity buffer, find the number of bits that are not set. + * This is used to compute the number of null elements in a nullable vector. + * + * @param validityBuffer + * @param valueCount + * @return number of bits not set. + */ + public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) { + if (valueCount == 0) { + return 0; + } + int count = 0; + final int sizeInBytes = getValidityBufferSize(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) { + return (byte) ((data.getByte(index) & 0xFF) >>> offset); + } + + public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte) ((data.getByte(index) << (8 - offset))); + } + + public static ArrowBuf loadValidityBuffer(final ArrowFieldNode fieldNode, + final ArrowBuf sourceValidityBuffer, + final BufferAllocator allocator) { + final int valueCount = fieldNode.getLength(); + ArrowBuf newBuffer = null; + /* either all NULLs or all non-NULLs */ + if (fieldNode.getNullCount() == 0 || fieldNode.getNullCount() == valueCount) { + newBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + newBuffer.setZero(0, newBuffer.capacity()); + if (fieldNode.getNullCount() != 0) { + /* all NULLs */ + return newBuffer; } - setValidityBit(validityBuffer, index, value); - if (index == (valueCount - 1)) { - validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + /* all non-NULLs */ + int fullBytesCount = valueCount / 8; + for (int i = 0; i < fullBytesCount; ++i) { + newBuffer.setByte(i, 0xFF); } - - return validityBuffer; - } - - /** - * Check if a bit at a given index is set or not. - * - * @param buffer - * @param index - * @return 1 if bit is set, 0 otherwise. - */ - public static int get(final ArrowBuf buffer, int index) { - final int byteIndex = index >> 3; - final byte b = buffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Compute the size of validity buffer required to manage a given number - * of elements in a vector. - * - * @param valueCount - * @return buffer size - */ - public static int getValidityBufferSize(int valueCount) { - return ((int) Math.ceil(valueCount / 8.0)); - } - - /** - * Given a validity buffer, find the number of bits that are not set. - * This is used to compute the number of null elements in a nullable vector. - * - * @param validityBuffer - * @param valueCount - * @return number of bits not set. - */ - public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) { - if (valueCount == 0) { return 0; } - int count = 0; - final int sizeInBytes = getValidityBufferSize(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + int remainder = valueCount % 8; + if (remainder > 0) { + byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7)); + newBuffer.setByte(fullBytesCount, bitMask); } - int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. + } else { + /* mixed byte pattern -- create another ArrowBuf associated with the + * target allocator */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; - } - - public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } + newBuffer = sourceValidityBuffer.retain(allocator); + } - public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } + return newBuffer; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java new file mode 100644 index 0000000000000..806beb5f8fedc --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java @@ -0,0 +1,336 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import java.math.BigDecimal; +import java.nio.charset.Charset; + +/** + * Helper class to generate test data for Nullable fixed and variable + * width scalar vectors. Previous implementations of java vector classes + * provided generateTestData(now deprecated) API to populate the vector + * with sample data. This class should be used for that purpose. + */ +public class GenerateSampleData { + + public static void generateTestData(final ValueVector vector, final int valueCount) { + if (vector instanceof NullableIntVector) { + writeIntData((NullableIntVector) vector, valueCount); + } else if (vector instanceof NullableDecimalVector) { + writeDecimalData((NullableDecimalVector) vector, valueCount); + } else if (vector instanceof NullableBitVector) { + writeBooleanData((NullableBitVector) vector, valueCount); + } else if (vector instanceof NullableVarCharVector) { + writeVarCharData((NullableVarCharVector) vector, valueCount); + } else if (vector instanceof NullableVarBinaryVector) { + writeVarBinaryData((NullableVarBinaryVector) vector, valueCount); + } else if (vector instanceof NullableBigIntVector) { + writeBigIntData((NullableBigIntVector) vector, valueCount); + } else if (vector instanceof NullableFloat4Vector) { + writeFloatData((NullableFloat4Vector) vector, valueCount); + } else if (vector instanceof NullableFloat8Vector) { + writeDoubleData((NullableFloat8Vector) vector, valueCount); + } else if (vector instanceof NullableDateDayVector) { + writeDateDayData((NullableDateDayVector) vector, valueCount); + } else if (vector instanceof NullableDateMilliVector) { + writeDateMilliData((NullableDateMilliVector) vector, valueCount); + } else if (vector instanceof NullableIntervalDayVector) { + writeIntervalDayData((NullableIntervalDayVector) vector, valueCount); + } else if (vector instanceof NullableIntervalYearVector) { + writeIntervalYearData((NullableIntervalYearVector) vector, valueCount); + } else if (vector instanceof NullableSmallIntVector) { + writeSmallIntData((NullableSmallIntVector) vector, valueCount); + } else if (vector instanceof NullableTinyIntVector) { + writeTinyIntData((NullableTinyIntVector) vector, valueCount); + } else if (vector instanceof NullableTimeMicroVector) { + writeTimeMicroData((NullableTimeMicroVector) vector, valueCount); + } else if (vector instanceof NullableTimeMilliVector) { + writeTimeMilliData((NullableTimeMilliVector) vector, valueCount); + } else if (vector instanceof NullableTimeNanoVector) { + writeTimeNanoData((NullableTimeNanoVector) vector, valueCount); + } else if (vector instanceof NullableTimeSecVector) { + writeTimeSecData((NullableTimeSecVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampSecVector) { + writeTimeStampData((NullableTimeStampSecVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMicroVector) { + writeTimeStampData((NullableTimeStampMicroVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMilliVector) { + writeTimeStampData((NullableTimeStampMilliVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampNanoVector) { + writeTimeStampData((NullableTimeStampNanoVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampSecTZVector) { + writeTimeStampData((NullableTimeStampSecTZVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMicroTZVector) { + writeTimeStampData((NullableTimeStampMicroTZVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMilliTZVector) { + writeTimeStampData((NullableTimeStampMilliTZVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampNanoTZVector) { + writeTimeStampData((NullableTimeStampNanoTZVector) vector, valueCount); + } + } + + private static void writeTimeStampData(NullableTimeStampVector vector, int valueCount) { + final long even = 100000; + final long odd = 200000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDecimalData(NullableDecimalVector vector, int valueCount) { + final BigDecimal even = new BigDecimal(0.0543278923); + final BigDecimal odd = new BigDecimal(2.0543278923); + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeIntData(NullableIntVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeBooleanData(NullableBitVector vector, int valueCount) { + final int even = 0; + final int odd = 1; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeIntervalYearData(NullableIntervalYearVector vector, int valueCount) { + final int even = 1; + final int odd = 2; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeIntervalDayData(NullableIntervalDayVector vector, int valueCount) { + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, 1, 50); + } else { + vector.setSafe(i, 2, 100); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTimeSecData(NullableTimeSecVector vector, int valueCount) { + final int even = 500; + final int odd = 900; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTimeMilliData(NullableTimeMilliVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTimeMicroData(NullableTimeMicroVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + + } + + private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDateDayData(NullableDateDayVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDateMilliData(NullableDateMilliVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeSmallIntData(NullableSmallIntVector vector, int valueCount) { + final short even = 10; + final short odd = 20; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTinyIntData(NullableTinyIntVector vector, int valueCount) { + final byte even = 1; + final byte odd = 2; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeBigIntData(NullableBigIntVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeFloatData(NullableFloat4Vector vector, int valueCount) { + final float even = 20.3f; + final float odd = 40.2f; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) { + final double even = 20.2373; + final double odd = 40.2378; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeVarBinaryData(NullableVarBinaryVector vector, int valueCount) { + Charset utf8Charset = Charset.forName("UTF-8"); + final byte[] even = "AAAAA1".getBytes(utf8Charset); + final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeVarCharData(NullableVarCharVector vector, int valueCount) { + Charset utf8Charset = Charset.forName("UTF-8"); + final byte[] even = "AAAAA1".getBytes(utf8Charset); + final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } +} + diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java index 253427333a2e0..eca6592c54ad2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,335 +34,335 @@ * maintained to track which elements in the vector are null. */ public class NullableBigIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableBigIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableBigIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), - allocator); - } - - /** - * Instantiate a NullableBigIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new BigIntReaderImpl(NullableBigIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.BIGINT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), + allocator); + } + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new BigIntReaderImpl(NullableBigIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIGINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBigIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableBigIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, BigIntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, BigIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, BigIntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BigIntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BigIntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableBigIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableBigIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableBigIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableBigIntVector to){ - this.to = to; - } - - @Override - public NullableBigIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBigIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableBigIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableBigIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBigIntVector to) { + this.to = to; + } + + @Override + public NullableBigIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java index eceaa61b57f41..bd363f5b7f1f3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,15 +6,15 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; @@ -39,150 +38,160 @@ * to a single bit in the underlying data stream backing the vector. */ public class NullableBitVector extends BaseNullableFixedWidthVector { - private final FieldReader reader; - - /** - * Instantiate a NullableBitVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableBitVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.BIT.getType()), - allocator); - } - - /** - * Instantiate a NullableBitVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, (byte)0); - reader = new BitReaderImpl(NullableBitVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.BIT; - } - - /** - * Sets the desired value capacity for the vector. This function doesn't - * allocate any memory for the vector. - * @param valueCount desired number of elements in the vector - */ - @Override - public void setInitialCapacity(int valueCount) { - final int size = getValidityBufferSizeFromCount(valueCount); - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - valueAllocationSizeInBytes = size; - validityAllocationSizeInBytes = size; - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity(){ - return (int)(validityBuffer.capacity() * 8L); - } - - /** - * Get the potential buffer size for a particular number of records. - * @param count desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int count) { - if (count == 0) { return 0; } - return 2 * getValidityBufferSizeFromCount(count); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - return getBufferSizeFor(valueCount); - } - - /** - * Slice this vector at desired index and length and transfer the - * corresponding data to the target vector. - * @param startIndex start position of the split in source vector. - * @param length length of the split. - * @param target destination vector - */ - public void splitAndTransferTo(int startIndex, int length, - BaseNullableFixedWidthVector target) { - compareTypes(target, "splitAndTransferTo"); - target.clear(); - target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, - validityBuffer, target.validityBuffer); - target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, - valueBuffer, target.valueBuffer); - - target.setValueCount(length); - } - - private ArrowBuf splitAndTransferBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target, - ArrowBuf sourceBuffer, ArrowBuf destBuffer) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { + private final FieldReader reader; + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIT.getType()), + allocator); + } + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, (byte) 0); + reader = new BitReaderImpl(NullableBitVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIT; + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final int size = getValidityBufferSizeFromCount(valueCount); + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = size; + validityAllocationSizeInBytes = size; + } + + /** + * Get the current value capacity for the vector + * + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + /** + * Get the potential buffer size for a particular number of records. + * + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { + return 0; + } + return 2 * getValidityBufferSizeFromCount(count); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + return getBufferSizeFor(valueCount); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, + validityBuffer, target.validityBuffer); + target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, + valueBuffer, target.valueBuffer); + + target.setValueCount(length); + } + + private ArrowBuf splitAndTransferBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target, + ArrowBuf sourceBuffer, ArrowBuf destBuffer) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { /* slice */ - if (destBuffer != null) { - destBuffer.release(); - } - destBuffer = destBuffer.slice(firstByteSource, byteSizeTarget); - destBuffer.retain(1); - } - else { + if (destBuffer != null) { + destBuffer.release(); + } + destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget); + destBuffer.retain(1); + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. * Each byte in the target is composed by a part in i-th byte, * another part in (i+1)-th byte. */ - destBuffer = allocator.buffer(byteSizeTarget); - destBuffer.readerIndex(0); - destBuffer.setZero(0, destBuffer.capacity()); + destBuffer = allocator.buffer(byteSizeTarget); + destBuffer.readerIndex(0); + destBuffer.setZero(0, destBuffer.capacity()); - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); - destBuffer.setByte(i, (b1 + b2)); - } + destBuffer.setByte(i, (b1 + b2)); + } /* Copying the last piece is done in the following manner: * if the source vector has 1 or more bytes remaining, we copy @@ -193,310 +202,315 @@ private ArrowBuf splitAndTransferBuffer(int startIndex, int length, * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, - firstByteSource + byteSizeTarget, offset); - - destBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, - firstByteSource + byteSizeTarget - 1, offset); - destBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - - return destBuffer; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - private int getBit(int index) { - final int byteIndex = index >> 3; - final byte b = valueBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return getBit(index); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableBitHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = getBit(index); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Boolean getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return new Boolean (getBit(index) != 0); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableBitVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - if (value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); - } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - if (holder.value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); - } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); - } - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, + firstByteSource + byteSizeTarget, offset); + + destBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + destBuffer.setByte(byteSizeTarget - 1, b1); + } } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, BitHolder holder) { + } + + return destBuffer; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + private int getBit(int index) { + final int byteIndex = index >> 3; + final byte b = valueBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return getBit(index); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBitHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = getBit(index); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Boolean getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return new Boolean(getBit(index) != 0); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBitVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); if (holder.value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); + BitVectorHelper.setValidityBitToOne(valueBuffer, index); } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); + BitVectorHelper.setValidityBit(valueBuffer, index, 0); } - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableBitHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, BitHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, BitHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BitHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BitHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableBitVector)to); - } - - private class TransferImpl implements TransferPair { - NullableBitVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableBitVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableBitVector to){ - this.to = to; - } - - @Override - public NullableBitVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBitVector) to); + } + + private class TransferImpl implements TransferPair { + NullableBitVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableBitVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBitVector to) { + this.to = to; + } + + @Override + public NullableBitVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java index e6b5b590f3f7e..c5ddc9e87be6a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,336 +35,336 @@ * maintained to track which elements in the vector are null. */ public class NullableDateDayVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableDateDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableDateDayVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), - allocator); - } - - /** - * Instantiate a NullableDateDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new DateDayReaderImpl(NullableDateDayVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.DATEDAY; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateDayReaderImpl(NullableDateDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateDayHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableDateDayHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, DateDayHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, DateDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, DateDayHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateDayHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateDayHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDateDayVector)to); - } - - private class TransferImpl implements TransferPair { - NullableDateDayVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableDateDayVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableDateDayVector to){ - this.to = to; - } - - @Override - public NullableDateDayVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateDayVector) to); + } + + private class TransferImpl implements TransferPair { + NullableDateDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableDateDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateDayVector to) { + this.to = to; + } + + @Override + public NullableDateDayVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java index 8e15100f70818..7e37055dab188 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,338 +37,339 @@ * maintained to track which elements in the vector are null. */ public class NullableDateMilliVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableDateMilliVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), - allocator); - } - - /** - * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new DateMilliReaderImpl(NullableDateMilliVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.DATEMILLI; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableDateMilliHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long millis = get(index); - final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, DateMilliHolder holder){ + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateMilliReaderImpl(NullableDateMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateMilliHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, DateMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, DateMilliHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateMilliHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateMilliHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDateMilliVector)to); - } - - private class TransferImpl implements TransferPair { - NullableDateMilliVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableDateMilliVector to){ - this.to = to; - } - - @Override - public NullableDateMilliVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateMilliVector) to); + } + + private class TransferImpl implements TransferPair { + NullableDateMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateMilliVector to) { + this.to = to; + } + + @Override + public NullableDateMilliVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index 2a611c6a2d5ba..dcc551094ae28 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,383 +37,391 @@ * maintained to track which elements in the vector are null. */ public class NullableDecimalVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 16; - private final FieldReader reader; - - private final int precision; - private final int scale; - - /** - * Instantiate a NullableDecimalVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableDecimalVector(String name, BufferAllocator allocator, - int precision, int scale) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), - allocator); - } - - /** - * Instantiate a NullableDecimalVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal)fieldType.getType(); - reader = new DecimalReaderImpl(NullableDecimalVector.this); - this.precision = arrowType.getPrecision(); - this.scale = arrowType.getScale(); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.DECIMAL; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public ArrowBuf get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableDecimalHolder holder) { - if (isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.buffer = valueBuffer; - holder.precision = precision; - holder.scale = scale; - holder.start = index * TYPE_WIDTH; - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public BigDecimal getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { - if (from.isSet(fromIndex) != 0) { - from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, - thisIndex * TYPE_WIDTH, TYPE_WIDTH); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param buffer ArrowBuf containing decimal value. - */ - public void set(int index, ArrowBuf buffer) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param start start index of data in the buffer - * @param buffer ArrowBuf containing decimal value. - */ - public void set(int index, int start, ArrowBuf buffer) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value BigDecimal containing decimal value. - */ - public void set(int index, BigDecimal value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - DecimalUtility.checkPrecisionAndScale(value, precision, scale); - DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, DecimalHolder holder){ + public static final byte TYPE_WIDTH = 16; + private final FieldReader reader; + + private final int precision; + private final int scale; + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, BufferAllocator allocator, + int precision, int scale) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), + allocator); + } + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal) fieldType.getType(); + reader = new DecimalReaderImpl(NullableDecimalVector.this); + this.precision = arrowType.getPrecision(); + this.scale = arrowType.getScale(); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DECIMAL; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDecimalHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.buffer = valueBuffer; + holder.precision = precision; + holder.scale = scale; + holder.start = index * TYPE_WIDTH; + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public BigDecimal getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { + if (from.isSet(fromIndex) != 0) { + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + /** + * Return scale for the decimal value + */ + public int getScale() { + return scale; + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, int start, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void set(int index, BigDecimal value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + DecimalUtility.checkPrecisionAndScale(value, precision, scale); + DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); - } - - /** - * Same as {@link #set(int, ArrowBuf)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param buffer ArrowBuf containing decimal value. - */ - public void setSafe(int index, ArrowBuf buffer) { - handleSafe(index); - set(index, buffer); - } - - /** - * Same as {@link #set(int, int, ArrowBuf)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param start start index of data in the buffer - * @param buffer ArrowBuf containing decimal value. - */ - public void setSafe(int index, int start, ArrowBuf buffer) { - handleSafe(index); - set(index, start, buffer); - } - - /** - * Same as {@link #set(int, BigDecimal)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value BigDecimal containing decimal value. - */ - public void setSafe(int index, BigDecimal value){ - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, DecimalHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, DecimalHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DecimalHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, ArrowBuf buffer) { + handleSafe(index); + set(index, buffer); + } + + /** + * Same as {@link #set(int, int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, start, buffer); + } + + /** + * Same as {@link #set(int, BigDecimal)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void setSafe(int index, BigDecimal value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DecimalHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void set(int index, int isSet, int start, ArrowBuf buffer) { + if (isSet > 0) { + set(index, start, buffer); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of the value in the buffer - * @param buffer buffer containing the value to be stored in the vector - */ - public void set(int index, int isSet, int start, ArrowBuf buffer) { - if (isSet > 0) { - set(index, start, buffer); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles - * the case when the position of new value is beyond the current value - * capacity of the vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of the value in the buffer - * @param buffer buffer containing the value to be stored in the vector - */ - public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { - handleSafe(index); - set(index, isSet, start, buffer); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDecimalVector)to); - } - - private class TransferImpl implements TransferPair { - NullableDecimalVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, - NullableDecimalVector.this.scale); - } - - public TransferImpl(NullableDecimalVector to){ - this.to = to; - } - - @Override - public NullableDecimalVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); - } - } + } + } + + /** + * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles + * the case when the position of new value is beyond the current value + * capacity of the vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, isSet, start, buffer); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDecimalVector) to); + } + + private class TransferImpl implements TransferPair { + NullableDecimalVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, + NullableDecimalVector.this.scale); + } + + public TransferImpl(NullableDecimalVector to) { + this.to = to; + } + + @Override + public NullableDecimalVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java index 3ba5cfcea5251..834f16c211cdb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,336 +34,336 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat4Vector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableFloat4Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), - allocator); - } - - /** - * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new Float4ReaderImpl(NullableFloat4Vector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.FLOAT4; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public float get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float4ReaderImpl(NullableFloat4Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public float get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat4Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Float getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getFloat(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableFloat4Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Float getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, float value) { - valueBuffer.setFloat(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, float value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, Float4Holder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, float value) { + valueBuffer.setFloat(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, float value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, float)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, float value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, Float4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, Float4Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float4Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, float)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, float value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float4Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, float value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, float value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, float)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, float value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static float get(final ArrowBuf buffer, final int index) { - return buffer.getFloat(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableFloat4Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableFloat4Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableFloat4Vector to){ - this.to = to; - } - - @Override - public NullableFloat4Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, float)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, float value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static float get(final ArrowBuf buffer, final int index) { + return buffer.getFloat(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat4Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableFloat4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat4Vector to) { + this.to = to; + } + + @Override + public NullableFloat4Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java index 2fb96a44fe78b..b054d9b05c06a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,336 +34,336 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat8Vector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableFloat8Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), - allocator); - } - - /** - * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new Float8ReaderImpl(NullableFloat8Vector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.FLOAT8; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public double get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float8ReaderImpl(NullableFloat8Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public double get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat8Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Double getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getDouble(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableFloat8Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Double getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, double value) { - valueBuffer.setDouble(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, double value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, Float8Holder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, double value) { + valueBuffer.setDouble(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, double value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, double)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, double value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, Float8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, Float8Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float8Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, double)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, double value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float8Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, double value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, double value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, double)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, double value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static double get(final ArrowBuf buffer, final int index) { - return buffer.getDouble(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableFloat8Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableFloat8Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableFloat8Vector to){ - this.to = to; - } - - @Override - public NullableFloat8Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, double)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, double value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static double get(final ArrowBuf buffer, final int index) { + return buffer.getDouble(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat8Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableFloat8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat8Vector to) { + this.to = to; + } + + @Override + public NullableFloat8Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 93deacbdfdc74..e5dfbcedcf90e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,336 +34,346 @@ * maintained to track which elements in the vector are null. */ public class NullableIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), - allocator); - } - - /** - * Instantiate a NullableIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntReaderImpl(NullableIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.INT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), + allocator); + } + + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntReaderImpl(NullableIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, IntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, IntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, IntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableIntVector to){ - this.to = to; - } - - @Override - public NullableIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + *

    + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntVector to) { + this.to = to; + } + + @Override + public NullableIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java index 174b722b04316..8d2fb5661d9eb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,394 +36,394 @@ * vector are null. */ public class NullableIntervalDayVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private static final byte MILLISECOND_OFFSET = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableIntervalDayVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), - allocator); - } - - /** - * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.INTERVALDAY; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public ArrowBuf get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - return null; - } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableIntervalDayHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } + private static final byte TYPE_WIDTH = 8; + private static final byte MILLISECOND_OFFSET = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + return null; + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalDayHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startIndex = index * TYPE_WIDTH; + holder.isSet = 1; + holder.days = valueBuffer.getInt(startIndex); + holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { final int startIndex = index * TYPE_WIDTH; - holder.isSet = 1; - holder.days = valueBuffer.getInt(startIndex); - holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Period getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final int startIndex = index * TYPE_WIDTH; - final int days = valueBuffer.getInt(startIndex); - final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); - final Period p = new Period(); - return p.plusDays(days).plusMillis(milliseconds); - } - } - - /** - * Get the Interval value at a given index as a {@link StringBuilder} object - * @param index position of the element - * @return String Builder object with Interval value as - * [days, hours, minutes, seconds, millis] - */ - public StringBuilder getAsStringBuilder(int index) { - if (isSet(index) == 0) { - return null; - }else{ - return getAsStringBuilderHelper(index); - } - } - - private StringBuilder getAsStringBuilderHelper(int index) { - final int startIndex = index * TYPE_WIDTH; - - final int days = valueBuffer.getInt(startIndex); - int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); - - final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); - - final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); - - final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); - - final String dayString = (Math.abs(days) == 1) ? " day " : " days "; - - return(new StringBuilder(). - append(days).append(dayString). - append(hours).append(":"). - append(minutes).append(":"). - append(seconds).append("."). - append(millis)); - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { - if (from.isSet(fromIndex) != 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); - from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, - thisIndex * TYPE_WIDTH, TYPE_WIDTH); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, ArrowBuf value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param days days for the interval - * @param milliseconds milliseconds for the interval - */ - public void set(int index, int days, int milliseconds){ - final int offsetIndex = index * TYPE_WIDTH; - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setInt(offsetIndex, days); - valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - set(index, holder.days, holder.milliseconds); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, IntervalDayHolder holder){ + final int days = valueBuffer.getInt(startIndex); + final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + final Period p = new Period(); + return p.plusDays(days).plusMillis(milliseconds); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [days, hours, minutes, seconds, millis] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + } else { + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + final int startIndex = index * TYPE_WIDTH; + + final int days = valueBuffer.getInt(startIndex); + int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + + final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + + final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + + final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + + final String dayString = (Math.abs(days) == 1) ? " day " : " days "; + + return (new StringBuilder(). + append(days).append(dayString). + append(hours).append(":"). + append(minutes).append(":"). + append(seconds).append("."). + append(millis)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + if (from.isSet(fromIndex) != 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, ArrowBuf value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void set(int index, int days, int milliseconds) { + final int offsetIndex = index * TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setInt(offsetIndex, days); + valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { set(index, holder.days, holder.milliseconds); - } - - /** - * Same as {@link #set(int, ArrowBuf)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, ArrowBuf value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param days days for the interval - * @param milliseconds milliseconds for the interval - */ - public void setSafe(int index, int days, int milliseconds) { - handleSafe(index); - set(index, days, milliseconds); - } - - /** - * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, IntervalDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, IntervalDayHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalDayHolder holder) { + set(index, holder.days, holder.milliseconds); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, ArrowBuf value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void setSafe(int index, int days, int milliseconds) { + handleSafe(index); + set(index, days, milliseconds); + } + + /** + * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalDayHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void set(int index, int isSet, int days, int milliseconds) { + if (isSet > 0) { + set(index, days, milliseconds); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param days days component of interval - * @param milliseconds millisecond component of interval - */ - public void set(int index, int isSet, int days, int milliseconds) { - if (isSet > 0) { - set(index, days, milliseconds); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param days days component of interval - * @param milliseconds millisecond component of interval - */ - public void setSafe(int index, int isSet, int days, int milliseconds) { - handleSafe(index); - set(index, isSet, days, milliseconds); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntervalDayVector)to); - } - - private class TransferImpl implements TransferPair { - NullableIntervalDayVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableIntervalDayVector to){ - this.to = to; - } - - @Override - public NullableIntervalDayVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void setSafe(int index, int isSet, int days, int milliseconds) { + handleSafe(index); + set(index, isSet, days, milliseconds); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalDayVector) to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalDayVector to) { + this.to = to; + } + + @Override + public NullableIntervalDayVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java index 303286d0ed30c..ac121d2f078ff 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,354 +34,354 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableIntervalYearVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableIntervalYearVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), - allocator); - } - - /** - * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.INTERVALYEAR; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableIntervalYearHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Period getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final int interval = get(index); - final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final Period p = new Period(); - return p.plusYears(years).plusMonths(months); - } - } - - /** - * Get the Interval value at a given index as a {@link StringBuilder} object - * @param index position of the element - * @return String Builder object with Interval value as - * [years, months] - */ - public StringBuilder getAsStringBuilder(int index) { - if (isSet(index) == 0) { - return null; - }else{ - return getAsStringBuilderHelper(index); - } - } - - private StringBuilder getAsStringBuilderHelper(int index) { - int value = valueBuffer.getInt(index * TYPE_WIDTH); - - final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); - - final String yearString = (Math.abs(years) == 1) ? " year " : " years "; - final String monthString = (Math.abs(months) == 1) ? " month " : " months "; - - return(new StringBuilder(). - append(years).append(yearString). - append(months).append(monthString)); - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, IntervalYearHolder holder){ + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALYEAR; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalYearHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int interval = valueBuffer.getInt(index * TYPE_WIDTH); + final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final Period p = new Period(); + return p.plusYears(years).plusMonths(months); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [years, months] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + } else { + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + int value = valueBuffer.getInt(index * TYPE_WIDTH); + + final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + + final String yearString = (Math.abs(years) == 1) ? " year " : " years "; + final String monthString = (Math.abs(months) == 1) ? " month " : " months "; + + return (new StringBuilder(). + append(years).append(yearString). + append(months).append(monthString)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, IntervalYearHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, IntervalYearHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalYearHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalYearHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntervalYearVector)to); - } - - private class TransferImpl implements TransferPair { - NullableIntervalYearVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableIntervalYearVector to){ - this.to = to; - } - - @Override - public NullableIntervalYearVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalYearVector) to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalYearVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalYearVector to) { + this.to = to; + } + + @Override + public NullableIntervalYearVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java index ed337188b5b0b..5dd1492a3e095 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,363 +34,363 @@ * maintained to track which elements in the vector are null. */ public class NullableSmallIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 2; - private final FieldReader reader; - - /** - * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableSmallIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), - allocator); - } - - /** - * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new SmallIntReaderImpl(NullableSmallIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.SMALLINT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public short get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), + allocator); + } + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new SmallIntReaderImpl(NullableSmallIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.SMALLINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public short get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableSmallIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Short getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getShort(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableSmallIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getShort(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Short getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); - } - - private void setValue(int index, short value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, short value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, SmallIntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + private void setValue(int index, short value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, short value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, short)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, short value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, SmallIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, SmallIntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, SmallIntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, short)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, short value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, SmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, SmallIntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, short value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, short value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, short)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, short value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static short get(final ArrowBuf buffer, final int index) { - return buffer.getShort(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableSmallIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableSmallIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableSmallIntVector to){ - this.to = to; - } - - @Override - public NullableSmallIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, short)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, short value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static short get(final ArrowBuf buffer, final int index) { + return buffer.getShort(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableSmallIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableSmallIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableSmallIntVector to) { + this.to = to; + } + + @Override + public NullableSmallIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java index d7ea322239741..25da4f68d506c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,335 +36,335 @@ * vector are null. */ public class NullableTimeMicroVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeMicroVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMEMICRO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMicroHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeMicroHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeMicroHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeMicroHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMicroHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMicroHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeMicroVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeMicroVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeMicroVector to){ - this.to = to; - } - - @Override - public NullableTimeMicroVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMicroVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMicroVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMicroVector to) { + this.to = to; + } + + @Override + public NullableTimeMicroVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java index 07d8abba4e57b..7d2d5d1f10fe2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,336 +36,336 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeMilliVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeMilliVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMEMILLI; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeMilliHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } - org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), - org.joda.time.DateTimeZone.UTC); - return ldt; - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeMilliHolder holder){ + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMilliHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } + org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), + org.joda.time.DateTimeZone.UTC); + return ldt; + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeMilliHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMilliHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMilliHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeMilliVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeMilliVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeMilliVector to){ - this.to = to; - } - - @Override - public NullableTimeMilliVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMilliVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMilliVector to) { + this.to = to; + } + + @Override + public NullableTimeMilliVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java index 947b2392f7a15..e93a63fe2a1f9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,335 +34,335 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeNanoVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeNanoVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMENANO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMENANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeNanoHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeNanoHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeNanoHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeNanoHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeNanoHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeNanoHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer,final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeNanoVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeNanoVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeNanoVector to){ - this.to = to; - } - - @Override - public NullableTimeNanoVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeNanoVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeNanoVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeNanoVector to) { + this.to = to; + } + + @Override + public NullableTimeNanoVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java index 0a3cfaac047eb..8329fa7813987 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,336 +34,336 @@ * maintained to track which elements in the vector are null. */ public class NullableTimeSecVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeSecVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeSecReaderImpl(NullableTimeSecVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESEC; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeSecReaderImpl(NullableTimeSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeSecHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeSecHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeSecHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeSecHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeSecHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeSecHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeSecVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeSecVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeSecVector to){ - this.to = to; - } - - @Override - public NullableTimeSecVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeSecVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeSecVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeSecVector to) { + this.to = to; + } + + @Override + public NullableTimeSecVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java index 0d25eff4e02b7..c7ad141549bcb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,186 +34,184 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMicroTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMICROTZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMicroTZHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMicroTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICROTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMicroTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMicroTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java index 41f7a1112f2df..5a34b194ce77f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,187 +34,186 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMicroVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMICRO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMicroHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */ - final long micros = get(index); - final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMicroHolder holder){ + final long micros = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMicroHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMicroVector )to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java index 6376b085e64e1..f9736aed7be62 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,185 +34,183 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMilliTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMILLITZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMilliTZHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMilliTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLITZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMilliTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMilliTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java index ff581d2ea2f03..fc38a6bd507de 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,185 +34,184 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMilliVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMILLI; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMilliHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long millis = get(index); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMilliHolder holder){ + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMilliHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMilliVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java index 6c4445d500b39..8eb48462c994b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,186 +34,184 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampNanoTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPNANOTZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampNanoTZHolder holder) { - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampNanoTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANOTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampNanoTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampNanoTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java index 096913ebf1cf0..6cc5c905783d7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,186 +34,185 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampNanoVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPNANO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampNanoHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long nanos = get(index); - final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampNanoHolder holder){ + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long nanos = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampNanoHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampNanoVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java index 8e5ba72ae1fe2..0593ac76dce57 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,185 +33,183 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampSecTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPSECTZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampSecTZHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampSecTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSECTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampSecTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampSecTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java index bc800e0e4f04f..1afdb30d0453e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,186 +34,185 @@ * maintained to track which elements in the vector are null. */ public class NullableTimeStampSecVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader() { - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPSEC; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampSecHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long secs = get(index); - final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampSecHolder holder){ + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long secs = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampSecHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampSecVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java index 1bf2abc384d92..d65cb83e15123 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,189 +29,191 @@ * maintained to track which elements in the vector are null. */ public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector { - protected static final byte TYPE_WIDTH = 8; - - /** - * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - protected void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + protected static final byte TYPE_WIDTH = 8; + + /** + * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + protected void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - public class TransferImpl implements TransferPair { - NullableTimeStampVector to; - - public TransferImpl(NullableTimeStampVector to) { - this.to = to; - } - - @Override - public NullableTimeStampVector getTo(){ - return to; - } - - @Override - public void transfer() { - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + public class TransferImpl implements TransferPair { + NullableTimeStampVector to; + + public TransferImpl(NullableTimeStampVector to) { + this.to = to; + } + + @Override + public NullableTimeStampVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java index ccbfa32b85f0c..106359ddd9666 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,363 +34,363 @@ * maintained to track which elements in the vector are null. */ public class NullableTinyIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 1; - private final FieldReader reader; - - /** - * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTinyIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), - allocator); - } - - /** - * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TinyIntReaderImpl(NullableTinyIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TINYINT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public byte get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), + allocator); + } + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TinyIntReaderImpl(NullableTinyIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TINYINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTinyIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTinyIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Byte getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - private void setValue(int index, int value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - private void setValue(int index, byte value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, byte value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TinyIntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, byte)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, byte value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TinyIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TinyIntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TinyIntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TinyIntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, byte value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, byte)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, byte value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static byte get(final ArrowBuf buffer, final int index) { - return buffer.getByte(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTinyIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTinyIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTinyIntVector to){ - this.to = to; - } - - @Override - public NullableTinyIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, byte)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static byte get(final ArrowBuf buffer, final int index) { + return buffer.getByte(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTinyIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTinyIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTinyIntVector to) { + this.to = to; + } + + @Override + public NullableTinyIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java index 4d20f800ba1fd..52f6c151bdff9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,286 +33,286 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt1Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 1; - private final FieldReader reader; - - public NullableUInt1Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), - allocator); - } - - public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt1ReaderImpl(NullableUInt1Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT1; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public byte get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + public NullableUInt1Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), + allocator); + } + + public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt1ReaderImpl(NullableUInt1Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT1; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt1Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt1Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Byte getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - private void setValue(int index, byte value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, byte value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt1Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, byte)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, byte value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt1Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt1Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt1Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt1Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, byte value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, byte value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt1Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt1Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt1Vector to){ - this.to = to; - } - - @Override - public NullableUInt1Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt1Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt1Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt1Vector to) { + this.to = to; + } + + @Override + public NullableUInt1Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java index b140b9f5cbc67..42ba6e139bf71 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,286 +34,286 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt2Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 2; - private final FieldReader reader; - - public NullableUInt2Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), - allocator); - } - - public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt2ReaderImpl(NullableUInt2Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT2; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public char get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + public NullableUInt2Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), + allocator); + } + + public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt2ReaderImpl(NullableUInt2Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT2; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public char get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt2Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Character getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getChar(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt2Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getChar(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Character getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); - } - - private void setValue(int index, char value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, char value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt2Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + private void setValue(int index, char value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, char value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, char)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, char value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt2Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt2Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt2Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, char)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, char value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt2Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, char value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, char value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, char value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt2Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt2Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt2Vector to){ - this.to = to; - } - - @Override - public NullableUInt2Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, char value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt2Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt2Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt2Vector to) { + this.to = to; + } + + @Override + public NullableUInt2Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java index fc8cb639581fb..1ec21490d8447 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,258 +34,258 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt4Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - public NullableUInt4Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), - allocator); - } - - public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt4ReaderImpl(NullableUInt4Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT4; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + public NullableUInt4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), + allocator); + } + + public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt4ReaderImpl(NullableUInt4Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt4Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt4Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt4Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt4Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt4Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt4Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt4Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt4Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt4Vector to){ - this.to = to; - } - - @Override - public NullableUInt4Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt4Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt4Vector to) { + this.to = to; + } + + @Override + public NullableUInt4Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java index 2110cf03b9f05..408a810cfa0af 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,258 +34,258 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt8Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - public NullableUInt8Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), - allocator); - } - - public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt8ReaderImpl(NullableUInt8Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT8; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + public NullableUInt8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), + allocator); + } + + public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt8ReaderImpl(NullableUInt8Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt8Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt8Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return get(index); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt8Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt8Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt8Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt8Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt8Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt8Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt8Vector to){ - this.to = to; - } - - @Override - public NullableUInt8Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt8Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt8Vector to) { + this.to = to; + } + + @Override + public NullableUInt8Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java index 330e06710fcdd..1dfe8912c52f9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,479 +36,293 @@ * to track which elements in the vector are null. */ public class NullableVarBinaryVector extends BaseNullableVariableWidthVector { - private final FieldReader reader; - - /** - * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableVarBinaryVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); - } - - /** - * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType); - reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.VARBINARY; - } - - - /****************************************************************** - * * - * vector value getter methods * - * * - ******************************************************************/ - - - /** - * Get the variable length element at specified index as byte array. - * - * @param index position of element to get - * @return array of bytes for non-null element, null otherwise - */ - public byte[] get(int index) { - assert index >= 0; - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - final byte[] result = new byte[dataLength]; - valueBuffer.getBytes(startOffset, result, 0, dataLength); - return result; - } - - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return byte array for non-null element, null otherwise - */ - public byte[] getObject(int index) { - byte[] b; - try { - b = get(index); - } catch (IllegalStateException e) { - return null; - } - return b; - } - - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return greater than 0 length for non-null element, 0 otherwise - */ - public int getValueLength(int index) { - assert index >= 0; - if(isSet(index) == 0) { - return 0; - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - return dataLength; - } - - /** - * Get the variable length element at specified index and sets the state - * in provided holder. - * - * @param index position of element to get - * @param holder data holder to be populated by this function - */ - public void get(int index, NullableVarBinaryHolder holder){ - assert index >= 0; - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - holder.isSet = 1; - holder.start = startOffset; - holder.end = dataLength; - holder.buffer = valueBuffer; - } - - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { - fillHoles(thisIndex); - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { - fillEmpties(thisIndex); - if (from.isSet(fromIndex) != 0) { - setSafe(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } - } - - - /** - * Set the variable length element at the specified index to the supplied - * byte array. This is same as using {@link #set(int, byte[], int, int)} - * with start as 0 and length as value.length - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void set(int index, byte[] value) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[])} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void setSafe(int index, byte[] value) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, value.length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the supplied - * byte array. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void set(int index, byte[] value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[], int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void setSafe(int index, byte[] value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the - * content in supplied ByteBuffer - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void set(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void setSafe(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, VarBinaryHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, VarBinaryHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, VarBinaryHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, NullableVarBinaryHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, NullableVarBinaryHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Sets the value length for an element. - * - * @param index position of the element to set - * @param length length of the element - */ - public void setValueLengthSafe(int index, int length) { - assert index >= 0; - handleSafe(index, length); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index, 0); - /* not really needed to set the bit to 0 as long as - * the buffer always starts from 0. - */ - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, isSet); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); - final ArrowBuf bb = buffer.slice(start, end); - valueBuffer.setBytes(startOffset, bb); - lastSet = index; - } - - /** - * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - handleSafe(index, end); - set(index, isSet, start, end, buffer); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableVarBinaryVector)to); - } - - private class TransferImpl implements TransferPair { - NullableVarBinaryVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableVarBinaryVector to){ - this.to = to; - } - - @Override - public NullableVarBinaryVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); - } - } + private final FieldReader reader; + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); + } + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARBINARY; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return byte array for non-null element, null otherwise + */ + public byte[] getObject(int index) { + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + return b; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.start = getstartOffset(index); + holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + holder.buffer = valueBuffer; + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + handleSafe(thisIndex, length); + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarBinaryVector) to); + } + + private class TransferImpl implements TransferPair { + NullableVarBinaryVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarBinaryVector to) { + this.to = to; + } + + @Override + public NullableVarBinaryVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); + } + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java index 0b03164f0f385..31e409cc6263f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,481 +37,295 @@ * to track which elements in the vector are null. */ public class NullableVarCharVector extends BaseNullableVariableWidthVector { - private final FieldReader reader; - - /** - * Instantiate a NullableVarCharVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableVarCharVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); - } - - /** - * Instantiate a NullableVarCharVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType); - reader = new VarCharReaderImpl(NullableVarCharVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.VARCHAR; - } - - - /****************************************************************** - * * - * vector value getter methods * - * * - ******************************************************************/ - - - /** - * Get the variable length element at specified index as byte array. - * - * @param index position of element to get - * @return array of bytes for non-null element, null otherwise - */ - public byte[] get(int index) { - assert index >= 0; - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - final byte[] result = new byte[dataLength]; - valueBuffer.getBytes(startOffset, result, 0, dataLength); - return result; - } - - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return Text object for non-null element, null otherwise - */ - public Text getObject(int index) { - Text result = new Text(); - byte[] b; - try { - b = get(index); - } catch (IllegalStateException e) { - return null; - } - result.set(b); - return result; - } - - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return greater than 0 length for non-null element, 0 otherwise - */ - public int getValueLength(int index) { - assert index >= 0; - if(isSet(index) == 0) { - return 0; - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - return dataLength; - } - - /** - * Get the variable length element at specified index and sets the state - * in provided holder. - * - * @param index position of element to get - * @param holder data holder to be populated by this function - */ - public void get(int index, NullableVarCharHolder holder){ - assert index >= 0; - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - holder.isSet = 1; - holder.start = startOffset; - holder.end = dataLength; - holder.buffer = valueBuffer; - } - - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { - fillHoles(thisIndex); - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { - fillEmpties(thisIndex); - if (from.isSet(fromIndex) != 0) { - setSafe(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } - } - - - /** - * Set the variable length element at the specified index to the supplied - * byte array. This is same as using {@link #set(int, byte[], int, int)} - * with start as 0 and length as value.length - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void set(int index, byte[] value) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[])} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void setSafe(int index, byte[] value) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, value.length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the supplied - * byte array. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void set(int index, byte[] value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[], int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void setSafe(int index, byte[] value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the - * content in supplied ByteBuffer - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void set(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void setSafe(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, VarCharHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, VarCharHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, VarCharHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, NullableVarCharHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, NullableVarCharHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Sets the value length for an element. - * - * @param index position of the element to set - * @param length length of the element - */ - public void setValueLengthSafe(int index, int length) { - assert index >= 0; - handleSafe(index, length); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index, 0); - /* not really needed to set the bit to 0 as long as - * the buffer always starts from 0. - */ - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, isSet); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); - final ArrowBuf bb = buffer.slice(start, end); - valueBuffer.setBytes(startOffset, bb); - lastSet = index; - } - - /** - * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - handleSafe(index, end); - set(index, isSet, start, end, buffer); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableVarCharVector)to); - } - - private class TransferImpl implements TransferPair { - NullableVarCharVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableVarCharVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableVarCharVector to){ - this.to = to; - } - - @Override - public NullableVarCharVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this); - } - } + private final FieldReader reader; + + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableVarCharVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); + } + + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarCharReaderImpl(NullableVarCharVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARCHAR; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return Text object for non-null element, null otherwise + */ + public Text getObject(int index) { + Text result = new Text(); + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + result.set(b); + return result; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarCharHolder holder) { + assert index >= 0; + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.start = getstartOffset(index); + holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + holder.buffer = valueBuffer; + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + handleSafe(thisIndex, length); + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarCharVector) to); + } + + private class TransferImpl implements TransferPair { + NullableVarCharVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableVarCharVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarCharVector to) { + this.to = to; + } + + @Override + public NullableVarCharVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this); + } + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index dc74afaa2e49c..0d3acf1646f53 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 288afc6bbec69..fd9677312ed33 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -22,13 +22,10 @@ import java.util.List; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.ValueVector.Accessor; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.schema.ArrowRecordBatch; import org.apache.arrow.vector.schema.ArrowVectorType; -import javax.annotation.Nullable; - public class VectorUnloader { private final VectorSchemaRoot root; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 3eab30a32d224..4648d078949a7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,8 +74,7 @@ public boolean allocateNewSafe() { e.printStackTrace(); clear(); return false; - } - finally { + } finally { if (!dataAlloc) { clear(); } @@ -84,7 +83,7 @@ public boolean allocateNewSafe() { } protected void allocateOffsetBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; offsetBuffer = allocator.buffer(curSize); offsetBuffer.readerIndex(0); offsetAllocationSizeInBytes = curSize; @@ -101,8 +100,8 @@ protected void reallocOffsetBuffer() { final int currentBufferCapacity = offsetBuffer.capacity(); long baseSize = offsetAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -112,13 +111,13 @@ protected void reallocOffsetBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; newBuf.setZero(halfNewCapacity, halfNewCapacity); offsetBuffer.release(1); offsetBuffer = newBuf; - offsetAllocationSizeInBytes = (int)newAllocationSize; + offsetAllocationSizeInBytes = (int) newAllocationSize; } @Override @@ -147,8 +146,8 @@ public int getValueCapacity() { return Math.min(vector.getValueCapacity(), offsetValueCapacity); } - private int getOffsetBufferValueCapacity() { - return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); + protected int getOffsetBufferValueCapacity() { + return (int) ((offsetBuffer.capacity() * 1.0) / OFFSET_WIDTH); } @Override @@ -183,8 +182,13 @@ public void clear() { @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, - vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false), + ArrowBuf.class); + } if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -208,15 +212,15 @@ public AddOrGetResult addOrGetVector(FieldType fieldT // returned vector must have the same field created = true; if (callBack != null && - // not a schema change if changing from ZeroVector to ZeroVector - (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { + // not a schema change if changing from ZeroVector to ZeroVector + (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { callBack.doWork(); } } if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", - fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); + fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); throw new SchemaChangeRuntimeException(msg); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index bb091f68c1807..6713b1c7871b5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,15 +6,15 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector.complex; @@ -125,7 +124,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers ArrowBuf bitBuffer = ownBuffers.get(0); validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); valueCount = fieldNode.getLength(); validityAllocationSizeInBytes = validityBuffer.capacity(); @@ -134,15 +133,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers @Override public List getFieldBuffers() { List result = new ArrayList<>(1); - - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - + setReaderAndWriterIndex(); result.add(validityBuffer); return result; } + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -182,6 +183,8 @@ public boolean allocateNewSafe() { */ boolean success = false; try { + /* we are doing a new allocation -- release the current buffers */ + clear(); /* allocate validity buffer */ allocateValidityBuffer(validityAllocationSizeInBytes); success = vector.allocateNewSafe(); @@ -196,7 +199,7 @@ public boolean allocateNewSafe() { } private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; validityBuffer = allocator.buffer(curSize); validityBuffer.readerIndex(0); validityAllocationSizeInBytes = curSize; @@ -213,8 +216,8 @@ private void reallocValidityBuffer() { final int currentBufferCapacity = validityBuffer.capacity(); long baseSize = validityAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -224,12 +227,12 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setZero(0, newBuf.capacity()); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); validityBuffer.release(1); validityBuffer = newBuf; - validityAllocationSizeInBytes = (int)newAllocationSize; + validityAllocationSizeInBytes = (int) newAllocationSize; } public FieldVector getDataVector() { @@ -282,7 +285,14 @@ public void clear() { @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class); + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), + ArrowBuf.class); + } if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -311,7 +321,7 @@ public AddOrGetResult addOrGetVector(FieldType type) // returned vector must have the same field if (!Objects.equals(vector.getField().getType(), type.getType())) { final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", - type.getType(), vector.getField().getType()); + type.getType(), vector.getField().getType()); throw new SchemaChangeRuntimeException(msg); } @@ -400,7 +410,7 @@ public int getValueCount() { } private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); + return (int) (validityBuffer.capacity() * 8L); } public void setNull(int index) { @@ -462,6 +472,7 @@ public void transfer() { dataPair.transfer(); to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; to.setValueCount(valueCount); + clear(); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java deleted file mode 100644 index 435d3d8ccf2e6..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.complex; - -import java.util.Collections; -import java.util.Iterator; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BaseValueVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.UInt4Vector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.ZeroVector; -import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.SchemaChangeRuntimeException; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; - -public abstract class LegacyBaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { - - public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; - public final static String DATA_VECTOR_NAME = "$data$"; - - protected final UInt4Vector offsets; - protected FieldVector vector; - protected final CallBack callBack; - - protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, DEFAULT_DATA_VECTOR, callBack); - } - - protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { - super(name, allocator); - this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); - this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); - this.callBack = callBack; - } - - @Override - public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - offsets.zeroVector(); - return success; - } - - @Override - public void reAlloc() { - offsets.reAlloc(); - vector.reAlloc(); - } - - @Override - public UInt4Vector getOffsetVector() { - return offsets; - } - - @Override - public FieldVector getDataVector() { - return vector; - } - - @Override - public void setInitialCapacity(int numRecords) { - offsets.setInitialCapacity(numRecords + 1); - vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); - } - - @Override - public int getValueCapacity() { - final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); - if (vector == DEFAULT_DATA_VECTOR) { - return offsetValueCapacity; - } - return Math.min(vector.getValueCapacity(), offsetValueCapacity); - } - - @Override - public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { - return 0; - } - return offsets.getBufferSize() + vector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(int valueCount) { - if (valueCount == 0) { - return 0; - } - - return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); - } - - @Override - public Iterator iterator() { - return Collections.singleton(getDataVector()).iterator(); - } - - @Override - public void clear() { - offsets.clear(); - vector.clear(); - super.clear(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); - if (clear) { - for (ArrowBuf buffer : buffers) { - buffer.retain(); - } - clear(); - } - return buffers; - } - - /** - * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 - */ - public int size() { - return vector == DEFAULT_DATA_VECTOR ? 0 : 1; - } - - public AddOrGetResult addOrGetVector(FieldType fieldType) { - boolean created = false; - if (vector instanceof ZeroVector) { - vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack); - // returned vector must have the same field - created = true; - if (callBack != null && - // not a schema change if changing from ZeroVector to ZeroVector - (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { - callBack.doWork(); - } - } - - if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { - final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", - fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); - throw new SchemaChangeRuntimeException(msg); - } - - return new AddOrGetResult<>((T) vector, created); - } - - protected void replaceDataVector(FieldVector v) { - vector.clear(); - vector = v; - } - - public abstract class LegacyBaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { - - @Override - public int getValueCount() { - return Math.max(offsets.getAccessor().getValueCount() - 1, 0); - } - - @Override - public int getInnerValueCount() { - return vector.getAccessor().getValueCount(); - } - - @Override - public int getInnerValueCountAt(int index) { - return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); - } - - @Override - public boolean isNull(int index) { - return false; - } - - @Override - public boolean isEmpty(int index) { - return false; - } - } - - public abstract class LegacyBaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { - - @Override - public int startNewValue(int index) { - while (offsets.getValueCapacity() <= index) { - offsets.reAlloc(); - } - int offset = offsets.getAccessor().get(index); - offsets.getMutator().setSafe(index + 1, offset); - setValueCount(index + 1); - return offset; - } - - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); - } - } - -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java deleted file mode 100644 index 0843907d9df82..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java +++ /dev/null @@ -1,298 +0,0 @@ -/******************************************************************************* - - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.arrow.vector.complex; - -import static java.util.Collections.singletonList; -import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.OutOfMemoryException; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringArrayList; -import org.apache.arrow.vector.util.SchemaChangeRuntimeException; -import org.apache.arrow.vector.util.TransferPair; - -public class LegacyFixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { - - public static LegacyFixedSizeListVector empty(String name, int size, BufferAllocator allocator) { - FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size)); - return new LegacyFixedSizeListVector(name, allocator, fieldType, null); - } - - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); - /* delegate */ - private final FixedSizeListVector fixedSizeListVector; - - // deprecated, use FieldType or static constructor instead - @Deprecated - public LegacyFixedSizeListVector(String name, - BufferAllocator allocator, - int listSize, - DictionaryEncoding dictionary, - CallBack schemaChangeCallback) { - super(name, allocator); - fixedSizeListVector = new FixedSizeListVector(name, allocator, listSize, dictionary, schemaChangeCallback); - } - - public LegacyFixedSizeListVector(String name, - BufferAllocator allocator, - FieldType fieldType, - CallBack schemaChangeCallback) { - super(name, allocator); - fixedSizeListVector = new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback); - } - - @Override - public Field getField() { - return fixedSizeListVector.getField(); - } - - @Override - public MinorType getMinorType() { - return MinorType.FIXED_SIZE_LIST; - } - - public int getListSize() { - return fixedSizeListVector.getListSize(); - } - - @Override - public void initializeChildrenFromFields(List children) { - fixedSizeListVector.initializeChildrenFromFields(children); - } - - @Override - public List getChildrenFromFields() { - return fixedSizeListVector.getChildrenFromFields(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - fixedSizeListVector.loadFieldBuffers(fieldNode, ownBuffers); - } - - @Override - public List getFieldBuffers() { - return fixedSizeListVector.getFieldBuffers(); - } - - @Override - public List getFieldInnerVectors() { - return fixedSizeListVector.getFieldInnerVectors(); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - @Override - public UnionFixedSizeListReader getReader() { - return fixedSizeListVector.getReader(); - } - - @Override - public void allocateNew() throws OutOfMemoryException { - fixedSizeListVector.allocateNew(); - } - - @Override - public boolean allocateNewSafe() { - return fixedSizeListVector.allocateNewSafe(); - } - - @Override - public void reAlloc() { - fixedSizeListVector.reAlloc(); - } - - public FieldVector getDataVector() { - return fixedSizeListVector.getDataVector(); - } - - @Override - public void setInitialCapacity(int numRecords) { - fixedSizeListVector.setInitialCapacity(numRecords); - } - - @Override - public int getValueCapacity() { - return fixedSizeListVector.getValueCapacity(); - } - - @Override - public int getBufferSize() { - return fixedSizeListVector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(int valueCount) { - return fixedSizeListVector.getBufferSizeFor(valueCount); - } - - @Override - public Iterator iterator() { - return fixedSizeListVector.iterator(); - } - - @Override - public void clear() { - fixedSizeListVector.clear(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - return fixedSizeListVector.getBuffers(clear); - } - - /** - * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 - */ - public int size() { - return fixedSizeListVector.size(); - } - - @Override - @SuppressWarnings("unchecked") - public AddOrGetResult addOrGetVector(FieldType type) { - return fixedSizeListVector.addOrGetVector(type); - } - - public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) { - fixedSizeListVector.copyFromSafe(inIndex, outIndex, from); - } - - public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) { - fixedSizeListVector.copyFrom(fromIndex, thisIndex, from); - } - - @Override - public UnionVector promoteToUnion() { - return fixedSizeListVector.promoteToUnion(); - } - - @Override - public long getValidityBufferAddress() { - return fixedSizeListVector.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getValidityBuffer() { - return fixedSizeListVector.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); - } - - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - return fixedSizeListVector.getObject(index); - } - - @Override - public boolean isNull(int index) { - return fixedSizeListVector.isNull(index); - } - - @Override - public int getNullCount() { - return fixedSizeListVector.getNullCount(); - } - - @Override - public int getValueCount() { - return fixedSizeListVector.getValueCount(); - } - } - - public class Mutator extends BaseValueVector.BaseMutator { - - public void setNull(int index) { - fixedSizeListVector.setNull(index); - } - - public void setNotNull(int index) { - fixedSizeListVector.setNotNull(index); - } - - @Override - public void setValueCount(int valueCount) { - fixedSizeListVector.setValueCount(valueCount); - } - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return fixedSizeListVector.getTransferPair(ref, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return fixedSizeListVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector target) { - return fixedSizeListVector.makeTransferPair(((LegacyFixedSizeListVector)target).fixedSizeListVector); - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java deleted file mode 100644 index 732699ada7a81..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java +++ /dev/null @@ -1,288 +0,0 @@ -/******************************************************************************* - - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.arrow.vector.complex; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.util.Collections.list; -import static java.util.Collections.singletonList; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.OutOfMemoryException; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.UnionListReader; -import org.apache.arrow.vector.complex.impl.UnionListWriter; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.complex.writer.FieldWriter; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.ArrowType.Null; -import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringArrayList; -import org.apache.arrow.vector.util.TransferPair; - -public class LegacyListVector extends LegacyBaseRepeatedValueVector implements FieldVector, PromotableVector { - - public static LegacyListVector empty(String name, BufferAllocator allocator) { - return new LegacyListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); - } - - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); - - private final ListVector listVector; - - // deprecated, use FieldType or static constructor instead - @Deprecated - public LegacyListVector(String name, BufferAllocator allocator, CallBack callBack) { - super(name, allocator, callBack); - listVector = new ListVector(name, allocator, callBack); - } - - // deprecated, use FieldType or static constructor instead - @Deprecated - public LegacyListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { - super(name, allocator, callBack); - listVector = new ListVector(name, allocator, dictionary, callBack); - } - - public LegacyListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); - listVector = new ListVector(name, allocator, fieldType, callBack); - } - - @Override - public void initializeChildrenFromFields(List children) { - listVector.initializeChildrenFromFields(children); - } - - @Override - public List getChildrenFromFields() { - return listVector.getChildrenFromFields(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - listVector.loadFieldBuffers(fieldNode, ownBuffers); - } - - @Override - public List getFieldBuffers() { - return listVector.getFieldBuffers(); - } - - @Override - public List getFieldInnerVectors() { - return listVector.getFieldInnerVectors(); - } - - public UnionListWriter getWriter() { - return listVector.getWriter(); - } - - @Override - public void allocateNew() throws OutOfMemoryException { - listVector.allocateNew(); - } - - @Override - public void reAlloc() { - listVector.reAlloc(); - } - - public void copyFromSafe(int inIndex, int outIndex, ListVector from) { - listVector.copyFrom(inIndex, outIndex, from); - } - - public void copyFrom(int inIndex, int outIndex, ListVector from) { - listVector.copyFrom(inIndex, outIndex, from); - } - - @Override - public FieldVector getDataVector() { - return listVector.getDataVector(); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return listVector.getTransferPair(ref, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return listVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector target) { - return listVector.makeTransferPair(((LegacyListVector)target).listVector); - } - - @Override - public long getValidityBufferAddress() { - return listVector.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public long getOffsetBufferAddress() { - return listVector.getOffsetBufferAddress(); - } - - @Override - public ArrowBuf getValidityBuffer() { - return listVector.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - return listVector.getOffsetBuffer(); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - @Override - public UnionListReader getReader() { - return listVector.getReader(); - } - - @Override - public boolean allocateNewSafe() { - return listVector.allocateNewSafe(); - } - - public AddOrGetResult addOrGetVector(FieldType fieldType) { - return listVector.addOrGetVector(fieldType); - } - - @Override - public int getBufferSize() { - return listVector.getBufferSize(); - } - - @Override - public Field getField() { - return listVector.getField(); - } - - @Override - public MinorType getMinorType() { - return MinorType.LIST; - } - - @Override - public void clear() { - listVector.clear(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - return listVector.getBuffers(clear); - } - - @Override - public UnionVector promoteToUnion() { - return listVector.promoteToUnion(); - } - - private int lastSet = 0; - - public class Accessor extends LegacyBaseRepeatedAccessor { - - @Override - public Object getObject(int index) { - return listVector.getObject(index); - } - - @Override - public boolean isNull(int index) { - return listVector.isNull(index); - } - - @Override - public int getNullCount() { - return listVector.getNullCount(); - } - } - - public class Mutator extends LegacyBaseRepeatedMutator { - public void setNotNull(int index) { - listVector.setNotNull(index); - } - - @Override - public int startNewValue(int index) { - return listVector.startNewValue(index); - } - - /** - * End the current value - * - * @param index index of the value to end - * @param size number of elements in the list that was written - */ - public void endValue(int index, int size) { - listVector.endValue(index, size); - } - - @Override - public void setValueCount(int valueCount) { - listVector.setValueCount(valueCount); - } - - public void setLastSet(int value) { - listVector.setLastSet(value); - } - - public int getLastSet() { - return listVector.getLastSet(); - } - } - -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java deleted file mode 100644 index 77556ab774f02..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.complex; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; -import com.google.common.primitives.Ints; - -import io.netty.buffer.ArrowBuf; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringHashMap; -import org.apache.arrow.vector.util.TransferPair; - -public class LegacyMapVector extends AbstractMapVector { - - /* delegate */ - private final MapVector mapVector; - - public static LegacyMapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); - return new LegacyMapVector(name, allocator, fieldType, null); - } - - private final Accessor accessor = new Accessor(); - private final Mutator mutator = new Mutator(); - - @Deprecated - public LegacyMapVector(String name, BufferAllocator allocator, CallBack callBack) { - super(name, allocator, callBack); - mapVector = new MapVector(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); - } - - public LegacyMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); - mapVector = new MapVector(name, allocator, fieldType, callBack); - } - - @Override - public FieldReader getReader() { - return mapVector.getReader(); - } - - public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { - mapVector.copyFromSafe(fromIndex, thisIndex, from); - } - - @Override - protected boolean supportsDirectRead() { - return true; - } - - public Iterator fieldNameIterator() { - return mapVector.fieldNameIterator(); - } - - @Override - public void setInitialCapacity(int numRecords) { - mapVector.setInitialCapacity(numRecords); - } - - @Override - public int getBufferSize() { - return mapVector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - return (int) mapVector.getBufferSizeFor(valueCount); - } - - @Override - public ArrowBuf getValidityBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return getTransferPair(name, allocator, null); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return mapVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return mapVector.makeTransferPair(((LegacyMapVector)to).mapVector); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return mapVector.getTransferPair(ref, allocator); - } - - @Override - public int getValueCapacity() { - return mapVector.getValueCapacity(); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - return mapVector.getObject(index); - } - - public void get(int index, ComplexHolder holder) { - mapVector.get(index, holder); - } - - @Override - public int getValueCount() { - return mapVector.getValueCount(); - } - } - - public ValueVector getVectorById(int id) { - return mapVector.getVectorById(id); - } - - public class Mutator extends BaseValueVector.BaseMutator { - - @Override - public void setValueCount(int valueCount) { - mapVector.setValueCount(valueCount); - } - - @Override - public void reset() { } - - @Override - public void generateTestData(int values) { } - } - - @Override - public void clear() { - mapVector.clear(); - } - - @Override - public Field getField() { - return mapVector.getField(); - } - - @Override - public MinorType getMinorType() { - return MinorType.MAP; - } - - @Override - public void close() { - mapVector.close(); - } - - public void initializeChildrenFromFields(List children) { - mapVector.initializeChildrenFromFields(children); - } - - public List getChildrenFromFields() { - return mapVector.getChildren(); - } - - public boolean isNull(int index) { return false; } - - public int getNullCount() { return 0; } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index ea577208cc238..afe86a692c3e1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,15 +6,15 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector.complex; @@ -60,7 +59,7 @@ public static ListVector empty(String name, BufferAllocator allocator) { return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); } - private ArrowBuf validityBuffer; + protected ArrowBuf validityBuffer; private UnionListReader reader; private CallBack callBack; private final FieldType fieldType; @@ -108,6 +107,13 @@ public List getChildrenFromFields() { return singletonList(getDataVector()); } + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 2) { @@ -129,20 +135,35 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers valueCount = fieldNode.getLength(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ @Override public List getFieldBuffers() { List result = new ArrayList<>(2); - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.readerIndex(0); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); - + setReaderAndWriterIndex(); result.add(validityBuffer); result.add(offsetBuffer); return result; } + /** + * Set the reader and writer indexes for the inner buffers. + */ + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + offsetBuffer.readerIndex(0); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + offsetBuffer.writerIndex(0); + } else { + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + } + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -153,16 +174,28 @@ public UnionListWriter getWriter() { return new UnionListWriter(this); } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() throws OutOfMemoryException { - if (!allocateNewSafe()) { - throw new OutOfMemoryException("Failure while allocating memory"); - } + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. + * + * @return false if memory allocation fails, true otherwise. + */ public boolean allocateNewSafe() { boolean success = false; try { + /* we are doing a new allocation -- release the current buffers */ + clear(); /* allocate validity buffer */ allocateValidityBuffer(validityAllocationSizeInBytes); /* allocate offset and data buffer */ @@ -177,13 +210,17 @@ public boolean allocateNewSafe() { } private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; validityBuffer = allocator.buffer(curSize); validityBuffer.readerIndex(0); validityAllocationSizeInBytes = curSize; validityBuffer.setZero(0, validityBuffer.capacity()); } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ @Override public void reAlloc() { /* reallocate the validity buffer */ @@ -201,8 +238,8 @@ private void reallocValidityBuffer() { final int currentBufferCapacity = validityBuffer.capacity(); long baseSize = validityAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -212,19 +249,34 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; newBuf.setZero(halfNewCapacity, halfNewCapacity); validityBuffer.release(1); validityBuffer = newBuf; - validityAllocationSizeInBytes = (int)newAllocationSize; + validityAllocationSizeInBytes = (int) newAllocationSize; } + /** + * Same as {@link #copyFrom(int, int, ListVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int inIndex, int outIndex, ListVector from) { copyFrom(inIndex, outIndex, from); } + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int inIndex, int outIndex, ListVector from) { FieldReader in = from.getReader(); in.setPosition(inIndex); @@ -233,6 +285,10 @@ public void copyFrom(int inIndex, int outIndex, ListVector from) { ComplexCopier.copy(in, out); } + /** + * Get the inner data vector for this list vector + * @return data vector + */ @Override public FieldVector getDataVector() { return vector; @@ -269,7 +325,9 @@ public long getOffsetBufferAddress() { } @Override - public ArrowBuf getValidityBuffer() { return validityBuffer; } + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } @Override public ArrowBuf getDataBuffer() { @@ -277,7 +335,9 @@ public ArrowBuf getDataBuffer() { } @Override - public ArrowBuf getOffsetBuffer() { return offsetBuffer; } + public ArrowBuf getOffsetBuffer() { + return offsetBuffer; + } private class TransferImpl implements TransferPair { @@ -297,15 +357,30 @@ public TransferImpl(ListVector to) { dataTransferPair = getDataVector().makeTransferPair(to.getDataVector()); } + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + */ @Override public void transfer() { + to.clear(); dataTransferPair.transfer(); to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer; to.lastSet = lastSet; - to.setValueCount(valueCount); + if (valueCount > 0) { + to.setValueCount(valueCount); + } + clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + */ @Override public void splitAndTransfer(int startIndex, int length) { final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); @@ -343,8 +418,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListVect } target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); target.validityBuffer.retain(1); - } - else { + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. @@ -369,15 +443,14 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListVect * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { + } else { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); @@ -408,9 +481,14 @@ public AddOrGetResult addOrGetVector(FieldType fieldT return result; } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { - if (getValueCount() == 0) { + if (valueCount == 0) { return 0; } final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; @@ -435,10 +513,28 @@ public void clear() { lastSet = 0; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, ObjectArrays.concat(new ArrowBuf[] {validityBuffer}, - vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, + ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, + vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); + } if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -459,6 +555,11 @@ public UnionVector promoteToUnion() { return vector; } + /** + * Get the element in the list vector at a particular index + * @param index position of the element + * @return Object at given position + */ @Override public Object getObject(int index) { if (isSet(index) == 0) { @@ -475,11 +576,23 @@ public Object getObject(int index) { return vals; } + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null, false otherwise + */ @Override public boolean isNull(int index) { return (isSet(index) == 0); } + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ public int isSet(int index) { final int byteIndex = index >> 3; final byte b = validityBuffer.getByte(byteIndex); @@ -487,29 +600,57 @@ public int isSet(int index) { return Long.bitCount(b & (1L << bitIndex)); } + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ @Override public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity()); + } + + private int getValidityAndOffsetValueCapacity() { + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); + return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); + } + + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + public void setNotNull(int index) { - if (index >= getValueCapacity()) { + while (index >= getValidityAndOffsetValueCapacity()) { reallocValidityAndOffsetBuffers(); } BitVectorHelper.setValidityBitToOne(validityBuffer, index); lastSet = index + 1; } + /** + * Start a new value in the list vector + * + * @param index index of the value to start + */ @Override public int startNewValue(int index) { - if (index >= getValueCapacity()) { + while (index >= getValidityAndOffsetValueCapacity()) { reallocValidityAndOffsetBuffers(); } for (int i = lastSet; i <= index; i++) { final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } - setNotNull(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); lastSet = index + 1; return offsetBuffer.getInt(lastSet * OFFSET_WIDTH); } @@ -525,12 +666,17 @@ public void endValue(int index, int size) { offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size); } + /** + * Sets the value count for the vector + * + * @param valueCount value count + */ @Override public void setValueCount(int valueCount) { this.valueCount = valueCount; if (valueCount > 0) { - while (valueCount > getValueCapacity()) { - /* realloc the inner buffers if needed */ + while (valueCount > getValidityAndOffsetValueCapacity()) { + /* check if validity and offset buffers need to be re-allocated */ reallocValidityAndOffsetBuffers(); } for (int i = lastSet; i < valueCount; i++) { @@ -542,6 +688,9 @@ public void setValueCount(int valueCount) { /* valueCount for the data vector is the current end offset */ final int childValueCount = (valueCount == 0) ? 0 : offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + /* set the value count of data vector and this will take care of + * checking whether data buffer needs to be reallocated. + */ vector.setValueCount(childValueCount); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index a1478c5a7213d..f95302f55f887 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -54,7 +54,7 @@ public static NullableMapVector empty(String name, BufferAllocator allocator) { private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); private final NullableMapWriter writer = new NullableMapWriter(this); - private ArrowBuf validityBuffer; + protected ArrowBuf validityBuffer; private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @@ -91,7 +91,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers ArrowBuf bitBuffer = ownBuffers.get(0); validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); valueCount = fieldNode.getLength(); validityAllocationSizeInBytes = validityBuffer.capacity(); } @@ -99,14 +99,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers @Override public List getFieldBuffers() { List result = new ArrayList<>(1); - - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + setReaderAndWriterIndex(); result.add(validityBuffer); return result; } + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -156,6 +159,7 @@ public void transfer() { target.clear(); target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; super.transfer(); + clear(); } @Override @@ -193,8 +197,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable } target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); target.validityBuffer.retain(1); - } - else { + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. @@ -219,15 +222,14 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { + } else { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); @@ -236,55 +238,108 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable } } + /** + * Get the value capacity of the internal validity buffer. + * @return number of elements that validity buffer can hold + */ private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); + return (int) (validityBuffer.capacity() * 8L); } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity() { return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity()); } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), + ArrowBuf.class); + } if (clear) { - validityBuffer.retain(1); + for (ArrowBuf buffer : buffers) { + buffer.retain(); + } + clear(); } - return ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(clear), ArrowBuf.class); + + return buffers; } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clearValidityBuffer(); super.close(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { clearValidityBuffer(); super.clear(); } + /** + * Release the validity buffer + */ private void clearValidityBuffer() { validityBuffer.release(); validityBuffer = allocator.getEmpty(); } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { - if (valueCount == 0) { return 0; } + if (valueCount == 0) { + return 0; + } return super.getBufferSize() + BitVectorHelper.getValidityBufferSize(valueCount); } + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } return super.getBufferSizeFor(valueCount) - + BitVectorHelper.getValidityBufferSize(valueCount); + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override @@ -302,7 +357,7 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - clearValidityBuffer(); + clear(); allocateValidityBuffer(validityAllocationSizeInBytes); success = super.allocateNewSafe(); } finally { @@ -315,7 +370,7 @@ public boolean allocateNewSafe() { } private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; validityBuffer = allocator.buffer(curSize); validityBuffer.readerIndex(0); validityAllocationSizeInBytes = curSize; @@ -333,8 +388,8 @@ private void reallocValidityBuffer() { final int currentBufferCapacity = validityBuffer.capacity(); long baseSize = validityAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -344,12 +399,12 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setZero(0, newBuf.capacity()); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); validityBuffer.release(1); validityBuffer = newBuf; - validityAllocationSizeInBytes = (int)newAllocationSize; + validityAllocationSizeInBytes = (int) newAllocationSize; } @Override @@ -431,7 +486,7 @@ public void setNull(int index) { @Override public void setValueCount(int valueCount) { assert valueCount >= 0; - while (valueCount > getValueCapacity()) { + while (valueCount > getValidityBufferValueCapacity()) { /* realloc the inner buffers if needed */ reallocValidityBuffer(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index d3c8b990394d8..8017b385d148a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -27,7 +27,8 @@ import java.io.File; import java.io.IOException; - +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -192,7 +193,7 @@ public VectorSchemaRoot read() throws IOException { private abstract class BufferReader { abstract protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException; - final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException { + ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException { readToken(START_ARRAY); ArrowBuf buf = read(allocator, count); readToken(END_ARRAY); @@ -201,8 +202,7 @@ final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOExcepti } private class BufferHelper { - BufferReader BIT = new BufferReader() { - + BufferReader BIT = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { final int bufferSize = BitVectorHelper.getValidityBufferSize(count); @@ -216,6 +216,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BitVectorHelper.setValidityBit(buf, i, parser.readValueAs(Boolean.class) ? 1 : 0); } + buf.writerIndex(bufferSize); return buf; } }; @@ -223,7 +224,8 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT1 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableTinyIntVector.TYPE_WIDTH); + final int size = count * NullableTinyIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); @@ -237,7 +239,8 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT2 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableSmallIntVector.TYPE_WIDTH); + final int size = count * NullableSmallIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); @@ -249,10 +252,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException }; BufferReader INT4 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableIntVector.TYPE_WIDTH); + final int size = count * NullableIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); @@ -264,10 +267,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException }; BufferReader INT8 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableBigIntVector.TYPE_WIDTH); + final int size = count * NullableBigIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); @@ -279,10 +282,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException }; BufferReader FLOAT4 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableFloat4Vector.TYPE_WIDTH); + final int size = count * NullableFloat4Vector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); @@ -294,10 +297,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException }; BufferReader FLOAT8 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableFloat8Vector.TYPE_WIDTH); + final int size = count * NullableFloat8Vector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); @@ -309,17 +312,18 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException }; BufferReader DECIMAL = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableDecimalVector.TYPE_WIDTH); + final int size = count * NullableDecimalVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); - final byte[] value = decodeHexSafe(parser.getValueAsString()); - DecimalUtility.writeByteArrayToArrowBuf(value, buf, i); + BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); + DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i); } + buf.writerIndex(size); return buf; } }; @@ -369,10 +373,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException return buf; } }; - } - private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferType, Types.MinorType type, int count) throws IOException { + private ArrowBuf readIntoBuffer(BufferAllocator allocator, ArrowVectorType bufferType, + Types.MinorType type, int count) throws IOException { ArrowBuf buf; BufferHelper helper = new BufferHelper(); @@ -461,6 +465,7 @@ private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferTyp } buf = reader.readBuffer(allocator, count); + assert buf != null; return buf; } @@ -506,12 +511,14 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json innerBufferValueCount = valueCount + 1; } - vectorBuffers[v] = readBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); + vectorBuffers[v] = readIntoBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); } - vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers)); + final int nullCount = BitVectorHelper.getNullCount(vectorBuffers[0], valueCount); + final ArrowFieldNode fieldNode = new ArrowFieldNode(valueCount, nullCount); + vector.loadFieldBuffers(fieldNode, Arrays.asList(vectorBuffers)); - // read child vectors, if any + /* read child vectors (if any) */ List fields = field.getChildren(); if (!fields.isEmpty()) { List vectorChildren = vector.getChildrenFromFields(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java index 7921cd34c773c..0c8507b51f598 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -159,6 +160,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE generator.writeObjectField("name", field.getName()); int valueCount = vector.getValueCount(); generator.writeObjectField("count", valueCount); + final int scale = (vector instanceof NullableDecimalVector) ? + ((NullableDecimalVector) vector).getScale() : 0; for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); ArrowBuf vectorBuffer = vectorBuffers.get(v); @@ -167,9 +170,9 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE for (int i = 0; i < bufferValueCount; i++) { if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || vector.getMinorType() == Types.MinorType.VARBINARY)) { - writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i); + writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale); } else { - writeValueToGenerator(vectorType, vectorBuffer, null, vector, i); + writeValueToGenerator(vectorType, vectorBuffer, null, vector, i, scale); } } generator.writeEndArray(); @@ -193,7 +196,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE } private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, - ArrowBuf offsetBuffer, FieldVector vector, int index) throws IOException { + ArrowBuf offsetBuffer, FieldVector vector, + final int index, final int scale) throws IOException { if (bufferType.equals(TYPE)) { generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH)); } else if (bufferType.equals(OFFSET)) { @@ -279,9 +283,9 @@ private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, break; } case DECIMAL: { - String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(buffer, - index)); - generator.writeString(hexString); + BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale); + // We write the unscaled value, because the scale is stored in the type metadata. + generator.writeString(decimalValue.unscaledValue().toString()); break; } default: diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index ede4f992291b4..aec7d0f327389 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import java.util.List; @@ -113,10 +114,23 @@ public void testTransfer() throws Exception { assertEquals(srcVector.getField(), destVector.getField()); // now check the values are transferred - assertEquals(srcVector.getValueCount(), destVector.getValueCount()); - for (int i = 0; i < srcVector.getValueCount(); i++) { - assertEquals("Different values at index " + i, srcVector.get(i), destVector.get(i)); - } + assertEquals(6, destVector.getValueCount()); + + assertFalse(destVector.isNull(0)); + assertEquals(5, destVector.getObject(0)); + + assertFalse(destVector.isNull(1)); + assertEquals(false, destVector.getObject(1)); + + assertTrue(destVector.isNull(2)); + + assertFalse(destVector.isNull(3)); + assertEquals(10, destVector.getObject(3)); + + assertTrue(destVector.isNull(4)); + + assertFalse(destVector.isNull(5)); + assertEquals(false, destVector.getObject(5)); } } } From ca3acdc138b1ac27c9111b236d33593988689a20 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Thu, 16 Nov 2017 00:07:13 -0500 Subject: [PATCH 052/177] ARROW-1821: [INTEGRATION] Add integration test case for when Field has zero null count and optional validity buffer Currently when a Field has null count = 0, C++ will omit the validity buffer as it is optional in this case. Testing for #1316 was failing because Java was not handling this properly. This PR adds an explicit test to ensure this is being tested and easier to locate possible issues. Author: Bryan Cutler Closes #1323 from BryanCutler/integration-test-optional-validity-buffer-ARROW-1821 and squashes the following commits: 4295660d [Bryan Cutler] expanded simple.json static test file to include field with null count = 0 --- integration/data/simple.json | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/integration/data/simple.json b/integration/data/simple.json index fb903e7ac4b63..bc8949912a48a 100644 --- a/integration/data/simple.json +++ b/integration/data/simple.json @@ -61,6 +61,54 @@ "DATA": ["aa", "", "", "bbb", "cccc"] } ] + }, + { + "count": 5, + "columns": [ + { + "name": "foo", + "count": 5, + "VALIDITY": [1, 1, 1, 1, 1], + "DATA": [1, 2, 3, 4, 5] + }, + { + "name": "bar", + "count": 5, + "VALIDITY": [1, 1, 1, 1, 1], + "DATA": [1.0, 2.0, 3.0, 4.0, 5.0] + }, + { + "name": "baz", + "count": 5, + "VALIDITY": [1, 1, 1, 1, 1], + "OFFSET": [0, 2, 3, 4, 7, 11], + "DATA": ["aa", "b", "c", "ddd", "eeee"] + } + ] + }, + { + "count": 5, + "columns": [ + { + "name": "foo", + "count": 5, + "VALIDITY": [0, 0, 0, 0, 0], + "DATA": [1, 2, 3, 4, 5] + }, + { + "name": "bar", + "count": 5, + "VALIDITY": [0, 0, 0, 0, 0], + "DATA": [1.0, 2.0, 3.0, 4.0, 5.0] + }, + { + "name": "baz", + "count": 5, + "VALIDITY": [0, 0, 0, 0, 0], + "OFFSET": [0, 0, 0, 0, 0, 0], + "DATA": ["", "", "", "", ""] + } + ] } ] } From ac26eb72c02d4ea7e8788ee3a8a3eeec6403b9bb Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Thu, 16 Nov 2017 18:15:06 -0800 Subject: [PATCH 053/177] ARROW-1829: [Plasma] Fixes to eviction policy. Some fixes to the bookkeeping for the eviction policy. Author: Robert Nishihara Closes #1317 from robertnishihara/evictionpolicy and squashes the following commits: db34d8e [Robert Nishihara] Add sanity checks. 0d3a6c8 [Robert Nishihara] Fixes to eviction policy. --- cpp/src/plasma/eviction_policy.cc | 34 +++++++++++++------------------ cpp/src/plasma/eviction_policy.h | 3 +-- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc index 6c2309f1709d2..a7758fd2c0e74 100644 --- a/cpp/src/plasma/eviction_policy.cc +++ b/cpp/src/plasma/eviction_policy.cc @@ -61,38 +61,32 @@ int64_t EvictionPolicy::choose_objects_to_evict(int64_t num_bytes_required, } /* Update the number of bytes used. */ memory_used_ -= bytes_evicted; + ARROW_CHECK(memory_used_ >= 0); return bytes_evicted; } void EvictionPolicy::object_created(const ObjectID& object_id) { auto entry = store_info_->objects[object_id].get(); cache_.add(object_id, entry->info.data_size + entry->info.metadata_size); + int64_t size = entry->info.data_size + entry->info.metadata_size; + memory_used_ += size; + ARROW_CHECK(memory_used_ <= store_info_->memory_capacity); } bool EvictionPolicy::require_space(int64_t size, std::vector* objects_to_evict) { /* Check if there is enough space to create the object. */ int64_t required_space = memory_used_ + size - store_info_->memory_capacity; - int64_t num_bytes_evicted; - if (required_space > 0) { - /* Try to free up at least as much space as we need right now but ideally - * up to 20% of the total capacity. */ - int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5); - ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects"; - /* Choose some objects to evict, and update the return pointers. */ - num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict); - ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting " - << objects_to_evict->size() << " objects to free up " - << num_bytes_evicted << " bytes."; - } else { - num_bytes_evicted = 0; - } - if (num_bytes_evicted >= required_space) { - /* We only increment the space used if there is enough space to create the - * object. */ - memory_used_ += size; - } - return num_bytes_evicted >= required_space; + /* Try to free up at least as much space as we need right now but ideally + * up to 20% of the total capacity. */ + int64_t space_to_free = std::max(required_space, store_info_->memory_capacity / 5); + ARROW_LOG(DEBUG) << "not enough space to create this object, so evicting objects"; + /* Choose some objects to evict, and update the return pointers. */ + int64_t num_bytes_evicted = choose_objects_to_evict(space_to_free, objects_to_evict); + ARROW_LOG(INFO) << "There is not enough space to create this object, so evicting " + << objects_to_evict->size() << " objects to free up " + << num_bytes_evicted << " bytes."; + return num_bytes_evicted >= required_space && num_bytes_evicted > 0; } void EvictionPolicy::begin_object_access(const ObjectID& object_id, diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h index de33dabcbafb3..cebf35b1c1b08 100644 --- a/cpp/src/plasma/eviction_policy.h +++ b/cpp/src/plasma/eviction_policy.h @@ -73,8 +73,7 @@ class EvictionPolicy { void object_created(const ObjectID& object_id); /// This method will be called when the Plasma store needs more space, perhaps - /// to create a new object. If the required amount of space cannot be freed up, - /// then a fatal error will be thrown. When this method is called, the eviction + /// to create a new object. When this method is called, the eviction /// policy will assume that the objects chosen to be evicted will in fact be /// evicted from the Plasma store by the caller. /// From cacbacd439919742a0b6fbec27ee73b5af29347f Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Thu, 16 Nov 2017 21:56:03 -0800 Subject: [PATCH 054/177] ARROW-1795: [Plasma] Create flag to make Plasma store use a single memory-mapped file. This adds the `-f` flag which tells the plasma store to use a single memory-mapped file. This is accomplished by simply calling `dlmemalign`/`dlfree` on the entire space at startup. Question: Why does the test pass? Given that `plasma_client` is constructed with a release delay of 64, shouldn't the store by unable to evict some objects? Yet they all seem to get evicted just fine. cc @pcmoritz @atumanov Author: Robert Nishihara Closes #1327 from robertnishihara/allocateupfront and squashes the following commits: 6f3b953 [Robert Nishihara] Augment test. 0daeae3 [Robert Nishihara] Remove hard-coded values and update test. a446fbd [Robert Nishihara] Add a test. c2374b6 [Robert Nishihara] Add flag to tell the plasma store to use a single memory-mapped file. --- cpp/src/plasma/store.cc | 27 ++++++++++++++++++++++----- python/pyarrow/tests/test_plasma.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc index 7094aed6f8e86..c6a19a5471883 100644 --- a/cpp/src/plasma/store.cc +++ b/cpp/src/plasma/store.cc @@ -676,12 +676,22 @@ class PlasmaStoreRunner { PlasmaStoreRunner() {} void Start(char* socket_name, int64_t system_memory, std::string directory, - bool hugepages_enabled) { + bool hugepages_enabled, bool use_one_memory_mapped_file) { // Create the event loop. loop_.reset(new EventLoop); store_.reset( new PlasmaStore(loop_.get(), system_memory, directory, hugepages_enabled)); plasma_config = store_->get_plasma_store_info(); + + // If the store is configured to use a single memory-mapped file, then we + // achieve that by mallocing and freeing a single large amount of space. + // that maximum allowed size up front. + if (use_one_memory_mapped_file) { + void* pointer = plasma::dlmemalign(BLOCK_SIZE, system_memory); + ARROW_CHECK(pointer != NULL); + plasma::dlfree(pointer); + } + int socket = bind_ipc_sock(socket_name, true); // TODO(pcm): Check return value. ARROW_CHECK(socket >= 0); @@ -716,14 +726,15 @@ void HandleSignal(int signal) { } void start_server(char* socket_name, int64_t system_memory, std::string plasma_directory, - bool hugepages_enabled) { + bool hugepages_enabled, bool use_one_memory_mapped_file) { // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write // to a client that has already died, the store could die. signal(SIGPIPE, SIG_IGN); g_runner.reset(new PlasmaStoreRunner()); signal(SIGTERM, HandleSignal); - g_runner->Start(socket_name, system_memory, plasma_directory, hugepages_enabled); + g_runner->Start(socket_name, system_memory, plasma_directory, hugepages_enabled, + use_one_memory_mapped_file); } } // namespace plasma @@ -733,9 +744,11 @@ int main(int argc, char* argv[]) { // Directory where plasma memory mapped files are stored. std::string plasma_directory; bool hugepages_enabled = false; + // True if a single large memory-mapped file should be created at startup. + bool use_one_memory_mapped_file = false; int64_t system_memory = -1; int c; - while ((c = getopt(argc, argv, "s:m:d:h")) != -1) { + while ((c = getopt(argc, argv, "s:m:d:hf")) != -1) { switch (c) { case 'd': plasma_directory = std::string(optarg); @@ -755,6 +768,9 @@ int main(int argc, char* argv[]) { << "GB of memory."; break; } + case 'f': + use_one_memory_mapped_file = true; + break; default: exit(-1); } @@ -808,5 +824,6 @@ int main(int argc, char* argv[]) { // available. plasma::dlmalloc_set_footprint_limit((size_t)system_memory); ARROW_LOG(DEBUG) << "starting server listening on " << socket_name; - plasma::start_server(socket_name, system_memory, plasma_directory, hugepages_enabled); + plasma::start_server(socket_name, system_memory, plasma_directory, hugepages_enabled, + use_one_memory_mapped_file); } diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py index b73d92d14fd43..b28bd60c48ad4 100644 --- a/python/pyarrow/tests/test_plasma.py +++ b/python/pyarrow/tests/test_plasma.py @@ -102,7 +102,8 @@ def assert_get_object_equal(unit_test, client1, client2, object_id, def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, use_valgrind=False, use_profiler=False, - stdout_file=None, stderr_file=None): + stdout_file=None, stderr_file=None, + use_one_memory_mapped_file=False): """Start a plasma store process. Args: use_valgrind (bool): True if the plasma store should be started inside @@ -113,6 +114,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, no redirection should happen, then this should be None. stderr_file: A file handle opened for writing to redirect stderr to. If no redirection should happen, then this should be None. + use_one_memory_mapped_file: If True, then the store will use only a + single memory-mapped file. Return: A tuple of the name of the plasma store socket and the process ID of the plasma store process. @@ -124,6 +127,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, command = [plasma_store_executable, "-s", plasma_store_name, "-m", str(plasma_store_memory)] + if use_one_memory_mapped_file: + command += ["-f"] if use_valgrind: pid = subprocess.Popen(["valgrind", "--track-origins=yes", @@ -147,10 +152,14 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, class TestPlasmaClient(object): def setup_method(self, test_method): + use_one_memory_mapped_file = (test_method == + self.test_use_one_memory_mapped_file) + import pyarrow.plasma as plasma # Start Plasma store. plasma_store_name, self.p = start_plasma_store( - use_valgrind=os.getenv("PLASMA_VALGRIND") == "1") + use_valgrind=os.getenv("PLASMA_VALGRIND") == "1", + use_one_memory_mapped_file=use_one_memory_mapped_file) # Connect to Plasma. self.plasma_client = plasma.connect(plasma_store_name, "", 64) # For the eviction test @@ -720,3 +729,19 @@ def test_subscribe_deletions(self): assert object_ids[i] == recv_objid assert -1 == recv_dsize assert -1 == recv_msize + + def test_use_one_memory_mapped_file(self): + # Fill the object store up with a large number of small objects and let + # them go out of scope. + for _ in range(100): + create_object( + self.plasma_client, + np.random.randint(1, DEFAULT_PLASMA_STORE_MEMORY // 20), 0) + # Create large objects that require the full object store size, and + # verify that they fit. + for _ in range(2): + create_object(self.plasma_client, DEFAULT_PLASMA_STORE_MEMORY, 0) + # Verify that an object that is too large does not fit. + with pytest.raises(pa.lib.PlasmaStoreFull): + create_object(self.plasma_client, DEFAULT_PLASMA_STORE_MEMORY + 1, + 0) From f2806fa518583907a129b2ecb0b7ec8758b69e17 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 17 Nov 2017 18:29:49 -0500 Subject: [PATCH 055/177] ARROW-1559: [C++] Add Unique kernel and refactor DictionaryBuilder to be a stateful kernel Only intended to implement selective categorical conversion in `to_pandas()` but it seems that there is a lot missing to do this in a clean fashion. Author: Wes McKinney Closes #1266 from xhochy/ARROW-1559 and squashes the following commits: 50249652 [Wes McKinney] Fix MSVC linker issue b6cb1ece [Wes McKinney] Export CastOptions 4ea3ce61 [Wes McKinney] Return NONE Datum in else branch of functions 4f969c6b [Wes McKinney] Move deprecation suppression after flag munging 7f557cc0 [Wes McKinney] Code review comments, disable C4996 warning (equivalent to -Wno-deprecated) in MSVC builds 84717461 [Wes McKinney] Do not compute hash table threshold on each iteration ae8f2339 [Wes McKinney] Fix double to int64_t conversion warning c1444a26 [Wes McKinney] Fix doxygen warnings 2de85961 [Wes McKinney] Add test cases for unique, dictionary_encode 383b46fd [Wes McKinney] Add Array methods for Unique, DictionaryEncode 0962f06b [Wes McKinney] Add cast method for Column, chunked_array and column factory functions 62c3cefd [Wes McKinney] Datum stubs 27151c47 [Wes McKinney] Implement Cast for chunked arrays, fix kernel implementation. Change kernel API to write to a single Datum 1bf2e2f4 [Wes McKinney] Fix bug with column using wrong type eaadc3e5 [Wes McKinney] Use macros to reduce code duplication in DoubleTableSize 6b4f8f3c [Wes McKinney] Fix datetime64->date32 casting error raised by refactor 2c77a19e [Wes McKinney] Some Decimal->Decimal128 renaming. Add DecimalType base class c07f91b3 [Wes McKinney] ARROW-1559: Add unique kernel --- LICENSE.txt | 59 + cpp/CMakeLists.txt | 1 + cpp/build-support/clang_format_exclusions.txt | 2 + cpp/cmake_modules/SetupCxxFlags.cmake | 9 + cpp/src/arrow/CMakeLists.txt | 4 +- cpp/src/arrow/array-test.cc | 356 +----- cpp/src/arrow/array.cc | 2 +- cpp/src/arrow/buffer.h | 3 +- cpp/src/arrow/builder-benchmark.cc | 43 - cpp/src/arrow/builder.cc | 456 +------ cpp/src/arrow/builder.h | 203 +-- cpp/src/arrow/compute/CMakeLists.txt | 4 +- cpp/src/arrow/compute/api.h | 4 +- cpp/src/arrow/compute/compute-benchmark.cc | 88 ++ cpp/src/arrow/compute/compute-test.cc | 369 +++++- cpp/src/arrow/compute/context.cc | 7 +- cpp/src/arrow/compute/kernel.h | 97 +- cpp/src/arrow/compute/kernels/CMakeLists.txt | 21 + cpp/src/arrow/compute/{ => kernels}/cast.cc | 317 ++--- cpp/src/arrow/compute/{ => kernels}/cast.h | 33 +- cpp/src/arrow/compute/kernels/hash.cc | 822 ++++++++++++ cpp/src/arrow/compute/kernels/hash.h | 106 ++ .../arrow/compute/kernels/util-internal.cc | 85 ++ cpp/src/arrow/compute/kernels/util-internal.h | 105 ++ cpp/src/arrow/pretty_print-test.cc | 5 +- cpp/src/arrow/python/arrow_to_pandas.cc | 15 +- cpp/src/arrow/python/numpy_to_arrow.cc | 59 +- cpp/src/arrow/table.h | 3 + cpp/src/arrow/test-util.h | 4 + cpp/src/arrow/type-test.cc | 6 +- cpp/src/arrow/type.h | 27 +- cpp/src/arrow/type_fwd.h | 6 + cpp/src/arrow/type_traits.h | 4 + cpp/src/arrow/util/CMakeLists.txt | 3 + cpp/src/arrow/util/variant.h | 1127 +++++++++++++++++ cpp/src/arrow/util/variant/CMakeLists.txt | 28 + cpp/src/arrow/util/variant/optional.h | 100 ++ .../arrow/util/variant/recursive_wrapper.h | 122 ++ cpp/src/arrow/util/variant/variant_cast.h | 112 ++ cpp/src/arrow/util/variant/variant_io.h | 72 ++ cpp/src/arrow/util/variant/variant_visitor.h | 69 + cpp/src/arrow/visitor.cc | 2 +- cpp/src/arrow/visitor_inline.h | 2 +- dev/release/rat_exclude_files.txt | 6 + python/doc/source/api.rst | 3 + python/doc/source/development.rst | 20 +- python/pyarrow/__init__.py | 2 +- python/pyarrow/array.pxi | 37 +- python/pyarrow/includes/libarrow.pxd | 49 + python/pyarrow/pandas_compat.py | 3 +- python/pyarrow/table.pxi | 129 +- python/pyarrow/tests/test_array.py | 39 + python/pyarrow/tests/test_parquet.py | 4 + python/pyarrow/types.pxi | 1 + 54 files changed, 3947 insertions(+), 1308 deletions(-) create mode 100644 cpp/src/arrow/compute/compute-benchmark.cc create mode 100644 cpp/src/arrow/compute/kernels/CMakeLists.txt rename cpp/src/arrow/compute/{ => kernels}/cast.cc (82%) rename cpp/src/arrow/compute/{ => kernels}/cast.h (69%) create mode 100644 cpp/src/arrow/compute/kernels/hash.cc create mode 100644 cpp/src/arrow/compute/kernels/hash.h create mode 100644 cpp/src/arrow/compute/kernels/util-internal.cc create mode 100644 cpp/src/arrow/compute/kernels/util-internal.h create mode 100644 cpp/src/arrow/util/variant.h create mode 100644 cpp/src/arrow/util/variant/CMakeLists.txt create mode 100644 cpp/src/arrow/util/variant/optional.h create mode 100644 cpp/src/arrow/util/variant/recursive_wrapper.h create mode 100644 cpp/src/arrow/util/variant/variant_cast.h create mode 100644 cpp/src/arrow/util/variant/variant_io.h create mode 100644 cpp/src/arrow/util/variant/variant_visitor.h diff --git a/LICENSE.txt b/LICENSE.txt index 00cb9ece232b0..038518a5d9b68 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -398,3 +398,62 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the mapbox/variant project, BSD 3-clause +license + +Copyright (c) MapBox +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +- Neither the name "MapBox" nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5f0c431d54aca..f4b7b29b9d3f3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -414,6 +414,7 @@ if (UNIX) (item MATCHES "xxhash.h") OR (item MATCHES "xxhash.cc") OR (item MATCHES "config.h") OR + (item MATCHES "util/variant") OR (item MATCHES "zmalloc.h") OR (item MATCHES "ae.h"))) LIST(APPEND FILTERED_LINT_FILES ${item}) diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/clang_format_exclusions.txt index 2d5d86d4e4cde..d31d8a00d2ab7 100644 --- a/cpp/build-support/clang_format_exclusions.txt +++ b/cpp/build-support/clang_format_exclusions.txt @@ -3,6 +3,8 @@ *pyarrow_api.h *python/config.h *python/platform.h +*util/variant.h +*util/variant/* *thirdparty/ae/* *xxhash.cc *xxhash.h diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 4b1950f7a17bd..6b0974b3d4c19 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -68,6 +68,9 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3") # Treat all compiler warnings as errors set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX") + + # MSVC version of -Wno-deprecated + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996") elseif ("${COMPILER_FAMILY}" STREQUAL "clang") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \ -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \ @@ -115,6 +118,9 @@ elseif ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "EVERYTHING") # /wdnnnn disables a warning where "nnnn" is a warning number # Treat all compiler warnings as errors set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX") + + # MSVC version of -Wno-deprecated + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996") elseif ("${COMPILER_FAMILY}" STREQUAL "clang") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic") # Treat all compiler warnings as errors @@ -134,6 +140,9 @@ else() # /wdnnnn disables a warning where "nnnn" is a warning number string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3") + + # MSVC version of -Wno-deprecated + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996") elseif ("${COMPILER_FAMILY}" STREQUAL "clang") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall") elseif ("${COMPILER_FAMILY}" STREQUAL "gcc") diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 69d5052330c3a..496e0da9d621e 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -50,8 +50,10 @@ endif() if (ARROW_COMPUTE) add_subdirectory(compute) set(ARROW_SRCS ${ARROW_SRCS} - compute/cast.cc compute/context.cc + compute/kernels/cast.cc + compute/kernels/hash.cc + compute/kernels/util-internal.cc ) endif() diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc index 15c75534e53ef..d894df1311e9e 100644 --- a/cpp/src/arrow/array-test.cc +++ b/cpp/src/arrow/array-test.cc @@ -402,10 +402,6 @@ typedef ::testing::TypesEquals(result_)); } -// ---------------------------------------------------------------------- -// Dictionary tests - -template -class TestDictionaryBuilder : public TestBuilder {}; - -typedef ::testing::Types - PrimitiveDictionaries; - -TYPED_TEST_CASE(TestDictionaryBuilder, PrimitiveDictionaries); - -TYPED_TEST(TestDictionaryBuilder, Basic) { - DictionaryBuilder builder(default_memory_pool()); - ASSERT_OK(builder.Append(static_cast(1))); - ASSERT_OK(builder.Append(static_cast(2))); - ASSERT_OK(builder.Append(static_cast(1))); - - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Build expected data - NumericBuilder dict_builder; - ASSERT_OK(dict_builder.Append(static_cast(1))); - ASSERT_OK(dict_builder.Append(static_cast(2))); - std::shared_ptr dict_array; - ASSERT_OK(dict_builder.Finish(&dict_array)); - auto dtype = std::make_shared(int8(), dict_array); - - Int8Builder int_builder; - ASSERT_OK(int_builder.Append(0)); - ASSERT_OK(int_builder.Append(1)); - ASSERT_OK(int_builder.Append(0)); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - -TYPED_TEST(TestDictionaryBuilder, ArrayConversion) { - NumericBuilder builder; - // DictionaryBuilder builder; - ASSERT_OK(builder.Append(static_cast(1))); - ASSERT_OK(builder.Append(static_cast(2))); - ASSERT_OK(builder.Append(static_cast(1))); - - std::shared_ptr intermediate_result; - ASSERT_OK(builder.Finish(&intermediate_result)); - DictionaryBuilder dictionary_builder(default_memory_pool()); - ASSERT_OK(dictionary_builder.AppendArray(*intermediate_result)); - std::shared_ptr result; - ASSERT_OK(dictionary_builder.Finish(&result)); - - // Build expected data - NumericBuilder dict_builder; - ASSERT_OK(dict_builder.Append(static_cast(1))); - ASSERT_OK(dict_builder.Append(static_cast(2))); - std::shared_ptr dict_array; - ASSERT_OK(dict_builder.Finish(&dict_array)); - auto dtype = std::make_shared(int8(), dict_array); - - Int8Builder int_builder; - ASSERT_OK(int_builder.Append(0)); - ASSERT_OK(int_builder.Append(1)); - ASSERT_OK(int_builder.Append(0)); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - -TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) { - using Scalar = typename TypeParam::c_type; - // Skip this test for (u)int8 - if (sizeof(Scalar) > 1) { - // Build the dictionary Array - DictionaryBuilder builder(default_memory_pool()); - // Build expected data - NumericBuilder dict_builder; - Int16Builder int_builder; - - // Fill with 1024 different values - for (int64_t i = 0; i < 1024; i++) { - ASSERT_OK(builder.Append(static_cast(i))); - ASSERT_OK(dict_builder.Append(static_cast(i))); - ASSERT_OK(int_builder.Append(static_cast(i))); - } - // Fill with an already existing value - for (int64_t i = 0; i < 1024; i++) { - ASSERT_OK(builder.Append(static_cast(1))); - ASSERT_OK(int_builder.Append(1)); - } - - // Finalize result - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Finalize expected data - std::shared_ptr dict_array; - ASSERT_OK(dict_builder.Finish(&dict_array)); - auto dtype = std::make_shared(int16(), dict_array); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); - } -} - -TEST(TestStringDictionaryBuilder, Basic) { - // Build the dictionary Array - StringDictionaryBuilder builder(default_memory_pool()); - ASSERT_OK(builder.Append("test")); - ASSERT_OK(builder.Append("test2")); - ASSERT_OK(builder.Append("test")); - - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Build expected data - StringBuilder str_builder; - ASSERT_OK(str_builder.Append("test")); - ASSERT_OK(str_builder.Append("test2")); - std::shared_ptr str_array; - ASSERT_OK(str_builder.Finish(&str_array)); - auto dtype = std::make_shared(int8(), str_array); - - Int8Builder int_builder; - ASSERT_OK(int_builder.Append(0)); - ASSERT_OK(int_builder.Append(1)); - ASSERT_OK(int_builder.Append(0)); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - -TEST(TestStringDictionaryBuilder, DoubleTableSize) { - // Build the dictionary Array - StringDictionaryBuilder builder(default_memory_pool()); - // Build expected data - StringBuilder str_builder; - Int16Builder int_builder; - - // Fill with 1024 different values - for (int64_t i = 0; i < 1024; i++) { - std::stringstream ss; - ss << "test" << i; - ASSERT_OK(builder.Append(ss.str())); - ASSERT_OK(str_builder.Append(ss.str())); - ASSERT_OK(int_builder.Append(static_cast(i))); - } - // Fill with an already existing value - for (int64_t i = 0; i < 1024; i++) { - ASSERT_OK(builder.Append("test1")); - ASSERT_OK(int_builder.Append(1)); - } - - // Finalize result - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Finalize expected data - std::shared_ptr str_array; - ASSERT_OK(str_builder.Finish(&str_array)); - auto dtype = std::make_shared(int16(), str_array); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - -TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) { - // Build the dictionary Array - DictionaryBuilder builder(arrow::fixed_size_binary(4), - default_memory_pool()); - std::vector test{12, 12, 11, 12}; - std::vector test2{12, 12, 11, 11}; - ASSERT_OK(builder.Append(test.data())); - ASSERT_OK(builder.Append(test2.data())); - ASSERT_OK(builder.Append(test.data())); - - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Build expected data - FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4)); - ASSERT_OK(fsb_builder.Append(test.data())); - ASSERT_OK(fsb_builder.Append(test2.data())); - std::shared_ptr fsb_array; - ASSERT_OK(fsb_builder.Finish(&fsb_array)); - auto dtype = std::make_shared(int8(), fsb_array); - - Int8Builder int_builder; - ASSERT_OK(int_builder.Append(0)); - ASSERT_OK(int_builder.Append(1)); - ASSERT_OK(int_builder.Append(0)); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - -TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) { - // Build the dictionary Array - DictionaryBuilder builder(arrow::fixed_size_binary(4), - default_memory_pool()); - // Build expected data - FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4)); - Int16Builder int_builder; - - // Fill with 1024 different values - for (int64_t i = 0; i < 1024; i++) { - std::vector value{12, 12, static_cast(i / 128), - static_cast(i % 128)}; - ASSERT_OK(builder.Append(value.data())); - ASSERT_OK(fsb_builder.Append(value.data())); - ASSERT_OK(int_builder.Append(static_cast(i))); - } - // Fill with an already existing value - std::vector known_value{12, 12, 0, 1}; - for (int64_t i = 0; i < 1024; i++) { - ASSERT_OK(builder.Append(known_value.data())); - ASSERT_OK(int_builder.Append(1)); - } - - // Finalize result - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Finalize expected data - std::shared_ptr fsb_array; - ASSERT_OK(fsb_builder.Finish(&fsb_array)); - auto dtype = std::make_shared(int16(), fsb_array); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - -TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) { - // Build the dictionary Array - DictionaryBuilder builder(arrow::fixed_size_binary(4), - default_memory_pool()); - // Build an array with different byte width - FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5)); - std::vector value{100, 1, 1, 1, 1}; - ASSERT_OK(fsb_builder.Append(value.data())); - std::shared_ptr fsb_array; - ASSERT_OK(fsb_builder.Finish(&fsb_array)); - - ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array)); -} - -TEST(TestDecimalDictionaryBuilder, Basic) { - // Build the dictionary Array - const auto& decimal_type = arrow::decimal(2, 0); - DictionaryBuilder builder(decimal_type, default_memory_pool()); - - // Test data - std::vector test{12, 12, 11, 12}; - for (const auto& value : test) { - ASSERT_OK(builder.Append(value.ToBytes().data())); - } - - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Build expected data - FixedSizeBinaryBuilder decimal_builder(decimal_type); - ASSERT_OK(decimal_builder.Append(Decimal128(12).ToBytes())); - ASSERT_OK(decimal_builder.Append(Decimal128(11).ToBytes())); - - std::shared_ptr decimal_array; - ASSERT_OK(decimal_builder.Finish(&decimal_array)); - auto dtype = arrow::dictionary(int8(), decimal_array); - - Int8Builder int_builder; - ASSERT_OK(int_builder.Append({0, 0, 1, 0})); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - -TEST(TestDecimalDictionaryBuilder, DoubleTableSize) { - const auto& decimal_type = arrow::decimal(21, 0); - - // Build the dictionary Array - DictionaryBuilder builder(decimal_type, default_memory_pool()); - - // Build expected data - FixedSizeBinaryBuilder fsb_builder(decimal_type); - Int16Builder int_builder; - - // Fill with 1024 different values - for (int64_t i = 0; i < 1024; i++) { - const uint8_t bytes[] = {0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 12, - 12, - static_cast(i / 128), - static_cast(i % 128)}; - ASSERT_OK(builder.Append(bytes)); - ASSERT_OK(fsb_builder.Append(bytes)); - ASSERT_OK(int_builder.Append(static_cast(i))); - } - // Fill with an already existing value - const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1}; - for (int64_t i = 0; i < 1024; i++) { - ASSERT_OK(builder.Append(known_value)); - ASSERT_OK(int_builder.Append(1)); - } - - // Finalize result - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - - // Finalize expected data - std::shared_ptr fsb_array; - ASSERT_OK(fsb_builder.Finish(&fsb_array)); - - auto dtype = std::make_shared(int16(), fsb_array); - std::shared_ptr int_array; - ASSERT_OK(int_builder.Finish(&int_array)); - - DictionaryArray expected(dtype, int_array); - ASSERT_TRUE(expected.Equals(result)); -} - // ---------------------------------------------------------------------- // List tests @@ -2766,9 +2415,8 @@ class DecimalTest : public ::testing::TestWithParam { template void TestCreate(int32_t precision, const DecimalVector& draw, const std::vector& valid_bytes, int64_t offset) const { - auto type = std::make_shared(precision, 4); - - auto builder = std::make_shared(type); + auto type = std::make_shared(precision, 4); + auto builder = std::make_shared(type); size_t null_count = 0; diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index f2dd75335df8b..4ceb071ac3338 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -92,7 +92,7 @@ static inline std::shared_ptr SliceData(const ArrayData& data, int64_ auto new_data = data.ShallowCopy(); new_data->length = length; new_data->offset = offset; - new_data->null_count = kUnknownNullCount; + new_data->null_count = data.null_count != 0 ? kUnknownNullCount : 0; return new_data; } diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index 7c5f6174fa6e9..450a4c78b5bbb 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -340,12 +340,13 @@ Status AllocateResizableBuffer(MemoryPool* pool, const int64_t size, #ifndef ARROW_NO_DEPRECATED_API /// \brief Create Buffer referencing std::string memory -/// \note Deprecated since 0.8.0 /// /// Warning: string instance must stay alive /// /// \param str std::string instance /// \return std::shared_ptr +/// +/// \note Deprecated Since 0.8.0 static inline std::shared_ptr GetBufferFromString(const std::string& str) { return std::make_shared(str); } diff --git a/cpp/src/arrow/builder-benchmark.cc b/cpp/src/arrow/builder-benchmark.cc index 7ac7fe3bed533..12dfbe8170846 100644 --- a/cpp/src/arrow/builder-benchmark.cc +++ b/cpp/src/arrow/builder-benchmark.cc @@ -115,47 +115,6 @@ static void BM_BuildAdaptiveUIntNoNulls( state.SetBytesProcessed(state.iterations() * data.size() * sizeof(int64_t)); } -static void BM_BuildDictionary(benchmark::State& state) { // NOLINT non-const reference - const int64_t iterations = 1024; - while (state.KeepRunning()) { - DictionaryBuilder builder(default_memory_pool()); - for (int64_t i = 0; i < iterations; i++) { - for (int64_t j = 0; j < i; j++) { - ABORT_NOT_OK(builder.Append(j)); - } - } - std::shared_ptr out; - ABORT_NOT_OK(builder.Finish(&out)); - } - state.SetBytesProcessed(state.iterations() * iterations * (iterations + 1) / 2 * - sizeof(int64_t)); -} - -static void BM_BuildStringDictionary( - benchmark::State& state) { // NOLINT non-const reference - const int64_t iterations = 1024; - // Pre-render strings - std::vector data; - for (int64_t i = 0; i < iterations; i++) { - std::stringstream ss; - ss << i; - data.push_back(ss.str()); - } - while (state.KeepRunning()) { - StringDictionaryBuilder builder(default_memory_pool()); - for (int64_t i = 0; i < iterations; i++) { - for (int64_t j = 0; j < i; j++) { - ABORT_NOT_OK(builder.Append(data[j])); - } - } - std::shared_ptr out; - ABORT_NOT_OK(builder.Finish(&out)); - } - // Assuming a string here needs on average 2 bytes - state.SetBytesProcessed(state.iterations() * iterations * (iterations + 1) / 2 * - sizeof(int32_t)); -} - static void BM_BuildBinaryArray(benchmark::State& state) { // NOLINT non-const reference const int64_t iterations = 1 << 20; @@ -179,8 +138,6 @@ BENCHMARK(BM_BuildAdaptiveIntNoNullsScalarAppend) ->Repetitions(3) ->Unit(benchmark::kMicrosecond); BENCHMARK(BM_BuildAdaptiveUIntNoNulls)->Repetitions(3)->Unit(benchmark::kMicrosecond); -BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond); -BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond); BENCHMARK(BM_BuildBinaryArray)->Repetitions(3)->Unit(benchmark::kMicrosecond); diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index d2d3dbdf7d13e..3e213fcd5ca70 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -40,7 +40,6 @@ namespace arrow { using internal::AdaptiveIntBuilderBase; -using internal::WrappedBinary; Status ArrayBuilder::AppendToBitmap(bool is_valid) { if (length_ == capacity_) { @@ -814,338 +813,6 @@ Status BooleanBuilder::Append(const std::vector& values) { return Status::OK(); } -// ---------------------------------------------------------------------- -// DictionaryBuilder - -template -DictionaryBuilder::DictionaryBuilder(const std::shared_ptr& type, - MemoryPool* pool) - : ArrayBuilder(type, pool), - hash_table_(new PoolBuffer(pool)), - hash_slots_(nullptr), - dict_builder_(type, pool), - values_builder_(pool), - byte_width_(-1) { - if (!::arrow::CpuInfo::initialized()) { - ::arrow::CpuInfo::Init(); - } -} - -DictionaryBuilder::DictionaryBuilder(const std::shared_ptr& type, - MemoryPool* pool) - : ArrayBuilder(type, pool), values_builder_(pool) { - if (!::arrow::CpuInfo::initialized()) { - ::arrow::CpuInfo::Init(); - } -} - -DictionaryBuilder::~DictionaryBuilder() {} - -template <> -DictionaryBuilder::DictionaryBuilder( - const std::shared_ptr& type, MemoryPool* pool) - : ArrayBuilder(type, pool), - hash_table_(new PoolBuffer(pool)), - hash_slots_(nullptr), - dict_builder_(type, pool), - values_builder_(pool), - byte_width_(static_cast(*type).byte_width()) { - if (!::arrow::CpuInfo::initialized()) { - ::arrow::CpuInfo::Init(); - } -} - -template -Status DictionaryBuilder::Init(int64_t elements) { - RETURN_NOT_OK(ArrayBuilder::Init(elements)); - - // Fill the initial hash table - RETURN_NOT_OK(hash_table_->Resize(sizeof(hash_slot_t) * kInitialHashTableSize)); - hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); - std::fill(hash_slots_, hash_slots_ + kInitialHashTableSize, kHashSlotEmpty); - hash_table_size_ = kInitialHashTableSize; - mod_bitmask_ = kInitialHashTableSize - 1; - - return values_builder_.Init(elements); -} - -Status DictionaryBuilder::Init(int64_t elements) { - RETURN_NOT_OK(ArrayBuilder::Init(elements)); - return values_builder_.Init(elements); -} - -template -Status DictionaryBuilder::Resize(int64_t capacity) { - if (capacity < kMinBuilderCapacity) { - capacity = kMinBuilderCapacity; - } - - if (capacity_ == 0) { - return Init(capacity); - } else { - return ArrayBuilder::Resize(capacity); - } -} - -Status DictionaryBuilder::Resize(int64_t capacity) { - if (capacity < kMinBuilderCapacity) { - capacity = kMinBuilderCapacity; - } - - if (capacity_ == 0) { - return Init(capacity); - } else { - return ArrayBuilder::Resize(capacity); - } -} - -template -Status DictionaryBuilder::FinishInternal(std::shared_ptr* out) { - std::shared_ptr dictionary; - RETURN_NOT_OK(dict_builder_.Finish(&dictionary)); - - RETURN_NOT_OK(values_builder_.FinishInternal(out)); - (*out)->type = std::make_shared((*out)->type, dictionary); - return Status::OK(); -} - -Status DictionaryBuilder::FinishInternal(std::shared_ptr* out) { - std::shared_ptr dictionary = std::make_shared(0); - - RETURN_NOT_OK(values_builder_.FinishInternal(out)); - (*out)->type = std::make_shared((*out)->type, dictionary); - return Status::OK(); -} - -template -Status DictionaryBuilder::Append(const Scalar& value) { - RETURN_NOT_OK(Reserve(1)); - // Based on DictEncoder::Put - int j = HashValue(value) & mod_bitmask_; - hash_slot_t index = hash_slots_[j]; - - // Find an empty slot - while (kHashSlotEmpty != index && SlotDifferent(index, value)) { - // Linear probing - ++j; - if (j == hash_table_size_) { - j = 0; - } - index = hash_slots_[j]; - } - - if (index == kHashSlotEmpty) { - // Not in the hash table, so we insert it now - index = static_cast(dict_builder_.length()); - hash_slots_[j] = index; - RETURN_NOT_OK(AppendDictionary(value)); - - if (ARROW_PREDICT_FALSE(static_cast(dict_builder_.length()) > - hash_table_size_ * kMaxHashTableLoad)) { - RETURN_NOT_OK(DoubleTableSize()); - } - } - - RETURN_NOT_OK(values_builder_.Append(index)); - - return Status::OK(); -} - -template -Status DictionaryBuilder::AppendArray(const Array& array) { - const auto& numeric_array = static_cast&>(array); - for (int64_t i = 0; i < array.length(); i++) { - if (array.IsNull(i)) { - RETURN_NOT_OK(AppendNull()); - } else { - RETURN_NOT_OK(Append(numeric_array.Value(i))); - } - } - return Status::OK(); -} - -Status DictionaryBuilder::AppendArray(const Array& array) { - for (int64_t i = 0; i < array.length(); i++) { - RETURN_NOT_OK(AppendNull()); - } - return Status::OK(); -} - -template <> -Status DictionaryBuilder::AppendArray(const Array& array) { - if (!type_->Equals(*array.type())) { - return Status::Invalid("Cannot append FixedSizeBinary array with non-matching type"); - } - - const auto& numeric_array = static_cast(array); - for (int64_t i = 0; i < array.length(); i++) { - if (array.IsNull(i)) { - RETURN_NOT_OK(AppendNull()); - } else { - RETURN_NOT_OK(Append(numeric_array.Value(i))); - } - } - return Status::OK(); -} - -template -Status DictionaryBuilder::AppendNull() { - return values_builder_.AppendNull(); -} - -Status DictionaryBuilder::AppendNull() { return values_builder_.AppendNull(); } - -template -Status DictionaryBuilder::DoubleTableSize() { - int new_size = hash_table_size_ * 2; - auto new_hash_table = std::make_shared(pool_); - - RETURN_NOT_OK(new_hash_table->Resize(sizeof(hash_slot_t) * new_size)); - int32_t* new_hash_slots = reinterpret_cast(new_hash_table->mutable_data()); - std::fill(new_hash_slots, new_hash_slots + new_size, kHashSlotEmpty); - int new_mod_bitmask = new_size - 1; - - for (int i = 0; i < hash_table_size_; ++i) { - hash_slot_t index = hash_slots_[i]; - - if (index == kHashSlotEmpty) { - continue; - } - - // Compute the hash value mod the new table size to start looking for an - // empty slot - Scalar value = GetDictionaryValue(static_cast(index)); - - // Find an empty slot in the new hash table - int j = HashValue(value) & new_mod_bitmask; - hash_slot_t slot = new_hash_slots[j]; - - while (kHashSlotEmpty != slot && SlotDifferent(slot, value)) { - ++j; - if (j == new_size) { - j = 0; - } - slot = new_hash_slots[j]; - } - - // Copy the old slot index to the new hash table - new_hash_slots[j] = index; - } - - hash_table_ = new_hash_table; - hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); - hash_table_size_ = new_size; - mod_bitmask_ = new_size - 1; - - return Status::OK(); -} - -template -typename DictionaryBuilder::Scalar DictionaryBuilder::GetDictionaryValue( - int64_t index) { - const Scalar* data = reinterpret_cast(dict_builder_.data()->data()); - return data[index]; -} - -template <> -const uint8_t* DictionaryBuilder::GetDictionaryValue(int64_t index) { - return dict_builder_.GetValue(index); -} - -template -int DictionaryBuilder::HashValue(const Scalar& value) { - return HashUtil::Hash(&value, sizeof(Scalar), 0); -} - -template <> -int DictionaryBuilder::HashValue(const Scalar& value) { - return HashUtil::Hash(value, byte_width_, 0); -} - -template -bool DictionaryBuilder::SlotDifferent(hash_slot_t index, const Scalar& value) { - const Scalar other = GetDictionaryValue(static_cast(index)); - return other != value; -} - -template <> -bool DictionaryBuilder::SlotDifferent(hash_slot_t index, - const Scalar& value) { - int32_t width = static_cast(*type_).byte_width(); - const Scalar other = GetDictionaryValue(static_cast(index)); - return memcmp(other, value, width) != 0; -} - -template -Status DictionaryBuilder::AppendDictionary(const Scalar& value) { - return dict_builder_.Append(value); -} - -#define BINARY_DICTIONARY_SPECIALIZATIONS(Type) \ - template <> \ - WrappedBinary DictionaryBuilder::GetDictionaryValue(int64_t index) { \ - int32_t v_len; \ - const uint8_t* v = dict_builder_.GetValue(static_cast(index), &v_len); \ - return WrappedBinary(v, v_len); \ - } \ - \ - template <> \ - Status DictionaryBuilder::AppendDictionary(const WrappedBinary& value) { \ - return dict_builder_.Append(value.ptr_, value.length_); \ - } \ - \ - template <> \ - Status DictionaryBuilder::AppendArray(const Array& array) { \ - const BinaryArray& binary_array = static_cast(array); \ - WrappedBinary value(nullptr, 0); \ - for (int64_t i = 0; i < array.length(); i++) { \ - if (array.IsNull(i)) { \ - RETURN_NOT_OK(AppendNull()); \ - } else { \ - value.ptr_ = binary_array.GetValue(i, &value.length_); \ - RETURN_NOT_OK(Append(value)); \ - } \ - } \ - return Status::OK(); \ - } \ - \ - template <> \ - int DictionaryBuilder::HashValue(const WrappedBinary& value) { \ - return HashUtil::Hash(value.ptr_, value.length_, 0); \ - } \ - \ - template <> \ - bool DictionaryBuilder::SlotDifferent(hash_slot_t index, \ - const WrappedBinary& value) { \ - int32_t other_length; \ - const uint8_t* other_value = \ - dict_builder_.GetValue(static_cast(index), &other_length); \ - return !(other_length == value.length_ && \ - 0 == memcmp(other_value, value.ptr_, value.length_)); \ - } - -BINARY_DICTIONARY_SPECIALIZATIONS(StringType); -BINARY_DICTIONARY_SPECIALIZATIONS(BinaryType); - -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; -template class DictionaryBuilder; - // ---------------------------------------------------------------------- // Decimal128Builder @@ -1446,7 +1113,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, BUILDER_CASE(STRING, StringBuilder); BUILDER_CASE(BINARY, BinaryBuilder); BUILDER_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryBuilder); - BUILDER_CASE(DECIMAL, DecimalBuilder); + BUILDER_CASE(DECIMAL, Decimal128Builder); case Type::LIST: { std::unique_ptr value_builder; std::shared_ptr value_type = @@ -1477,125 +1144,4 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, } } -#define DICTIONARY_BUILDER_CASE(ENUM, BuilderType) \ - case Type::ENUM: \ - out->reset(new BuilderType(type, pool)); \ - return Status::OK(); - -Status MakeDictionaryBuilder(MemoryPool* pool, const std::shared_ptr& type, - std::shared_ptr* out) { - switch (type->id()) { - DICTIONARY_BUILDER_CASE(NA, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(UINT8, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(INT8, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(UINT16, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(INT16, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(UINT32, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(INT32, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(UINT64, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(INT64, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(DATE32, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(DATE64, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(TIME32, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(TIME64, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(TIMESTAMP, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(FLOAT, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(DOUBLE, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(STRING, StringDictionaryBuilder); - DICTIONARY_BUILDER_CASE(BINARY, BinaryDictionaryBuilder); - DICTIONARY_BUILDER_CASE(FIXED_SIZE_BINARY, DictionaryBuilder); - DICTIONARY_BUILDER_CASE(DECIMAL, DictionaryBuilder); - default: - return Status::NotImplemented(type->ToString()); - } -} - -#define DICTIONARY_ARRAY_CASE(ENUM, BuilderType) \ - case Type::ENUM: \ - builder = std::make_shared(type, pool); \ - RETURN_NOT_OK(static_cast(*builder).AppendArray(input)); \ - RETURN_NOT_OK(builder->Finish(out)); \ - return Status::OK(); - -Status EncodeArrayToDictionary(const Array& input, MemoryPool* pool, - std::shared_ptr* out) { - const std::shared_ptr& type = input.data()->type; - std::shared_ptr builder; - switch (type->id()) { - DICTIONARY_ARRAY_CASE(NA, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(UINT8, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(INT8, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(UINT16, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(INT16, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(UINT32, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(INT32, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(UINT64, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(INT64, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(DATE32, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(DATE64, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(TIME32, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(TIME64, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(TIMESTAMP, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(FLOAT, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(DOUBLE, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(STRING, StringDictionaryBuilder); - DICTIONARY_ARRAY_CASE(BINARY, BinaryDictionaryBuilder); - DICTIONARY_ARRAY_CASE(FIXED_SIZE_BINARY, DictionaryBuilder); - DICTIONARY_ARRAY_CASE(DECIMAL, DictionaryBuilder); - default: - std::stringstream ss; - ss << "Cannot encode array of type " << type->ToString(); - ss << " to dictionary"; - return Status::NotImplemented(ss.str()); - } -} -#define DICTIONARY_COLUMN_CASE(ENUM, BuilderType) \ - case Type::ENUM: \ - builder = std::make_shared(type, pool); \ - chunks = input.data(); \ - for (auto chunk : chunks->chunks()) { \ - RETURN_NOT_OK(static_cast(*builder).AppendArray(*chunk)); \ - } \ - RETURN_NOT_OK(builder->Finish(&arr)); \ - *out = std::make_shared(input.name(), arr); \ - return Status::OK(); - -/// \brief Encodes a column to a suitable dictionary type -/// \param input Column to be encoded -/// \param pool MemoryPool to allocate the dictionary -/// \param out The new column -/// \return Status -Status EncodeColumnToDictionary(const Column& input, MemoryPool* pool, - std::shared_ptr* out) { - const std::shared_ptr& type = input.type(); - std::shared_ptr builder; - std::shared_ptr arr; - std::shared_ptr chunks; - switch (type->id()) { - DICTIONARY_COLUMN_CASE(UINT8, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(INT8, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(UINT16, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(INT16, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(UINT32, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(INT32, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(UINT64, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(INT64, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(DATE32, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(DATE64, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(TIME32, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(TIME64, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(TIMESTAMP, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(FLOAT, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(DOUBLE, DictionaryBuilder); - DICTIONARY_COLUMN_CASE(STRING, StringDictionaryBuilder); - DICTIONARY_COLUMN_CASE(BINARY, BinaryDictionaryBuilder); - DICTIONARY_COLUMN_CASE(FIXED_SIZE_BINARY, DictionaryBuilder); - default: - std::stringstream ss; - ss << "Cannot encode column of type " << type->ToString(); - ss << " to dictionary"; - return Status::NotImplemented(ss.str()); - } -} - } // namespace arrow diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index bc25d0d211934..32741b53ac438 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -123,6 +123,18 @@ class ARROW_EXPORT ArrayBuilder { std::shared_ptr type() const { return type_; } + // Unsafe operations (don't check capacity/don't resize) + + // Append to null bitmap. + void UnsafeAppendToBitmap(bool is_valid) { + if (is_valid) { + BitUtil::SetBit(null_bitmap_data_, length_); + } else { + ++null_count_; + } + ++length_; + } + protected: ArrayBuilder() {} @@ -143,18 +155,6 @@ class ARROW_EXPORT ArrayBuilder { void Reset(); - // Unsafe operations (don't check capacity/don't resize) - - // Append to null bitmap. - void UnsafeAppendToBitmap(bool is_valid) { - if (is_valid) { - BitUtil::SetBit(null_bitmap_data_, length_); - } else { - ++null_count_; - } - ++length_; - } - // Vector append. Treat each zero byte as a nullzero. If valid_bytes is null // assume all of length bits are valid. void UnsafeAppendToBitmap(const uint8_t* valid_bytes, int64_t length); @@ -810,191 +810,12 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder { std::vector> field_builders_; }; -// ---------------------------------------------------------------------- -// Dictionary builder - -// Based on Apache Parquet-cpp's DictEncoder - -// Initially 1024 elements -static constexpr int kInitialHashTableSize = 1 << 10; - -typedef int32_t hash_slot_t; -static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits::max(); - -// The maximum load factor for the hash table before resizing. -static constexpr double kMaxHashTableLoad = 0.7; - -namespace internal { - -// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here. -struct WrappedBinary { - WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {} - - const uint8_t* ptr_; - int32_t length_; -}; - -template -struct DictionaryScalar { - using type = typename T::c_type; -}; - -template <> -struct DictionaryScalar { - using type = WrappedBinary; -}; - -template <> -struct DictionaryScalar { - using type = WrappedBinary; -}; - -template <> -struct DictionaryScalar { - using type = uint8_t const*; -}; - -} // namespace internal - -/// \brief Array builder for created encoded DictionaryArray from dense array -/// data -template -class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder { - public: - using Scalar = typename internal::DictionaryScalar::type; - - ~DictionaryBuilder() {} - - DictionaryBuilder(const std::shared_ptr& type, MemoryPool* pool); - - template - explicit DictionaryBuilder( - typename std::enable_if::is_parameter_free, MemoryPool*>::type pool) - : DictionaryBuilder(TypeTraits::type_singleton(), pool) {} - - /// \brief Append a scalar value - Status Append(const Scalar& value); - - /// \brief Append a scalar null value - Status AppendNull(); - - /// \brief Append a whole dense array to the builder - Status AppendArray(const Array& array); - - Status Init(int64_t elements) override; - Status Resize(int64_t capacity) override; - Status FinishInternal(std::shared_ptr* out) override; - - protected: - Status DoubleTableSize(); - Scalar GetDictionaryValue(int64_t index); - int HashValue(const Scalar& value); - bool SlotDifferent(hash_slot_t slot, const Scalar& value); - Status AppendDictionary(const Scalar& value); - - std::shared_ptr hash_table_; - int32_t* hash_slots_; - - /// Size of the table. Must be a power of 2. - int hash_table_size_; - - // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j % - // hash_table_size_, but uses far fewer CPU cycles - int mod_bitmask_; - - typename TypeTraits::BuilderType dict_builder_; - AdaptiveIntBuilder values_builder_; - int32_t byte_width_; -}; - -template <> -class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder { - public: - ~DictionaryBuilder(); - - DictionaryBuilder(const std::shared_ptr& type, MemoryPool* pool); - explicit DictionaryBuilder(MemoryPool* pool); - - /// \brief Append a scalar null value - Status AppendNull(); - - /// \brief Append a whole dense array to the builder - Status AppendArray(const Array& array); - - Status Init(int64_t elements) override; - Status Resize(int64_t capacity) override; - Status FinishInternal(std::shared_ptr* out) override; - - protected: - AdaptiveIntBuilder values_builder_; -}; - -class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder { - public: - using DictionaryBuilder::Append; - using DictionaryBuilder::DictionaryBuilder; - - Status Append(const uint8_t* value, int32_t length) { - return Append(internal::WrappedBinary(value, length)); - } - - Status Append(const char* value, int32_t length) { - return Append( - internal::WrappedBinary(reinterpret_cast(value), length)); - } - - Status Append(const std::string& value) { - return Append(internal::WrappedBinary(reinterpret_cast(value.c_str()), - static_cast(value.size()))); - } -}; - -/// \brief Dictionary array builder with convenience methods for strings -class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder { - public: - using DictionaryBuilder::Append; - using DictionaryBuilder::DictionaryBuilder; - - Status Append(const uint8_t* value, int32_t length) { - return Append(internal::WrappedBinary(value, length)); - } - - Status Append(const char* value, int32_t length) { - return Append( - internal::WrappedBinary(reinterpret_cast(value), length)); - } - - Status Append(const std::string& value) { - return Append(internal::WrappedBinary(reinterpret_cast(value.c_str()), - static_cast(value.size()))); - } -}; - // ---------------------------------------------------------------------- // Helper functions Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, std::unique_ptr* out); -Status ARROW_EXPORT MakeDictionaryBuilder(MemoryPool* pool, - const std::shared_ptr& type, - std::shared_ptr* out); - -/// \brief Convert Array to encoded DictionaryArray form -/// -/// \param[in] input The Array to be encoded -/// \param[in] pool MemoryPool to allocate memory for the hash table -/// \param[out] out Array encoded to DictionaryArray -Status ARROW_EXPORT EncodeArrayToDictionary(const Array& input, MemoryPool* pool, - std::shared_ptr* out); - -/// \brief Convert a Column's data internally to DictionaryArray -/// -/// \param[in] input The ChunkedArray to be encoded -/// \param[in] pool MemoryPool to allocate memory for the hash table -/// \param[out] out Column with data converted to DictionaryArray -Status ARROW_EXPORT EncodeColumnToDictionary(const Column& input, MemoryPool* pool, - std::shared_ptr* out); } // namespace arrow #endif // ARROW_BUILDER_H_ diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt index 4589afb9574d3..d4369ed27b7c4 100644 --- a/cpp/src/arrow/compute/CMakeLists.txt +++ b/cpp/src/arrow/compute/CMakeLists.txt @@ -18,7 +18,6 @@ # Headers: top level install(FILES api.h - cast.h context.h kernel.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute") @@ -36,3 +35,6 @@ install( ####################################### ADD_ARROW_TEST(compute-test) +ADD_ARROW_BENCHMARK(compute-benchmark) + +add_subdirectory(kernels) diff --git a/cpp/src/arrow/compute/api.h b/cpp/src/arrow/compute/api.h index da7df1cbbfafc..b3700b4c5813c 100644 --- a/cpp/src/arrow/compute/api.h +++ b/cpp/src/arrow/compute/api.h @@ -18,8 +18,10 @@ #ifndef ARROW_COMPUTE_API_H #define ARROW_COMPUTE_API_H -#include "arrow/compute/cast.h" #include "arrow/compute/context.h" #include "arrow/compute/kernel.h" +#include "arrow/compute/kernels/cast.h" +#include "arrow/compute/kernels/hash.h" + #endif // ARROW_COMPUTE_API_H diff --git a/cpp/src/arrow/compute/compute-benchmark.cc b/cpp/src/arrow/compute/compute-benchmark.cc new file mode 100644 index 0000000000000..974fffcd6d99d --- /dev/null +++ b/cpp/src/arrow/compute/compute-benchmark.cc @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "benchmark/benchmark.h" + +#include + +#include "arrow/builder.h" +#include "arrow/memory_pool.h" +#include "arrow/test-util.h" + +#include "arrow/compute/context.h" +#include "arrow/compute/kernels/hash.h" + +namespace arrow { +namespace compute { + +static void BM_BuildDictionary(benchmark::State& state) { // NOLINT non-const reference + const int64_t iterations = 1024; + + std::vector values; + std::vector is_valid; + for (int64_t i = 0; i < iterations; i++) { + for (int64_t j = 0; j < i; j++) { + is_valid.push_back((i + j) % 9 == 0); + values.push_back(j); + } + } + + std::shared_ptr arr; + ArrayFromVector(is_valid, values, &arr); + + FunctionContext ctx; + + while (state.KeepRunning()) { + Datum out; + ABORT_NOT_OK(DictionaryEncode(&ctx, Datum(arr), &out)); + } + state.SetBytesProcessed(state.iterations() * values.size() * sizeof(int64_t)); +} + +static void BM_BuildStringDictionary( + benchmark::State& state) { // NOLINT non-const reference + const int64_t iterations = 1024 * 64; + // Pre-render strings + std::vector data; + + int64_t total_bytes = 0; + for (int64_t i = 0; i < iterations; i++) { + std::stringstream ss; + ss << i; + auto val = ss.str(); + data.push_back(val); + total_bytes += static_cast(val.size()); + } + + std::shared_ptr arr; + ArrayFromVector(data, &arr); + + FunctionContext ctx; + + while (state.KeepRunning()) { + Datum out; + ABORT_NOT_OK(DictionaryEncode(&ctx, Datum(arr), &out)); + } + // Assuming a string here needs on average 2 bytes + state.SetBytesProcessed(state.iterations() * total_bytes); +} + +BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond); + +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index 61d53c4d50b33..5eada398d52fc 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -37,10 +37,12 @@ #include "arrow/type.h" #include "arrow/type_traits.h" -#include "arrow/compute/cast.h" #include "arrow/compute/context.h" #include "arrow/compute/kernel.h" +#include "arrow/compute/kernels/cast.h" +#include "arrow/compute/kernels/hash.h" +using std::shared_ptr; using std::vector; namespace arrow { @@ -54,6 +56,18 @@ class ComputeFixture { FunctionContext ctx_; }; +template +shared_ptr _MakeArray(const shared_ptr& type, const vector& values, + const vector& is_valid) { + shared_ptr result; + if (is_valid.size() > 0) { + ArrayFromVector(type, is_valid, values, &result); + } else { + ArrayFromVector(type, values, &result); + } + return result; +} + // ---------------------------------------------------------------------- // Cast @@ -65,17 +79,17 @@ static void AssertBufferSame(const Array& left, const Array& right, int buffer_i class TestCast : public ComputeFixture, public TestBase { public: void CheckPass(const Array& input, const Array& expected, - const std::shared_ptr& out_type, const CastOptions& options) { - std::shared_ptr result; + const shared_ptr& out_type, const CastOptions& options) { + shared_ptr result; ASSERT_OK(Cast(&ctx_, input, out_type, options, &result)); ASSERT_ARRAYS_EQUAL(expected, *result); } template - void CheckFails(const std::shared_ptr& in_type, - const std::vector& in_values, const std::vector& is_valid, - const std::shared_ptr& out_type, const CastOptions& options) { - std::shared_ptr input, result; + void CheckFails(const shared_ptr& in_type, const vector& in_values, + const vector& is_valid, const shared_ptr& out_type, + const CastOptions& options) { + shared_ptr input, result; if (is_valid.size() > 0) { ArrayFromVector(in_type, is_valid, in_values, &input); } else { @@ -84,19 +98,18 @@ class TestCast : public ComputeFixture, public TestBase { ASSERT_RAISES(Invalid, Cast(&ctx_, *input, out_type, options, &result)); } - void CheckZeroCopy(const Array& input, const std::shared_ptr& out_type) { - std::shared_ptr result; + void CheckZeroCopy(const Array& input, const shared_ptr& out_type) { + shared_ptr result; ASSERT_OK(Cast(&ctx_, input, out_type, {}, &result)); AssertBufferSame(input, *result, 0); AssertBufferSame(input, *result, 1); } template - void CheckCase(const std::shared_ptr& in_type, - const std::vector& in_values, const std::vector& is_valid, - const std::shared_ptr& out_type, - const std::vector& out_values, const CastOptions& options) { - std::shared_ptr input, expected; + void CheckCase(const shared_ptr& in_type, const vector& in_values, + const vector& is_valid, const shared_ptr& out_type, + const vector& out_values, const CastOptions& options) { + shared_ptr input, expected; if (is_valid.size() > 0) { ArrayFromVector(in_type, is_valid, in_values, &input); ArrayFromVector(out_type, is_valid, out_values, &expected); @@ -117,10 +130,10 @@ TEST_F(TestCast, SameTypeZeroCopy) { vector is_valid = {true, false, true, true, true}; vector v1 = {0, 1, 2, 3, 4}; - std::shared_ptr arr; + shared_ptr arr; ArrayFromVector(int32(), is_valid, v1, &arr); - std::shared_ptr result; + shared_ptr result; ASSERT_OK(Cast(&this->ctx_, *arr, int32(), {}, &result)); AssertBufferSame(*arr, *result, 0); @@ -185,7 +198,7 @@ TEST_F(TestCast, OverflowInNullSlot) { vector v11 = {0, 70000, 2000, 1000, 0}; vector e11 = {0, 0, 2000, 1000, 0}; - std::shared_ptr expected; + shared_ptr expected; ArrayFromVector(int16(), is_valid, e11, &expected); auto buf = std::make_shared(reinterpret_cast(v11.data()), @@ -280,8 +293,8 @@ TEST_F(TestCast, TimestampToTimestamp) { auto CheckTimestampCast = [this]( const CastOptions& options, TimeUnit::type from_unit, TimeUnit::type to_unit, - const std::vector& from_values, const std::vector& to_values, - const std::vector& is_valid) { + const vector& from_values, const vector& to_values, + const vector& is_valid) { CheckCase( timestamp(from_unit), from_values, is_valid, timestamp(to_unit), to_values, options); @@ -315,8 +328,8 @@ TEST_F(TestCast, TimestampToTimestamp) { CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::NANO, v6, e6, is_valid); // Zero copy - std::shared_ptr arr; vector v7 = {0, 70000, 2000, 1000, 0}; + shared_ptr arr; ArrayFromVector(timestamp(TimeUnit::SECOND), is_valid, v7, &arr); CheckZeroCopy(*arr, timestamp(TimeUnit::SECOND)); @@ -456,8 +469,8 @@ TEST_F(TestCast, TimeToTime) { time64(TimeUnit::MICRO), v6, is_valid, time64(TimeUnit::NANO), e6, options); // Zero copy - std::shared_ptr arr; vector v7 = {0, 70000, 2000, 1000, 0}; + shared_ptr arr; ArrayFromVector(time64(TimeUnit::MICRO), is_valid, v7, &arr); CheckZeroCopy(*arr, time64(TimeUnit::MICRO)); @@ -516,9 +529,9 @@ TEST_F(TestCast, DateToDate) { e1, options); // Zero copy - std::shared_ptr arr; vector v2 = {0, 70000, 2000, 1000, 0}; vector v3 = {0, 70000, 2000, 1000, 0}; + shared_ptr arr; ArrayFromVector(date32(), is_valid, v2, &arr); CheckZeroCopy(*arr, date32()); @@ -561,22 +574,54 @@ TEST_F(TestCast, ToDouble) { options); } +TEST_F(TestCast, ChunkedArray) { + vector values1 = {0, 1, 2}; + vector values2 = {3, 4, 5}; + + auto type = int16(); + auto out_type = int64(); + + auto a1 = _MakeArray(type, values1, {}); + auto a2 = _MakeArray(type, values2, {}); + + ArrayVector arrays = {a1, a2}; + auto carr = std::make_shared(arrays); + + CastOptions options; + + Datum out; + ASSERT_OK(Cast(&this->ctx_, Datum(carr), out_type, options, &out)); + ASSERT_EQ(Datum::CHUNKED_ARRAY, out.kind()); + + auto out_carr = out.chunked_array(); + + vector ex_values1 = {0, 1, 2}; + vector ex_values2 = {3, 4, 5}; + auto a3 = _MakeArray(out_type, ex_values1, {}); + auto a4 = _MakeArray(out_type, ex_values2, {}); + + ArrayVector ex_arrays = {a3, a4}; + auto ex_carr = std::make_shared(ex_arrays); + + ASSERT_TRUE(out.chunked_array()->Equals(*ex_carr)); +} + TEST_F(TestCast, UnsupportedTarget) { vector is_valid = {true, false, true, true, true}; vector v1 = {0, 1, 2, 3, 4}; - std::shared_ptr arr; + shared_ptr arr; ArrayFromVector(int32(), is_valid, v1, &arr); - std::shared_ptr result; + shared_ptr result; ASSERT_RAISES(NotImplemented, Cast(&this->ctx_, *arr, utf8(), {}, &result)); } TEST_F(TestCast, DateTimeZeroCopy) { vector is_valid = {true, false, true, true, true}; - std::shared_ptr arr; vector v1 = {0, 70000, 2000, 1000, 0}; + shared_ptr arr; ArrayFromVector(int32(), is_valid, v1, &arr); CheckZeroCopy(*arr, time32(TimeUnit::SECOND)); @@ -596,7 +641,7 @@ TEST_F(TestCast, FromNull) { NullArray arr(length); - std::shared_ptr result; + shared_ptr result; ASSERT_OK(Cast(&ctx_, arr, int32(), {}, &result)); ASSERT_EQ(length, result->length()); @@ -614,7 +659,7 @@ TEST_F(TestCast, PreallocatedMemory) { const int64_t length = 5; - std::shared_ptr arr; + shared_ptr arr; vector v1 = {0, 70000, 2000, 1000, 0}; vector e1 = {0, 70000, 2000, 1000, 0}; ArrayFromVector(int32(), is_valid, v1, &arr); @@ -626,19 +671,20 @@ TEST_F(TestCast, PreallocatedMemory) { auto out_data = std::make_shared(out_type, length); - std::shared_ptr out_values; + shared_ptr out_values; ASSERT_OK(this->ctx_.Allocate(length * sizeof(int64_t), &out_values)); out_data->buffers.push_back(nullptr); out_data->buffers.push_back(out_values); - ASSERT_OK(kernel->Call(&this->ctx_, *arr, out_data.get())); + Datum out(out_data); + ASSERT_OK(kernel->Call(&this->ctx_, *arr->data(), &out)); // Buffer address unchanged ASSERT_EQ(out_values.get(), out_data->buffers[1].get()); - std::shared_ptr result = MakeArray(out_data); - std::shared_ptr expected; + shared_ptr result = MakeArray(out_data); + shared_ptr expected; ArrayFromVector(int64(), is_valid, e1, &expected); ASSERT_ARRAYS_EQUAL(*expected, *result); @@ -656,13 +702,268 @@ TYPED_TEST_CASE(TestDictionaryCast, TestTypes); TYPED_TEST(TestDictionaryCast, Basic) { CastOptions options; - std::shared_ptr plain_array = + shared_ptr plain_array = + TestBase::MakeRandomArray::ArrayType>(10, 2); + + Datum out; + ASSERT_OK(DictionaryEncode(&this->ctx_, Datum(plain_array->data()), &out)); + + this->CheckPass(*MakeArray(out.array()), *plain_array, plain_array->type(), options); +} + +/*TYPED_TEST(TestDictionaryCast, Reverse) { + CastOptions options; + shared_ptr plain_array = TestBase::MakeRandomArray::ArrayType>(10, 2); - std::shared_ptr dict_array; + shared_ptr dict_array; ASSERT_OK(EncodeArrayToDictionary(*plain_array, this->pool_, &dict_array)); - this->CheckPass(*dict_array, *plain_array, plain_array->type(), options); + this->CheckPass(*plain_array, *dict_array, dict_array->type(), options); +}*/ + +// ---------------------------------------------------------------------- +// Dictionary tests + +template +void CheckUnique(FunctionContext* ctx, const shared_ptr& type, + const vector& in_values, const vector& in_is_valid, + const vector& out_values, const vector& out_is_valid) { + shared_ptr input = _MakeArray(type, in_values, in_is_valid); + shared_ptr expected = _MakeArray(type, out_values, out_is_valid); + + shared_ptr result; + ASSERT_OK(Unique(ctx, Datum(input), &result)); + ASSERT_ARRAYS_EQUAL(*expected, *result); +} + +template +void CheckDictEncode(FunctionContext* ctx, const shared_ptr& type, + const vector& in_values, const vector& in_is_valid, + const vector& out_values, const vector& out_is_valid, + const vector& out_indices) { + shared_ptr input = _MakeArray(type, in_values, in_is_valid); + shared_ptr ex_dict = _MakeArray(type, out_values, out_is_valid); + shared_ptr ex_indices = + _MakeArray(int32(), out_indices, in_is_valid); + + DictionaryArray expected(dictionary(int32(), ex_dict), ex_indices); + + Datum datum_out; + ASSERT_OK(DictionaryEncode(ctx, Datum(input), &datum_out)); + shared_ptr result = MakeArray(datum_out.array()); + + ASSERT_ARRAYS_EQUAL(expected, *result); +} + +class TestHashKernel : public ComputeFixture, public TestBase {}; + +template +class TestHashKernelPrimitive : public ComputeFixture, public TestBase {}; + +typedef ::testing::Types + PrimitiveDictionaries; + +TYPED_TEST_CASE(TestHashKernelPrimitive, PrimitiveDictionaries); + +TYPED_TEST(TestHashKernelPrimitive, Unique) { + using T = typename TypeParam::c_type; + auto type = TypeTraits::type_singleton(); + CheckUnique(&this->ctx_, type, {2, 1, 2, 1}, {true, false, true, true}, + {2, 1}, {}); +} + +TYPED_TEST(TestHashKernelPrimitive, DictEncode) { + using T = typename TypeParam::c_type; + auto type = TypeTraits::type_singleton(); + CheckDictEncode(&this->ctx_, type, {2, 1, 2, 1, 2, 3}, + {true, false, true, true, true, true}, {2, 1, 3}, {}, + {0, 0, 0, 1, 0, 2}); +} + +TYPED_TEST(TestHashKernelPrimitive, PrimitiveResizeTable) { + using T = typename TypeParam::c_type; + // Skip this test for (u)int8 + if (sizeof(Scalar) == 1) { + return; + } + + const int64_t kTotalValues = 10000; + const int64_t kRepeats = 10; + + vector values; + vector uniques; + vector indices; + for (int64_t i = 0; i < kTotalValues * kRepeats; i++) { + const auto val = static_cast(i % kTotalValues); + values.push_back(val); + + if (i < kTotalValues) { + uniques.push_back(val); + } + indices.push_back(static_cast(i % kTotalValues)); + } + + auto type = TypeTraits::type_singleton(); + CheckUnique(&this->ctx_, type, values, {}, uniques, {}); + + CheckDictEncode(&this->ctx_, type, values, {}, uniques, {}, indices); +} + +TEST_F(TestHashKernel, UniqueTimeTimestamp) { + CheckUnique(&this->ctx_, time32(TimeUnit::SECOND), {2, 1, 2, 1}, + {true, false, true, true}, {2, 1}, {}); + + CheckUnique(&this->ctx_, time64(TimeUnit::NANO), {2, 1, 2, 1}, + {true, false, true, true}, {2, 1}, {}); + + CheckUnique(&this->ctx_, timestamp(TimeUnit::NANO), + {2, 1, 2, 1}, {true, false, true, true}, {2, 1}, + {}); +} + +TEST_F(TestHashKernel, UniqueBinary) { + CheckUnique(&this->ctx_, binary(), + {"test", "", "test2", "test"}, + {true, false, true, true}, {"test", "test2"}, {}); + + CheckUnique(&this->ctx_, utf8(), {"test", "", "test2", "test"}, + {true, false, true, true}, {"test", "test2"}, {}); +} + +TEST_F(TestHashKernel, DictEncodeBinary) { + CheckDictEncode( + &this->ctx_, binary(), {"test", "", "test2", "test", "baz"}, + {true, false, true, true, true}, {"test", "test2", "baz"}, {}, {0, 0, 1, 0, 2}); + + CheckDictEncode( + &this->ctx_, utf8(), {"test", "", "test2", "test", "baz"}, + {true, false, true, true, true}, {"test", "test2", "baz"}, {}, {0, 0, 1, 0, 2}); +} + +TEST_F(TestHashKernel, BinaryResizeTable) { + const int64_t kTotalValues = 10000; + const int64_t kRepeats = 10; + + vector values; + vector uniques; + vector indices; + for (int64_t i = 0; i < kTotalValues * kRepeats; i++) { + int64_t index = i % kTotalValues; + std::stringstream ss; + ss << "test" << index; + std::string val = ss.str(); + + values.push_back(val); + + if (i < kTotalValues) { + uniques.push_back(val); + } + indices.push_back(static_cast(i % kTotalValues)); + } + + CheckUnique(&this->ctx_, binary(), values, {}, uniques, {}); + CheckDictEncode(&this->ctx_, binary(), values, {}, uniques, {}, + indices); + + CheckUnique(&this->ctx_, utf8(), values, {}, uniques, {}); + CheckDictEncode(&this->ctx_, utf8(), values, {}, uniques, {}, + indices); +} + +TEST_F(TestHashKernel, UniqueFixedSizeBinary) { + CheckUnique( + &this->ctx_, fixed_size_binary(5), {"aaaaa", "", "bbbbb", "aaaaa"}, + {true, false, true, true}, {"aaaaa", "bbbbb"}, {}); +} + +TEST_F(TestHashKernel, DictEncodeFixedSizeBinary) { + CheckDictEncode( + &this->ctx_, fixed_size_binary(5), {"bbbbb", "", "bbbbb", "aaaaa", "ccccc"}, + {true, false, true, true, true}, {"bbbbb", "aaaaa", "ccccc"}, {}, {0, 0, 0, 1, 2}); +} + +TEST_F(TestHashKernel, FixedSizeBinaryResizeTable) { + const int64_t kTotalValues = 10000; + const int64_t kRepeats = 10; + + vector values; + vector uniques; + vector indices; + for (int64_t i = 0; i < kTotalValues * kRepeats; i++) { + int64_t index = i % kTotalValues; + std::stringstream ss; + ss << "test" << static_cast(index / 128) << static_cast(index % 128); + std::string val = ss.str(); + + values.push_back(val); + + if (i < kTotalValues) { + uniques.push_back(val); + } + indices.push_back(static_cast(i % kTotalValues)); + } + + auto type = fixed_size_binary(6); + CheckUnique(&this->ctx_, type, values, {}, uniques, + {}); + CheckDictEncode(&this->ctx_, type, values, {}, + uniques, {}, indices); +} + +TEST_F(TestHashKernel, UniqueDecimal) { + vector values{12, 12, 11, 12}; + vector expected{12, 11}; + + CheckUnique(&this->ctx_, decimal(2, 0), values, + {true, false, true, true}, expected, {}); +} + +TEST_F(TestHashKernel, DictEncodeDecimal) { + vector values{12, 12, 11, 12, 13}; + vector expected{12, 11, 13}; + + CheckDictEncode(&this->ctx_, decimal(2, 0), values, + {true, false, true, true, true}, expected, + {}, {0, 0, 1, 0, 2}); +} + +TEST_F(TestHashKernel, ChunkedArrayInvoke) { + vector values1 = {"foo", "bar", "foo"}; + vector values2 = {"bar", "baz", "quuux", "foo"}; + + auto type = utf8(); + auto a1 = _MakeArray(type, values1, {}); + auto a2 = _MakeArray(type, values2, {}); + + vector dict_values = {"foo", "bar", "baz", "quuux"}; + auto ex_dict = _MakeArray(type, dict_values, {}); + + ArrayVector arrays = {a1, a2}; + auto carr = std::make_shared(arrays); + + // Unique + shared_ptr result; + ASSERT_OK(Unique(&this->ctx_, Datum(carr), &result)); + ASSERT_ARRAYS_EQUAL(*ex_dict, *result); + + // Dictionary encode + auto dict_type = dictionary(int32(), ex_dict); + + auto i1 = _MakeArray(int32(), {0, 1, 0}, {}); + auto i2 = _MakeArray(int32(), {1, 2, 3, 0}, {}); + + ArrayVector dict_arrays = {std::make_shared(dict_type, i1), + std::make_shared(dict_type, i2)}; + auto dict_carr = std::make_shared(dict_arrays); + + Datum encoded_out; + ASSERT_OK(DictionaryEncode(&this->ctx_, Datum(carr), &encoded_out)); + ASSERT_EQ(Datum::CHUNKED_ARRAY, encoded_out.kind()); + + ASSERT_TRUE(encoded_out.chunked_array()->Equals(*dict_carr)); } } // namespace compute diff --git a/cpp/src/arrow/compute/context.cc b/cpp/src/arrow/compute/context.cc index 792dc4f386939..63aa341a3d718 100644 --- a/cpp/src/arrow/compute/context.cc +++ b/cpp/src/arrow/compute/context.cc @@ -20,11 +20,16 @@ #include #include "arrow/buffer.h" +#include "arrow/util/cpu-info.h" namespace arrow { namespace compute { -FunctionContext::FunctionContext(MemoryPool* pool) : pool_(pool) {} +FunctionContext::FunctionContext(MemoryPool* pool) : pool_(pool) { + if (!::arrow::CpuInfo::initialized()) { + ::arrow::CpuInfo::Init(); + } +} MemoryPool* FunctionContext::memory_pool() const { return pool_; } diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h index 4e072a7c143ed..0037245d610b4 100644 --- a/cpp/src/arrow/compute/kernel.h +++ b/cpp/src/arrow/compute/kernel.h @@ -18,7 +18,14 @@ #ifndef ARROW_COMPUTE_KERNEL_H #define ARROW_COMPUTE_KERNEL_H +#include +#include + #include "arrow/array.h" +#include "arrow/table.h" +#include "arrow/util/macros.h" +#include "arrow/util/variant.h" +#include "arrow/util/visibility.h" namespace arrow { namespace compute { @@ -32,11 +39,99 @@ class ARROW_EXPORT OpKernel { virtual ~OpKernel() = default; }; +/// \brief Placeholder for Scalar values until we implement these +struct ARROW_EXPORT Scalar { + ~Scalar() {} + + ARROW_DISALLOW_COPY_AND_ASSIGN(Scalar); +}; + +/// \class Datum +/// \brief Variant type for various Arrow C++ data structures +struct ARROW_EXPORT Datum { + enum type { NONE, SCALAR, ARRAY, CHUNKED_ARRAY, RECORD_BATCH, TABLE, COLLECTION }; + + util::variant, std::shared_ptr, + std::shared_ptr, std::shared_ptr, + std::shared_ptr, std::vector> + value; + + /// \brief Empty datum, to be populated elsewhere + Datum() : value(nullptr) {} + + explicit Datum(const std::shared_ptr& value) : value(value) {} + + explicit Datum(const std::shared_ptr& value) : value(value) {} + + explicit Datum(const std::shared_ptr& value) : Datum(value->data()) {} + + explicit Datum(const std::shared_ptr& value) : value(value) {} + + explicit Datum(const std::shared_ptr& value) : value(value) {} + + explicit Datum(const std::shared_ptr
    & value) : value(value) {} + + explicit Datum(const std::vector& value) : value(value) {} + + ~Datum() {} + + Datum(const Datum& other) noexcept { this->value = other.value; } + + Datum::type kind() const { + switch (this->value.which()) { + case 0: + return Datum::NONE; + case 1: + return Datum::SCALAR; + case 2: + return Datum::ARRAY; + case 3: + return Datum::CHUNKED_ARRAY; + case 4: + return Datum::RECORD_BATCH; + case 5: + return Datum::TABLE; + case 6: + return Datum::COLLECTION; + default: + return Datum::NONE; + } + } + + std::shared_ptr array() const { + return util::get>(this->value); + } + + std::shared_ptr chunked_array() const { + return util::get>(this->value); + } + + const std::vector collection() const { + return util::get>(this->value); + } + + bool is_arraylike() const { + return this->kind() == Datum::ARRAY || this->kind() == Datum::CHUNKED_ARRAY; + } + + /// \brief The value type of the variant, if any + /// + /// \return nullptr if no type + std::shared_ptr type() const { + if (this->kind() == Datum::ARRAY) { + return util::get>(this->value)->type; + } else if (this->kind() == Datum::CHUNKED_ARRAY) { + return util::get>(this->value)->type(); + } + return nullptr; + } +}; + /// \class UnaryKernel /// \brief An array-valued function of a single input argument class ARROW_EXPORT UnaryKernel : public OpKernel { public: - virtual Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) = 0; + virtual Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) = 0; }; } // namespace compute diff --git a/cpp/src/arrow/compute/kernels/CMakeLists.txt b/cpp/src/arrow/compute/kernels/CMakeLists.txt new file mode 100644 index 0000000000000..715e6c6612fe4 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/CMakeLists.txt @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +install(FILES + cast.h + hash.h + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/compute/kernels") diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc similarity index 82% rename from cpp/src/arrow/compute/cast.cc rename to cpp/src/arrow/compute/kernels/cast.cc index 114ab9af0d0d0..6a42ec8b2b807 100644 --- a/cpp/src/arrow/compute/cast.cc +++ b/cpp/src/arrow/compute/kernels/cast.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/compute/cast.h" +#include "arrow/compute/kernels/cast.h" #include #include @@ -26,6 +26,7 @@ #include #include #include +#include #include "arrow/array.h" #include "arrow/buffer.h" @@ -39,6 +40,7 @@ #include "arrow/compute/context.h" #include "arrow/compute/kernel.h" +#include "arrow/compute/kernels/util-internal.h" #ifdef ARROW_EXTRA_ERROR_CONTEXT @@ -71,29 +73,6 @@ namespace compute { constexpr int64_t kMillisecondsInDay = 86400000; -template -inline const T* GetValues(const ArrayData& data, int i) { - return reinterpret_cast(data.buffers[i]->data()) + data.offset; -} - -template -inline T* GetMutableValues(const ArrayData* data, int i) { - return reinterpret_cast(data->buffers[i]->mutable_data()) + data->offset; -} - -namespace { - -void CopyData(const Array& input, ArrayData* output) { - auto in_data = input.data(); - output->length = in_data->length; - output->null_count = input.null_count(); - output->buffers = in_data->buffers; - output->offset = in_data->offset; - output->child_data = in_data->child_data; -} - -} // namespace - // ---------------------------------------------------------------------- // Zero copy casts @@ -128,8 +107,8 @@ struct CastFunctor {}; // Indicated no computation required template struct CastFunctor::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { CopyData(input, output); } }; @@ -140,8 +119,8 @@ struct CastFunctor::value> template struct CastFunctor::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { // Simply initialize data to 0 auto buf = output->buffers[1]; DCHECK_EQ(output->offset, 0); @@ -151,8 +130,8 @@ struct CastFunctor struct CastFunctor { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) {} + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) {} }; // ---------------------------------------------------------------------- @@ -160,19 +139,17 @@ struct CastFunctor { // Cast from Boolean to other numbers template -struct CastFunctor::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { +struct CastFunctor> { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { using c_type = typename T::c_type; constexpr auto kOne = static_cast(1); constexpr auto kZero = static_cast(0); - auto in_data = input.data(); - internal::BitmapReader bit_reader(in_data->buffers[1]->data(), in_data->offset, - in_data->length); + internal::BitmapReader bit_reader(input.buffers[1]->data(), input.offset, + input.length); auto out = GetMutableValues(output, 1); - for (int64_t i = 0; i < input.length(); ++i) { + for (int64_t i = 0; i < input.length; ++i) { *out++ = bit_reader.IsSet() ? kOne : kZero; bit_reader.Next(); } @@ -216,14 +193,14 @@ template struct CastFunctor::value && std::is_base_of::value && !std::is_same::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { using in_type = typename I::c_type; DCHECK_EQ(output->offset, 0); - const in_type* in_data = GetValues(*input.data(), 1); + const in_type* in_data = GetValues(input, 1); uint8_t* out_data = GetMutableValues(output, 1); - for (int64_t i = 0; i < input.length(); ++i) { + for (int64_t i = 0; i < input.length; ++i) { BitUtil::SetBitTo(out_data, i, (*in_data++) != 0); } } @@ -232,25 +209,26 @@ struct CastFunctor::v template struct CastFunctor::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { using in_type = typename I::c_type; using out_type = typename O::c_type; DCHECK_EQ(output->offset, 0); - auto in_offset = input.offset(); + auto in_offset = input.offset; - const in_type* in_data = GetValues(*input.data(), 1); + const in_type* in_data = GetValues(input, 1); auto out_data = GetMutableValues(output, 1); if (!options.allow_int_overflow) { constexpr in_type kMax = static_cast(std::numeric_limits::max()); constexpr in_type kMin = static_cast(std::numeric_limits::min()); - if (input.null_count() > 0) { - internal::BitmapReader is_valid_reader(input.data()->buffers[0]->data(), - in_offset, input.length()); - for (int64_t i = 0; i < input.length(); ++i) { + // Null count may be -1 if the input array had been sliced + if (input.null_count != 0) { + internal::BitmapReader is_valid_reader(input.buffers[0]->data(), in_offset, + input.length); + for (int64_t i = 0; i < input.length; ++i) { if (ARROW_PREDICT_FALSE(is_valid_reader.IsSet() && (*in_data > kMax || *in_data < kMin))) { ctx->SetStatus(Status::Invalid("Integer value out of bounds")); @@ -259,7 +237,7 @@ struct CastFunctor kMax || *in_data < kMin)) { ctx->SetStatus(Status::Invalid("Integer value out of bounds")); } @@ -267,7 +245,7 @@ struct CastFunctor(*in_data++); } } @@ -278,14 +256,14 @@ template struct CastFunctor::value && !is_integer_downcast::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { using in_type = typename I::c_type; using out_type = typename O::c_type; - const in_type* in_data = GetValues(*input.data(), 1); + const in_type* in_data = GetValues(input, 1); auto out_data = GetMutableValues(output, 1); - for (int64_t i = 0; i < input.length(); ++i) { + for (int64_t i = 0; i < input.length; ++i) { *out_data++ = static_cast(*in_data++); } } @@ -296,34 +274,52 @@ struct CastFunctor void ShiftTime(FunctionContext* ctx, const CastOptions& options, const bool is_multiply, - const int64_t factor, const Array& input, ArrayData* output) { - const in_type* in_data = GetValues(*input.data(), 1); + const int64_t factor, const ArrayData& input, ArrayData* output) { + const in_type* in_data = GetValues(input, 1); auto out_data = GetMutableValues(output, 1); if (factor == 1) { - for (int64_t i = 0; i < input.length(); i++) { + for (int64_t i = 0; i < input.length; i++) { out_data[i] = static_cast(in_data[i]); } } else if (is_multiply) { - for (int64_t i = 0; i < input.length(); i++) { + for (int64_t i = 0; i < input.length; i++) { out_data[i] = static_cast(in_data[i] * factor); } } else { if (options.allow_time_truncate) { - for (int64_t i = 0; i < input.length(); i++) { + for (int64_t i = 0; i < input.length; i++) { out_data[i] = static_cast(in_data[i] / factor); } } else { - for (int64_t i = 0; i < input.length(); i++) { - out_data[i] = static_cast(in_data[i] / factor); - if (input.IsValid(i) && (out_data[i] * factor != in_data[i])) { - std::stringstream ss; - ss << "Casting from " << input.type()->ToString() << " to " - << output->type->ToString() << " would lose data: " << in_data[i]; - ctx->SetStatus(Status::Invalid(ss.str())); - break; +#define RAISE_INVALID_CAST(VAL) \ + std::stringstream ss; \ + ss << "Casting from " << input.type->ToString() << " to " << output->type->ToString() \ + << " would lose data: " << VAL; \ + ctx->SetStatus(Status::Invalid(ss.str())); + + if (input.null_count != 0) { + internal::BitmapReader bit_reader(input.buffers[0]->data(), input.offset, + input.length); + for (int64_t i = 0; i < input.length; i++) { + out_data[i] = static_cast(in_data[i] / factor); + if (bit_reader.IsSet() && (out_data[i] * factor != in_data[i])) { + RAISE_INVALID_CAST(in_data[i]); + break; + } + bit_reader.Next(); + } + } else { + for (int64_t i = 0; i < input.length; i++) { + out_data[i] = static_cast(in_data[i] / factor); + if (out_data[i] * factor != in_data[i]) { + RAISE_INVALID_CAST(in_data[i]); + break; + } } } + +#undef RAISE_INVALID_CAST } } } @@ -342,10 +338,10 @@ const std::pair kTimeConversionTable[4][4] = { template <> struct CastFunctor { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { // If units are the same, zero copy, otherwise convert - const auto& in_type = static_cast(*input.type()); + const auto& in_type = static_cast(*input.type); const auto& out_type = static_cast(*output->type); if (in_type.unit() == out_type.unit()) { @@ -364,9 +360,9 @@ struct CastFunctor { template <> struct CastFunctor { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { - const auto& in_type = static_cast(*input.type()); + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { + const auto& in_type = static_cast(*input.type); static const int64_t kTimestampToDateFactors[4] = { 86400LL, // SECOND @@ -382,9 +378,9 @@ struct CastFunctor { template <> struct CastFunctor { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { - const auto& in_type = static_cast(*input.type()); + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { + const auto& in_type = static_cast(*input.type); std::pair conversion = kTimeConversionTable[static_cast(in_type.unit())] @@ -393,17 +389,21 @@ struct CastFunctor { ShiftTime(ctx, options, conversion.first, conversion.second, input, output); + internal::BitmapReader bit_reader(input.buffers[0]->data(), input.offset, + input.length); + // Ensure that intraday milliseconds have been zeroed out auto out_data = GetMutableValues(output, 1); - for (int64_t i = 0; i < input.length(); ++i) { + for (int64_t i = 0; i < input.length; ++i) { const int64_t remainder = out_data[i] % kMillisecondsInDay; - if (ARROW_PREDICT_FALSE(!options.allow_time_truncate && input.IsValid(i) && + if (ARROW_PREDICT_FALSE(!options.allow_time_truncate && bit_reader.IsSet() && remainder > 0)) { ctx->SetStatus( Status::Invalid("Timestamp value had non-zero intraday milliseconds")); break; } out_data[i] -= remainder; + bit_reader.Next(); } } }; @@ -415,13 +415,13 @@ template struct CastFunctor::value && std::is_base_of::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { using in_t = typename I::c_type; using out_t = typename O::c_type; // If units are the same, zero copy, otherwise convert - const auto& in_type = static_cast(*input.type()); + const auto& in_type = static_cast(*input.type); const auto& out_type = static_cast(*output->type); if (in_type.unit() == out_type.unit()) { @@ -443,16 +443,16 @@ struct CastFunctor struct CastFunctor { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { ShiftTime(ctx, options, true, kMillisecondsInDay, input, output); } }; template <> struct CastFunctor { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { ShiftTime(ctx, options, false, kMillisecondsInDay, input, output); } }; @@ -487,10 +487,11 @@ template struct CastFunctor< T, DictionaryType, typename std::enable_if::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { - const DictionaryArray& dict_array = static_cast(input); - const DictionaryType& type = static_cast(*input.type()); + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { + DictionaryArray dict_array(input.ShallowCopy()); + + const DictionaryType& type = static_cast(*input.type); const DataType& values_type = *type.dictionary()->type(); const FixedSizeBinaryArray& dictionary = static_cast(*type.dictionary()); @@ -558,10 +559,11 @@ Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices, template struct CastFunctor::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { - const DictionaryArray& dict_array = static_cast(input); - const DictionaryType& type = static_cast(*input.type()); + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { + DictionaryArray dict_array(input.ShallowCopy()); + + const DictionaryType& type = static_cast(*input.type); const DataType& values_type = *type.dictionary()->type(); const BinaryArray& dictionary = static_cast(*type.dictionary()); @@ -617,12 +619,13 @@ void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary, template struct CastFunctor::value>::type> { - void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input, - ArrayData* output) { + void operator()(FunctionContext* ctx, const CastOptions& options, + const ArrayData& input, ArrayData* output) { using c_type = typename T::c_type; - const DictionaryArray& dict_array = static_cast(input); - const DictionaryType& type = static_cast(*input.type()); + DictionaryArray dict_array(input.ShallowCopy()); + + const DictionaryType& type = static_cast(*input.type); const DataType& values_type = *type.dictionary()->type(); // Check if values and output type match @@ -657,24 +660,23 @@ struct CastFunctor CastFunction; -static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input, +static Status AllocateIfNotPreallocated(FunctionContext* ctx, const ArrayData& input, bool can_pre_allocate_values, ArrayData* out) { - const int64_t length = input.length(); - - out->null_count = input.null_count(); + const int64_t length = input.length; + out->null_count = input.null_count; // Propagate bitmap unless we are null type - std::shared_ptr validity_bitmap = input.data()->buffers[0]; - if (input.type_id() == Type::NA) { + std::shared_ptr validity_bitmap = input.buffers[0]; + if (input.type->id() == Type::NA) { int64_t bitmap_size = BitUtil::BytesForBits(length); RETURN_NOT_OK(ctx->Allocate(bitmap_size, &validity_bitmap)); memset(validity_bitmap->mutable_data(), 0, bitmap_size); - } else if (input.offset() != 0) { - RETURN_NOT_OK(CopyBitmap(ctx->memory_pool(), validity_bitmap->data(), input.offset(), + } else if (input.offset != 0) { + RETURN_NOT_OK(CopyBitmap(ctx->memory_pool(), validity_bitmap->data(), input.offset, length, &validity_bitmap)); } @@ -727,17 +729,28 @@ static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input class CastKernel : public UnaryKernel { public: CastKernel(const CastOptions& options, const CastFunction& func, bool is_zero_copy, - bool can_pre_allocate_values) + bool can_pre_allocate_values, const std::shared_ptr& out_type) : options_(options), func_(func), is_zero_copy_(is_zero_copy), - can_pre_allocate_values_(can_pre_allocate_values) {} + can_pre_allocate_values_(can_pre_allocate_values), + out_type_(out_type) {} + + Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override { + ArrayData* result; + + if (out->kind() == Datum::NONE) { + out->value = std::make_shared(out_type_, input.length); + } + + result = out->array().get(); - Status Call(FunctionContext* ctx, const Array& input, ArrayData* out) override { if (!is_zero_copy_) { - RETURN_NOT_OK(AllocateIfNotPreallocated(ctx, input, can_pre_allocate_values_, out)); + RETURN_NOT_OK( + AllocateIfNotPreallocated(ctx, input, can_pre_allocate_values_, result)); } - func_(ctx, options_, input, out); + func_(ctx, options_, input, result); + RETURN_IF_ERROR(ctx); return Status::OK(); } @@ -747,18 +760,19 @@ class CastKernel : public UnaryKernel { CastFunction func_; bool is_zero_copy_; bool can_pre_allocate_values_; + std::shared_ptr out_type_; }; -#define CAST_CASE(InType, OutType) \ - case OutType::type_id: \ - is_zero_copy = is_zero_copy_cast::value; \ - can_pre_allocate_values = \ - !(!is_binary_like(InType::type_id) && is_binary_like(OutType::type_id)); \ - func = [](FunctionContext* ctx, const CastOptions& options, const Array& input, \ - ArrayData* out) { \ - CastFunctor func; \ - func(ctx, options, input, out); \ - }; \ +#define CAST_CASE(InType, OutType) \ + case OutType::type_id: \ + is_zero_copy = is_zero_copy_cast::value; \ + can_pre_allocate_values = \ + !(!is_binary_like(InType::type_id) && is_binary_like(OutType::type_id)); \ + func = [](FunctionContext* ctx, const CastOptions& options, const ArrayData& input, \ + ArrayData* out) { \ + CastFunctor func; \ + func(ctx, options, input, out); \ + }; \ break; #define NUMERIC_CASES(FN, IN_TYPE) \ @@ -832,26 +846,26 @@ class CastKernel : public UnaryKernel { FN(IN_TYPE, FloatType); \ FN(IN_TYPE, DoubleType); \ FN(IN_TYPE, FixedSizeBinaryType); \ - FN(IN_TYPE, DecimalType); \ + FN(IN_TYPE, Decimal128Type); \ FN(IN_TYPE, BinaryType); \ FN(IN_TYPE, StringType); -#define GET_CAST_FUNCTION(CASE_GENERATOR, InType) \ - static std::unique_ptr Get##InType##CastFunc( \ - const std::shared_ptr& out_type, const CastOptions& options) { \ - CastFunction func; \ - bool is_zero_copy = false; \ - bool can_pre_allocate_values = true; \ - switch (out_type->id()) { \ - CASE_GENERATOR(CAST_CASE, InType); \ - default: \ - break; \ - } \ - if (func != nullptr) { \ - return std::unique_ptr( \ - new CastKernel(options, func, is_zero_copy, can_pre_allocate_values)); \ - } \ - return nullptr; \ +#define GET_CAST_FUNCTION(CASE_GENERATOR, InType) \ + static std::unique_ptr Get##InType##CastFunc( \ + const std::shared_ptr& out_type, const CastOptions& options) { \ + CastFunction func; \ + bool is_zero_copy = false; \ + bool can_pre_allocate_values = true; \ + switch (out_type->id()) { \ + CASE_GENERATOR(CAST_CASE, InType); \ + default: \ + break; \ + } \ + if (func != nullptr) { \ + return std::unique_ptr(new CastKernel( \ + options, func, is_zero_copy, can_pre_allocate_values, out_type)); \ + } \ + return nullptr; \ } GET_CAST_FUNCTION(NULL_CASES, NullType); @@ -912,18 +926,27 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr& return Status::OK(); } -Status Cast(FunctionContext* ctx, const Array& array, +Status Cast(FunctionContext* ctx, const Datum& value, const std::shared_ptr& out_type, const CastOptions& options, - std::shared_ptr* out) { + Datum* out) { // Dynamic dispatch to obtain right cast function std::unique_ptr func; - RETURN_NOT_OK(GetCastFunction(*array.type(), out_type, options, &func)); + RETURN_NOT_OK(GetCastFunction(*value.type(), out_type, options, &func)); - // Data structure for output - auto out_data = std::make_shared(out_type, array.length()); + std::vector result; + RETURN_NOT_OK(detail::InvokeUnaryArrayKernel(ctx, func.get(), value, &result)); - RETURN_NOT_OK(func->Call(ctx, array, out_data.get())); - *out = MakeArray(out_data); + *out = detail::WrapDatumsLike(value, result); + return Status::OK(); +} + +Status Cast(FunctionContext* ctx, const Array& array, + const std::shared_ptr& out_type, const CastOptions& options, + std::shared_ptr* out) { + Datum datum_out; + RETURN_NOT_OK(Cast(ctx, Datum(array.data()), out_type, options, &datum_out)); + DCHECK_EQ(Datum::ARRAY, datum_out.kind()); + *out = MakeArray(datum_out.array()); return Status::OK(); } diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/kernels/cast.h similarity index 69% rename from cpp/src/arrow/compute/cast.h rename to cpp/src/arrow/compute/kernels/cast.h index d7bde20d607db..b75bb7b6c15f4 100644 --- a/cpp/src/arrow/compute/cast.h +++ b/cpp/src/arrow/compute/kernels/cast.h @@ -15,25 +15,26 @@ // specific language governing permissions and limitations // under the License. -#ifndef ARROW_COMPUTE_CAST_H -#define ARROW_COMPUTE_CAST_H +#ifndef ARROW_COMPUTE_KERNELS_CAST_H +#define ARROW_COMPUTE_KERNELS_CAST_H #include #include "arrow/status.h" #include "arrow/util/visibility.h" +#include "arrow/compute/kernel.h" + namespace arrow { class Array; +class ChunkedArray; +class Column; class DataType; namespace compute { -class FunctionContext; -class UnaryKernel; - -struct CastOptions { +struct ARROW_EXPORT CastOptions { CastOptions() : allow_int_overflow(false), allow_time_truncate(false) {} bool allow_int_overflow; @@ -48,7 +49,7 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr& /// \brief Cast from one array type to another /// \param[in] context the FunctionContext -/// \param[in] array array to cast +/// \param[in] value array to cast /// \param[in] to_type type to cast to /// \param[in] options casting options /// \param[out] out resulting array @@ -56,11 +57,25 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr& /// \since 0.7.0 /// \note API not yet finalized ARROW_EXPORT -Status Cast(FunctionContext* context, const Array& array, +Status Cast(FunctionContext* context, const Array& value, const std::shared_ptr& to_type, const CastOptions& options, std::shared_ptr* out); +/// \brief Cast from one value to another +/// \param[in] context the FunctionContext +/// \param[in] value datum to cast +/// \param[in] to_type type to cast to +/// \param[in] options casting options +/// \param[out] out resulting datum +/// +/// \since 0.8.0 +/// \note API not yet finalized +ARROW_EXPORT +Status Cast(FunctionContext* context, const Datum& value, + const std::shared_ptr& to_type, const CastOptions& options, + Datum* out); + } // namespace compute } // namespace arrow -#endif // ARROW_COMPUTE_CAST_H +#endif // ARROW_COMPUTE_KERNELS_CAST_H diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc new file mode 100644 index 0000000000000..3af41609fef28 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/hash.cc @@ -0,0 +1,822 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/compute/kernels/hash.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/builder.h" +#include "arrow/compute/context.h" +#include "arrow/compute/kernel.h" +#include "arrow/compute/kernels/util-internal.h" +#include "arrow/util/hash-util.h" + +namespace arrow { +namespace compute { + +namespace { + +// Initially 1024 elements +static constexpr int64_t kInitialHashTableSize = 1 << 10; + +typedef int32_t hash_slot_t; +static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits::max(); + +// The maximum load factor for the hash table before resizing. +static constexpr double kMaxHashTableLoad = 0.7; + +enum class SIMDMode : char { NOSIMD, SSE4, AVX2 }; + +#define CHECK_IMPLEMENTED(KERNEL, FUNCNAME, TYPE) \ + if (!KERNEL) { \ + std::stringstream ss; \ + ss << FUNCNAME << " not implemented for " << type->ToString(); \ + return Status::NotImplemented(ss.str()); \ + } + +Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr* out) { + auto hash_table = std::make_shared(pool); + + RETURN_NOT_OK(hash_table->Resize(sizeof(hash_slot_t) * size)); + int32_t* slots = reinterpret_cast(hash_table->mutable_data()); + std::fill(slots, slots + size, kHashSlotEmpty); + + *out = hash_table; + return Status::OK(); +} + +// This is a slight design concession -- some hash actions have the possibility +// of failure. Rather than introduce extra error checking into all actions, we +// will raise an internal exception so that only the actions where errors can +// occur will experience the extra overhead +class HashException : public std::exception { + public: + explicit HashException(const std::string& msg, StatusCode code = StatusCode::Invalid) + : msg_(msg), code_(code) {} + + ~HashException() throw() {} + + const char* what() const throw() override; + + StatusCode code() const { return code_; } + + private: + std::string msg_; + StatusCode code_; +}; + +const char* HashException::what() const throw() { return msg_.c_str(); } + +class HashTable { + public: + HashTable(const std::shared_ptr& type, MemoryPool* pool) + : type_(type), + pool_(pool), + initialized_(false), + hash_table_(nullptr), + hash_slots_(nullptr), + hash_table_size_(0), + mod_bitmask_(0) {} + + virtual ~HashTable() {} + + virtual Status Append(const ArrayData& input) = 0; + virtual Status Flush(Datum* out) = 0; + virtual Status GetDictionary(std::shared_ptr* out) = 0; + + protected: + Status Init(int64_t elements); + + std::shared_ptr type_; + MemoryPool* pool_; + bool initialized_; + + // The hash table contains integer indices that reference the set of observed + // distinct values + std::shared_ptr hash_table_; + hash_slot_t* hash_slots_; + + /// Size of the table. Must be a power of 2. + int64_t hash_table_size_; + + /// Size at which we decide to resize + int64_t hash_table_load_threshold_; + + // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j % + // hash_table_size_, but uses far fewer CPU cycles + int64_t mod_bitmask_; +}; + +Status HashTable::Init(int64_t elements) { + DCHECK_EQ(elements, BitUtil::NextPower2(elements)); + RETURN_NOT_OK(NewHashTable(elements, pool_, &hash_table_)); + hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); + hash_table_size_ = elements; + hash_table_load_threshold_ = + static_cast(static_cast(elements) * kMaxHashTableLoad); + mod_bitmask_ = elements - 1; + initialized_ = true; + return Status::OK(); +} + +template +class HashTableKernel : public HashTable {}; + +// Types of hash actions +// +// unique: append to dictionary when not found, no-op with slot +// dictionary-encode: append to dictionary when not found, append slot # +// match: raise or set null when not found, otherwise append slot # +// isin: set false when not found, otherwise true +// value counts: append to dictionary when not found, increment count for slot + +template +class HashDictionary {}; + +// ---------------------------------------------------------------------- +// Hash table pass for nulls + +template +class HashTableKernel> : public HashTable { + public: + using HashTable::HashTable; + + Status Init() { + // No-op, do not even need to initialize hash table + return Status::OK(); + } + + Status Append(const ArrayData& arr) override { + if (!initialized_) { + RETURN_NOT_OK(Init()); + } + auto action = static_cast(this); + RETURN_NOT_OK(action->Reserve(arr.length)); + for (int64_t i = 0; i < arr.length; ++i) { + action->ObserveNull(); + } + return Status::OK(); + } + + Status GetDictionary(std::shared_ptr* out) override { + // TODO(wesm): handle null being a valid dictionary value + auto null_array = std::make_shared(0); + *out = null_array->data(); + return Status::OK(); + } +}; + +// ---------------------------------------------------------------------- +// Hash table pass for primitive types + +template +struct HashDictionary> { + using T = typename Type::c_type; + + explicit HashDictionary(MemoryPool* pool) + : pool(pool), buffer(std::make_shared(pool)), size(0), capacity(0) {} + + Status Init() { + this->size = 0; + return Resize(kInitialHashTableSize); + } + + Status DoubleSize() { return Resize(this->size * 2); } + + Status Resize(const int64_t elements) { + RETURN_NOT_OK(this->buffer->Resize(elements * sizeof(T))); + + this->capacity = elements; + this->values = reinterpret_cast(this->buffer->mutable_data()); + return Status::OK(); + } + + MemoryPool* pool; + std::shared_ptr buffer; + T* values; + int64_t size; + int64_t capacity; +}; + +#define GENERIC_HASH_PASS(HASH_INNER_LOOP) \ + if (arr.null_count != 0) { \ + internal::BitmapReader valid_reader(arr.buffers[0]->data(), arr.offset, arr.length); \ + for (int64_t i = 0; i < arr.length; ++i) { \ + const bool is_null = valid_reader.IsNotSet(); \ + valid_reader.Next(); \ + \ + if (is_null) { \ + action->ObserveNull(); \ + continue; \ + } \ + \ + HASH_INNER_LOOP(); \ + } \ + } else { \ + for (int64_t i = 0; i < arr.length; ++i) { \ + HASH_INNER_LOOP(); \ + } \ + } + +#define DOUBLE_TABLE_SIZE(SETUP_CODE, COMPUTE_HASH) \ + do { \ + int64_t new_size = hash_table_size_ * 2; \ + \ + std::shared_ptr new_hash_table; \ + RETURN_NOT_OK(NewHashTable(new_size, pool_, &new_hash_table)); \ + int32_t* new_hash_slots = \ + reinterpret_cast(new_hash_table->mutable_data()); \ + int64_t new_mod_bitmask = new_size - 1; \ + \ + SETUP_CODE; \ + \ + for (int i = 0; i < hash_table_size_; ++i) { \ + hash_slot_t index = hash_slots_[i]; \ + \ + if (index == kHashSlotEmpty) { \ + continue; \ + } \ + \ + COMPUTE_HASH; \ + while (kHashSlotEmpty != new_hash_slots[j]) { \ + ++j; \ + if (ARROW_PREDICT_FALSE(j == hash_table_size_)) { \ + j = 0; \ + } \ + } \ + \ + new_hash_slots[j] = index; \ + } \ + \ + hash_table_ = new_hash_table; \ + hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); \ + hash_table_size_ = new_size; \ + hash_table_load_threshold_ = \ + static_cast(static_cast(new_size) * kMaxHashTableLoad); \ + mod_bitmask_ = new_size - 1; \ + } while (false) + +template +class HashTableKernel> : public HashTable { + public: + using T = typename Type::c_type; + + HashTableKernel(const std::shared_ptr& type, MemoryPool* pool) + : HashTable(type, pool), dict_(pool) {} + + Status Init() { + RETURN_NOT_OK(dict_.Init()); + return HashTable::Init(kInitialHashTableSize); + } + + Status Append(const ArrayData& arr) override { + if (!initialized_) { + RETURN_NOT_OK(Init()); + } + + const T* values = GetValues(arr, 1); + auto action = static_cast(this); + + RETURN_NOT_OK(action->Reserve(arr.length)); + +#define HASH_INNER_LOOP() \ + const T value = values[i]; \ + int64_t j = HashValue(value) & mod_bitmask_; \ + hash_slot_t slot = hash_slots_[j]; \ + \ + while (kHashSlotEmpty != slot && dict_.values[slot] != value) { \ + ++j; \ + if (ARROW_PREDICT_FALSE(j == hash_table_size_)) { \ + j = 0; \ + } \ + slot = hash_slots_[j]; \ + } \ + \ + if (slot == kHashSlotEmpty) { \ + if (!Action::allow_expand) { \ + throw HashException("Encountered new dictionary value"); \ + } \ + \ + slot = static_cast(dict_.size); \ + hash_slots_[j] = slot; \ + dict_.values[dict_.size++] = value; \ + \ + action->ObserveNotFound(slot); \ + \ + if (ARROW_PREDICT_FALSE(dict_.size > hash_table_load_threshold_)) { \ + RETURN_NOT_OK(action->DoubleSize()); \ + } \ + } else { \ + action->ObserveFound(slot); \ + } + + GENERIC_HASH_PASS(HASH_INNER_LOOP); + +#undef HASH_INNER_LOOP + + return Status::OK(); + } + + Status GetDictionary(std::shared_ptr* out) override { + // TODO(wesm): handle null being in the dictionary + auto dict_data = dict_.buffer; + RETURN_NOT_OK(dict_data->Resize(dict_.size * sizeof(T), false)); + + BufferVector buffers = {nullptr, dict_data}; + *out = std::make_shared(type_, dict_.size, std::move(buffers), 0); + return Status::OK(); + } + + protected: + int64_t HashValue(const T& value) const { + // TODO(wesm): Use faster hash function for C types + return HashUtil::Hash(&value, sizeof(T), 0); + } + + Status DoubleTableSize() { +#define PRIMITIVE_INNER_LOOP \ + const T value = dict_.values[index]; \ + int64_t j = HashValue(value) & new_mod_bitmask; + + DOUBLE_TABLE_SIZE(, PRIMITIVE_INNER_LOOP); + +#undef PRIMITIVE_INNER_LOOP + + return dict_.Resize(hash_table_size_); + } + + HashDictionary dict_; +}; + +// ---------------------------------------------------------------------- +// Hash table pass for variable-length binary types + +template +class HashTableKernel> : public HashTable { + public: + HashTableKernel(const std::shared_ptr& type, MemoryPool* pool) + : HashTable(type, pool), dict_offsets_(pool), dict_data_(pool), dict_size_(0) {} + + Status Init() { + RETURN_NOT_OK(dict_offsets_.Resize(kInitialHashTableSize)); + + // We append the end offset after each append to the dictionary, so this + // sets the initial condition for the length-0 case + // + // initial offsets (dict size == 0): 0 + // after 1st dict entry of length 3: 0 3 + // after 2nd dict entry of length 4: 0 3 7 + RETURN_NOT_OK(dict_offsets_.Append(0)); + return HashTable::Init(kInitialHashTableSize); + } + + Status Append(const ArrayData& arr) override { + if (!initialized_) { + RETURN_NOT_OK(Init()); + } + + const int32_t* offsets = GetValues(arr, 1); + const uint8_t* data = GetValues(arr, 2); + + auto action = static_cast(this); + RETURN_NOT_OK(action->Reserve(arr.length)); + +#define HASH_INNER_LOOP() \ + const int32_t position = offsets[i]; \ + const int32_t length = offsets[i + 1] - position; \ + const uint8_t* value = data + position; \ + \ + int64_t j = HashValue(value, length) & mod_bitmask_; \ + hash_slot_t slot = hash_slots_[j]; \ + \ + const int32_t* dict_offsets = dict_offsets_.data(); \ + const uint8_t* dict_data = dict_data_.data(); \ + while (kHashSlotEmpty != slot && \ + !((dict_offsets[slot + 1] - dict_offsets[slot]) == length && \ + 0 == memcmp(value, dict_data + dict_offsets[slot], length))) { \ + ++j; \ + if (ARROW_PREDICT_FALSE(j == hash_table_size_)) { \ + j = 0; \ + } \ + slot = hash_slots_[j]; \ + } \ + \ + if (slot == kHashSlotEmpty) { \ + if (!Action::allow_expand) { \ + throw HashException("Encountered new dictionary value"); \ + } \ + \ + slot = dict_size_++; \ + hash_slots_[j] = slot; \ + \ + RETURN_NOT_OK(dict_data_.Append(value, length)); \ + RETURN_NOT_OK(dict_offsets_.Append(static_cast(dict_data_.length()))); \ + \ + action->ObserveNotFound(slot); \ + \ + if (ARROW_PREDICT_FALSE(dict_size_ > hash_table_load_threshold_)) { \ + RETURN_NOT_OK(action->DoubleSize()); \ + } \ + } else { \ + action->ObserveFound(slot); \ + } + + GENERIC_HASH_PASS(HASH_INNER_LOOP); + +#undef HASH_INNER_LOOP + + return Status::OK(); + } + + Status GetDictionary(std::shared_ptr* out) override { + // TODO(wesm): handle null being in the dictionary + BufferVector buffers = {nullptr, nullptr, nullptr}; + + RETURN_NOT_OK(dict_offsets_.Finish(&buffers[1])); + RETURN_NOT_OK(dict_data_.Finish(&buffers[2])); + + *out = std::make_shared(type_, dict_size_, std::move(buffers), 0); + return Status::OK(); + } + + protected: + int64_t HashValue(const uint8_t* data, int32_t length) const { + return HashUtil::Hash(data, length, 0); + } + + Status DoubleTableSize() { +#define VARBYTES_SETUP \ + const int32_t* dict_offsets = dict_offsets_.data(); \ + const uint8_t* dict_data = dict_data_.data() + +#define VARBYTES_COMPUTE_HASH \ + const int32_t length = dict_offsets[index + 1] - dict_offsets[index]; \ + const uint8_t* value = dict_data + dict_offsets[index]; \ + int64_t j = HashValue(value, length) & new_mod_bitmask + + DOUBLE_TABLE_SIZE(VARBYTES_SETUP, VARBYTES_COMPUTE_HASH); + +#undef VARBYTES_SETUP +#undef VARBYTES_COMPUTE_HASH + + return Status::OK(); + } + + TypedBufferBuilder dict_offsets_; + TypedBufferBuilder dict_data_; + int32_t dict_size_; +}; + +// ---------------------------------------------------------------------- +// Hash table pass for fixed size binary types + +template +class HashTableKernel> + : public HashTable { + public: + HashTableKernel(const std::shared_ptr& type, MemoryPool* pool) + : HashTable(type, pool), dict_data_(pool), dict_size_(0) { + const auto& fw_type = static_cast(*type); + byte_width_ = fw_type.bit_width() / 8; + } + + Status Init() { + RETURN_NOT_OK(dict_data_.Resize(kInitialHashTableSize * byte_width_)); + return HashTable::Init(kInitialHashTableSize); + } + + Status Append(const ArrayData& arr) override { + if (!initialized_) { + RETURN_NOT_OK(Init()); + } + + const uint8_t* data = GetValues(arr, 1); + + auto action = static_cast(this); + RETURN_NOT_OK(action->Reserve(arr.length)); + +#define HASH_INNER_LOOP() \ + const uint8_t* value = data + i * byte_width_; \ + int64_t j = HashValue(value) & mod_bitmask_; \ + hash_slot_t slot = hash_slots_[j]; \ + \ + const uint8_t* dict_data = dict_data_.data(); \ + while (kHashSlotEmpty != slot && \ + !(0 == memcmp(value, dict_data + slot * byte_width_, byte_width_))) { \ + ++j; \ + if (ARROW_PREDICT_FALSE(j == hash_table_size_)) { \ + j = 0; \ + } \ + slot = hash_slots_[j]; \ + } \ + \ + if (slot == kHashSlotEmpty) { \ + if (!Action::allow_expand) { \ + throw HashException("Encountered new dictionary value"); \ + } \ + \ + slot = dict_size_++; \ + hash_slots_[j] = slot; \ + \ + RETURN_NOT_OK(dict_data_.Append(value, byte_width_)); \ + \ + action->ObserveNotFound(slot); \ + \ + if (ARROW_PREDICT_FALSE(dict_size_ > hash_table_load_threshold_)) { \ + RETURN_NOT_OK(action->DoubleSize()); \ + } \ + } else { \ + action->ObserveFound(slot); \ + } + + GENERIC_HASH_PASS(HASH_INNER_LOOP); + +#undef HASH_INNER_LOOP + + return Status::OK(); + } + + Status GetDictionary(std::shared_ptr* out) override { + // TODO(wesm): handle null being in the dictionary + BufferVector buffers = {nullptr, nullptr}; + RETURN_NOT_OK(dict_data_.Finish(&buffers[1])); + + *out = std::make_shared(type_, dict_size_, std::move(buffers), 0); + return Status::OK(); + } + + protected: + int64_t HashValue(const uint8_t* data) const { + return HashUtil::Hash(data, byte_width_, 0); + } + + Status DoubleTableSize() { +#define FIXED_BYTES_SETUP const uint8_t* dict_data = dict_data_.data() + +#define FIXED_BYTES_COMPUTE_HASH \ + int64_t j = HashValue(dict_data + index * byte_width_) & new_mod_bitmask + + DOUBLE_TABLE_SIZE(FIXED_BYTES_SETUP, FIXED_BYTES_COMPUTE_HASH); + +#undef FIXED_BYTES_SETUP +#undef FIXED_BYTES_COMPUTE_HASH + + return Status::OK(); + } + + int32_t byte_width_; + TypedBufferBuilder dict_data_; + int32_t dict_size_; +}; + +// ---------------------------------------------------------------------- +// Unique implementation + +template +class UniqueImpl : public HashTableKernel> { + public: + static constexpr bool allow_expand = true; + using Base = HashTableKernel>; + using Base::Base; + + Status Reserve(const int64_t length) { return Status::OK(); } + + void ObserveFound(const hash_slot_t slot) {} + void ObserveNull() {} + void ObserveNotFound(const hash_slot_t slot) {} + + Status DoubleSize() { return Base::DoubleTableSize(); } + + Status Append(const ArrayData& input) override { return Base::Append(input); } + + Status Flush(Datum* out) override { + // No-op + return Status::OK(); + } +}; + +// ---------------------------------------------------------------------- +// Dictionary encode implementation + +template +class DictEncodeImpl : public HashTableKernel> { + public: + static constexpr bool allow_expand = true; + using Base = HashTableKernel; + + DictEncodeImpl(const std::shared_ptr& type, MemoryPool* pool) + : Base(type, pool), indices_builder_(pool) {} + + Status Reserve(const int64_t length) { return indices_builder_.Reserve(length); } + + void ObserveNull() { indices_builder_.UnsafeAppendToBitmap(false); } + + void ObserveFound(const hash_slot_t slot) { indices_builder_.UnsafeAppend(slot); } + + void ObserveNotFound(const hash_slot_t slot) { return ObserveFound(slot); } + + Status DoubleSize() { return Base::DoubleTableSize(); } + + Status Flush(Datum* out) override { + std::shared_ptr result; + RETURN_NOT_OK(indices_builder_.FinishInternal(&result)); + out->value = std::move(result); + return Status::OK(); + } + + using Base::Append; + + private: + Int32Builder indices_builder_; +}; + +// ---------------------------------------------------------------------- +// Kernel wrapper for generic hash table kernels + +class HashKernelImpl : public HashKernel { + public: + explicit HashKernelImpl(std::unique_ptr hasher) + : hasher_(std::move(hasher)) {} + + Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override { + RETURN_NOT_OK(Append(ctx, input)); + return Flush(out); + } + + Status Append(FunctionContext* ctx, const ArrayData& input) override { + std::lock_guard guard(lock_); + try { + RETURN_NOT_OK(hasher_->Append(input)); + } catch (const HashException& e) { + return Status(e.code(), e.what()); + } + return Status::OK(); + } + + Status Flush(Datum* out) override { return hasher_->Flush(out); } + + Status GetDictionary(std::shared_ptr* out) override { + return hasher_->GetDictionary(out); + } + + private: + std::mutex lock_; + std::unique_ptr hasher_; +}; + +} // namespace + +Status GetUniqueKernel(FunctionContext* ctx, const std::shared_ptr& type, + std::unique_ptr* out) { + std::unique_ptr hasher; + +#define UNIQUE_CASE(InType) \ + case InType::type_id: \ + hasher.reset(new UniqueImpl(type, ctx->memory_pool())); \ + break + + switch (type->id()) { + UNIQUE_CASE(NullType); + // UNIQUE_CASE(BooleanType); + UNIQUE_CASE(UInt8Type); + UNIQUE_CASE(Int8Type); + UNIQUE_CASE(UInt16Type); + UNIQUE_CASE(Int16Type); + UNIQUE_CASE(UInt32Type); + UNIQUE_CASE(Int32Type); + UNIQUE_CASE(UInt64Type); + UNIQUE_CASE(Int64Type); + UNIQUE_CASE(FloatType); + UNIQUE_CASE(DoubleType); + UNIQUE_CASE(Date32Type); + UNIQUE_CASE(Date64Type); + UNIQUE_CASE(Time32Type); + UNIQUE_CASE(Time64Type); + UNIQUE_CASE(TimestampType); + UNIQUE_CASE(BinaryType); + UNIQUE_CASE(StringType); + UNIQUE_CASE(FixedSizeBinaryType); + UNIQUE_CASE(Decimal128Type); + default: + break; + } + +#undef UNIQUE_CASE + + CHECK_IMPLEMENTED(hasher, "unique", type); + out->reset(new HashKernelImpl(std::move(hasher))); + return Status::OK(); +} + +Status GetDictionaryEncodeKernel(FunctionContext* ctx, + const std::shared_ptr& type, + std::unique_ptr* out) { + std::unique_ptr hasher; + +#define DICTIONARY_ENCODE_CASE(InType) \ + case InType::type_id: \ + hasher.reset(new DictEncodeImpl(type, ctx->memory_pool())); \ + break + + switch (type->id()) { + DICTIONARY_ENCODE_CASE(NullType); + // DICTIONARY_ENCODE_CASE(BooleanType); + DICTIONARY_ENCODE_CASE(UInt8Type); + DICTIONARY_ENCODE_CASE(Int8Type); + DICTIONARY_ENCODE_CASE(UInt16Type); + DICTIONARY_ENCODE_CASE(Int16Type); + DICTIONARY_ENCODE_CASE(UInt32Type); + DICTIONARY_ENCODE_CASE(Int32Type); + DICTIONARY_ENCODE_CASE(UInt64Type); + DICTIONARY_ENCODE_CASE(Int64Type); + DICTIONARY_ENCODE_CASE(FloatType); + DICTIONARY_ENCODE_CASE(DoubleType); + DICTIONARY_ENCODE_CASE(Date32Type); + DICTIONARY_ENCODE_CASE(Date64Type); + DICTIONARY_ENCODE_CASE(Time32Type); + DICTIONARY_ENCODE_CASE(Time64Type); + DICTIONARY_ENCODE_CASE(TimestampType); + DICTIONARY_ENCODE_CASE(BinaryType); + DICTIONARY_ENCODE_CASE(StringType); + DICTIONARY_ENCODE_CASE(FixedSizeBinaryType); + DICTIONARY_ENCODE_CASE(Decimal128Type); + default: + break; + } + +#undef DICTIONARY_ENCODE_CASE + + CHECK_IMPLEMENTED(hasher, "dictionary-encode", type); + out->reset(new HashKernelImpl(std::move(hasher))); + return Status::OK(); +} + +namespace { + +Status InvokeHash(FunctionContext* ctx, HashKernel* func, const Datum& value, + std::vector* kernel_outputs, + std::shared_ptr* dictionary) { + RETURN_NOT_OK(detail::InvokeUnaryArrayKernel(ctx, func, value, kernel_outputs)); + + std::shared_ptr dict_data; + RETURN_NOT_OK(func->GetDictionary(&dict_data)); + *dictionary = MakeArray(dict_data); + return Status::OK(); +} + +} // namespace + +Status Unique(FunctionContext* ctx, const Datum& value, std::shared_ptr* out) { + std::unique_ptr func; + RETURN_NOT_OK(GetUniqueKernel(ctx, value.type(), &func)); + + std::vector dummy_outputs; + return InvokeHash(ctx, func.get(), value, &dummy_outputs, out); +} + +Status DictionaryEncode(FunctionContext* ctx, const Datum& value, Datum* out) { + std::unique_ptr func; + RETURN_NOT_OK(GetDictionaryEncodeKernel(ctx, value.type(), &func)); + + std::shared_ptr dictionary; + std::vector indices_outputs; + RETURN_NOT_OK(InvokeHash(ctx, func.get(), value, &indices_outputs, &dictionary)); + + // Create the dictionary type + DCHECK_EQ(indices_outputs[0].kind(), Datum::ARRAY); + std::shared_ptr dict_type = + ::arrow::dictionary(indices_outputs[0].array()->type, dictionary); + + // Create DictionaryArray for each piece yielded by the kernel invocations + std::vector> dict_chunks; + for (const Datum& datum : indices_outputs) { + dict_chunks.emplace_back( + std::make_shared(dict_type, MakeArray(datum.array()))); + } + + *out = detail::WrapArraysLike(value, dict_chunks); + return Status::OK(); +} + +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/hash.h b/cpp/src/arrow/compute/kernels/hash.h new file mode 100644 index 0000000000000..05f242949893f --- /dev/null +++ b/cpp/src/arrow/compute/kernels/hash.h @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_COMPUTE_KERNELS_HASH_H +#define ARROW_COMPUTE_KERNELS_HASH_H + +#include +#include + +#include "arrow/compute/kernel.h" +#include "arrow/status.h" +#include "arrow/type_fwd.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace compute { + +class FunctionContext; + +/// \brief Invoke hash table kernel on input array, returning any output +/// values. Implementations should be thread-safe +class ARROW_EXPORT HashKernel : public UnaryKernel { + public: + virtual Status Append(FunctionContext* ctx, const ArrayData& input) = 0; + virtual Status Flush(Datum* out) = 0; + virtual Status GetDictionary(std::shared_ptr* out) = 0; +}; + +/// \since 0.8.0 +/// \note API not yet finalized +ARROW_EXPORT +Status GetUniqueKernel(FunctionContext* ctx, const std::shared_ptr& type, + std::unique_ptr* kernel); + +ARROW_EXPORT +Status GetDictionaryEncodeKernel(FunctionContext* ctx, + const std::shared_ptr& type, + std::unique_ptr* kernel); + +/// \brief Compute unique elements from an array-like object +/// \param[in] context the FunctionContext +/// \param[in] datum array-like input +/// \param[out] out result as Array +/// +/// \since 0.8.0 +/// \note API not yet finalized +ARROW_EXPORT +Status Unique(FunctionContext* context, const Datum& datum, std::shared_ptr* out); + +/// \brief Dictionary-encode values in an array-like object +/// \param[in] context the FunctionContext +/// \param[in] data array-like input +/// \param[out] out result with same shape and type as input +/// +/// \since 0.8.0 +/// \note API not yet finalized +ARROW_EXPORT +Status DictionaryEncode(FunctionContext* context, const Datum& data, Datum* out); + +// TODO(wesm): Define API for incremental dictionary encoding + +// TODO(wesm): Define API for regularizing DictionaryArray objects with +// different dictionaries + +// class DictionaryEncoder { +// public: +// virtual Encode(const Datum& data, Datum* out) = 0; +// }; + +// +// ARROW_EXPORT +// Status DictionaryEncode(FunctionContext* context, const Datum& data, +// const Array& prior_dictionary, Datum* out); + +// TODO(wesm): Implement these next +// ARROW_EXPORT +// Status Match(FunctionContext* context, const Datum& values, const Datum& member_set, +// Datum* out); + +// ARROW_EXPORT +// Status IsIn(FunctionContext* context, const Datum& values, const Datum& member_set, +// Datum* out); + +// ARROW_EXPORT +// Status CountValues(FunctionContext* context, const Datum& values, +// std::shared_ptr* out_uniques, +// std::shared_ptr* out_counts); + +} // namespace compute +} // namespace arrow + +#endif // ARROW_COMPUTE_KERNELS_HASH_H diff --git a/cpp/src/arrow/compute/kernels/util-internal.cc b/cpp/src/arrow/compute/kernels/util-internal.cc new file mode 100644 index 0000000000000..df68637e088f7 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/util-internal.cc @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/compute/kernels/util-internal.h" + +#include + +#include "arrow/array.h" +#include "arrow/status.h" +#include "arrow/util/logging.h" + +#include "arrow/compute/context.h" +#include "arrow/compute/kernel.h" + +namespace arrow { +namespace compute { +namespace detail { + +Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel, + const Datum& value, std::vector* outputs) { + if (value.kind() == Datum::ARRAY) { + Datum output; + RETURN_NOT_OK(kernel->Call(ctx, *value.array(), &output)); + outputs->push_back(output); + } else if (value.kind() == Datum::CHUNKED_ARRAY) { + const ChunkedArray& array = *value.chunked_array(); + for (int i = 0; i < array.num_chunks(); i++) { + Datum output; + RETURN_NOT_OK(kernel->Call(ctx, *(array.chunk(i)->data()), &output)); + outputs->push_back(output); + } + } else { + return Status::Invalid("Input Datum was not array-like"); + } + return Status::OK(); +} + +Datum WrapArraysLike(const Datum& value, + const std::vector>& arrays) { + // Create right kind of datum + if (value.kind() == Datum::ARRAY) { + return Datum(arrays[0]->data()); + } else if (value.kind() == Datum::CHUNKED_ARRAY) { + return Datum(std::make_shared(arrays)); + } else { + DCHECK(false) << "unhandled datum kind"; + return Datum(); + } +} + +Datum WrapDatumsLike(const Datum& value, const std::vector& datums) { + // Create right kind of datum + if (value.kind() == Datum::ARRAY) { + DCHECK_EQ(1, datums.size()); + return Datum(datums[0].array()); + } else if (value.kind() == Datum::CHUNKED_ARRAY) { + std::vector> arrays; + for (const Datum& datum : datums) { + DCHECK_EQ(Datum::ARRAY, datum.kind()); + arrays.emplace_back(MakeArray(datum.array())); + } + return Datum(std::make_shared(arrays)); + } else { + DCHECK(false) << "unhandled datum kind"; + return Datum(); + } +} + +} // namespace detail +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/util-internal.h b/cpp/src/arrow/compute/kernels/util-internal.h new file mode 100644 index 0000000000000..70c506286ede6 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/util-internal.h @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H +#define ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H + +#include + +#include "arrow/compute/kernel.h" +#include "arrow/type_fwd.h" + +namespace arrow { +namespace compute { + +class FunctionContext; + +template +using is_number = std::is_base_of; + +template +using enable_if_primitive_ctype = + typename std::enable_if::value>::type; + +template +using enable_if_date = typename std::enable_if::value>::type; + +template +using enable_if_time = typename std::enable_if::value>::type; + +template +using enable_if_timestamp = + typename std::enable_if::value>::type; + +template +using enable_if_has_c_type = + typename std::enable_if::value || + std::is_base_of::value || + std::is_base_of::value || + std::is_base_of::value>::type; + +template +using enable_if_null = typename std::enable_if::value>::type; + +template +using enable_if_binary = + typename std::enable_if::value>::type; + +template +using enable_if_fixed_size_binary = + typename std::enable_if::value>::type; + +template +using enable_if_list = typename std::enable_if::value>::type; + +template +using enable_if_number = typename std::enable_if::value>::type; + +template +inline const T* GetValues(const ArrayData& data, int i) { + return reinterpret_cast(data.buffers[i]->data()) + data.offset; +} + +template +inline T* GetMutableValues(const ArrayData* data, int i) { + return reinterpret_cast(data->buffers[i]->mutable_data()) + data->offset; +} + +static inline void CopyData(const ArrayData& input, ArrayData* output) { + output->length = input.length; + output->null_count = input.null_count; + output->buffers = input.buffers; + output->offset = input.offset; + output->child_data = input.child_data; +} + +namespace detail { + +Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel, + const Datum& value, std::vector* outputs); + +Datum WrapArraysLike(const Datum& value, + const std::vector>& arrays); + +Datum WrapDatumsLike(const Datum& value, const std::vector& datums); + +} // namespace detail + +} // namespace compute +} // namespace arrow + +#endif // ARROW_COMPUTE_KERNELS_UTIL_INTERNAL_H diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc index 8b9a24fecd332..bf29d6a03cd93 100644 --- a/cpp/src/arrow/pretty_print-test.cc +++ b/cpp/src/arrow/pretty_print-test.cc @@ -107,14 +107,13 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) { CheckArray(*array, 0, ex); } -TEST_F(TestPrettyPrint, DecimalType) { +TEST_F(TestPrettyPrint, Decimal128Type) { int32_t p = 19; int32_t s = 4; auto type = decimal(p, s); - DecimalBuilder builder(type); - + Decimal128Builder builder(type); Decimal128 val; ASSERT_OK(Decimal128::FromString("123.4567", &val)); diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 2b0f964391cb1..8814fc190ab3d 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -42,6 +42,8 @@ #include "arrow/util/parallel.h" #include "arrow/visitor_inline.h" +#include "arrow/compute/api.h" + #include "arrow/python/builtin_convert.h" #include "arrow/python/common.h" #include "arrow/python/config.h" @@ -57,6 +59,8 @@ namespace py { using internal::kPandasTimestampNull; using internal::kNanosecondsInDay; +using compute::Datum; + // ---------------------------------------------------------------------- // Utility code @@ -1028,8 +1032,13 @@ class CategoricalBlock : public PandasBlock { std::shared_ptr converted_col; if (options_.strings_to_categorical && (col->type()->id() == Type::STRING || col->type()->id() == Type::BINARY)) { - RETURN_NOT_OK(EncodeColumnToDictionary(static_cast(*col), pool_, - &converted_col)); + compute::FunctionContext ctx(pool_); + + Datum out; + RETURN_NOT_OK(compute::DictionaryEncode(&ctx, Datum(col->data()), &out)); + DCHECK_EQ(out.kind(), Datum::CHUNKED_ARRAY); + converted_col = + std::make_shared(field(col->name(), out.type()), out.chunked_array()); } else { converted_col = col; } @@ -1646,7 +1655,7 @@ class ArrowDeserializer { CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT) CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE) CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING) - CONVERTVALUES_LISTSLIKE_CASE(DecimalType, DECIMAL) + CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL) CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST) default: { std::stringstream ss; diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index a6c28af9b7443..0d2df93151913 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -42,8 +42,8 @@ #include "arrow/util/macros.h" #include "arrow/visitor_inline.h" -#include "arrow/compute/cast.h" #include "arrow/compute/context.h" +#include "arrow/compute/kernels/cast.h" #include "arrow/python/builtin_convert.h" #include "arrow/python/common.h" @@ -466,13 +466,14 @@ Status NumPyConverter::Convert() { namespace { -Status CastBuffer(const std::shared_ptr& input, const int64_t length, - const std::shared_ptr& in_type, +Status CastBuffer(const std::shared_ptr& in_type, + const std::shared_ptr& input, const int64_t length, + const std::shared_ptr& valid_bitmap, const int64_t null_count, const std::shared_ptr& out_type, MemoryPool* pool, std::shared_ptr* out) { // Must cast - std::vector> buffers = {nullptr, input}; - auto tmp_data = std::make_shared(in_type, length, buffers, 0); + std::vector> buffers = {valid_bitmap, input}; + auto tmp_data = std::make_shared(in_type, length, buffers, null_count); std::shared_ptr tmp_array = MakeArray(tmp_data); std::shared_ptr casted_array; @@ -488,6 +489,21 @@ Status CastBuffer(const std::shared_ptr& input, const int64_t length, return Status::OK(); } +template +Status StaticCastBuffer(const Buffer& input, const int64_t length, MemoryPool* pool, + std::shared_ptr* out) { + auto result = std::make_shared(pool); + RETURN_NOT_OK(result->Resize(sizeof(ToType) * length)); + + auto in_values = reinterpret_cast(input.data()); + auto out_values = reinterpret_cast(result->mutable_data()); + for (int64_t i = 0; i < length; ++i) { + *out_values++ = static_cast(*in_values++); + } + *out = result; + return Status::OK(); +} + template void CopyStrided(T* input_data, int64_t length, int64_t stride, T2* output_data) { // Passing input_data as non-const is a concession to PyObject* @@ -531,7 +547,7 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); if (!input_type->Equals(*type_)) { - RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data)); + RETURN_NOT_OK(CastBuffer(input_type, *data, length_, nullptr, 0, type_, pool_, data)); } return Status::OK(); @@ -567,27 +583,32 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d *data = std::make_shared(reinterpret_cast(arr_)); } - // If we have inbound datetime64[D] data, this needs to be downcasted - // separately here from int64_t to int32_t, because this data is not - // supported in compute::Cast + std::shared_ptr input_type; + auto date_dtype = reinterpret_cast(dtype_->c_metadata); - if (dtype_->type_num == NPY_DATETIME && date_dtype->meta.base == NPY_FR_D) { - auto date32_buffer = std::make_shared(pool_); - RETURN_NOT_OK(date32_buffer->Resize(sizeof(int32_t) * length_)); + if (dtype_->type_num == NPY_DATETIME) { + const int64_t null_count = ValuesToBitmap(arr_, null_bitmap_data_); - auto datetime64_values = reinterpret_cast((*data)->data()); - auto date32_values = reinterpret_cast(date32_buffer->mutable_data()); - for (int64_t i = 0; i < length_; ++i) { + // If we have inbound datetime64[D] data, this needs to be downcasted + // separately here from int64_t to int32_t, because this data is not + // supported in compute::Cast + if (date_dtype->meta.base == NPY_FR_D) { // TODO(wesm): How pedantic do we really want to be about checking for int32 // overflow here? - *date32_values++ = static_cast(*datetime64_values++); + Status s = StaticCastBuffer(**data, length_, pool_, data); + RETURN_NOT_OK(s); + } else { + RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); + if (!input_type->Equals(*type_)) { + RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count, + type_, pool_, data)); + } } - *data = date32_buffer; } else { - std::shared_ptr input_type; RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); if (!input_type->Equals(*type_)) { - RETURN_NOT_OK(CastBuffer(*data, length_, input_type, type_, pool_, data)); + RETURN_NOT_OK( + CastBuffer(input_type, *data, length_, nullptr, 0, type_, pool_, data)); } } diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h index d3145ff107ae0..2cff32f74ef14 100644 --- a/cpp/src/arrow/table.h +++ b/cpp/src/arrow/table.h @@ -63,6 +63,9 @@ class ARROW_EXPORT ChunkedArray { ArrayVector chunks_; int64_t length_; int64_t null_count_; + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(ChunkedArray); }; /// \brief An immutable column data structure consisting of a field (type diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h index 9b875ce116a29..77f489ab17797 100644 --- a/cpp/src/arrow/test-util.h +++ b/cpp/src/arrow/test-util.h @@ -365,6 +365,10 @@ Status MakeArray(const std::vector& valid_bytes, const std::vector& } \ } while (false) +#define DECL_T() typedef typename TestFixture::T T; + +#define DECL_TYPE() typedef typename TestFixture::Type Type; + void AssertArraysEqual(const Array& expected, const Array& actual) { ASSERT_ARRAYS_EQUAL(expected, actual); } diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc index 3242fadd50dd6..48982cad42435 100644 --- a/cpp/src/arrow/type-test.cc +++ b/cpp/src/arrow/type-test.cc @@ -400,7 +400,7 @@ TEST(TestStructType, Basics) { } TEST(TypesTest, TestDecimal128Small) { - DecimalType t1(8, 4); + Decimal128Type t1(8, 4); ASSERT_EQ(t1.id(), Type::DECIMAL); ASSERT_EQ(t1.precision(), 8); @@ -414,7 +414,7 @@ TEST(TypesTest, TestDecimal128Small) { } TEST(TypesTest, TestDecimal128Medium) { - DecimalType t1(12, 5); + Decimal128Type t1(12, 5); ASSERT_EQ(t1.id(), Type::DECIMAL); ASSERT_EQ(t1.precision(), 12); @@ -428,7 +428,7 @@ TEST(TypesTest, TestDecimal128Medium) { } TEST(TypesTest, TestDecimal128Large) { - DecimalType t1(27, 7); + Decimal128Type t1(27, 7); ASSERT_EQ(t1.id(), Type::DECIMAL); ASSERT_EQ(t1.precision(), 27); diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index d86e7ef57bec6..70f275c0fa495 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -498,25 +498,34 @@ class ARROW_EXPORT StructType : public NestedType { std::vector GetBufferLayout() const override; }; -class ARROW_EXPORT Decimal128Type : public FixedSizeBinaryType { +class ARROW_EXPORT DecimalBaseType : public FixedSizeBinaryType { + public: + explicit DecimalBaseType(int32_t byte_width, int32_t precision, int32_t scale) + : FixedSizeBinaryType(byte_width, Type::DECIMAL), + precision_(precision), + scale_(scale) {} + + int32_t precision() const { return precision_; } + int32_t scale() const { return scale_; } + + protected: + int32_t precision_; + int32_t scale_; +}; + +class ARROW_EXPORT Decimal128Type : public DecimalBaseType { public: static constexpr Type::type type_id = Type::DECIMAL; explicit Decimal128Type(int32_t precision, int32_t scale) - : FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {} + : DecimalBaseType(16, precision, scale) {} Status Accept(TypeVisitor* visitor) const override; std::string ToString() const override; std::string name() const override { return "decimal"; } - - int32_t precision() const { return precision_; } - int32_t scale() const { return scale_; } - - private: - int32_t precision_; - int32_t scale_; }; +// TODO(wesm): Remove this using DecimalType = Decimal128Type; struct UnionMode { diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h index b8b3c5aa5adb7..9d8a23ce0714f 100644 --- a/cpp/src/arrow/type_fwd.h +++ b/cpp/src/arrow/type_fwd.h @@ -28,10 +28,16 @@ class Status; class DataType; class Array; +struct ArrayData; class ArrayBuilder; class Field; class Tensor; +class ChunkedArray; +class Column; +class RecordBatch; +class Table; + class Buffer; class MemoryPool; class RecordBatch; diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index 6707f3759581d..4bfce9b5f0c53 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -430,6 +430,10 @@ static inline bool is_binary_like(Type::type type_id) { return false; } +static inline bool is_dictionary(Type::type type_id) { + return type_id == Type::DICTIONARY; +} + } // namespace arrow #endif // ARROW_TYPE_TRAITS_H diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 7810a3be46da5..29b18a9357ad0 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -42,6 +42,7 @@ install(FILES sse-util.h stl.h type_traits.h + variant.h visibility.h DESTINATION include/arrow/util) @@ -72,3 +73,5 @@ ADD_ARROW_TEST(decimal-test) ADD_ARROW_TEST(key-value-metadata-test) ADD_ARROW_TEST(rle-encoding-test) ADD_ARROW_TEST(stl-util-test) + +add_subdirectory(variant) diff --git a/cpp/src/arrow/util/variant.h b/cpp/src/arrow/util/variant.h new file mode 100644 index 0000000000000..923a8685f0cdb --- /dev/null +++ b/cpp/src/arrow/util/variant.h @@ -0,0 +1,1127 @@ +// Copyright (c) MapBox +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// - Neither the name "MapBox" nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef ARROW_UTIL_VARIANT_H +#define ARROW_UTIL_VARIANT_H + +#include +#include // size_t +#include // operator new +#include // runtime_error +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// clang-format off +// [[deprecated]] is only available in C++14, use this for the time being +#if __cplusplus <= 201103L +# ifdef __GNUC__ +# define ARROW_VARIANT_DEPRECATED __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ARROW_VARIANT_DEPRECATED __declspec(deprecated) +# else +# define ARROW_VARIANT_DEPRECATED +# endif +#else +# define ARROW_VARIANT_DEPRECATED [[deprecated]] +#endif + + +#ifdef _MSC_VER +// https://msdn.microsoft.com/en-us/library/bw1hbe6y.aspx +# ifdef NDEBUG +# define VARIANT_INLINE __forceinline +# else +# define VARIANT_INLINE //__declspec(noinline) +# endif +#else +# ifdef NDEBUG +# define VARIANT_INLINE //inline __attribute__((always_inline)) +# else +# define VARIANT_INLINE __attribute__((noinline)) +# endif +#endif +// clang-format on + +// Exceptions +#if defined( __EXCEPTIONS) || defined( _MSC_VER) +#define HAS_EXCEPTIONS +#endif + +#define VARIANT_MAJOR_VERSION 1 +#define VARIANT_MINOR_VERSION 1 +#define VARIANT_PATCH_VERSION 0 + +#define VARIANT_VERSION (VARIANT_MAJOR_VERSION * 100000) + (VARIANT_MINOR_VERSION * 100) + (VARIANT_PATCH_VERSION) + +namespace arrow { +namespace util { + +// XXX This should derive from std::logic_error instead of std::runtime_error. +// See https://github.com/mapbox/variant/issues/48 for details. +class bad_variant_access : public std::runtime_error +{ + +public: + explicit bad_variant_access(const std::string& what_arg) + : runtime_error(what_arg) {} + + explicit bad_variant_access(const char* what_arg) + : runtime_error(what_arg) {} + +}; // class bad_variant_access + +template +struct ARROW_VARIANT_DEPRECATED static_visitor +{ + using result_type = R; + +protected: + static_visitor() {} + ~static_visitor() {} +}; + +#if !defined(ARROW_VARIANT_MINIMIZE_SIZE) +using type_index_t = unsigned int; +#else +#if defined(ARROW_VARIANT_OPTIMIZE_FOR_SPEED) +using type_index_t = std::uint_fast8_t; +#else +using type_index_t = std::uint_least8_t; +#endif +#endif + +namespace detail { + +static constexpr type_index_t invalid_value = type_index_t(-1); + +template +struct direct_type; + +template +struct direct_type +{ + static constexpr type_index_t index = std::is_same::value + ? sizeof...(Types) + : direct_type::index; +}; + +template +struct direct_type +{ + static constexpr type_index_t index = invalid_value; +}; + +#if __cpp_lib_logical_traits >= 201510L + +using std::conjunction; +using std::disjunction; + +#else + +template +struct conjunction : std::true_type {}; + +template +struct conjunction : B1 {}; + +template +struct conjunction : std::conditional::type {}; + +template +struct conjunction : std::conditional, B1>::type {}; + +template +struct disjunction : std::false_type {}; + +template +struct disjunction : B1 {}; + +template +struct disjunction : std::conditional::type {}; + +template +struct disjunction : std::conditional>::type {}; + +#endif + +template +struct convertible_type; + +template +struct convertible_type +{ + static constexpr type_index_t index = std::is_convertible::value + ? disjunction...>::value ? invalid_value : sizeof...(Types) + : convertible_type::index; +}; + +template +struct convertible_type +{ + static constexpr type_index_t index = invalid_value; +}; + +template +struct value_traits +{ + using value_type = typename std::remove_const::type>::type; + using value_type_wrapper = recursive_wrapper; + static constexpr type_index_t direct_index = direct_type::index; + static constexpr bool is_direct = direct_index != invalid_value; + static constexpr type_index_t index_direct_or_wrapper = is_direct ? direct_index : direct_type::index; + static constexpr bool is_direct_or_wrapper = index_direct_or_wrapper != invalid_value; + static constexpr type_index_t index = is_direct_or_wrapper ? index_direct_or_wrapper : convertible_type::index; + static constexpr bool is_valid = index != invalid_value; + static constexpr type_index_t tindex = is_valid ? sizeof...(Types)-index : 0; + using target_type = typename std::tuple_element>::type; +}; + +template +struct enable_if_type +{ + using type = R; +}; + +template +struct result_of_unary_visit +{ + using type = typename std::result_of::type; +}; + +template +struct result_of_unary_visit::type> +{ + using type = typename F::result_type; +}; + +template +struct result_of_binary_visit +{ + using type = typename std::result_of::type; +}; + +template +struct result_of_binary_visit::type> +{ + using type = typename F::result_type; +}; + +template +struct static_max; + +template +struct static_max +{ + static const type_index_t value = arg; +}; + +template +struct static_max +{ + static const type_index_t value = arg1 >= arg2 ? static_max::value : static_max::value; +}; + +template +struct variant_helper; + +template +struct variant_helper +{ + VARIANT_INLINE static void destroy(const type_index_t type_index, void* data) + { + if (type_index == sizeof...(Types)) + { + reinterpret_cast(data)->~T(); + } + else + { + variant_helper::destroy(type_index, data); + } + } + + VARIANT_INLINE static void move(const type_index_t old_type_index, void* old_value, void* new_value) + { + if (old_type_index == sizeof...(Types)) + { + new (new_value) T(std::move(*reinterpret_cast(old_value))); + } + else + { + variant_helper::move(old_type_index, old_value, new_value); + } + } + + VARIANT_INLINE static void copy(const type_index_t old_type_index, const void* old_value, void* new_value) + { + if (old_type_index == sizeof...(Types)) + { + new (new_value) T(*reinterpret_cast(old_value)); + } + else + { + variant_helper::copy(old_type_index, old_value, new_value); + } + } +}; + +template <> +struct variant_helper<> +{ + VARIANT_INLINE static void destroy(const type_index_t, void*) {} + VARIANT_INLINE static void move(const type_index_t, void*, void*) {} + VARIANT_INLINE static void copy(const type_index_t, const void*, void*) {} +}; + +template +struct unwrapper +{ + static T const& apply_const(T const& obj) { return obj; } + static T& apply(T& obj) { return obj; } +}; + +template +struct unwrapper> +{ + static auto apply_const(recursive_wrapper const& obj) + -> typename recursive_wrapper::type const& + { + return obj.get(); + } + static auto apply(recursive_wrapper& obj) + -> typename recursive_wrapper::type& + { + return obj.get(); + } +}; + +template +struct unwrapper> +{ + static auto apply_const(std::reference_wrapper const& obj) + -> typename std::reference_wrapper::type const& + { + return obj.get(); + } + static auto apply(std::reference_wrapper& obj) + -> typename std::reference_wrapper::type& + { + return obj.get(); + } +}; + +template +struct dispatcher; + +template +struct dispatcher +{ + VARIANT_INLINE static R apply_const(V const& v, F&& f) + { + if (v.template is()) + { + return f(unwrapper::apply_const(v.template get_unchecked())); + } + else + { + return dispatcher::apply_const(v, std::forward(f)); + } + } + + VARIANT_INLINE static R apply(V& v, F&& f) + { + if (v.template is()) + { + return f(unwrapper::apply(v.template get_unchecked())); + } + else + { + return dispatcher::apply(v, std::forward(f)); + } + } +}; + +template +struct dispatcher +{ + VARIANT_INLINE static R apply_const(V const& v, F&& f) + { + return f(unwrapper::apply_const(v.template get_unchecked())); + } + + VARIANT_INLINE static R apply(V& v, F&& f) + { + return f(unwrapper::apply(v.template get_unchecked())); + } +}; + +template +struct binary_dispatcher_rhs; + +template +struct binary_dispatcher_rhs +{ + VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f) + { + if (rhs.template is()) // call binary functor + { + return f(unwrapper::apply_const(lhs.template get_unchecked()), + unwrapper::apply_const(rhs.template get_unchecked())); + } + else + { + return binary_dispatcher_rhs::apply_const(lhs, rhs, std::forward(f)); + } + } + + VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f) + { + if (rhs.template is()) // call binary functor + { + return f(unwrapper::apply(lhs.template get_unchecked()), + unwrapper::apply(rhs.template get_unchecked())); + } + else + { + return binary_dispatcher_rhs::apply(lhs, rhs, std::forward(f)); + } + } +}; + +template +struct binary_dispatcher_rhs +{ + VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f) + { + return f(unwrapper::apply_const(lhs.template get_unchecked()), + unwrapper::apply_const(rhs.template get_unchecked())); + } + + VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f) + { + return f(unwrapper::apply(lhs.template get_unchecked()), + unwrapper::apply(rhs.template get_unchecked())); + } +}; + +template +struct binary_dispatcher_lhs; + +template +struct binary_dispatcher_lhs +{ + VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f) + { + if (lhs.template is()) // call binary functor + { + return f(unwrapper::apply_const(lhs.template get_unchecked()), + unwrapper::apply_const(rhs.template get_unchecked())); + } + else + { + return binary_dispatcher_lhs::apply_const(lhs, rhs, std::forward(f)); + } + } + + VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f) + { + if (lhs.template is()) // call binary functor + { + return f(unwrapper::apply(lhs.template get_unchecked()), + unwrapper::apply(rhs.template get_unchecked())); + } + else + { + return binary_dispatcher_lhs::apply(lhs, rhs, std::forward(f)); + } + } +}; + +template +struct binary_dispatcher_lhs +{ + VARIANT_INLINE static R apply_const(V const& lhs, V const& rhs, F&& f) + { + return f(unwrapper::apply_const(lhs.template get_unchecked()), + unwrapper::apply_const(rhs.template get_unchecked())); + } + + VARIANT_INLINE static R apply(V& lhs, V& rhs, F&& f) + { + return f(unwrapper::apply(lhs.template get_unchecked()), + unwrapper::apply(rhs.template get_unchecked())); + } +}; + +template +struct binary_dispatcher; + +template +struct binary_dispatcher +{ + VARIANT_INLINE static R apply_const(V const& v0, V const& v1, F&& f) + { + if (v0.template is()) + { + if (v1.template is()) + { + return f(unwrapper::apply_const(v0.template get_unchecked()), + unwrapper::apply_const(v1.template get_unchecked())); // call binary functor + } + else + { + return binary_dispatcher_rhs::apply_const(v0, v1, std::forward(f)); + } + } + else if (v1.template is()) + { + return binary_dispatcher_lhs::apply_const(v0, v1, std::forward(f)); + } + return binary_dispatcher::apply_const(v0, v1, std::forward(f)); + } + + VARIANT_INLINE static R apply(V& v0, V& v1, F&& f) + { + if (v0.template is()) + { + if (v1.template is()) + { + return f(unwrapper::apply(v0.template get_unchecked()), + unwrapper::apply(v1.template get_unchecked())); // call binary functor + } + else + { + return binary_dispatcher_rhs::apply(v0, v1, std::forward(f)); + } + } + else if (v1.template is()) + { + return binary_dispatcher_lhs::apply(v0, v1, std::forward(f)); + } + return binary_dispatcher::apply(v0, v1, std::forward(f)); + } +}; + +template +struct binary_dispatcher +{ + VARIANT_INLINE static R apply_const(V const& v0, V const& v1, F&& f) + { + return f(unwrapper::apply_const(v0.template get_unchecked()), + unwrapper::apply_const(v1.template get_unchecked())); // call binary functor + } + + VARIANT_INLINE static R apply(V& v0, V& v1, F&& f) + { + return f(unwrapper::apply(v0.template get_unchecked()), + unwrapper::apply(v1.template get_unchecked())); // call binary functor + } +}; + +// comparator functors +struct equal_comp +{ + template + bool operator()(T const& lhs, T const& rhs) const + { + return lhs == rhs; + } +}; + +struct less_comp +{ + template + bool operator()(T const& lhs, T const& rhs) const + { + return lhs < rhs; + } +}; + +template +class comparer +{ +public: + explicit comparer(Variant const& lhs) noexcept + : lhs_(lhs) {} + comparer& operator=(comparer const&) = delete; + // visitor + template + bool operator()(T const& rhs_content) const + { + T const& lhs_content = lhs_.template get_unchecked(); + return Comp()(lhs_content, rhs_content); + } + +private: + Variant const& lhs_; +}; + +// hashing visitor +struct hasher +{ + template + std::size_t operator()(const T& hashable) const + { + return std::hash{}(hashable); + } +}; + +} // namespace detail + +struct no_init {}; + +template +class variant +{ + static_assert(sizeof...(Types) > 0, "Template parameter type list of variant can not be empty."); + static_assert(!detail::disjunction...>::value, "Variant can not hold reference types. Maybe use std::reference_wrapper?"); + static_assert(!detail::disjunction...>::value, "Variant can not hold array types."); + static_assert(sizeof...(Types) < std::numeric_limits::max(), "Internal index type must be able to accommodate all alternatives."); +private: + static const std::size_t data_size = detail::static_max::value; + static const std::size_t data_align = detail::static_max::value; +public: + struct adapted_variant_tag; + using types = std::tuple; +private: + using first_type = typename std::tuple_element<0, types>::type; + using data_type = typename std::aligned_storage::type; + using helper_type = detail::variant_helper; + + type_index_t type_index; + data_type data; + +public: + VARIANT_INLINE variant() noexcept(std::is_nothrow_default_constructible::value) + : type_index(sizeof...(Types)-1) + { + static_assert(std::is_default_constructible::value, "First type in variant must be default constructible to allow default construction of variant."); + new (&data) first_type(); + } + + VARIANT_INLINE variant(no_init) noexcept + : type_index(detail::invalid_value) {} + + // http://isocpp.org/blog/2012/11/universal-references-in-c11-scott-meyers + template , + typename Enable = typename std::enable_if, typename Traits::value_type>::value>::type > + VARIANT_INLINE variant(T&& val) noexcept(std::is_nothrow_constructible::value) + : type_index(Traits::index) + { + new (&data) typename Traits::target_type(std::forward(val)); + } + + VARIANT_INLINE variant(variant const& old) + : type_index(old.type_index) + { + helper_type::copy(old.type_index, &old.data, &data); + } + + VARIANT_INLINE variant(variant&& old) + noexcept(detail::conjunction...>::value) + : type_index(old.type_index) + { + helper_type::move(old.type_index, &old.data, &data); + } + +private: + VARIANT_INLINE void copy_assign(variant const& rhs) + { + helper_type::destroy(type_index, &data); + type_index = detail::invalid_value; + helper_type::copy(rhs.type_index, &rhs.data, &data); + type_index = rhs.type_index; + } + + VARIANT_INLINE void move_assign(variant&& rhs) + { + helper_type::destroy(type_index, &data); + type_index = detail::invalid_value; + helper_type::move(rhs.type_index, &rhs.data, &data); + type_index = rhs.type_index; + } + +public: + VARIANT_INLINE variant& operator=(variant&& other) + { + move_assign(std::move(other)); + return *this; + } + + VARIANT_INLINE variant& operator=(variant const& other) + { + copy_assign(other); + return *this; + } + + // conversions + // move-assign + template + VARIANT_INLINE variant& operator=(T&& rhs) noexcept + { + variant temp(std::forward(rhs)); + move_assign(std::move(temp)); + return *this; + } + + // copy-assign + template + VARIANT_INLINE variant& operator=(T const& rhs) + { + variant temp(rhs); + copy_assign(temp); + return *this; + } + + template ::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE bool is() const + { + return type_index == detail::direct_type::index; + } + + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE bool is() const + { + return type_index == detail::direct_type, Types...>::index; + } + + VARIANT_INLINE bool valid() const + { + return type_index != detail::invalid_value; + } + + template + VARIANT_INLINE void set(Args&&... args) + { + helper_type::destroy(type_index, &data); + type_index = detail::invalid_value; + new (&data) T(std::forward(args)...); + type_index = detail::direct_type::index; + } + + // get_unchecked() + template ::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T& get_unchecked() + { + return *reinterpret_cast(&data); + } + +#ifdef HAS_EXCEPTIONS + // get() + template ::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T& get() + { + if (type_index == detail::direct_type::index) + { + return *reinterpret_cast(&data); + } + else + { + throw bad_variant_access("in get()"); + } + } +#endif + + template ::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T const& get_unchecked() const + { + return *reinterpret_cast(&data); + } + +#ifdef HAS_EXCEPTIONS + template ::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T const& get() const + { + if (type_index == detail::direct_type::index) + { + return *reinterpret_cast(&data); + } + else + { + throw bad_variant_access("in get()"); + } + } +#endif + + // get_unchecked() - T stored as recursive_wrapper + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T& get_unchecked() + { + return (*reinterpret_cast*>(&data)).get(); + } + +#ifdef HAS_EXCEPTIONS + // get() - T stored as recursive_wrapper + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T& get() + { + if (type_index == detail::direct_type, Types...>::index) + { + return (*reinterpret_cast*>(&data)).get(); + } + else + { + throw bad_variant_access("in get()"); + } + } +#endif + + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T const& get_unchecked() const + { + return (*reinterpret_cast const*>(&data)).get(); + } + +#ifdef HAS_EXCEPTIONS + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T const& get() const + { + if (type_index == detail::direct_type, Types...>::index) + { + return (*reinterpret_cast const*>(&data)).get(); + } + else + { + throw bad_variant_access("in get()"); + } + } +#endif + + // get_unchecked() - T stored as std::reference_wrapper + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T& get_unchecked() + { + return (*reinterpret_cast*>(&data)).get(); + } + +#ifdef HAS_EXCEPTIONS + // get() - T stored as std::reference_wrapper + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T& get() + { + if (type_index == detail::direct_type, Types...>::index) + { + return (*reinterpret_cast*>(&data)).get(); + } + else + { + throw bad_variant_access("in get()"); + } + } +#endif + + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T const& get_unchecked() const + { + return (*reinterpret_cast const*>(&data)).get(); + } + +#ifdef HAS_EXCEPTIONS + template , Types...>::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE T const& get() const + { + if (type_index == detail::direct_type, Types...>::index) + { + return (*reinterpret_cast const*>(&data)).get(); + } + else + { + throw bad_variant_access("in get()"); + } + } +#endif + + // This function is deprecated because it returns an internal index field. + // Use which() instead. + ARROW_VARIANT_DEPRECATED VARIANT_INLINE type_index_t get_type_index() const + { + return type_index; + } + + VARIANT_INLINE int which() const noexcept + { + return static_cast(sizeof...(Types) - type_index - 1); + } + + template ::index != detail::invalid_value)>::type* = nullptr> + VARIANT_INLINE static constexpr int which() noexcept + { + return static_cast(sizeof...(Types)-detail::direct_type::index - 1); + } + + // visitor + // unary + template ::type> + auto VARIANT_INLINE static visit(V const& v, F&& f) + -> decltype(detail::dispatcher::apply_const(v, std::forward(f))) + { + return detail::dispatcher::apply_const(v, std::forward(f)); + } + // non-const + template ::type> + auto VARIANT_INLINE static visit(V& v, F&& f) + -> decltype(detail::dispatcher::apply(v, std::forward(f))) + { + return detail::dispatcher::apply(v, std::forward(f)); + } + + // binary + // const + template ::type> + auto VARIANT_INLINE static binary_visit(V const& v0, V const& v1, F&& f) + -> decltype(detail::binary_dispatcher::apply_const(v0, v1, std::forward(f))) + { + return detail::binary_dispatcher::apply_const(v0, v1, std::forward(f)); + } + // non-const + template ::type> + auto VARIANT_INLINE static binary_visit(V& v0, V& v1, F&& f) + -> decltype(detail::binary_dispatcher::apply(v0, v1, std::forward(f))) + { + return detail::binary_dispatcher::apply(v0, v1, std::forward(f)); + } + + // match + // unary + template + auto VARIANT_INLINE match(Fs&&... fs) const + -> decltype(variant::visit(*this, ::arrow::util::make_visitor(std::forward(fs)...))) + { + return variant::visit(*this, ::arrow::util::make_visitor(std::forward(fs)...)); + } + // non-const + template + auto VARIANT_INLINE match(Fs&&... fs) + -> decltype(variant::visit(*this, ::arrow::util::make_visitor(std::forward(fs)...))) + { + return variant::visit(*this, ::arrow::util::make_visitor(std::forward(fs)...)); + } + + ~variant() noexcept // no-throw destructor + { + helper_type::destroy(type_index, &data); + } + + // comparison operators + // equality + VARIANT_INLINE bool operator==(variant const& rhs) const + { + assert(valid() && rhs.valid()); + if (this->which() != rhs.which()) + { + return false; + } + detail::comparer visitor(*this); + return visit(rhs, visitor); + } + + VARIANT_INLINE bool operator!=(variant const& rhs) const + { + return !(*this == rhs); + } + + // less than + VARIANT_INLINE bool operator<(variant const& rhs) const + { + assert(valid() && rhs.valid()); + if (this->which() != rhs.which()) + { + return this->which() < rhs.which(); + } + detail::comparer visitor(*this); + return visit(rhs, visitor); + } + VARIANT_INLINE bool operator>(variant const& rhs) const + { + return rhs < *this; + } + VARIANT_INLINE bool operator<=(variant const& rhs) const + { + return !(*this > rhs); + } + VARIANT_INLINE bool operator>=(variant const& rhs) const + { + return !(*this < rhs); + } +}; + +// unary visitor interface +// const +template +auto VARIANT_INLINE apply_visitor(F&& f, V const& v) -> decltype(V::visit(v, std::forward(f))) +{ + return V::visit(v, std::forward(f)); +} + +// non-const +template +auto VARIANT_INLINE apply_visitor(F&& f, V& v) -> decltype(V::visit(v, std::forward(f))) +{ + return V::visit(v, std::forward(f)); +} + +// binary visitor interface +// const +template +auto VARIANT_INLINE apply_visitor(F&& f, V const& v0, V const& v1) -> decltype(V::binary_visit(v0, v1, std::forward(f))) +{ + return V::binary_visit(v0, v1, std::forward(f)); +} + +// non-const +template +auto VARIANT_INLINE apply_visitor(F&& f, V& v0, V& v1) -> decltype(V::binary_visit(v0, v1, std::forward(f))) +{ + return V::binary_visit(v0, v1, std::forward(f)); +} + +// getter interface + +#ifdef HAS_EXCEPTIONS +template +auto get(T& var)->decltype(var.template get()) +{ + return var.template get(); +} +#endif + +template +ResultType& get_unchecked(T& var) +{ + return var.template get_unchecked(); +} + +#ifdef HAS_EXCEPTIONS +template +auto get(T const& var)->decltype(var.template get()) +{ + return var.template get(); +} +#endif + +template +ResultType const& get_unchecked(T const& var) +{ + return var.template get_unchecked(); +} +// variant_size +template +struct variant_size; + +//variable templates is c++14 +//template +//constexpr std::size_t variant_size_v = variant_size::value; + +template +struct variant_size + : variant_size {}; + +template +struct variant_size + : variant_size {}; + +template +struct variant_size + : variant_size {}; + +template +struct variant_size> + : std::integral_constant {}; + +// variant_alternative +template +struct variant_alternative; + +#if defined(__clang__) +#if __has_builtin(__type_pack_element) +#define has_type_pack_element +#endif +#endif + +#if defined(has_type_pack_element) +template +struct variant_alternative> +{ + static_assert(sizeof...(Types) > Index , "Index out of range"); + using type = __type_pack_element; +}; +#else +template +struct variant_alternative> + : variant_alternative> +{ + static_assert(sizeof...(Types) > Index -1 , "Index out of range"); +}; + +template +struct variant_alternative<0, variant> +{ + using type = First; +}; + +#endif + +template +using variant_alternative_t = typename variant_alternative::type; + +template +struct variant_alternative + : std::add_const> {}; + +template +struct variant_alternative + : std::add_volatile> {}; + +template +struct variant_alternative + : std::add_cv> {}; + +} // namespace util +} // namespace arrow + +#endif // ARROW_UTIL_VARIANT_H diff --git a/cpp/src/arrow/util/variant/CMakeLists.txt b/cpp/src/arrow/util/variant/CMakeLists.txt new file mode 100644 index 0000000000000..0ebb2516246ed --- /dev/null +++ b/cpp/src/arrow/util/variant/CMakeLists.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +####################################### +# arrow_util_variant +####################################### + +install(FILES + optional.h + recursive_wrapper.h + variant_cast.h + variant_io.h + variant_visitor.h + DESTINATION include/arrow/util/variant) diff --git a/cpp/src/arrow/util/variant/optional.h b/cpp/src/arrow/util/variant/optional.h new file mode 100644 index 0000000000000..4c6671061fe80 --- /dev/null +++ b/cpp/src/arrow/util/variant/optional.h @@ -0,0 +1,100 @@ +// Copyright (c) MapBox +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// - Neither the name "MapBox" nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef ARROW_UTIL_VARIANT_OPTIONAL_H +#define ARROW_UTIL_VARIANT_OPTIONAL_H + +#pragma message("This implementation of optional is deprecated. See https://github.com/mapbox/variant/issues/64.") + +#include +#include + +#include + +namespace arrow { +namespace util { + +template +class optional +{ + static_assert(!std::is_reference::value, "optional doesn't support references"); + + struct none_type + { + }; + + variant variant_; + +public: + optional() = default; + + optional(optional const& rhs) + { + if (this != &rhs) + { // protect against invalid self-assignment + variant_ = rhs.variant_; + } + } + + optional(T const& v) { variant_ = v; } + + explicit operator bool() const noexcept { return variant_.template is(); } + + T const& get() const { return variant_.template get(); } + T& get() { return variant_.template get(); } + + T const& operator*() const { return this->get(); } + T operator*() { return this->get(); } + + optional& operator=(T const& v) + { + variant_ = v; + return *this; + } + + optional& operator=(optional const& rhs) + { + if (this != &rhs) + { + variant_ = rhs.variant_; + } + return *this; + } + + template + void emplace(Args&&... args) + { + variant_ = T{std::forward(args)...}; + } + + void reset() { variant_ = none_type{}; } + +}; // class optional + +} // namespace util +} // namespace arrow + +#endif // ARROW_UTIL_VARIANT_OPTIONAL_H diff --git a/cpp/src/arrow/util/variant/recursive_wrapper.h b/cpp/src/arrow/util/variant/recursive_wrapper.h new file mode 100644 index 0000000000000..c9d9385394b38 --- /dev/null +++ b/cpp/src/arrow/util/variant/recursive_wrapper.h @@ -0,0 +1,122 @@ +#ifndef ARROW_UTIL_VARIANT_RECURSIVE_WRAPPER_H +#define ARROW_UTIL_VARIANT_RECURSIVE_WRAPPER_H + +// Based on variant/recursive_wrapper.h from boost. +// +// Original license: +// +// Copyright (c) 2002-2003 +// Eric Friedman, Itay Maman +// +// Distributed under the Boost Software License, Version 1.0. (See +// accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +namespace arrow { +namespace util { + +template +class recursive_wrapper +{ + + T* p_; + + void assign(T const& rhs) + { + this->get() = rhs; + } + +public: + using type = T; + + /** + * Default constructor default initializes the internally stored value. + * For POD types this means nothing is done and the storage is + * uninitialized. + * + * @throws std::bad_alloc if there is insufficient memory for an object + * of type T. + * @throws any exception thrown by the default constructur of T. + */ + recursive_wrapper() + : p_(new T){} + + ~recursive_wrapper() noexcept { delete p_; } + + recursive_wrapper(recursive_wrapper const& operand) + : p_(new T(operand.get())) {} + + recursive_wrapper(T const& operand) + : p_(new T(operand)) {} + + recursive_wrapper(recursive_wrapper&& operand) + : p_(new T(std::move(operand.get()))) {} + + recursive_wrapper(T&& operand) + : p_(new T(std::move(operand))) {} + + inline recursive_wrapper& operator=(recursive_wrapper const& rhs) + { + assign(rhs.get()); + return *this; + } + + inline recursive_wrapper& operator=(T const& rhs) + { + assign(rhs); + return *this; + } + + inline void swap(recursive_wrapper& operand) noexcept + { + T* temp = operand.p_; + operand.p_ = p_; + p_ = temp; + } + + recursive_wrapper& operator=(recursive_wrapper&& rhs) noexcept + { + swap(rhs); + return *this; + } + + recursive_wrapper& operator=(T&& rhs) + { + get() = std::move(rhs); + return *this; + } + + T& get() + { + assert(p_); + return *get_pointer(); + } + + T const& get() const + { + assert(p_); + return *get_pointer(); + } + + T* get_pointer() { return p_; } + + const T* get_pointer() const { return p_; } + + operator T const&() const { return this->get(); } + + operator T&() { return this->get(); } + +}; // class recursive_wrapper + +template +inline void swap(recursive_wrapper& lhs, recursive_wrapper& rhs) noexcept +{ + lhs.swap(rhs); +} +} // namespace util +} // namespace arrow + +#endif // ARROW_UTIL_VARIANT_RECURSIVE_WRAPPER_H diff --git a/cpp/src/arrow/util/variant/variant_cast.h b/cpp/src/arrow/util/variant/variant_cast.h new file mode 100644 index 0000000000000..558f1d9a60c20 --- /dev/null +++ b/cpp/src/arrow/util/variant/variant_cast.h @@ -0,0 +1,112 @@ +// Copyright (c) MapBox +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// - Neither the name "MapBox" nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef ARROW_UTIL_VARIANT_CAST_H +#define ARROW_UTIL_VARIANT_CAST_H + +#include + +namespace arrow { +namespace util { + +namespace detail { + +template +class static_caster +{ +public: + template + T& operator()(V& v) const + { + return static_cast(v); + } +}; + +template +class dynamic_caster +{ +public: + using result_type = T&; + template + T& operator()(V& v, typename std::enable_if::value>::type* = nullptr) const + { + throw std::bad_cast(); + } + template + T& operator()(V& v, typename std::enable_if::value>::type* = nullptr) const + { + return dynamic_cast(v); + } +}; + +template +class dynamic_caster +{ +public: + using result_type = T*; + template + T* operator()(V& v, typename std::enable_if::value>::type* = nullptr) const + { + return nullptr; + } + template + T* operator()(V& v, typename std::enable_if::value>::type* = nullptr) const + { + return dynamic_cast(&v); + } +}; +} + +template +typename detail::dynamic_caster::result_type +dynamic_variant_cast(V& v) +{ + return arrow::util::apply_visitor(detail::dynamic_caster(), v); +} + +template +typename detail::dynamic_caster::result_type +dynamic_variant_cast(const V& v) +{ + return arrow::util::apply_visitor(detail::dynamic_caster(), v); +} + +template +T& static_variant_cast(V& v) +{ + return arrow::util::apply_visitor(detail::static_caster(), v); +} + +template +const T& static_variant_cast(const V& v) +{ + return arrow::util::apply_visitor(detail::static_caster(), v); +} + +} // namespace util +} // namespace arrow + +#endif // ARROW_UTIL_VARIANT_CAST_H diff --git a/cpp/src/arrow/util/variant/variant_io.h b/cpp/src/arrow/util/variant/variant_io.h new file mode 100644 index 0000000000000..5541a81f7035f --- /dev/null +++ b/cpp/src/arrow/util/variant/variant_io.h @@ -0,0 +1,72 @@ +// Copyright (c) MapBox +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// - Neither the name "MapBox" nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef ARROW_UTIL_VARIANT_IO_H +#define ARROW_UTIL_VARIANT_IO_H + +#include + +#include + +namespace arrow { +namespace util { + +namespace detail { +// operator<< helper +template +class printer +{ +public: + explicit printer(Out& out) + : out_(out) {} + printer& operator=(printer const&) = delete; + + // visitor + template + void operator()(T const& operand) const + { + out_ << operand; + } + +private: + Out& out_; +}; +} + +// operator<< +template +VARIANT_INLINE std::basic_ostream& +operator<<(std::basic_ostream& out, variant const& rhs) +{ + detail::printer> visitor(out); + apply_visitor(visitor, rhs); + return out; +} + +} // namespace util +} // namespace arrow + +#endif // ARROW_UTIL_VARIANT_IO_H diff --git a/cpp/src/arrow/util/variant/variant_visitor.h b/cpp/src/arrow/util/variant/variant_visitor.h new file mode 100644 index 0000000000000..66b1dfea3d7c9 --- /dev/null +++ b/cpp/src/arrow/util/variant/variant_visitor.h @@ -0,0 +1,69 @@ +// Copyright (c) MapBox +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// - Neither the name "MapBox" nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef ARROW_UTIL_VARIANT_VISITOR_HPP +#define ARROW_UTIL_VARIANT_VISITOR_HPP + +#include + +namespace arrow { +namespace util { + +template +struct visitor; + +template +struct visitor : Fn +{ + using Fn::operator(); + + template + visitor(T&& fn) : Fn(std::forward(fn)) {} +}; + +template +struct visitor : Fn, visitor +{ + using Fn::operator(); + using visitor::operator(); + + template + visitor(T&& fn, Ts&&... fns) + : Fn(std::forward(fn)) + , visitor(std::forward(fns)...) {} +}; + +template +visitor::type...> make_visitor(Fns&&... fns) +{ + return visitor::type...> + (std::forward(fns)...); +} + +} // namespace util +} // namespace arrow + +#endif // ARROW_UTIL_VARIANT_VISITOR_HPP diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc index 3739e89f3d019..47dba6cd8ddf2 100644 --- a/cpp/src/arrow/visitor.cc +++ b/cpp/src/arrow/visitor.cc @@ -90,7 +90,7 @@ TYPE_VISITOR_DEFAULT(Time32Type); TYPE_VISITOR_DEFAULT(Time64Type); TYPE_VISITOR_DEFAULT(TimestampType); TYPE_VISITOR_DEFAULT(IntervalType); -TYPE_VISITOR_DEFAULT(DecimalType); +TYPE_VISITOR_DEFAULT(Decimal128Type); TYPE_VISITOR_DEFAULT(ListType); TYPE_VISITOR_DEFAULT(StructType); TYPE_VISITOR_DEFAULT(UnionType); diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h index 72c82a3bb871b..41b0108aeb116 100644 --- a/cpp/src/arrow/visitor_inline.h +++ b/cpp/src/arrow/visitor_inline.h @@ -97,7 +97,7 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) { ARRAY_VISIT_INLINE(TimestampType); ARRAY_VISIT_INLINE(Time32Type); ARRAY_VISIT_INLINE(Time64Type); - ARRAY_VISIT_INLINE(DecimalType); + ARRAY_VISIT_INLINE(Decimal128Type); ARRAY_VISIT_INLINE(ListType); ARRAY_VISIT_INLINE(StructType); ARRAY_VISIT_INLINE(UnionType); diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index db3b3aa5fac9c..bf962bcd49e42 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -10,6 +10,12 @@ cpp/src/arrow/io/mman.h cpp/src/arrow/util/random.h cpp/src/arrow/status.cc cpp/src/arrow/status.h +cpp/src/arrow/util/variant.h +cpp/src/arrow/util/variant/optional.h +cpp/src/arrow/util/variant/recursive_wrapper.h +cpp/src/arrow/util/variant/variant_cast.h +cpp/src/arrow/util/variant/variant_io.h +cpp/src/arrow/util/variant/variant_visitor.h cpp/build-support/asan_symbolize.py cpp/build-support/cpplint.py cpp/build-support/clang_format_exclusions.txt diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst index c52024044a62d..8f2f23d9fd470 100644 --- a/python/doc/source/api.rst +++ b/python/doc/source/api.rst @@ -132,6 +132,7 @@ Array Types .. autosummary:: :toctree: generated/ + array Array BooleanArray DictionaryArray @@ -168,6 +169,8 @@ Tables and Record Batches .. autosummary:: :toctree: generated/ + column + chunked_array ChunkedArray Column RecordBatch diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst index 7ef6a722be1dc..885613725a10a 100644 --- a/python/doc/source/development.rst +++ b/python/doc/source/development.rst @@ -84,7 +84,7 @@ from conda-forge: conda create -y -q -n pyarrow-dev \ python=3.6 numpy six setuptools cython pandas pytest \ cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \ - brotli jemalloc lz4-c zstd -c conda-forge + gflags brotli jemalloc lz4-c zstd -c conda-forge source activate pyarrow-dev @@ -256,17 +256,11 @@ First, starting from fresh clones of Apache Arrow and parquet-cpp: .. code-block:: shell - conda create -n arrow-dev cmake git boost-cpp ^ - flatbuffers snappy zlib brotli thrift-cpp rapidjson ^ - -c conda-forge - activate arrow-dev - -As one git housekeeping item, we must run this command in our Arrow clone: - -.. code-block:: shell - - cd arrow - git config core.symlinks true + conda create -y -q -n pyarrow-dev ^ + python=3.6 numpy six setuptools cython pandas pytest ^ + cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib ^ + gflags brotli lz4-c zstd -c conda-forge + activate pyarrow-dev Now, we build and install Arrow C++ libraries @@ -280,7 +274,7 @@ Now, we build and install Arrow C++ libraries -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DCMAKE_BUILD_TYPE=Release ^ -DARROW_BUILD_TESTS=on ^ - -DARROW_CXXFLAGS="/WX" ^ + -DARROW_CXXFLAGS="/WX /MP" ^ -DARROW_PYTHON=on .. cmake --build . --target INSTALL --config Release cd ..\.. diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 09bf6b35f396a..c8ded2d3cb88d 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -43,7 +43,7 @@ Schema, schema, Array, Tensor, - array, + array, chunked_array, column, from_numpy_dtype, NullArray, NumericArray, IntegerArray, FloatingPointArray, diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 2ef592ff758fc..cca9425881b00 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -77,7 +77,7 @@ cdef _ndarray_to_array(object values, object mask, DataType type, return pyarrow_wrap_array(chunked_out.get().chunk(0)) -cdef DataType _ensure_type(object type): +cdef inline DataType _ensure_type(object type): if type is None: return None elif not isinstance(type, DataType): @@ -228,6 +228,15 @@ cdef CFunctionContext* _context() nogil: return _global_ctx.ctx.get() +cdef wrap_datum(const CDatum& datum): + if datum.kind() == DatumType_ARRAY: + return pyarrow_wrap_array(MakeArray(datum.array())) + elif datum.kind() == DatumType_CHUNKED_ARRAY: + return pyarrow_wrap_chunked_array(datum.chunked_array()) + else: + raise ValueError("Unable to wrap Datum in a Python object") + + cdef class Array: cdef void init(self, const shared_ptr[CArray]& sp_array): @@ -270,6 +279,29 @@ cdef class Array: return pyarrow_wrap_array(result) + def unique(self): + """ + Compute distinct elements in array + """ + cdef shared_ptr[CArray] result + + with nogil: + check_status(Unique(_context(), CDatum(self.sp_array), &result)) + + return pyarrow_wrap_array(result) + + def dictionary_encode(self): + """ + Compute dictionary-encoded representation of array + """ + cdef CDatum out + + with nogil: + check_status(DictionaryEncode(_context(), CDatum(self.sp_array), + &out)) + + return wrap_datum(out) + @staticmethod def from_pandas(obj, mask=None, type=None, MemoryPool memory_pool=None): """ @@ -702,6 +734,9 @@ cdef class DictionaryArray(Array): return box_scalar(dictionary.type, dictionary.sp_array, index.as_py()) + def dictionary_encode(self): + return self + property dictionary: def __get__(self): diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 11cc6b3ff2664..dbfd89cc378cb 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -90,6 +90,14 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_string ToString() + cdef cppclass CArrayData" arrow::ArrayData": + shared_ptr[CDataType] type + int64_t length + int64_t null_count + int64_t offset + vector[shared_ptr[CBuffer]] buffers + vector[shared_ptr[CArrayData]] child_data + cdef cppclass CArray" arrow::Array": shared_ptr[CDataType] type() @@ -102,9 +110,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_bool Equals(const CArray& arr) c_bool IsNull(int i) + shared_ptr[CArrayData] data() + shared_ptr[CArray] Slice(int64_t offset) shared_ptr[CArray] Slice(int64_t offset, int64_t length) + shared_ptr[CArray] MakeArray(const shared_ptr[CArrayData]& data) + CStatus DebugPrint(const CArray& arr, int indent) cdef cppclass CFixedWidthType" arrow::FixedWidthType"(CDataType): @@ -363,6 +375,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CStatus ValidateArray(const CArray& array) cdef cppclass CChunkedArray" arrow::ChunkedArray": + CChunkedArray(const vector[shared_ptr[CArray]]& arrays) int64_t length() int64_t null_count() int num_chunks() @@ -376,8 +389,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CColumn(const shared_ptr[CField]& field, const vector[shared_ptr[CArray]]& chunks) + CColumn(const shared_ptr[CField]& field, + const shared_ptr[CChunkedArray]& data) + c_bool Equals(const CColumn& other) + shared_ptr[CField] field() + int64_t length() int64_t null_count() const c_string& name() @@ -776,11 +794,42 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil: c_bool allow_int_overflow c_bool allow_time_truncate + enum DatumType" arrow::compute::Datum::type": + DatumType_NONE" arrow::compute::Datum::NONE" + DatumType_SCALAR" arrow::compute::Datum::SCALAR" + DatumType_ARRAY" arrow::compute::Datum::ARRAY" + DatumType_CHUNKED_ARRAY" arrow::compute::Datum::CHUNKED_ARRAY" + DatumType_RECORD_BATCH" arrow::compute::Datum::RECORD_BATCH" + DatumType_TABLE" arrow::compute::Datum::TABLE" + DatumType_COLLECTION" arrow::compute::Datum::COLLECTION" + + cdef cppclass CDatum" arrow::compute::Datum": + CDatum() + CDatum(const shared_ptr[CArray]& value) + CDatum(const shared_ptr[CChunkedArray]& value) + CDatum(const shared_ptr[CRecordBatch]& value) + CDatum(const shared_ptr[CTable]& value) + + DatumType kind() + + shared_ptr[CArrayData] array() + shared_ptr[CChunkedArray] chunked_array() + CStatus Cast(CFunctionContext* context, const CArray& array, const shared_ptr[CDataType]& to_type, const CCastOptions& options, shared_ptr[CArray]* out) + CStatus Cast(CFunctionContext* context, const CDatum& value, + const shared_ptr[CDataType]& to_type, + const CCastOptions& options, CDatum* out) + + CStatus Unique(CFunctionContext* context, const CDatum& value, + shared_ptr[CArray]* out) + + CStatus DictionaryEncode(CFunctionContext* context, const CDatum& value, + CDatum* out) + cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CDataType] GetPrimitiveType(Type type) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 41eaf0bac4525..0aab9a41b8025 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -362,7 +362,8 @@ def backwards_compatible_index_name(raw_name, logical_name): return logical_name -def table_to_blockmanager(options, table, memory_pool, nthreads=1): +def table_to_blockmanager(options, table, memory_pool, nthreads=1, + categoricals=None): import pandas.core.internals as _int import pyarrow.lib as lib diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 5ba5f83d22f90..1a9d23db4ebdc 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -108,6 +108,15 @@ cdef class ChunkedArray: return pyarrow_wrap_array(self.chunked_array.chunk(i)) + property chunks: + + def __get__(self): + cdef int i + chunks = [] + for i in range(self.num_chunks): + chunks.append(self.chunk(i)) + return chunks + def iterchunks(self): for i in range(self.num_chunks): yield self.chunk(i) @@ -122,6 +131,74 @@ cdef class ChunkedArray: return result +def chunked_array(arrays, type=None): + """ + Construct chunked array from list of array-like objects + + Parameters + ---------- + arrays : list of Array or values coercible to arrays + type : DataType or string coercible to DataType + + Returns + ------- + ChunkedArray + """ + cdef: + Array arr + vector[shared_ptr[CArray]] c_arrays + shared_ptr[CChunkedArray] sp_chunked_array + + for x in arrays: + if isinstance(x, Array): + arr = x + if type is not None: + assert x.type == type + else: + arr = array(x, type=type) + + c_arrays.push_back(arr.sp_array) + + sp_chunked_array.reset(new CChunkedArray(c_arrays)) + return pyarrow_wrap_chunked_array(sp_chunked_array) + + +def column(object field_or_name, arr): + """ + Create Column object from field/string and array-like data + """ + cdef: + Field boxed_field + Array _arr + ChunkedArray _carr + shared_ptr[CColumn] sp_column + + if isinstance(arr, list): + arr = chunked_array(arr) + elif not isinstance(arr, (Array, ChunkedArray)): + arr = array(arr) + + if isinstance(field_or_name, Field): + boxed_field = field_or_name + if arr.type != boxed_field.type: + raise ValueError('Passed field type does not match array') + else: + boxed_field = field(field_or_name, arr.type) + + if isinstance(arr, Array): + _arr = arr + sp_column.reset(new CColumn(boxed_field.sp_field, _arr.sp_array)) + elif isinstance(arr, ChunkedArray): + _carr = arr + sp_column.reset(new CColumn(boxed_field.sp_field, + _carr.sp_chunked_array)) + else: + raise ValueError("Unsupported type for column(...): {}" + .format(type(arr))) + + return pyarrow_wrap_column(sp_column) + + cdef class Column: """ Named vector of elements of equal type. @@ -143,25 +220,47 @@ cdef class Column: result = StringIO() result.write(object.__repr__(self)) data = self.data - for i in range(len(data)): - result.write('\nchunk {0}: {1}'.format(i, repr(data.chunk(0)))) + for i, chunk in enumerate(data.chunks): + result.write('\nchunk {0}: {1}'.format(i, repr(chunk))) return result.getvalue() @staticmethod - def from_array(object field_or_name, Array arr): - cdef Field boxed_field + def from_array(*args): + return column(*args) - if isinstance(field_or_name, Field): - boxed_field = field_or_name - if arr.type != boxed_field.type: - raise ValueError('Passed field type does not match array') - else: - boxed_field = field(field_or_name, arr.type) + def cast(self, object target_type, safe=True): + """ + Cast column values to another data type + + Parameters + ---------- + target_type : DataType + Type to cast to + safe : boolean, default True + Check for overflows or other unsafe conversions + + Returns + ------- + casted : Column + """ + cdef: + CCastOptions options + shared_ptr[CArray] result + DataType type + CDatum out + + type = _ensure_type(target_type) - cdef shared_ptr[CColumn] sp_column - sp_column.reset(new CColumn(boxed_field.sp_field, arr.sp_array)) - return pyarrow_wrap_column(sp_column) + options.allow_int_overflow = not safe + options.allow_time_truncate = not safe + + with nogil: + check_status(Cast(_context(), CDatum(self.column.data()), + type.sp_type, options, &out)) + + casted_data = pyarrow_wrap_chunked_array(out.chunked_array()) + return column(self.name, casted_data) def to_pandas(self, strings_to_categorical=False, zero_copy_only=False): """ @@ -241,6 +340,10 @@ cdef class Column: self._check_nullptr() return self.column.length() + @property + def field(self): + return pyarrow_wrap_field(self.column.field()) + @property def shape(self): """ diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index 7dc93c28ea7a4..c061e6820753e 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -295,6 +295,18 @@ def test_cast_integers_safe(): in_arr.cast(out_type) +def test_cast_column(): + arrays = [pa.array([1, 2, 3]), pa.array([4, 5, 6])] + + col = pa.column('foo', arrays) + + target = pa.float64() + casted = col.cast(target) + + expected = pa.column('foo', [x.cast(target) for x in arrays]) + assert casted.equals(expected) + + def test_cast_integers_unsafe(): # We let NumPy do the unsafe casting unsafe_cases = [ @@ -350,6 +362,33 @@ def test_cast_signed_to_unsigned(): _check_cast_case(case) +def test_unique_simple(): + cases = [ + (pa.array([1, 2, 3, 1, 2, 3]), pa.array([1, 2, 3])), + (pa.array(['foo', None, 'bar', 'foo']), + pa.array(['foo', 'bar'])) + ] + for arr, expected in cases: + result = arr.unique() + assert result.equals(expected) + + +def test_dictionary_encode_simple(): + cases = [ + (pa.array([1, 2, 3, None, 1, 2, 3]), + pa.DictionaryArray.from_arrays( + pa.array([0, 1, 2, None, 0, 1, 2], type='int32'), + [1, 2, 3])), + (pa.array(['foo', None, 'bar', 'foo']), + pa.DictionaryArray.from_arrays( + pa.array([0, None, 1, 0], type='int32'), + ['foo', 'bar'])) + ] + for arr, expected in cases: + result = arr.dictionary_encode() + assert result.equals(expected) + + def test_simple_type_construction(): result = pa.lib.TimestampType() with pytest.raises(TypeError): diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 6ba4fd2fad8ea..1df80acc0d6cf 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1436,6 +1436,7 @@ def test_large_table_int32_overflow(): _write_table(table, f) +@parquet def test_index_column_name_duplicate(tmpdir): data = { 'close': { @@ -1460,6 +1461,7 @@ def test_index_column_name_duplicate(tmpdir): tm.assert_frame_equal(result_df, dfx) +@parquet def test_backwards_compatible_index_naming(): expected_string = b"""\ carat cut color clarity depth table price x y z @@ -1482,6 +1484,7 @@ def test_backwards_compatible_index_naming(): tm.assert_frame_equal(result, expected) +@parquet def test_backwards_compatible_index_multi_level_named(): expected_string = b"""\ carat cut color clarity depth table price x y z @@ -1507,6 +1510,7 @@ def test_backwards_compatible_index_multi_level_named(): tm.assert_frame_equal(result, expected) +@parquet def test_backwards_compatible_index_multi_level_some_named(): expected_string = b"""\ carat cut color clarity depth table price x y z diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index edf0d8a305a02..abd678bac6e58 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -1108,6 +1108,7 @@ def union(children_fields, mode): return pyarrow_wrap_data_type(union_type) + cdef dict _type_aliases = { 'null': null, 'i1': int8, From eb7be48ef80fcecea11be29320b244df82f1cc06 Mon Sep 17 00:00:00 2001 From: Manuel Date: Fri, 17 Nov 2017 21:14:16 -0500 Subject: [PATCH 056/177] ARROW-1805: [Python] Ignore special private files when traversing ParquetDataset When exploring dataset, we should ignore _SUCCESS, _metadata and _common_metadata files to determine if the current directory is valid Author: Manuel Closes #1314 from manuvaldes/ARROW-1805 and squashes the following commits: c11e2435 [Manuel] ignore files when exploring partitioned dataset --- python/pyarrow/parquet.py | 2 +- python/pyarrow/tests/test_parquet.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index 9e0749bb35c9e..3023e1771e404 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -573,7 +573,7 @@ def _visit_level(self, level, base_path, part_keys): filtered_files.sort() filtered_directories.sort() - if len(files) > 0 and len(filtered_directories) > 0: + if len(filtered_files) > 0 and len(filtered_directories) > 0: raise ValueError('Found files in an intermediate ' 'directory: {0}'.format(base_path)) elif len(filtered_directories) > 0: diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 1df80acc0d6cf..522815fce16c8 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1027,8 +1027,11 @@ def _visit_level(base_dir, level, part_keys): with fs.open(file_path, 'wb') as f: _write_table(part_table, f) assert fs.exists(file_path) + + _touch(pjoin(level_dir, '_SUCCESS')) else: _visit_level(level_dir, level + 1, this_part_keys) + _touch(pjoin(level_dir, '_SUCCESS')) _visit_level(base_dir, 0, []) @@ -1101,6 +1104,11 @@ def _filter_partition(df, part_keys): return df[predicate].drop(to_drop, axis=1) +def _touch(path): + with open(path, 'wb'): + pass + + @parquet def test_read_multiple_files(tmpdir): import pyarrow.parquet as pq @@ -1128,8 +1136,7 @@ def test_read_multiple_files(tmpdir): paths.append(path) # Write a _SUCCESS.crc file - with open(pjoin(dirpath, '_SUCCESS.crc'), 'wb') as f: - f.write(b'0') + _touch(pjoin(dirpath, '_SUCCESS.crc')) def read_multiple_files(paths, columns=None, nthreads=None, **kwargs): dataset = pq.ParquetDataset(paths, **kwargs) From 202e6503cd2e941a2df2ccacedf09611d8ad2e0f Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 18 Nov 2017 00:41:01 -0500 Subject: [PATCH 057/177] ARROW-1791: Limit generated data range to physical limits for temporal types cc @trxcllnt, do these time ranges seem reasonable? Author: Wes McKinney Closes #1328 from wesm/ARROW-1791 and squashes the following commits: 57a117aa [Wes McKinney] Add comment about nanosecond timestamp range e20ad490 [Wes McKinney] Also restrict time and timestamp generated integer ranges 784e7a95 [Wes McKinney] Restrict generated date range in integration tests --- integration/integration_test.py | 41 +++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/integration/integration_test.py b/integration/integration_test.py index 205176eccc11a..46d010608118d 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -231,10 +231,19 @@ class DateType(IntegerType): DAY = 0 MILLISECOND = 1 + # 1/1/1 to 12/31/9999 + _ranges = { + DAY: [-719162, 2932896], + MILLISECOND: [-62135596800000, 253402214400000] + } + def __init__(self, name, unit, nullable=True): bit_width = 32 if unit == self.DAY else 64 + + min_value, max_value = self._ranges[unit] super(DateType, self).__init__( - name, True, bit_width, nullable=nullable + name, True, bit_width, nullable=nullable, + min_value=min_value, max_value=max_value ) self.unit = unit @@ -262,10 +271,19 @@ class TimeType(IntegerType): 'ns': 64 } + _ranges = { + 's': [0, 86400], + 'ms': [0, 86400000], + 'us': [0, 86400000000], + 'ns': [0, 86400000000000] + } + def __init__(self, name, unit='s', nullable=True): - super(TimeType, self).__init__( - name, True, self.BIT_WIDTHS[unit], nullable=nullable - ) + min_val, max_val = self._ranges[unit] + super(TimeType, self).__init__(name, True, self.BIT_WIDTHS[unit], + nullable=nullable, + min_value=min_val, + max_value=max_val) self.unit = unit def _get_type(self): @@ -278,8 +296,21 @@ def _get_type(self): class TimestampType(IntegerType): + # 1/1/1 to 12/31/9999 + _ranges = { + 's': [-62135596800, 253402214400], + 'ms': [-62135596800000, 253402214400000], + 'us': [-62135596800000000, 253402214400000000], + + # Physical range for int64, ~584 years and change + 'ns': [np.iinfo('int64').min, np.iinfo('int64').max] + } + def __init__(self, name, unit='s', tz=None, nullable=True): - super(TimestampType, self).__init__(name, True, 64, nullable=nullable) + min_val, max_val = self._ranges[unit] + super(TimestampType, self).__init__(name, True, 64, nullable=nullable, + min_value=min_val, + max_value=max_val) self.unit = unit self.tz = tz From 952ec05b19fdaff8548746b6b43618ab044cd298 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sat, 18 Nov 2017 00:42:12 -0500 Subject: [PATCH 058/177] ARROW-1773: [C++] Add casts from date/time types to compatible signed integers This closes [ARROW-1773](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-1773). Author: Licht-T Author: Wes McKinney Closes #1310 from Licht-T/feature-date-time-to-int and squashes the following commits: 1e6061a4 [Wes McKinney] Add zero-copy cast from timestamp to int64 039609a3 [Wes McKinney] Add C++ unit test, clean up Python tests 69a53991 [Licht-T] ENH: Add test for casting from/to Date or Time to/from int 9a64e61d [Licht-T] ENH: Add cast methods from Date/Time to int --- cpp/src/arrow/compute/compute-test.cc | 23 +++++++++++- cpp/src/arrow/compute/kernels/cast.cc | 27 +++++++++----- python/pyarrow/tests/test_array.py | 53 +++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 11 deletions(-) diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index 5eada398d52fc..58a991c60f24b 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -334,6 +334,9 @@ TEST_F(TestCast, TimestampToTimestamp) { &arr); CheckZeroCopy(*arr, timestamp(TimeUnit::SECOND)); + // ARROW-1773, cast to integer + CheckZeroCopy(*arr, int64()); + // Divide, truncate vector v8 = {0, 100123, 200456, 1123, 2456}; vector e8 = {0, 100, 200, 1, 2}; @@ -432,7 +435,7 @@ TEST_F(TestCast, TimestampToDate32_Date64) { timestamp(TimeUnit::SECOND), v_second_nofail, is_valid, date32(), v_day, options); } -TEST_F(TestCast, TimeToTime) { +TEST_F(TestCast, TimeToCompatible) { CastOptions options; vector is_valid = {true, false, true, true, true}; @@ -474,6 +477,16 @@ TEST_F(TestCast, TimeToTime) { ArrayFromVector(time64(TimeUnit::MICRO), is_valid, v7, &arr); CheckZeroCopy(*arr, time64(TimeUnit::MICRO)); + // ARROW-1773: cast to int64 + CheckZeroCopy(*arr, int64()); + + vector v7_2 = {0, 70000, 2000, 1000, 0}; + ArrayFromVector(time32(TimeUnit::SECOND), is_valid, v7_2, &arr); + CheckZeroCopy(*arr, time32(TimeUnit::SECOND)); + + // ARROW-1773: cast to int64 + CheckZeroCopy(*arr, int32()); + // Divide, truncate vector v8 = {0, 100123, 200456, 1123, 2456}; vector e8 = {0, 100, 200, 1, 2}; @@ -515,7 +528,7 @@ TEST_F(TestCast, TimeToTime) { options); } -TEST_F(TestCast, DateToDate) { +TEST_F(TestCast, DateToCompatible) { CastOptions options; vector is_valid = {true, false, true, true, true}; @@ -535,9 +548,15 @@ TEST_F(TestCast, DateToDate) { ArrayFromVector(date32(), is_valid, v2, &arr); CheckZeroCopy(*arr, date32()); + // ARROW-1773: zero copy cast to integer + CheckZeroCopy(*arr, int32()); + ArrayFromVector(date64(), is_valid, v3, &arr); CheckZeroCopy(*arr, date64()); + // ARROW-1773: zero copy cast to integer + CheckZeroCopy(*arr, int64()); + // Divide, truncate vector v8 = {0, 100 * F + 123, 200 * F + 456, F + 123, 2 * F + 456}; vector e8 = {0, 100, 200, 1, 2}; diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc index 6a42ec8b2b807..c866054eadd8a 100644 --- a/cpp/src/arrow/compute/kernels/cast.cc +++ b/cpp/src/arrow/compute/kernels/cast.cc @@ -91,10 +91,14 @@ struct is_zero_copy_cast< // From integers to date/time types with zero copy template struct is_zero_copy_cast< - O, I, typename std::enable_if::value && - (std::is_base_of::value || - std::is_base_of::value || - std::is_base_of::value)>::type> { + O, I, + typename std::enable_if< + (std::is_base_of::value && + (std::is_base_of::value || std::is_base_of::value || + std::is_base_of::value)) || + (std::is_base_of::value && + (std::is_base_of::value || std::is_base_of::value || + std::is_base_of::value))>::type> { using O_T = typename O::c_type; using I_T = typename I::c_type; @@ -809,24 +813,29 @@ class CastKernel : public UnaryKernel { #define DATE32_CASES(FN, IN_TYPE) \ FN(Date32Type, Date32Type); \ - FN(Date32Type, Date64Type); + FN(Date32Type, Date64Type); \ + FN(Date32Type, Int32Type); #define DATE64_CASES(FN, IN_TYPE) \ FN(Date64Type, Date64Type); \ - FN(Date64Type, Date32Type); + FN(Date64Type, Date32Type); \ + FN(Date64Type, Int64Type); #define TIME32_CASES(FN, IN_TYPE) \ FN(Time32Type, Time32Type); \ - FN(Time32Type, Time64Type); + FN(Time32Type, Time64Type); \ + FN(Time32Type, Int32Type); #define TIME64_CASES(FN, IN_TYPE) \ FN(Time64Type, Time32Type); \ - FN(Time64Type, Time64Type); + FN(Time64Type, Time64Type); \ + FN(Time64Type, Int64Type); #define TIMESTAMP_CASES(FN, IN_TYPE) \ FN(TimestampType, TimestampType); \ FN(TimestampType, Date32Type); \ - FN(TimestampType, Date64Type); + FN(TimestampType, Date64Type); \ + FN(TimestampType, Int64Type); #define DICTIONARY_CASES(FN, IN_TYPE) \ FN(IN_TYPE, NullType); \ diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index c061e6820753e..b7b0b18333677 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -389,6 +389,59 @@ def test_dictionary_encode_simple(): assert result.equals(expected) +def test_cast_time32_to_int(): + arr = pa.array(np.array([0, 1, 2], dtype='int32'), + type=pa.time32('s')) + expected = pa.array([0, 1, 2], type='i4') + + result = arr.cast('i4') + assert result.equals(expected) + + +def test_cast_time64_to_int(): + arr = pa.array(np.array([0, 1, 2], dtype='int64'), + type=pa.time64('us')) + expected = pa.array([0, 1, 2], type='i8') + + result = arr.cast('i8') + assert result.equals(expected) + + +def test_cast_timestamp_to_int(): + arr = pa.array(np.array([0, 1, 2], dtype='int64'), + type=pa.timestamp('us')) + expected = pa.array([0, 1, 2], type='i8') + + result = arr.cast('i8') + assert result.equals(expected) + + +def test_cast_date32_to_int(): + arr = pa.array([0, 1, 2], type='i4') + + result1 = arr.cast('date32') + result2 = result1.cast('i4') + + expected1 = pa.array([ + datetime.date(1970, 1, 1), + datetime.date(1970, 1, 2), + datetime.date(1970, 1, 3) + ]).cast('date32') + + assert result1.equals(expected1) + assert result2.equals(arr) + + +def test_cast_date64_to_int(): + arr = pa.array(np.array([0, 1, 2], dtype='int64'), + type=pa.date64()) + expected = pa.array([0, 1, 2], type='i8') + + result = arr.cast('i8') + + assert result.equals(expected) + + def test_simple_type_construction(): result = pa.lib.TimestampType() with pytest.raises(TypeError): From 37214ef35cf5ecaf195c0796df50371e9c574928 Mon Sep 17 00:00:00 2001 From: Li Jin Date: Sat, 18 Nov 2017 16:22:08 -0500 Subject: [PATCH 059/177] ARROW-1827: [Java] Add checkstyle file and license template Author: Li Jin Closes #1326 from icexelloss/ARROW-1827 and squashes the following commits: a6ea0321 [Li Jin] Make license consistent 28348721 [Li Jin] Minor edit ab705a14 [Li Jin] Add newline 291dbf93 [Li Jin] Add checkstyle config and header --- java/.gitattributes | 3 + java/dev/checkstyle/checkstyle.license | 17 ++ java/dev/checkstyle/checkstyle.xml | 254 +++++++++++++++++++++++++ java/dev/checkstyle/suppressions.xml | 31 +++ java/pom.xml | 4 +- 5 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 java/.gitattributes create mode 100644 java/dev/checkstyle/checkstyle.license create mode 100644 java/dev/checkstyle/checkstyle.xml create mode 100644 java/dev/checkstyle/suppressions.xml diff --git a/java/.gitattributes b/java/.gitattributes new file mode 100644 index 0000000000000..cb02d822677ed --- /dev/null +++ b/java/.gitattributes @@ -0,0 +1,3 @@ +.gitattributes export-ignore +.gitignore export-ignore +/dev export-ignore diff --git a/java/dev/checkstyle/checkstyle.license b/java/dev/checkstyle/checkstyle.license new file mode 100644 index 0000000000000..c06c90cd287af --- /dev/null +++ b/java/dev/checkstyle/checkstyle.license @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/java/dev/checkstyle/checkstyle.xml b/java/dev/checkstyle/checkstyle.xml new file mode 100644 index 0000000000000..14dbede169f19 --- /dev/null +++ b/java/dev/checkstyle/checkstyle.xml @@ -0,0 +1,254 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/dev/checkstyle/suppressions.xml b/java/dev/checkstyle/suppressions.xml new file mode 100644 index 0000000000000..36697256d69d9 --- /dev/null +++ b/java/dev/checkstyle/suppressions.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + diff --git a/java/pom.xml b/java/pom.xml index 0a0f2e0ce8f65..c479d651f6b5f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -304,7 +304,9 @@ - google_checks.xml + dev/checkstyle/checkstyle.xml + dev/checkstyle/checkstyle.license + dev/checkstyle/suppressions.xml UTF-8 true ${checkstyle.failOnViolation} From 9f9dc5b7025744458e50713fa460803bf52f5d33 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 18 Nov 2017 16:25:56 -0500 Subject: [PATCH 060/177] ARROW-1575: [Python] Add tests for pyarrow.column factory function Author: Wes McKinney Closes #1329 from wesm/ARROW-1575 and squashes the following commits: d9f14a12 [Wes McKinney] Add tests for pyarrow.column factory function --- python/pyarrow/table.pxi | 9 ++++ python/pyarrow/tests/test_table.py | 79 +++++++++++++++++------------- 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 1a9d23db4ebdc..591f3297587a3 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -166,6 +166,15 @@ def chunked_array(arrays, type=None): def column(object field_or_name, arr): """ Create Column object from field/string and array-like data + + Parameters + ---------- + field_or_name : string or Field + arr : Array, list of Arrays, or ChunkedArray + + Returns + ------- + column : Column """ cdef: Field boxed_field diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 4282224660a1a..cd05fb8e1fce5 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -21,42 +21,55 @@ import pandas as pd import pytest -from pyarrow.compat import unittest import pyarrow as pa -class TestColumn(unittest.TestCase): - - def test_basics(self): - data = [ - pa.array([-10, -5, 0, 5, 10]) - ] - table = pa.Table.from_arrays(data, names=['a']) - column = table.column(0) - assert column.name == 'a' - assert column.length() == 5 - assert len(column) == 5 - assert column.shape == (5,) - assert column.to_pylist() == [-10, -5, 0, 5, 10] - - def test_from_array(self): - arr = pa.array([0, 1, 2, 3, 4]) - - col1 = pa.Column.from_array('foo', arr) - col2 = pa.Column.from_array(pa.field('foo', arr.type), arr) - - assert col1.equals(col2) - - def test_pandas(self): - data = [ - pa.array([-10, -5, 0, 5, 10]) - ] - table = pa.Table.from_arrays(data, names=['a']) - column = table.column(0) - series = column.to_pandas() - assert series.name == 'a' - assert series.shape == (5,) - assert series.iloc[0] == -10 +def test_column_basics(): + data = [ + pa.array([-10, -5, 0, 5, 10]) + ] + table = pa.Table.from_arrays(data, names=['a']) + column = table.column(0) + assert column.name == 'a' + assert column.length() == 5 + assert len(column) == 5 + assert column.shape == (5,) + assert column.to_pylist() == [-10, -5, 0, 5, 10] + + +def test_column_factory_function(): + # ARROW-1575 + arr = pa.array([0, 1, 2, 3, 4]) + arr2 = pa.array([5, 6, 7, 8]) + + col1 = pa.Column.from_array('foo', arr) + col2 = pa.Column.from_array(pa.field('foo', arr.type), arr) + + assert col1.equals(col2) + + col3 = pa.column('foo', [arr, arr2]) + chunked_arr = pa.chunked_array([arr, arr2]) + col4 = pa.column('foo', chunked_arr) + assert col3.equals(col4) + + col5 = pa.column('foo', arr.to_pandas()) + assert col5.equals(pa.column('foo', arr)) + + # Type mismatch + with pytest.raises(ValueError): + pa.Column.from_array(pa.field('foo', pa.string()), arr) + + +def test_column_to_pandas(): + data = [ + pa.array([-10, -5, 0, 5, 10]) + ] + table = pa.Table.from_arrays(data, names=['a']) + column = table.column(0) + series = column.to_pandas() + assert series.name == 'a' + assert series.shape == (5,) + assert series.iloc[0] == -10 def test_recordbatch_basics(): From d92735e3956f61ba647f88409e1bf39b8b07ba72 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Sun, 19 Nov 2017 20:02:17 -0500 Subject: [PATCH 061/177] =?UTF-8?q?ARROW-1834:=20[Doc]=C2=A0Build=20docume?= =?UTF-8?q?ntation=20in=20separate=20build=20folders?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author: Korn, Uwe Closes #1332 from xhochy/ARROW-1834 and squashes the following commits: e9c02533 [Korn, Uwe] ARROW-1834: [Doc] Build documentation in separate build folders --- dev/gen_apidocs/Dockerfile | 2 +- dev/gen_apidocs/create_documents.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile index eaeb548184d61..ca4718e637842 100644 --- a/dev/gen_apidocs/Dockerfile +++ b/dev/gen_apidocs/Dockerfile @@ -26,7 +26,7 @@ RUN apt-add-repository -y ppa:ubuntu-toolchain-r/test && \ git \ gcc-4.9 \ g++-4.9 \ - build-essential + build-essential # This will install conda in /home/ubuntu/miniconda RUN wget -O /tmp/miniconda.sh \ https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ diff --git a/dev/gen_apidocs/create_documents.sh b/dev/gen_apidocs/create_documents.sh index 762b85222bd4d..566d9cee79c85 100755 --- a/dev/gen_apidocs/create_documents.sh +++ b/dev/gen_apidocs/create_documents.sh @@ -45,9 +45,9 @@ source activate pyarrow-dev export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX -rm -rf arrow/cpp/build -mkdir arrow/cpp/build -pushd arrow/cpp/build +rm -rf arrow/cpp/build_docs +mkdir arrow/cpp/build_docs +pushd arrow/cpp/build_docs cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DARROW_PYTHON=on \ @@ -59,9 +59,9 @@ make install popd # Build Parquet C++ -rm -rf parquet-cpp/build -mkdir parquet-cpp/build -pushd parquet-cpp/build +rm -rf parquet-cpp/build_docs +mkdir parquet-cpp/build_docs +pushd parquet-cpp/build_docs cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ -DPARQUET_BUILD_BENCHMARKS=off \ From b3a3a743866e2cbeff6fa20c77fa378db13232f6 Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Mon, 20 Nov 2017 09:22:54 -0500 Subject: [PATCH 062/177] ARROW-1693: [JS] Expand JavaScript implementation, build system, fix integration tests This PR adds a workaround for reading the metadata layout for C++ dictionary-encoded vectors. I added tests that validate against the C++/Java integration suite. In order to make the new tests pass, I had to update the generated flatbuffers format and add a few types the JS version didn't have yet (Bool, Date32, and Timestamp). It also uses the new `isDelta` flag on DictionaryBatches to determine whether the DictionaryBatch vector should replace or append to the existing dictionary. I also added a script for generating test arrow files from the C++ and Java implementations, so we don't break the tests updating the format in the future. I saved the generated Arrow files in with the tests because I didn't see a way to pipe the JSON test data through the C++/Java json-to-arrow commands without writing to a file. If I missed something and we can do it all in-memory, I'd be happy to make that change! This PR is marked WIP because I added an [integration test](https://github.com/apache/arrow/commit/6e98874d9f4bfae7758f8f731212ae7ceb3f1321#diff-18c6be12406c482092d4b1f7bd70a8e1R22) that validates the JS reader reads C++ and Java files the same way, but unfortunately it doesn't. Debugging, I noticed a number of other differences between the buffer layout metadata between the C++ and Java versions. If we go ahead with @jacques-n [comment in ARROW-1693](https://issues.apache.org/jira/browse/ARROW-1693?focusedCommentId=16244812&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16244812) and remove/ignore the metadata, this test should pass too. cc @TheNeuralBit Author: Paul Taylor Author: Wes McKinney Closes #1294 from trxcllnt/generate-js-test-files and squashes the following commits: f907d5a7 [Paul Taylor] fix aggressive closure-compiler mangling in the ES5 UMD bundle 57c7df45 [Paul Taylor] remove arrow files from perf tests 5972349c [Paul Taylor] update performance tests to use generated test data 14be77f4 [Paul Taylor] fix Date64Vector TypedArray, enable datetime integration tests 5660eb34 [Wes McKinney] Use openjdk8 for integration tests, jdk7 for main Java CI job 019e8e24 [Paul Taylor] update closure compiler with full support for ESModules, and remove closure-compiler-scripts 48111290 [Paul Taylor] Add support for reading Arrow buffers < MetadataVersion 4 c72134a5 [Paul Taylor] compile JS source in integration tests c83a700d [Wes McKinney] Hack until ARROW-1837 resolved. Constrain unsigned integers max to signed max for bit width fd3ed475 [Wes McKinney] Uppercase hex values 224e041c [Wes McKinney] Remove hard-coded file name to prevent primitive JSON file from being clobbered 0882d8e9 [Paul Taylor] separate JS unit tests from integration tests in CI 1f6a81b4 [Paul Taylor] add missing mkdirp for test json data 19136fbf [Paul Taylor] remove test data files in favor of auto-generating them in CI 9f195682 [Paul Taylor] Generate test files when the test run if they don't exist 0cdb74e0 [Paul Taylor] Add a cli arg to integration_test.py generate test JSON files for JS cc744564 [Paul Taylor] resolve LICENSE.txt conflict 33916230 [Paul Taylor] move js license to top-level license.txt d0b61f49 [Paul Taylor] add validate package script back in, make npm-release.sh suitable for ASF release process 7e3be574 [Paul Taylor] Copy license.txt and notice.txt into target dirs from arrow root. c8125d2d [Paul Taylor] Update readme to reflect new Table.from signature 49ac3398 [Paul Taylor] allow unrecognized cli args in gulpfile 3c52587e [Paul Taylor] re-enable node_js job in travis cb142f11 [Paul Taylor] add npm release script, remove unused package scripts d51793dd [Paul Taylor] run tests on src folder for accurate jest coverage statistics c087f482 [Paul Taylor] generate test data in build scripts 1d814d00 [Paul Taylor] excise test data csvs 14d48964 [Paul Taylor] stringify Struct Array cells 1f004968 [Paul Taylor] rename FixedWidthListVector to FixedWidthNumericVector be73c918 [Paul Taylor] add BinaryVector, change ListVector to always return an Array 02fb3006 [Paul Taylor] compare iterator results in integration tests e67a66a1 [Paul Taylor] remove/ignore test snapshots (getting too big) de7d96a3 [Paul Taylor] regenerate test arrows from master a6d3c83e [Paul Taylor] enable integration tests 44889fbe [Paul Taylor] report errors generating test arrows fd68d510 [Paul Taylor] always increment validity buffer index while reading 562eba7d [Paul Taylor] update test snapshots d4399a8a [Paul Taylor] update integration tests, add custom jest vector matcher 8d44dcd7 [Paul Taylor] update tests 6d2c03d4 [Paul Taylor] clean arrows folders before regenerating test data 4166a9ff [Paul Taylor] hard-code reader to Arrow spec and ignore field layout metadata c60305d6 [Paul Taylor] refactor: flatten vector folder, add more types ba984c61 [Paul Taylor] update dependencies 5eee3eaa [Paul Taylor] add integration tests to compare how JS reads cpp vs. java arrows d4ff57aa [Paul Taylor] update test snapshots 407b9f5b [Paul Taylor] update reader/table tests for new generated arrows 85497069 [Paul Taylor] update cli args to execute partial test runs for debugging eefc256d [Paul Taylor] remove old test arrows, add new generated test arrows 0cd31ab9 [Paul Taylor] add generate-arrows script to tests 3ff71384 [Paul Taylor] Add bool, date, time, timestamp, and ARROW-1693 workaround in reader 4a34247c [Paul Taylor] export Row type 141194e7 [Paul Taylor] use fieldNode.length as vector length c45718e7 [Paul Taylor] support new DictionaryBatch isDelta flag 9d8fef97 [Paul Taylor] split DateVector into Date32 and Date64 types 8592ff3c [Paul Taylor] update generated format flatbuffers --- .travis.yml | 21 +- LICENSE.txt | 95 + ci/travis_script_integration.sh | 19 + ci/travis_script_js.sh | 15 +- integration/integration_test.py | 56 +- js/.gitignore | 5 + js/DEVELOP.md | 251 ++- js/LICENSE | 39 - js/README.md | 12 +- js/closure-compiler-scripts/flatbuffers.js | 1204 ----------- js/closure-compiler-scripts/text-encoding.js | 648 ------ js/closure-compiler-scripts/tslib.js | 151 -- js/gulp/argv.js | 7 +- js/gulp/arrow-task.js | 5 +- js/gulp/closure-task.js | 16 +- js/gulp/test-task.js | 130 +- js/gulp/typescript-task.js | 26 +- js/gulp/uglify-task.js | 2 +- js/gulp/util.js | 2 +- js/gulpfile.js | 10 +- js/lerna.json | 4 +- js/{prepublish.sh => npm-release.sh} | 8 +- js/package.json | 38 +- js/perf/arrows/file/dictionary.arrow | Bin 2522 -> 0 bytes js/perf/arrows/file/simple.arrow | Bin 1642 -> 0 bytes js/perf/arrows/file/struct.arrow | Bin 2354 -> 0 bytes js/perf/arrows/multi/count/records.arrow | Bin 224 -> 0 bytes js/perf/arrows/multi/count/schema.arrow | Bin 184 -> 0 bytes js/perf/arrows/multi/latlong/records.arrow | Bin 352 -> 0 bytes js/perf/arrows/multi/latlong/schema.arrow | Bin 264 -> 0 bytes js/perf/arrows/multi/origins/records.arrow | Bin 224 -> 0 bytes js/perf/arrows/multi/origins/schema.arrow | Bin 1604 -> 0 bytes js/perf/arrows/stream/dictionary.arrow | Bin 1776 -> 0 bytes js/perf/arrows/stream/simple.arrow | Bin 1188 -> 0 bytes js/perf/arrows/stream/struct.arrow | Bin 1884 -> 0 bytes js/perf/config.js | 24 +- js/perf/index.js | 24 +- js/src/Arrow.ts | 98 +- js/src/format/{File_generated.ts => File.ts} | 2 +- .../format}/File_generated.js | 12 +- .../{Message_generated.ts => Message.ts} | 33 +- .../format}/Message_generated.js | 43 +- .../format/{Schema_generated.ts => Schema.ts} | 19 +- .../format}/Schema_generated.js | 46 +- js/src/reader/arrow.ts | 197 +- js/src/reader/dictionary.ts | 36 - js/src/reader/file.ts | 82 - js/src/reader/format.ts | 112 + js/src/reader/message.ts | 63 - js/src/reader/stream.ts | 43 - js/src/reader/vector.ts | 423 ++-- js/src/types/arrow.ts | 88 - js/src/types/fixedsizelist.ts | 35 - js/src/types/list.ts | 35 - js/src/types/table/from.ts | 34 - js/src/types/table/row.ts | 61 - js/src/types/table/struct.ts | 63 - js/src/types/table/table.ts | 30 - js/src/types/table/toString.ts | 40 - js/src/types/vector/bool.ts | 55 - js/src/types/vector/date.ts | 29 - js/src/types/vector/long.ts | 35 - js/src/types/vector/typed.ts | 57 - js/src/types/vector/virtual.ts | 129 -- js/src/vector/arrow.ts | 245 +++ js/src/{types => vector}/dictionary.ts | 28 +- js/src/vector/list.ts | 74 + js/src/vector/numeric.ts | 168 ++ js/src/vector/struct.ts | 127 ++ js/src/vector/table.ts | 59 + js/src/{types => }/vector/traits.ts | 12 +- js/src/vector/types.ts | 43 + js/src/{types => vector}/utf8.ts | 4 +- js/src/{types/types.ts => vector/vector.ts} | 56 +- js/src/vector/virtual.ts | 129 ++ js/test/Arrow.ts | 57 +- js/test/__snapshots__/reader-tests.ts.snap | 497 ----- js/test/__snapshots__/table-tests.ts.snap | 1815 ----------------- js/test/arrows/file/dictionary.arrow | Bin 2522 -> 0 bytes js/test/arrows/file/dictionary2.arrow | Bin 2762 -> 0 bytes js/test/arrows/file/multi_dictionary.arrow | Bin 3482 -> 0 bytes js/test/arrows/file/simple.arrow | Bin 1642 -> 0 bytes js/test/arrows/file/struct.arrow | Bin 2354 -> 0 bytes js/test/arrows/multi/count/records.arrow | Bin 224 -> 0 bytes js/test/arrows/multi/count/schema.arrow | Bin 184 -> 0 bytes js/test/arrows/multi/latlong/records.arrow | Bin 352 -> 0 bytes js/test/arrows/multi/latlong/schema.arrow | Bin 264 -> 0 bytes js/test/arrows/multi/origins/records.arrow | Bin 224 -> 0 bytes js/test/arrows/multi/origins/schema.arrow | Bin 1604 -> 0 bytes js/test/arrows/stream/dictionary.arrow | Bin 1776 -> 0 bytes js/test/arrows/stream/simple.arrow | Bin 1188 -> 0 bytes js/test/arrows/stream/struct.arrow | Bin 1884 -> 0 bytes js/test/integration-tests.ts | 114 ++ js/test/reader-tests.ts | 69 +- js/test/table-tests.ts | 175 +- js/test/test-config.ts | 50 +- js/test/tsconfig.json | 2 +- js/test/vector-tests.ts | 109 +- js/tsconfig/tsconfig.es5.cls.json | 4 +- 99 files changed, 2468 insertions(+), 6081 deletions(-) delete mode 100644 js/LICENSE delete mode 100644 js/closure-compiler-scripts/flatbuffers.js delete mode 100644 js/closure-compiler-scripts/text-encoding.js delete mode 100644 js/closure-compiler-scripts/tslib.js rename js/{prepublish.sh => npm-release.sh} (85%) delete mode 100644 js/perf/arrows/file/dictionary.arrow delete mode 100644 js/perf/arrows/file/simple.arrow delete mode 100644 js/perf/arrows/file/struct.arrow delete mode 100644 js/perf/arrows/multi/count/records.arrow delete mode 100644 js/perf/arrows/multi/count/schema.arrow delete mode 100644 js/perf/arrows/multi/latlong/records.arrow delete mode 100644 js/perf/arrows/multi/latlong/schema.arrow delete mode 100644 js/perf/arrows/multi/origins/records.arrow delete mode 100644 js/perf/arrows/multi/origins/schema.arrow delete mode 100644 js/perf/arrows/stream/dictionary.arrow delete mode 100644 js/perf/arrows/stream/simple.arrow delete mode 100644 js/perf/arrows/stream/struct.arrow rename js/src/format/{File_generated.ts => File.ts} (99%) rename js/{closure-compiler-scripts => src/format}/File_generated.js (95%) rename js/src/format/{Message_generated.ts => Message.ts} (94%) rename js/{closure-compiler-scripts => src/format}/Message_generated.js (93%) rename js/src/format/{Schema_generated.ts => Schema.ts} (99%) rename js/{closure-compiler-scripts => src/format}/Schema_generated.js (98%) delete mode 100644 js/src/reader/dictionary.ts delete mode 100644 js/src/reader/file.ts create mode 100644 js/src/reader/format.ts delete mode 100644 js/src/reader/message.ts delete mode 100644 js/src/reader/stream.ts delete mode 100644 js/src/types/arrow.ts delete mode 100644 js/src/types/fixedsizelist.ts delete mode 100644 js/src/types/list.ts delete mode 100644 js/src/types/table/from.ts delete mode 100644 js/src/types/table/row.ts delete mode 100644 js/src/types/table/struct.ts delete mode 100644 js/src/types/table/table.ts delete mode 100644 js/src/types/table/toString.ts delete mode 100644 js/src/types/vector/bool.ts delete mode 100644 js/src/types/vector/date.ts delete mode 100644 js/src/types/vector/long.ts delete mode 100644 js/src/types/vector/typed.ts delete mode 100644 js/src/types/vector/virtual.ts create mode 100644 js/src/vector/arrow.ts rename js/src/{types => vector}/dictionary.ts (62%) create mode 100644 js/src/vector/list.ts create mode 100644 js/src/vector/numeric.ts create mode 100644 js/src/vector/struct.ts create mode 100644 js/src/vector/table.ts rename js/src/{types => }/vector/traits.ts (88%) create mode 100644 js/src/vector/types.ts rename js/src/{types => vector}/utf8.ts (94%) rename js/src/{types/types.ts => vector/vector.ts} (56%) create mode 100644 js/src/vector/virtual.ts delete mode 100644 js/test/__snapshots__/reader-tests.ts.snap delete mode 100644 js/test/__snapshots__/table-tests.ts.snap delete mode 100644 js/test/arrows/file/dictionary.arrow delete mode 100644 js/test/arrows/file/dictionary2.arrow delete mode 100644 js/test/arrows/file/multi_dictionary.arrow delete mode 100644 js/test/arrows/file/simple.arrow delete mode 100644 js/test/arrows/file/struct.arrow delete mode 100644 js/test/arrows/multi/count/records.arrow delete mode 100644 js/test/arrows/multi/count/schema.arrow delete mode 100644 js/test/arrows/multi/latlong/records.arrow delete mode 100644 js/test/arrows/multi/latlong/schema.arrow delete mode 100644 js/test/arrows/multi/origins/records.arrow delete mode 100644 js/test/arrows/multi/origins/schema.arrow delete mode 100644 js/test/arrows/stream/dictionary.arrow delete mode 100644 js/test/arrows/stream/simple.arrow delete mode 100644 js/test/arrows/stream/struct.arrow create mode 100644 js/test/integration-tests.ts diff --git a/.travis.yml b/.travis.yml index 9917a261451c2..9c714a68948e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,7 +87,7 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh - language: java os: linux - jdk: openjdk8 + jdk: openjdk7 script: - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - language: java @@ -103,23 +103,24 @@ matrix: - language: java os: linux env: ARROW_TEST_GROUP=integration - jdk: openjdk7 + jdk: openjdk8 before_script: - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - export CC="clang-4.0" - export CXX="clang++-4.0" + - nvm install node - $TRAVIS_BUILD_DIR/ci/travis_lint.sh + - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh - # TODO(wesm): Re-enable after issues in ARROW-1409 resolved - # - language: node_js - # os: linux - # node_js: node - # before_script: - # - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh - # script: - # - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh + - language: node_js + os: linux + node_js: node + before_script: + - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh + script: + - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh - compiler: gcc language: cpp os: linux diff --git a/LICENSE.txt b/LICENSE.txt index 038518a5d9b68..84e6a4e2a2a97 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -457,3 +457,98 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the mapbox/variant project, BSD 3-clause +license + +Copyright (c) MapBox +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +- Neither the name "MapBox" nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh index be025512f0b88..105a0cc6086df 100755 --- a/ci/travis_script_integration.sh +++ b/ci/travis_script_integration.sh @@ -44,3 +44,22 @@ conda install -y pip numpy six python integration_test.py --debug popd + +pushd $ARROW_JS_DIR + +# lint and compile JS source +npm run lint +npm run build +# create initial test data +npm run test:createTestData +# run once to write the snapshots +npm test -- -t ts -u --integration +# run again to test all builds against the snapshots +npm test -- --integration +# run tests against source to generate coverage data +npm run test:coverage -- --integration +# Uncomment to upload to coveralls +# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js; + + +popd \ No newline at end of file diff --git a/ci/travis_script_js.sh b/ci/travis_script_js.sh index 9f77dec8c53a7..1871b4265cd01 100755 --- a/ci/travis_script_js.sh +++ b/ci/travis_script_js.sh @@ -17,16 +17,15 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex -JS_DIR=${TRAVIS_BUILD_DIR}/js +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh -pushd $JS_DIR +pushd $ARROW_JS_DIR -npm run validate - -# Uncomment to use coveralls -# npm run test:coverage -# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js; +npm run lint +npm run build +# run the non-snapshot unit tests +npm test popd diff --git a/integration/integration_test.py b/integration/integration_test.py index 46d010608118d..a063fb36f49d5 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -28,6 +28,7 @@ import subprocess import tempfile import uuid +import errno import numpy as np @@ -198,9 +199,18 @@ def __init__(self, name, is_signed, bit_width, nullable=True, self.min_value = min_value self.max_value = max_value - @property - def numpy_type(self): - return ('int' if self.is_signed else 'uint') + str(self.bit_width) + def _get_generated_data_bounds(self): + signed_iinfo = np.iinfo('int' + str(self.bit_width)) + if self.is_signed: + min_value, max_value = signed_iinfo.min, signed_iinfo.max + else: + # ARROW-1837 Remove this hack and restore full unsigned integer + # range + min_value, max_value = 0, signed_iinfo.max + + lower_bound = max(min_value, self.min_value) + upper_bound = min(max_value, self.max_value) + return lower_bound, upper_bound def _get_type(self): return OrderedDict([ @@ -210,9 +220,7 @@ def _get_type(self): ]) def generate_column(self, size, name=None): - iinfo = np.iinfo(self.numpy_type) - lower_bound = max(iinfo.min, self.min_value) - upper_bound = min(iinfo.max, self.max_value) + lower_bound, upper_bound = self._get_generated_data_bounds() return self.generate_range(size, lower_bound, upper_bound, name=name) def generate_range(self, size, lower, upper, name=None): @@ -521,7 +529,7 @@ def get_json(self): class BinaryColumn(PrimitiveColumn): def _encode_value(self, x): - return frombytes(binascii.hexlify(x)) + return frombytes(binascii.hexlify(x).upper()) def _get_buffers(self): offset = 0 @@ -785,7 +793,7 @@ def _generate_file(name, fields, batch_sizes, dictionaries=None): return JsonFile(name, schema, batches, dictionaries) -def generate_primitive_case(batch_sizes): +def generate_primitive_case(batch_sizes, name='primitive'): types = ['bool', 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64', 'binary', 'utf8'] @@ -796,7 +804,7 @@ def generate_primitive_case(batch_sizes): fields.append(get_field(type_ + "_nullable", type_, True)) fields.append(get_field(type_ + "_nonnullable", type_, False)) - return _generate_file("primitive", fields, batch_sizes) + return _generate_file(name, fields, batch_sizes) def generate_decimal_case(): @@ -874,8 +882,8 @@ def _temp_path(): return file_objs = [ - generate_primitive_case([7, 10]), - generate_primitive_case([0, 0, 0]), + generate_primitive_case([17, 20], name='primitive'), + generate_primitive_case([0, 0, 0], name='primitive_zerolength'), generate_decimal_case(), generate_datetime_case(), generate_nested_case(), @@ -1079,11 +1087,33 @@ def run_all_tests(debug=False): print('-- All tests passed!') +def write_js_test_json(directory): + generate_nested_case().write(os.path.join(directory, 'nested.json')) + generate_decimal_case().write(os.path.join(directory, 'decimal.json')) + generate_datetime_case().write(os.path.join(directory, 'datetime.json')) + (generate_dictionary_case() + .write(os.path.join(directory, 'dictionary.json'))) + (generate_primitive_case([7, 10]) + .write(os.path.join(directory, 'primitive.json'))) + (generate_primitive_case([0, 0, 0]) + .write(os.path.join(directory, 'primitive-empty.json'))) + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Arrow integration test CLI') + parser.add_argument('--write_generated_json', dest='generated_json_path', + action='store', default=False, + help='Generate test JSON') parser.add_argument('--debug', dest='debug', action='store_true', default=False, help='Run executables in debug mode as relevant') - args = parser.parse_args() - run_all_tests(debug=args.debug) + if args.generated_json_path: + try: + os.makedirs(args.generated_json_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + write_js_test_json(args.generated_json_path) + else: + run_all_tests(debug=args.debug) diff --git a/js/.gitignore b/js/.gitignore index 88c612d8faf37..f705f2510f93f 100644 --- a/js/.gitignore +++ b/js/.gitignore @@ -83,3 +83,8 @@ package-lock.json # compilation targets dist targets + +# test data files +test/data/ +# jest snapshots (too big) +test/__snapshots__/ \ No newline at end of file diff --git a/js/DEVELOP.md b/js/DEVELOP.md index 5b4ac14ed3466..9f586e1b39bcb 100644 --- a/js/DEVELOP.md +++ b/js/DEVELOP.md @@ -50,91 +50,220 @@ Once generated, the flatbuffers format code needs to be adjusted for our TS and 1. Generate the flatbuffers TypeScript source from the Arrow project root directory: ```sh + cd $ARROW_HOME + flatc --ts -o ./js/src/format ./format/*.fbs + + cd ./js/src/format + + # Delete Tensor_generated.js (skip this when we support Tensors) + rm ./Tensor_generated.ts + + # Remove "_generated" suffix from TS files + mv ./File_generated.ts .File.ts + mv ./Schema_generated.ts .Schema.ts + mv ./Message_generated.ts .Message.ts ``` -1. Change all the `flatbuffers` imports to +1. Remove Tensor import from `Schema.ts` +1. Fix all the `flatbuffers` imports ```ts - import { flatbuffers } from "flatbuffers" + import { flatbuffers } from "./flatbuffers" // <-- change + import { flatbuffers } from "flatbuffers" // <-- to this ``` -1. Delete `Tensor_generated.ts` (remove this step once we support Tensors) -1. Remove Tensor import from `Schema_generated.ts` -1. Add `/* tslint:disable:class-name */` to the top of `Schema_generated.ts` +1. Remove `_generated` from the ES6 imports of the generated files + ```ts + import * as NS16187549871986683199 from "./Schema_generated"; // <-- change + import * as NS16187549871986683199 from "./Schema"; // <------- to this + ``` +1. Add `/* tslint:disable:class-name */` to the top of `Schema.ts` +1. Execute `npm run lint` to fix all the linting errors ## JavaScript (for Google Closure Compiler builds) 1. Generate the flatbuffers JS source from the Arrow project root directory ```sh - flatc --js -o ./js/closure-compiler-scripts ./format/*.fbs - ``` -1. Delete `Tensor_generated.js` (remove this step once we support Tensors) -1. Add `goog.module` declarations to the top of each generated file + cd $ARROW_HOME - Each file starts with a header that looks like this: - ```js - // automatically generated by the FlatBuffers compiler, do not modify + flatc --js --no-js-exports -o ./js/src/format ./format/*.fbs - /** - * @const - * @namespace - */ - var org = org || {}; - ``` + cd ./js/src/format + + # Delete Tensor_generated.js (skip this when we support Tensors) + rm Tensor_generated.js - Update the header of each file to explicitly declare its module. + # append an ES6 export to Schema_generated.js + echo "$(cat Schema_generated.js) + export { org }; + " > Schema_generated.js - `Schema_generated.js`: + # import Schema's "org" namespace and + # append an ES6 export to File_generated.js + echo "import { org } from './Schema'; + $(cat File_generated.js) + export { org }; + " > File_generated.js + + # import Schema's "org" namespace and + # append an ES6 export to Message_generated.js + echo "import { org } from './Schema'; + $(cat Message_generated.js) + export { org }; + " > Message_generated.js + ``` +1. Fixup the generated JS enums with the reverse value-to-key mappings to match TypeScript + `Message_generated.js` ```js - // automatically generated by the FlatBuffers compiler, do not modify - goog.module("module$targets$es5$cls$format$Schema_generated"); - goog.module.declareLegacyNamespace(); + // Replace this + org.apache.arrow.flatbuf.MessageHeader = { + NONE: 0, + Schema: 1, + DictionaryBatch: 2, + RecordBatch: 3, + Tensor: 4 + }; + // With this + org.apache.arrow.flatbuf.MessageHeader = { + NONE: 0, 0: 'NONE', + Schema: 1, 1: 'Schema', + DictionaryBatch: 2, 2: 'DictionaryBatch', + RecordBatch: 3, 3: 'RecordBatch', + Tensor: 4, 4: 'Tensor' + }; + ``` + `Schema_generated.js` + ```js + /** + * @enum + */ + org.apache.arrow.flatbuf.MetadataVersion = { + /** + * 0.1.0 + */ + V1: 0, 0: 'V1', + + /** + * 0.2.0 + */ + V2: 1, 1: 'V2', + + /** + * 0.3.0 -> 0.7.1 + */ + V3: 2, 2: 'V3', + + /** + * >= 0.8.0 + */ + V4: 3, 3: 'V4' + }; /** - * @const - * @namespace - */ - var org = org || {}; - ``` + * @enum + */ + org.apache.arrow.flatbuf.UnionMode = { + Sparse: 0, 0: 'Sparse', + Dense: 1, 1: 'Dense', + }; - `File_generated.js`: + /** + * @enum + */ + org.apache.arrow.flatbuf.Precision = { + HALF: 0, 0: 'HALF', + SINGLE: 1, 1: 'SINGLE', + DOUBLE: 2, 2: 'DOUBLE', + }; - ```js - // automatically generated by the FlatBuffers compiler, do not modify - goog.module("module$targets$es5$cls$format$File_generated"); - goog.module.declareLegacyNamespace(); - var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated"); /** - * @const - * @namespace - */ - var org = Schema_.org; - ``` + * @enum + */ + org.apache.arrow.flatbuf.DateUnit = { + DAY: 0, 0: 'DAY', + MILLISECOND: 1, 1: 'MILLISECOND', + }; - `Message_generated.js`: + /** + * @enum + */ + org.apache.arrow.flatbuf.TimeUnit = { + SECOND: 0, 0: 'SECOND', + MILLISECOND: 1, 1: 'MILLISECOND', + MICROSECOND: 2, 2: 'MICROSECOND', + NANOSECOND: 3, 3: 'NANOSECOND', + }; - ```js - // automatically generated by the FlatBuffers compiler, do not modify - goog.module("module$targets$es5$cls$format$Message_generated"); - goog.module.declareLegacyNamespace(); - var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated"); /** - * @const - * @namespace - */ - var org = Schema_.org; - ``` + * @enum + */ + org.apache.arrow.flatbuf.IntervalUnit = { + YEAR_MONTH: 0, 0: 'YEAR_MONTH', + DAY_TIME: 1, 1: 'DAY_TIME', + }; -1. Replace the last line's export declaration + /** + * ---------------------------------------------------------------------- + * Top-level Type value, enabling extensible type-specific metadata. We can + * add new logical types to Type without breaking backwards compatibility + * + * @enum + */ + org.apache.arrow.flatbuf.Type = { + NONE: 0, 0: 'NONE', + Null: 1, 1: 'Null', + Int: 2, 2: 'Int', + FloatingPoint: 3, 3: 'FloatingPoint', + Binary: 4, 4: 'Binary', + Utf8: 5, 5: 'Utf8', + Bool: 6, 6: 'Bool', + Decimal: 7, 7: 'Decimal', + Date: 8, 8: 'Date', + Time: 9, 9: 'Time', + Timestamp: 10, 10: 'Timestamp', + Interval: 11, 11: 'Interval', + List: 12, 12: 'List', + Struct_: 13, 13: 'Struct_', + Union: 14, 14: 'Union', + FixedSizeBinary: 15, 15: 'FixedSizeBinary', + FixedSizeList: 16, 16: 'FixedSizeList', + Map: 17, 17: 'Map' + }; - The last line of each file is: + /** + * ---------------------------------------------------------------------- + * The possible types of a vector + * + * @enum + */ + org.apache.arrow.flatbuf.VectorType = { + /** + * used in List type, Dense Union and variable length primitive types (String, Binary) + */ + OFFSET: 0, 0: 'OFFSET', - ```js - // Exports for Node.js and RequireJS - this.org = org; - ``` + /** + * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector + */ + DATA: 1, 1: 'DATA', - This should instead read: + /** + * Bit vector indicating if each value is null + */ + VALIDITY: 2, 2: 'VALIDITY', - ```js - // Exports for Node.js and RequireJS - exports.org = org; - ``` \ No newline at end of file + /** + * Type vector used in Union type + */ + TYPE: 3, 3: 'TYPE' + }; + + /** + * ---------------------------------------------------------------------- + * Endianness of the platform producing the data + * + * @enum + */ + org.apache.arrow.flatbuf.Endianness = { + Little: 0, 0: 'Little', + Big: 1, 1: 'Big', + }; + ``` diff --git a/js/LICENSE b/js/LICENSE deleted file mode 100644 index 02e7948085055..0000000000000 --- a/js/LICENSE +++ /dev/null @@ -1,39 +0,0 @@ -## 3rd-party licenses for code that has been adapted for the Arrow JavaScript - library - --------------------------------------------------------------------------------- - -This project includes code from the FlatBuffers project - -Copyright 2014 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the tslib project - -Copyright 2015 Microsoft Corporation. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - diff --git a/js/README.md b/js/README.md index bee3a9c7d42db..e07be6bc0aaaf 100644 --- a/js/README.md +++ b/js/README.md @@ -22,7 +22,7 @@ [![Build Status](https://travis-ci.org/apache/arrow.svg?branch=master)](https://travis-ci.org/apache/arrow) [![Coverage Status](https://coveralls.io/repos/github/apache/arrow/badge.svg)](https://coveralls.io/github/apache/arrow) -Arrow is a set of technologies that enable big-data systems to process and move data fast. +Arrow is a set of technologies that enable big-data systems to process and transfer data quickly. ## install [apache-arrow from npm](https://www.npmjs.com/package/apache-arrow) @@ -50,7 +50,7 @@ import { readFileSync } from 'fs'; import { Table } from 'apache-arrow'; const arrow = readFileSync('simple.arrow'); -const table = Table.from(arrow); +const table = Table.from([arrow]); console.log(table.toString()); @@ -70,7 +70,7 @@ null, null, null import { readFileSync } from 'fs'; import { Table } from 'apache-arrow'; -const table = Table.from(...[ +const table = Table.from([ 'latlong/schema.arrow', 'latlong/records.arrow' ].map((file) => readFileSync(file))); @@ -93,12 +93,12 @@ console.log(table.toString()); import { readFileSync } from 'fs'; import { Table } from 'apache-arrow'; -const table = Table.from(...[ +const table = Table.from([ 'latlong/schema.arrow', 'latlong/records.arrow' ].map(readFileSync)); -const column = table.getColumn('origin_lat'); +const column = table.col('origin_lat'); const typed = column.slice(); assert(typed instanceof Float32Array); @@ -135,7 +135,7 @@ MapD.open(host, port) // Create Arrow Table from results Table.from(schema, records)) .map((table) => - // Stringify the table to CSV + // Stringify the table to CSV with row numbers table.toString({ index: true })) .subscribe((csvStr) => console.log(csvStr)); diff --git a/js/closure-compiler-scripts/flatbuffers.js b/js/closure-compiler-scripts/flatbuffers.js deleted file mode 100644 index e51a4a039536f..0000000000000 --- a/js/closure-compiler-scripts/flatbuffers.js +++ /dev/null @@ -1,1204 +0,0 @@ -/** - * closure-compiler-friendly flatbuffers - * copied from node_modules/flatbuffers/js/flatbuffers.js - * update as needed - */ - - /// @file -/// @addtogroup flatbuffers_javascript_api -/// @{ -/// @cond FLATBUFFERS_INTERNAL - -goog.module("module$flatbuffers"); -goog.module.declareLegacyNamespace(); -/** - * @fileoverview - * - * Need to suppress 'global this' error so the Node.js export line doesn't cause - * closure compile to error out. - * @suppress {globalThis} - */ - -/** - * @const - * @namespace - */ -var flatbuffers = {}; - -/** - * @typedef {number} - */ -flatbuffers.Offset; - -/** - * @typedef {{ - * bb: flatbuffers.ByteBuffer, - * bb_pos: number - * }} - */ -flatbuffers.Table; - -/** - * @type {number} - * @const - */ -flatbuffers.SIZEOF_SHORT = 2; - -/** - * @type {number} - * @const - */ -flatbuffers.SIZEOF_INT = 4; - -/** - * @type {number} - * @const - */ -flatbuffers.FILE_IDENTIFIER_LENGTH = 4; - -/** - * @enum {number} - */ -flatbuffers.Encoding = { - UTF8_BYTES: 1, - UTF16_STRING: 2 -}; - -/** - * @type {Int32Array} - * @const - */ -flatbuffers.int32 = new Int32Array(2); - -/** - * @type {Float32Array} - * @const - */ -flatbuffers.float32 = new Float32Array(flatbuffers.int32.buffer); - -/** - * @type {Float64Array} - * @const - */ -flatbuffers.float64 = new Float64Array(flatbuffers.int32.buffer); - -/** - * @type {boolean} - * @const - */ -flatbuffers.isLittleEndian = new Uint16Array(new Uint8Array([1, 0]).buffer)[0] === 1; - -//////////////////////////////////////////////////////////////////////////////// - -/** - * @constructor - * @param {number} low - * @param {number} high - */ -flatbuffers.Long = function(low, high) { - /** - * @type {number} - * @const - */ - this.low = low | 0; - - /** - * @type {number} - * @const - */ - this.high = high | 0; -}; - -/** - * @param {number} low - * @param {number} high - * @returns {flatbuffers.Long} - */ -flatbuffers.Long.create = function(low, high) { - // Special-case zero to avoid GC overhead for default values - return low == 0 && high == 0 ? flatbuffers.Long.ZERO : new flatbuffers.Long(low, high); -}; - -/** - * @returns {number} - */ -flatbuffers.Long.prototype.toFloat64 = function() { - return (this.low >>> 0) + this.high * 0x100000000; -}; - -/** - * @param {flatbuffers.Long} other - * @returns {boolean} - */ -flatbuffers.Long.prototype.equals = function(other) { - return this.low == other.low && this.high == other.high; -}; - -/** - * @type {flatbuffers.Long} - * @const - */ -flatbuffers.Long.ZERO = new flatbuffers.Long(0, 0); - -/// @endcond -//////////////////////////////////////////////////////////////////////////////// -/** - * Create a FlatBufferBuilder. - * - * @constructor - * @param {number=} opt_initial_size - */ -flatbuffers.Builder = function(opt_initial_size) { - if (!opt_initial_size) { - var initial_size = 1024; - } else { - var initial_size = opt_initial_size; - } - - /** - * @type {flatbuffers.ByteBuffer} - * @private - */ - this.bb = flatbuffers.ByteBuffer.allocate(initial_size); - - /** - * Remaining space in the ByteBuffer. - * - * @type {number} - * @private - */ - this.space = initial_size; - - /** - * Minimum alignment encountered so far. - * - * @type {number} - * @private - */ - this.minalign = 1; - - /** - * The vtable for the current table. - * - * @type {Array.} - * @private - */ - this.vtable = null; - - /** - * The amount of fields we're actually using. - * - * @type {number} - * @private - */ - this.vtable_in_use = 0; - - /** - * Whether we are currently serializing a table. - * - * @type {boolean} - * @private - */ - this.isNested = false; - - /** - * Starting offset of the current struct/table. - * - * @type {number} - * @private - */ - this.object_start = 0; - - /** - * List of offsets of all vtables. - * - * @type {Array.} - * @private - */ - this.vtables = []; - - /** - * For the current vector being built. - * - * @type {number} - * @private - */ - this.vector_num_elems = 0; - - /** - * False omits default values from the serialized data - * - * @type {boolean} - * @private - */ - this.force_defaults = false; -}; - -/** - * In order to save space, fields that are set to their default value - * don't get serialized into the buffer. Forcing defaults provides a - * way to manually disable this optimization. - * - * @param {boolean} forceDefaults true always serializes default values - */ -flatbuffers.Builder.prototype.forceDefaults = function(forceDefaults) { - this.force_defaults = forceDefaults; -}; - -/** - * Get the ByteBuffer representing the FlatBuffer. Only call this after you've - * called finish(). The actual data starts at the ByteBuffer's current position, - * not necessarily at 0. - * - * @returns {flatbuffers.ByteBuffer} - */ -flatbuffers.Builder.prototype.dataBuffer = function() { - return this.bb; -}; - -/** - * Get the bytes representing the FlatBuffer. Only call this after you've - * called finish(). - * - * @returns {Uint8Array} - */ -flatbuffers.Builder.prototype.asUint8Array = function() { - return this.bb.bytes().subarray(this.bb.position(), this.bb.position() + this.offset()); -}; - -/// @cond FLATBUFFERS_INTERNAL -/** - * Prepare to write an element of `size` after `additional_bytes` have been - * written, e.g. if you write a string, you need to align such the int length - * field is aligned to 4 bytes, and the string data follows it directly. If all - * you need to do is alignment, `additional_bytes` will be 0. - * - * @param {number} size This is the of the new element to write - * @param {number} additional_bytes The padding size - */ -flatbuffers.Builder.prototype.prep = function(size, additional_bytes) { - // Track the biggest thing we've ever aligned to. - if (size > this.minalign) { - this.minalign = size; - } - - // Find the amount of alignment needed such that `size` is properly - // aligned after `additional_bytes` - var align_size = ((~(this.bb.capacity() - this.space + additional_bytes)) + 1) & (size - 1); - - // Reallocate the buffer if needed. - while (this.space < align_size + size + additional_bytes) { - var old_buf_size = this.bb.capacity(); - this.bb = flatbuffers.Builder.growByteBuffer(this.bb); - this.space += this.bb.capacity() - old_buf_size; - } - - this.pad(align_size); -}; - -/** - * @param {number} byte_size - */ -flatbuffers.Builder.prototype.pad = function(byte_size) { - for (var i = 0; i < byte_size; i++) { - this.bb.writeInt8(--this.space, 0); - } -}; - -/** - * @param {number} value - */ -flatbuffers.Builder.prototype.writeInt8 = function(value) { - this.bb.writeInt8(this.space -= 1, value); -}; - -/** - * @param {number} value - */ -flatbuffers.Builder.prototype.writeInt16 = function(value) { - this.bb.writeInt16(this.space -= 2, value); -}; - -/** - * @param {number} value - */ -flatbuffers.Builder.prototype.writeInt32 = function(value) { - this.bb.writeInt32(this.space -= 4, value); -}; - -/** - * @param {flatbuffers.Long} value - */ -flatbuffers.Builder.prototype.writeInt64 = function(value) { - this.bb.writeInt64(this.space -= 8, value); -}; - -/** - * @param {number} value - */ -flatbuffers.Builder.prototype.writeFloat32 = function(value) { - this.bb.writeFloat32(this.space -= 4, value); -}; - -/** - * @param {number} value - */ -flatbuffers.Builder.prototype.writeFloat64 = function(value) { - this.bb.writeFloat64(this.space -= 8, value); -}; -/// @endcond - -/** - * Add an `int8` to the buffer, properly aligned, and grows the buffer (if necessary). - * @param {number} value The `int8` to add the the buffer. - */ -flatbuffers.Builder.prototype.addInt8 = function(value) { - this.prep(1, 0); - this.writeInt8(value); -}; - -/** - * Add an `int16` to the buffer, properly aligned, and grows the buffer (if necessary). - * @param {number} value The `int16` to add the the buffer. - */ -flatbuffers.Builder.prototype.addInt16 = function(value) { - this.prep(2, 0); - this.writeInt16(value); -}; - -/** - * Add an `int32` to the buffer, properly aligned, and grows the buffer (if necessary). - * @param {number} value The `int32` to add the the buffer. - */ -flatbuffers.Builder.prototype.addInt32 = function(value) { - this.prep(4, 0); - this.writeInt32(value); -}; - -/** - * Add an `int64` to the buffer, properly aligned, and grows the buffer (if necessary). - * @param {flatbuffers.Long} value The `int64` to add the the buffer. - */ -flatbuffers.Builder.prototype.addInt64 = function(value) { - this.prep(8, 0); - this.writeInt64(value); -}; - -/** - * Add a `float32` to the buffer, properly aligned, and grows the buffer (if necessary). - * @param {number} value The `float32` to add the the buffer. - */ -flatbuffers.Builder.prototype.addFloat32 = function(value) { - this.prep(4, 0); - this.writeFloat32(value); -}; - -/** - * Add a `float64` to the buffer, properly aligned, and grows the buffer (if necessary). - * @param {number} value The `float64` to add the the buffer. - */ -flatbuffers.Builder.prototype.addFloat64 = function(value) { - this.prep(8, 0); - this.writeFloat64(value); -}; - -/// @cond FLATBUFFERS_INTERNAL -/** - * @param {number} voffset - * @param {number} value - * @param {number} defaultValue - */ -flatbuffers.Builder.prototype.addFieldInt8 = function(voffset, value, defaultValue) { - if (this.force_defaults || value != defaultValue) { - this.addInt8(value); - this.slot(voffset); - } -}; - -/** - * @param {number} voffset - * @param {number} value - * @param {number} defaultValue - */ -flatbuffers.Builder.prototype.addFieldInt16 = function(voffset, value, defaultValue) { - if (this.force_defaults || value != defaultValue) { - this.addInt16(value); - this.slot(voffset); - } -}; - -/** - * @param {number} voffset - * @param {number} value - * @param {number} defaultValue - */ -flatbuffers.Builder.prototype.addFieldInt32 = function(voffset, value, defaultValue) { - if (this.force_defaults || value != defaultValue) { - this.addInt32(value); - this.slot(voffset); - } -}; - -/** - * @param {number} voffset - * @param {flatbuffers.Long} value - * @param {flatbuffers.Long} defaultValue - */ -flatbuffers.Builder.prototype.addFieldInt64 = function(voffset, value, defaultValue) { - if (this.force_defaults || !value.equals(defaultValue)) { - this.addInt64(value); - this.slot(voffset); - } -}; - -/** - * @param {number} voffset - * @param {number} value - * @param {number} defaultValue - */ -flatbuffers.Builder.prototype.addFieldFloat32 = function(voffset, value, defaultValue) { - if (this.force_defaults || value != defaultValue) { - this.addFloat32(value); - this.slot(voffset); - } -}; - -/** - * @param {number} voffset - * @param {number} value - * @param {number} defaultValue - */ -flatbuffers.Builder.prototype.addFieldFloat64 = function(voffset, value, defaultValue) { - if (this.force_defaults || value != defaultValue) { - this.addFloat64(value); - this.slot(voffset); - } -}; - -/** - * @param {number} voffset - * @param {flatbuffers.Offset} value - * @param {flatbuffers.Offset} defaultValue - */ -flatbuffers.Builder.prototype.addFieldOffset = function(voffset, value, defaultValue) { - if (this.force_defaults || value != defaultValue) { - this.addOffset(value); - this.slot(voffset); - } -}; - -/** - * Structs are stored inline, so nothing additional is being added. `d` is always 0. - * - * @param {number} voffset - * @param {flatbuffers.Offset} value - * @param {flatbuffers.Offset} defaultValue - */ -flatbuffers.Builder.prototype.addFieldStruct = function(voffset, value, defaultValue) { - if (value != defaultValue) { - this.nested(value); - this.slot(voffset); - } -}; - -/** - * Structures are always stored inline, they need to be created right - * where they're used. You'll get this assertion failure if you - * created it elsewhere. - * - * @param {flatbuffers.Offset} obj The offset of the created object - */ -flatbuffers.Builder.prototype.nested = function(obj) { - if (obj != this.offset()) { - throw new Error('FlatBuffers: struct must be serialized inline.'); - } -}; - -/** - * Should not be creating any other object, string or vector - * while an object is being constructed - */ -flatbuffers.Builder.prototype.notNested = function() { - if (this.isNested) { - throw new Error('FlatBuffers: object serialization must not be nested.'); - } -}; - -/** - * Set the current vtable at `voffset` to the current location in the buffer. - * - * @param {number} voffset - */ -flatbuffers.Builder.prototype.slot = function(voffset) { - this.vtable[voffset] = this.offset(); -}; - -/** - * @returns {flatbuffers.Offset} Offset relative to the end of the buffer. - */ -flatbuffers.Builder.prototype.offset = function() { - return this.bb.capacity() - this.space; -}; - -/** - * Doubles the size of the backing ByteBuffer and copies the old data towards - * the end of the new buffer (since we build the buffer backwards). - * - * @param {flatbuffers.ByteBuffer} bb The current buffer with the existing data - * @returns {flatbuffers.ByteBuffer} A new byte buffer with the old data copied - * to it. The data is located at the end of the buffer. - * - * uint8Array.set() formally takes {Array|ArrayBufferView}, so to pass - * it a uint8Array we need to suppress the type check: - * @suppress {checkTypes} - */ -flatbuffers.Builder.growByteBuffer = function(bb) { - var old_buf_size = bb.capacity(); - - // Ensure we don't grow beyond what fits in an int. - if (old_buf_size & 0xC0000000) { - throw new Error('FlatBuffers: cannot grow buffer beyond 2 gigabytes.'); - } - - var new_buf_size = old_buf_size << 1; - var nbb = flatbuffers.ByteBuffer.allocate(new_buf_size); - nbb.setPosition(new_buf_size - old_buf_size); - nbb.bytes().set(bb.bytes(), new_buf_size - old_buf_size); - return nbb; -}; -/// @endcond - -/** - * Adds on offset, relative to where it will be written. - * - * @param {flatbuffers.Offset} offset The offset to add. - */ -flatbuffers.Builder.prototype.addOffset = function(offset) { - this.prep(flatbuffers.SIZEOF_INT, 0); // Ensure alignment is already done. - this.writeInt32(this.offset() - offset + flatbuffers.SIZEOF_INT); -}; - -/// @cond FLATBUFFERS_INTERNAL -/** - * Start encoding a new object in the buffer. Users will not usually need to - * call this directly. The FlatBuffers compiler will generate helper methods - * that call this method internally. - * - * @param {number} numfields - */ -flatbuffers.Builder.prototype.startObject = function(numfields) { - this.notNested(); - if (this.vtable == null) { - this.vtable = []; - } - this.vtable_in_use = numfields; - for (var i = 0; i < numfields; i++) { - this.vtable[i] = 0; // This will push additional elements as needed - } - this.isNested = true; - this.object_start = this.offset(); -}; - -/** - * Finish off writing the object that is under construction. - * - * @returns {flatbuffers.Offset} The offset to the object inside `dataBuffer` - */ -flatbuffers.Builder.prototype.endObject = function() { - if (this.vtable == null || !this.isNested) { - throw new Error('FlatBuffers: endObject called without startObject'); - } - - this.addInt32(0); - var vtableloc = this.offset(); - - // Write out the current vtable. - for (var i = this.vtable_in_use - 1; i >= 0; i--) { - // Offset relative to the start of the table. - this.addInt16(this.vtable[i] != 0 ? vtableloc - this.vtable[i] : 0); - } - - var standard_fields = 2; // The fields below: - this.addInt16(vtableloc - this.object_start); - this.addInt16((this.vtable_in_use + standard_fields) * flatbuffers.SIZEOF_SHORT); - - // Search for an existing vtable that matches the current one. - var existing_vtable = 0; -outer_loop: - for (var i = 0; i < this.vtables.length; i++) { - var vt1 = this.bb.capacity() - this.vtables[i]; - var vt2 = this.space; - var len = this.bb.readInt16(vt1); - if (len == this.bb.readInt16(vt2)) { - for (var j = flatbuffers.SIZEOF_SHORT; j < len; j += flatbuffers.SIZEOF_SHORT) { - if (this.bb.readInt16(vt1 + j) != this.bb.readInt16(vt2 + j)) { - continue outer_loop; - } - } - existing_vtable = this.vtables[i]; - break; - } - } - - if (existing_vtable) { - // Found a match: - // Remove the current vtable. - this.space = this.bb.capacity() - vtableloc; - - // Point table to existing vtable. - this.bb.writeInt32(this.space, existing_vtable - vtableloc); - } else { - // No match: - // Add the location of the current vtable to the list of vtables. - this.vtables.push(this.offset()); - - // Point table to current vtable. - this.bb.writeInt32(this.bb.capacity() - vtableloc, this.offset() - vtableloc); - } - - this.isNested = false; - return vtableloc; -}; -/// @endcond - -/** - * Finalize a buffer, poiting to the given `root_table`. - * - * @param {flatbuffers.Offset} root_table - * @param {string=} opt_file_identifier - */ -flatbuffers.Builder.prototype.finish = function(root_table, opt_file_identifier) { - if (opt_file_identifier) { - var file_identifier = opt_file_identifier; - this.prep(this.minalign, flatbuffers.SIZEOF_INT + - flatbuffers.FILE_IDENTIFIER_LENGTH); - if (file_identifier.length != flatbuffers.FILE_IDENTIFIER_LENGTH) { - throw new Error('FlatBuffers: file identifier must be length ' + - flatbuffers.FILE_IDENTIFIER_LENGTH); - } - for (var i = flatbuffers.FILE_IDENTIFIER_LENGTH - 1; i >= 0; i--) { - this.writeInt8(file_identifier.charCodeAt(i)); - } - } - this.prep(this.minalign, flatbuffers.SIZEOF_INT); - this.addOffset(root_table); - this.bb.setPosition(this.space); -}; - -/// @cond FLATBUFFERS_INTERNAL -/** - * This checks a required field has been set in a given table that has - * just been constructed. - * - * @param {flatbuffers.Offset} table - * @param {number} field - */ -flatbuffers.Builder.prototype.requiredField = function(table, field) { - var table_start = this.bb.capacity() - table; - var vtable_start = table_start - this.bb.readInt32(table_start); - var ok = this.bb.readInt16(vtable_start + field) != 0; - - // If this fails, the caller will show what field needs to be set. - if (!ok) { - throw new Error('FlatBuffers: field ' + field + ' must be set'); - } -}; - -/** - * Start a new array/vector of objects. Users usually will not call - * this directly. The FlatBuffers compiler will create a start/end - * method for vector types in generated code. - * - * @param {number} elem_size The size of each element in the array - * @param {number} num_elems The number of elements in the array - * @param {number} alignment The alignment of the array - */ -flatbuffers.Builder.prototype.startVector = function(elem_size, num_elems, alignment) { - this.notNested(); - this.vector_num_elems = num_elems; - this.prep(flatbuffers.SIZEOF_INT, elem_size * num_elems); - this.prep(alignment, elem_size * num_elems); // Just in case alignment > int. -}; - -/** - * Finish off the creation of an array and all its elements. The array must be - * created with `startVector`. - * - * @returns {flatbuffers.Offset} The offset at which the newly created array - * starts. - */ -flatbuffers.Builder.prototype.endVector = function() { - this.writeInt32(this.vector_num_elems); - return this.offset(); -}; -/// @endcond - -/** - * Encode the string `s` in the buffer using UTF-8. If a Uint8Array is passed - * instead of a string, it is assumed to contain valid UTF-8 encoded data. - * - * @param {string|Uint8Array} s The string to encode - * @return {flatbuffers.Offset} The offset in the buffer where the encoded string starts - */ -flatbuffers.Builder.prototype.createString = function(s) { - if (s instanceof Uint8Array) { - var utf8 = s; - } else { - var utf8 = []; - var i = 0; - - while (i < s.length) { - var codePoint; - - // Decode UTF-16 - var a = s.charCodeAt(i++); - if (a < 0xD800 || a >= 0xDC00) { - codePoint = a; - } else { - var b = s.charCodeAt(i++); - codePoint = (a << 10) + b + (0x10000 - (0xD800 << 10) - 0xDC00); - } - - // Encode UTF-8 - if (codePoint < 0x80) { - utf8.push(codePoint); - } else { - if (codePoint < 0x800) { - utf8.push(((codePoint >> 6) & 0x1F) | 0xC0); - } else { - if (codePoint < 0x10000) { - utf8.push(((codePoint >> 12) & 0x0F) | 0xE0); - } else { - utf8.push( - ((codePoint >> 18) & 0x07) | 0xF0, - ((codePoint >> 12) & 0x3F) | 0x80); - } - utf8.push(((codePoint >> 6) & 0x3F) | 0x80); - } - utf8.push((codePoint & 0x3F) | 0x80); - } - } - } - - this.addInt8(0); - this.startVector(1, utf8.length, 1); - this.bb.setPosition(this.space -= utf8.length); - for (var i = 0, offset = this.space, bytes = this.bb.bytes(); i < utf8.length; i++) { - bytes[offset++] = utf8[i]; - } - return this.endVector(); -}; - -/** - * A helper function to avoid generated code depending on this file directly. - * - * @param {number} low - * @param {number} high - * @returns {flatbuffers.Long} - */ -flatbuffers.Builder.prototype.createLong = function(low, high) { - return flatbuffers.Long.create(low, high); -}; -//////////////////////////////////////////////////////////////////////////////// -/// @cond FLATBUFFERS_INTERNAL -/** - * Create a new ByteBuffer with a given array of bytes (`Uint8Array`). - * - * @constructor - * @param {Uint8Array} bytes - */ -flatbuffers.ByteBuffer = function(bytes) { - /** - * @type {Uint8Array} - * @private - */ - this.bytes_ = bytes; - - /** - * @type {number} - * @private - */ - this.position_ = 0; -}; - -/** - * Create and allocate a new ByteBuffer with a given size. - * - * @param {number} byte_size - * @returns {flatbuffers.ByteBuffer} - */ -flatbuffers.ByteBuffer.allocate = function(byte_size) { - return new flatbuffers.ByteBuffer(new Uint8Array(byte_size)); -}; - -/** - * Get the underlying `Uint8Array`. - * - * @returns {Uint8Array} - */ -flatbuffers.ByteBuffer.prototype.bytes = function() { - return this.bytes_; -}; - -/** - * Get the buffer's position. - * - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.position = function() { - return this.position_; -}; - -/** - * Set the buffer's position. - * - * @param {number} position - */ -flatbuffers.ByteBuffer.prototype.setPosition = function(position) { - this.position_ = position; -}; - -/** - * Get the buffer's capacity. - * - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.capacity = function() { - return this.bytes_.length; -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readInt8 = function(offset) { - return this.readUint8(offset) << 24 >> 24; -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readUint8 = function(offset) { - return this.bytes_[offset]; -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readInt16 = function(offset) { - return this.readUint16(offset) << 16 >> 16; -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readUint16 = function(offset) { - return this.bytes_[offset] | this.bytes_[offset + 1] << 8; -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readInt32 = function(offset) { - return this.bytes_[offset] | this.bytes_[offset + 1] << 8 | this.bytes_[offset + 2] << 16 | this.bytes_[offset + 3] << 24; -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readUint32 = function(offset) { - return this.readInt32(offset) >>> 0; -}; - -/** - * @param {number} offset - * @returns {flatbuffers.Long} - */ -flatbuffers.ByteBuffer.prototype.readInt64 = function(offset) { - return new flatbuffers.Long(this.readInt32(offset), this.readInt32(offset + 4)); -}; - -/** - * @param {number} offset - * @returns {flatbuffers.Long} - */ -flatbuffers.ByteBuffer.prototype.readUint64 = function(offset) { - return new flatbuffers.Long(this.readUint32(offset), this.readUint32(offset + 4)); -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readFloat32 = function(offset) { - flatbuffers.int32[0] = this.readInt32(offset); - return flatbuffers.float32[0]; -}; - -/** - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.readFloat64 = function(offset) { - flatbuffers.int32[flatbuffers.isLittleEndian ? 0 : 1] = this.readInt32(offset); - flatbuffers.int32[flatbuffers.isLittleEndian ? 1 : 0] = this.readInt32(offset + 4); - return flatbuffers.float64[0]; -}; - -/** - * @param {number} offset - * @param {number|boolean} value - */ -flatbuffers.ByteBuffer.prototype.writeInt8 = function(offset, value) { - this.bytes_[offset] = /** @type {number} */(value); -}; - -/** - * @param {number} offset - * @param {number} value - */ -flatbuffers.ByteBuffer.prototype.writeUint8 = function(offset, value) { - this.bytes_[offset] = value; -}; - -/** - * @param {number} offset - * @param {number} value - */ -flatbuffers.ByteBuffer.prototype.writeInt16 = function(offset, value) { - this.bytes_[offset] = value; - this.bytes_[offset + 1] = value >> 8; -}; - -/** - * @param {number} offset - * @param {number} value - */ -flatbuffers.ByteBuffer.prototype.writeUint16 = function(offset, value) { - this.bytes_[offset] = value; - this.bytes_[offset + 1] = value >> 8; -}; - -/** - * @param {number} offset - * @param {number} value - */ -flatbuffers.ByteBuffer.prototype.writeInt32 = function(offset, value) { - this.bytes_[offset] = value; - this.bytes_[offset + 1] = value >> 8; - this.bytes_[offset + 2] = value >> 16; - this.bytes_[offset + 3] = value >> 24; -}; - -/** - * @param {number} offset - * @param {number} value - */ -flatbuffers.ByteBuffer.prototype.writeUint32 = function(offset, value) { - this.bytes_[offset] = value; - this.bytes_[offset + 1] = value >> 8; - this.bytes_[offset + 2] = value >> 16; - this.bytes_[offset + 3] = value >> 24; -}; - -/** - * @param {number} offset - * @param {flatbuffers.Long} value - */ -flatbuffers.ByteBuffer.prototype.writeInt64 = function(offset, value) { - this.writeInt32(offset, value.low); - this.writeInt32(offset + 4, value.high); -}; - -/** - * @param {number} offset - * @param {flatbuffers.Long} value - */ -flatbuffers.ByteBuffer.prototype.writeUint64 = function(offset, value) { - this.writeUint32(offset, value.low); - this.writeUint32(offset + 4, value.high); -}; - -/** - * @param {number} offset - * @param {number} value - */ -flatbuffers.ByteBuffer.prototype.writeFloat32 = function(offset, value) { - flatbuffers.float32[0] = value; - this.writeInt32(offset, flatbuffers.int32[0]); -}; - -/** - * @param {number} offset - * @param {number} value - */ -flatbuffers.ByteBuffer.prototype.writeFloat64 = function(offset, value) { - flatbuffers.float64[0] = value; - this.writeInt32(offset, flatbuffers.int32[flatbuffers.isLittleEndian ? 0 : 1]); - this.writeInt32(offset + 4, flatbuffers.int32[flatbuffers.isLittleEndian ? 1 : 0]); -}; - -/** - * Look up a field in the vtable, return an offset into the object, or 0 if the - * field is not present. - * - * @param {number} bb_pos - * @param {number} vtable_offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.__offset = function(bb_pos, vtable_offset) { - var vtable = bb_pos - this.readInt32(bb_pos); - return vtable_offset < this.readInt16(vtable) ? this.readInt16(vtable + vtable_offset) : 0; -}; - -/** - * Initialize any Table-derived type to point to the union at the given offset. - * - * @param {flatbuffers.Table} t - * @param {number} offset - * @returns {flatbuffers.Table} - */ -flatbuffers.ByteBuffer.prototype.__union = function(t, offset) { - t.bb_pos = offset + this.readInt32(offset); - t.bb = this; - return t; -}; - -/** - * Create a JavaScript string from UTF-8 data stored inside the FlatBuffer. - * This allocates a new string and converts to wide chars upon each access. - * - * To avoid the conversion to UTF-16, pass flatbuffers.Encoding.UTF8_BYTES as - * the "optionalEncoding" argument. This is useful for avoiding conversion to - * and from UTF-16 when the data will just be packaged back up in another - * FlatBuffer later on. - * - * @param {number} offset - * @param {flatbuffers.Encoding=} opt_encoding Defaults to UTF16_STRING - * @returns {string|Uint8Array} - */ -flatbuffers.ByteBuffer.prototype.__string = function(offset, opt_encoding) { - offset += this.readInt32(offset); - - var length = this.readInt32(offset); - var result = ''; - var i = 0; - - offset += flatbuffers.SIZEOF_INT; - - if (opt_encoding === flatbuffers.Encoding.UTF8_BYTES) { - return this.bytes_.subarray(offset, offset + length); - } - - while (i < length) { - var codePoint; - - // Decode UTF-8 - var a = this.readUint8(offset + i++); - if (a < 0xC0) { - codePoint = a; - } else { - var b = this.readUint8(offset + i++); - if (a < 0xE0) { - codePoint = - ((a & 0x1F) << 6) | - (b & 0x3F); - } else { - var c = this.readUint8(offset + i++); - if (a < 0xF0) { - codePoint = - ((a & 0x0F) << 12) | - ((b & 0x3F) << 6) | - (c & 0x3F); - } else { - var d = this.readUint8(offset + i++); - codePoint = - ((a & 0x07) << 18) | - ((b & 0x3F) << 12) | - ((c & 0x3F) << 6) | - (d & 0x3F); - } - } - } - - // Encode UTF-16 - if (codePoint < 0x10000) { - result += String.fromCharCode(codePoint); - } else { - codePoint -= 0x10000; - result += String.fromCharCode( - (codePoint >> 10) + 0xD800, - (codePoint & ((1 << 10) - 1)) + 0xDC00); - } - } - - return result; -}; - -/** - * Retrieve the relative offset stored at "offset" - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.__indirect = function(offset) { - return offset + this.readInt32(offset); -}; - -/** - * Get the start of data of a vector whose offset is stored at "offset" in this object. - * - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.__vector = function(offset) { - return offset + this.readInt32(offset) + flatbuffers.SIZEOF_INT; // data starts after the length -}; - -/** - * Get the length of a vector whose offset is stored at "offset" in this object. - * - * @param {number} offset - * @returns {number} - */ -flatbuffers.ByteBuffer.prototype.__vector_len = function(offset) { - return this.readInt32(offset + this.readInt32(offset)); -}; - -/** - * @param {string} ident - * @returns {boolean} - */ -flatbuffers.ByteBuffer.prototype.__has_identifier = function(ident) { - if (ident.length != flatbuffers.FILE_IDENTIFIER_LENGTH) { - throw new Error('FlatBuffers: file identifier must be length ' + - flatbuffers.FILE_IDENTIFIER_LENGTH); - } - for (var i = 0; i < flatbuffers.FILE_IDENTIFIER_LENGTH; i++) { - if (ident.charCodeAt(i) != this.readInt8(this.position_ + flatbuffers.SIZEOF_INT + i)) { - return false; - } - } - return true; -}; - -/** - * A helper function to avoid generated code depending on this file directly. - * - * @param {number} low - * @param {number} high - * @returns {flatbuffers.Long} - */ -flatbuffers.ByteBuffer.prototype.createLong = function(low, high) { - return flatbuffers.Long.create(low, high); -}; - -// Exports for Node.js and RequireJS -exports.flatbuffers = flatbuffers; - -/// @endcond -/// @} diff --git a/js/closure-compiler-scripts/text-encoding.js b/js/closure-compiler-scripts/text-encoding.js deleted file mode 100644 index 398883ab9b4be..0000000000000 --- a/js/closure-compiler-scripts/text-encoding.js +++ /dev/null @@ -1,648 +0,0 @@ -/** - * closure-compiler-friendly text-encoding-utf-8 - * copied from node_modules/text-encoding-utf-8/lib/encoding.cjs.js - * update as needed - */ - - // This is free and unencumbered software released into the public domain. -// See LICENSE.md for more information. - -// -// Utilities -// - -goog.module("module$text_encoding_utf_8"); -goog.module.declareLegacyNamespace(); -/** - * @param {number} a The number to test. - * @param {number} min The minimum value in the range, inclusive. - * @param {number} max The maximum value in the range, inclusive. - * @return {boolean} True if a >= min and a <= max. - */ -function inRange(a, min, max) { - return min <= a && a <= max; -} - -/** - * @param {*} o - * @return {Object} - */ -function ToDictionary(o) { - if (o === undefined) return {}; - if (o === Object(o)) return o; - throw TypeError('Could not convert argument to dictionary'); -} - -/** - * @param {string} string Input string of UTF-16 code units. - * @return {!Array.} Code points. - */ -function stringToCodePoints(string) { - // https://heycam.github.io/webidl/#dfn-obtain-unicode - - // 1. Let S be the DOMString value. - var s = String(string); - - // 2. Let n be the length of S. - var n = s.length; - - // 3. Initialize i to 0. - var i = 0; - - // 4. Initialize U to be an empty sequence of Unicode characters. - var u = []; - - // 5. While i < n: - while (i < n) { - - // 1. Let c be the code unit in S at index i. - var c = s.charCodeAt(i); - - // 2. Depending on the value of c: - - // c < 0xD800 or c > 0xDFFF - if (c < 0xD800 || c > 0xDFFF) { - // Append to U the Unicode character with code point c. - u.push(c); - } - - // 0xDC00 ≤ c ≤ 0xDFFF - else if (0xDC00 <= c && c <= 0xDFFF) { - // Append to U a U+FFFD REPLACEMENT CHARACTER. - u.push(0xFFFD); - } - - // 0xD800 ≤ c ≤ 0xDBFF - else if (0xD800 <= c && c <= 0xDBFF) { - // 1. If i = n−1, then append to U a U+FFFD REPLACEMENT - // CHARACTER. - if (i === n - 1) { - u.push(0xFFFD); - } - // 2. Otherwise, i < n−1: - else { - // 1. Let d be the code unit in S at index i+1. - var d = string.charCodeAt(i + 1); - - // 2. If 0xDC00 ≤ d ≤ 0xDFFF, then: - if (0xDC00 <= d && d <= 0xDFFF) { - // 1. Let a be c & 0x3FF. - var a = c & 0x3FF; - - // 2. Let b be d & 0x3FF. - var b = d & 0x3FF; - - // 3. Append to U the Unicode character with code point - // 2^16+2^10*a+b. - u.push(0x10000 + (a << 10) + b); - - // 4. Set i to i+1. - i += 1; - } - - // 3. Otherwise, d < 0xDC00 or d > 0xDFFF. Append to U a - // U+FFFD REPLACEMENT CHARACTER. - else { - u.push(0xFFFD); - } - } - } - - // 3. Set i to i+1. - i += 1; - } - - // 6. Return U. - return u; -} - -/** - * @param {!Array.} code_points Array of code points. - * @return {string} string String of UTF-16 code units. - */ -function codePointsToString(code_points) { - var s = ''; - for (var i = 0; i < code_points.length; ++i) { - var cp = code_points[i]; - if (cp <= 0xFFFF) { - s += String.fromCharCode(cp); - } else { - cp -= 0x10000; - s += String.fromCharCode((cp >> 10) + 0xD800, - (cp & 0x3FF) + 0xDC00); - } - } - return s; -} - - -// -// Implementation of Encoding specification -// https://encoding.spec.whatwg.org/ -// - -// -// 3. Terminology -// - -/** - * End-of-stream is a special token that signifies no more tokens - * are in the stream. - * @const - */ var end_of_stream = -1; - -/** - * A stream represents an ordered sequence of tokens. - * - * @constructor - * @param {!(Array.|Uint8Array)} tokens Array of tokens that provide the - * stream. - */ -function Stream(tokens) { - /** @type {!Array.} */ - this.tokens = [].slice.call(tokens); -} - -Stream.prototype = { - /** - * @return {boolean} True if end-of-stream has been hit. - */ - endOfStream: function() { - return !this.tokens.length; - }, - - /** - * When a token is read from a stream, the first token in the - * stream must be returned and subsequently removed, and - * end-of-stream must be returned otherwise. - * - * @return {number} Get the next token from the stream, or - * end_of_stream. - */ - read: function() { - if (!this.tokens.length) - return end_of_stream; - return this.tokens.shift(); - }, - - /** - * When one or more tokens are prepended to a stream, those tokens - * must be inserted, in given order, before the first token in the - * stream. - * - * @param {(number|!Array.)} token The token(s) to prepend to the stream. - */ - prepend: function(token) { - if (Array.isArray(token)) { - var tokens = /**@type {!Array.}*/(token); - while (tokens.length) - this.tokens.unshift(tokens.pop()); - } else { - this.tokens.unshift(token); - } - }, - - /** - * When one or more tokens are pushed to a stream, those tokens - * must be inserted, in given order, after the last token in the - * stream. - * - * @param {(number|!Array.)} token The tokens(s) to prepend to the stream. - */ - push: function(token) { - if (Array.isArray(token)) { - var tokens = /**@type {!Array.}*/(token); - while (tokens.length) - this.tokens.push(tokens.shift()); - } else { - this.tokens.push(token); - } - } -}; - -// -// 4. Encodings -// - -// 4.1 Encoders and decoders - -/** @const */ -var finished = -1; - -/** - * @param {boolean} fatal If true, decoding errors raise an exception. - * @param {number=} opt_code_point Override the standard fallback code point. - * @return {number} The code point to insert on a decoding error. - */ -function decoderError(fatal, opt_code_point) { - if (fatal) - throw TypeError('Decoder error'); - return opt_code_point || 0xFFFD; -} - -// -// 7. API -// - -/** @const */ var DEFAULT_ENCODING = 'utf-8'; - -// 7.1 Interface TextDecoder - -/** - * @constructor - * @param {string=} encoding The label of the encoding; - * defaults to 'utf-8'. - * @param {Object=} options - */ -function TextDecoder(encoding, options) { - if (!(this instanceof TextDecoder)) { - return new TextDecoder(encoding, options); - } - encoding = encoding !== undefined ? String(encoding).toLowerCase() : DEFAULT_ENCODING; - if (encoding !== DEFAULT_ENCODING) { - throw new Error('Encoding not supported. Only utf-8 is supported'); - } - options = ToDictionary(options); - - /** @private @type {boolean} */ - this._streaming = false; - /** @private @type {boolean} */ - this._BOMseen = false; - /** @private @type {?Decoder} */ - this._decoder = null; - /** @private @type {boolean} */ - this._fatal = Boolean(options['fatal']); - /** @private @type {boolean} */ - this._ignoreBOM = Boolean(options['ignoreBOM']); - - Object.defineProperty(this, 'encoding', {value: 'utf-8'}); - Object.defineProperty(this, 'fatal', {value: this._fatal}); - Object.defineProperty(this, 'ignoreBOM', {value: this._ignoreBOM}); -} - -TextDecoder.prototype = { - /** - * @param {ArrayBufferView=} input The buffer of bytes to decode. - * @param {Object=} options - * @return {string} The decoded string. - */ - decode: function decode(input, options) { - var bytes; - if (typeof input === 'object' && input instanceof ArrayBuffer) { - bytes = new Uint8Array(input); - } else if (typeof input === 'object' && 'buffer' in input && - input.buffer instanceof ArrayBuffer) { - bytes = new Uint8Array(input.buffer, - input.byteOffset, - input.byteLength); - } else { - bytes = new Uint8Array(0); - } - - options = ToDictionary(options); - - if (!this._streaming) { - this._decoder = new UTF8Decoder({fatal: this._fatal}); - this._BOMseen = false; - } - this._streaming = Boolean(options['stream']); - - var input_stream = new Stream(bytes); - - var code_points = []; - - /** @type {?(number|!Array.)} */ - var result; - - while (!input_stream.endOfStream()) { - result = this._decoder.handler(input_stream, input_stream.read()); - if (result === finished) - break; - if (result === null) - continue; - if (Array.isArray(result)) - code_points.push.apply(code_points, /**@type {!Array.}*/(result)); - else - code_points.push(result); - } - if (!this._streaming) { - do { - result = this._decoder.handler(input_stream, input_stream.read()); - if (result === finished) - break; - if (result === null) - continue; - if (Array.isArray(result)) - code_points.push.apply(code_points, /**@type {!Array.}*/(result)); - else - code_points.push(result); - } while (!input_stream.endOfStream()); - this._decoder = null; - } - - if (code_points.length) { - // If encoding is one of utf-8, utf-16be, and utf-16le, and - // ignore BOM flag and BOM seen flag are unset, run these - // subsubsteps: - if (['utf-8'].indexOf(this.encoding) !== -1 && - !this._ignoreBOM && !this._BOMseen) { - // If token is U+FEFF, set BOM seen flag. - if (code_points[0] === 0xFEFF) { - this._BOMseen = true; - code_points.shift(); - } else { - // Otherwise, if token is not end-of-stream, set BOM seen - // flag and append token to output. - this._BOMseen = true; - } - } - } - - return codePointsToString(code_points); - } -}; - -// 7.2 Interface TextEncoder - -/** - * @constructor - * @param {string=} encoding The label of the encoding; - * defaults to 'utf-8'. - * @param {Object=} options - */ -function TextEncoder(encoding, options) { - if (!(this instanceof TextEncoder)) - return new TextEncoder(encoding, options); - encoding = encoding !== undefined ? String(encoding).toLowerCase() : DEFAULT_ENCODING; - if (encoding !== DEFAULT_ENCODING) { - throw new Error('Encoding not supported. Only utf-8 is supported'); - } - options = ToDictionary(options); - - /** @private @type {boolean} */ - this._streaming = false; - /** @private @type {?Encoder} */ - this._encoder = null; - /** @private @type {{fatal: boolean}} */ - this._options = {fatal: Boolean(options['fatal'])}; - - Object.defineProperty(this, 'encoding', {value: 'utf-8'}); -} - -TextEncoder.prototype = { - /** - * @param {string=} opt_string The string to encode. - * @param {Object=} options - * @return {Uint8Array} Encoded bytes, as a Uint8Array. - */ - encode: function encode(opt_string, options) { - opt_string = opt_string ? String(opt_string) : ''; - options = ToDictionary(options); - - // NOTE: This option is nonstandard. None of the encodings - // permitted for encoding (i.e. UTF-8, UTF-16) are stateful, - // so streaming is not necessary. - if (!this._streaming) - this._encoder = new UTF8Encoder(this._options); - this._streaming = Boolean(options['stream']); - - var bytes = []; - var input_stream = new Stream(stringToCodePoints(opt_string)); - /** @type {?(number|!Array.)} */ - var result; - while (!input_stream.endOfStream()) { - result = this._encoder.handler(input_stream, input_stream.read()); - if (result === finished) - break; - if (Array.isArray(result)) - bytes.push.apply(bytes, /**@type {!Array.}*/(result)); - else - bytes.push(result); - } - if (!this._streaming) { - while (true) { - result = this._encoder.handler(input_stream, input_stream.read()); - if (result === finished) - break; - if (Array.isArray(result)) - bytes.push.apply(bytes, /**@type {!Array.}*/(result)); - else - bytes.push(result); - } - this._encoder = null; - } - return new Uint8Array(bytes); - } -}; - -// -// 8. The encoding -// - -// 8.1 utf-8 - -/** - * @constructor - * @implements {Decoder} - * @param {{fatal: boolean}} options - */ -function UTF8Decoder(options) { - var fatal = options.fatal; - - // utf-8's decoder's has an associated utf-8 code point, utf-8 - // bytes seen, and utf-8 bytes needed (all initially 0), a utf-8 - // lower boundary (initially 0x80), and a utf-8 upper boundary - // (initially 0xBF). - var /** @type {number} */ utf8_code_point = 0, - /** @type {number} */ utf8_bytes_seen = 0, - /** @type {number} */ utf8_bytes_needed = 0, - /** @type {number} */ utf8_lower_boundary = 0x80, - /** @type {number} */ utf8_upper_boundary = 0xBF; - - /** - * @param {Stream} stream The stream of bytes being decoded. - * @param {number} bite The next byte read from the stream. - * @return {?(number|!Array.)} The next code point(s) - * decoded, or null if not enough data exists in the input - * stream to decode a complete code point. - */ - this.handler = function(stream, bite) { - // 1. If byte is end-of-stream and utf-8 bytes needed is not 0, - // set utf-8 bytes needed to 0 and return error. - if (bite === end_of_stream && utf8_bytes_needed !== 0) { - utf8_bytes_needed = 0; - return decoderError(fatal); - } - - // 2. If byte is end-of-stream, return finished. - if (bite === end_of_stream) - return finished; - - // 3. If utf-8 bytes needed is 0, based on byte: - if (utf8_bytes_needed === 0) { - - // 0x00 to 0x7F - if (inRange(bite, 0x00, 0x7F)) { - // Return a code point whose value is byte. - return bite; - } - - // 0xC2 to 0xDF - if (inRange(bite, 0xC2, 0xDF)) { - // Set utf-8 bytes needed to 1 and utf-8 code point to byte - // − 0xC0. - utf8_bytes_needed = 1; - utf8_code_point = bite - 0xC0; - } - - // 0xE0 to 0xEF - else if (inRange(bite, 0xE0, 0xEF)) { - // 1. If byte is 0xE0, set utf-8 lower boundary to 0xA0. - if (bite === 0xE0) - utf8_lower_boundary = 0xA0; - // 2. If byte is 0xED, set utf-8 upper boundary to 0x9F. - if (bite === 0xED) - utf8_upper_boundary = 0x9F; - // 3. Set utf-8 bytes needed to 2 and utf-8 code point to - // byte − 0xE0. - utf8_bytes_needed = 2; - utf8_code_point = bite - 0xE0; - } - - // 0xF0 to 0xF4 - else if (inRange(bite, 0xF0, 0xF4)) { - // 1. If byte is 0xF0, set utf-8 lower boundary to 0x90. - if (bite === 0xF0) - utf8_lower_boundary = 0x90; - // 2. If byte is 0xF4, set utf-8 upper boundary to 0x8F. - if (bite === 0xF4) - utf8_upper_boundary = 0x8F; - // 3. Set utf-8 bytes needed to 3 and utf-8 code point to - // byte − 0xF0. - utf8_bytes_needed = 3; - utf8_code_point = bite - 0xF0; - } - - // Otherwise - else { - // Return error. - return decoderError(fatal); - } - - // Then (byte is in the range 0xC2 to 0xF4) set utf-8 code - // point to utf-8 code point << (6 × utf-8 bytes needed) and - // return continue. - utf8_code_point = utf8_code_point << (6 * utf8_bytes_needed); - return null; - } - - // 4. If byte is not in the range utf-8 lower boundary to utf-8 - // upper boundary, run these substeps: - if (!inRange(bite, utf8_lower_boundary, utf8_upper_boundary)) { - - // 1. Set utf-8 code point, utf-8 bytes needed, and utf-8 - // bytes seen to 0, set utf-8 lower boundary to 0x80, and set - // utf-8 upper boundary to 0xBF. - utf8_code_point = utf8_bytes_needed = utf8_bytes_seen = 0; - utf8_lower_boundary = 0x80; - utf8_upper_boundary = 0xBF; - - // 2. Prepend byte to stream. - stream.prepend(bite); - - // 3. Return error. - return decoderError(fatal); - } - - // 5. Set utf-8 lower boundary to 0x80 and utf-8 upper boundary - // to 0xBF. - utf8_lower_boundary = 0x80; - utf8_upper_boundary = 0xBF; - - // 6. Increase utf-8 bytes seen by one and set utf-8 code point - // to utf-8 code point + (byte − 0x80) << (6 × (utf-8 bytes - // needed − utf-8 bytes seen)). - utf8_bytes_seen += 1; - utf8_code_point += (bite - 0x80) << (6 * (utf8_bytes_needed - utf8_bytes_seen)); - - // 7. If utf-8 bytes seen is not equal to utf-8 bytes needed, - // continue. - if (utf8_bytes_seen !== utf8_bytes_needed) - return null; - - // 8. Let code point be utf-8 code point. - var code_point = utf8_code_point; - - // 9. Set utf-8 code point, utf-8 bytes needed, and utf-8 bytes - // seen to 0. - utf8_code_point = utf8_bytes_needed = utf8_bytes_seen = 0; - - // 10. Return a code point whose value is code point. - return code_point; - }; -} - -/** - * @constructor - * @implements {Encoder} - * @param {{fatal: boolean}} options - */ -function UTF8Encoder(options) { - var fatal = options.fatal; - /** - * @param {Stream} stream Input stream. - * @param {number} code_point Next code point read from the stream. - * @return {(number|!Array.)} Byte(s) to emit. - */ - this.handler = function(stream, code_point) { - // 1. If code point is end-of-stream, return finished. - if (code_point === end_of_stream) - return finished; - - // 2. If code point is in the range U+0000 to U+007F, return a - // byte whose value is code point. - if (inRange(code_point, 0x0000, 0x007f)) - return code_point; - - // 3. Set count and offset based on the range code point is in: - var count, offset; - // U+0080 to U+07FF: 1 and 0xC0 - if (inRange(code_point, 0x0080, 0x07FF)) { - count = 1; - offset = 0xC0; - } - // U+0800 to U+FFFF: 2 and 0xE0 - else if (inRange(code_point, 0x0800, 0xFFFF)) { - count = 2; - offset = 0xE0; - } - // U+10000 to U+10FFFF: 3 and 0xF0 - else if (inRange(code_point, 0x10000, 0x10FFFF)) { - count = 3; - offset = 0xF0; - } - - // 4.Let bytes be a byte sequence whose first byte is (code - // point >> (6 × count)) + offset. - var bytes = [(code_point >> (6 * count)) + offset]; - - // 5. Run these substeps while count is greater than 0: - while (count > 0) { - - // 1. Set temp to code point >> (6 × (count − 1)). - var temp = code_point >> (6 * (count - 1)); - - // 2. Append to bytes 0x80 | (temp & 0x3F). - bytes.push(0x80 | (temp & 0x3F)); - - // 3. Decrease count by one. - count -= 1; - } - - // 6. Return bytes bytes, in order. - return bytes; - }; -} - -exports.TextEncoder = TextEncoder; -exports.TextDecoder = TextDecoder; diff --git a/js/closure-compiler-scripts/tslib.js b/js/closure-compiler-scripts/tslib.js deleted file mode 100644 index b5a722a652c34..0000000000000 --- a/js/closure-compiler-scripts/tslib.js +++ /dev/null @@ -1,151 +0,0 @@ -/** - * closure-compiler-friendly tslib - * copied from node_modules/tslib/tslib.js - * update as needed - */ - -var extendStatics = Object.setPrototypeOf || - ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || - function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; }; - -function __extends(d, b) { - extendStatics(d, b); - function __() { this.constructor = d; } - d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); -}; - -var __assign = Object.assign || function (t) { - for (var s, i = 1, n = arguments.length; i < n; i++) { - s = arguments[i]; - for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p]; - } - return t; -}; - -function __rest(s, e) { - var t = {}; - for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0) - t[p] = s[p]; - if (s != null && typeof Object.getOwnPropertySymbols === "function") - for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) if (e.indexOf(p[i]) < 0) - t[p[i]] = s[p[i]]; - return t; -}; - -function __decorate(decorators, target, key, desc) { - var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d; - if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc); - else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r; - return c > 3 && r && Object.defineProperty(target, key, r), r; -}; - -function __param(paramIndex, decorator) { - return function (target, key) { decorator(target, key, paramIndex); } -}; - -function __metadata(metadataKey, metadataValue) { - if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(metadataKey, metadataValue); -}; - -function __awaiter(thisArg, _arguments, P, generator) { - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; - -function __generator(thisArg, body) { - var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; - return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; - function verb(n) { return function (v) { return step([n, v]); }; } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while (_) try { - if (f = 1, y && (t = y[op[0] & 2 ? "return" : op[0] ? "throw" : "next"]) && !(t = t.call(y, op[1])).done) return t; - if (y = 0, t) op = [0, t.value]; - switch (op[0]) { - case 0: case 1: t = op; break; - case 4: _.label++; return { value: op[1], done: false }; - case 5: _.label++; y = op[1]; op = [0]; continue; - case 7: op = _.ops.pop(); _.trys.pop(); continue; - default: - if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } - if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } - if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } - if (t[2]) _.ops.pop(); - _.trys.pop(); continue; - } - op = body.call(thisArg, _); - } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } - if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; - } -}; - -function __exportStar(m, exports) { - for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p]; -}; - -function __values(o) { - var m = typeof Symbol === "function" && o[Symbol.iterator], i = 0; - if (m) return m.call(o); - return { - next: function () { - if (o && i >= o.length) o = void 0; - return { value: o && o[i++], done: !o }; - } - }; -}; - -function __read(o, n) { - var m = typeof Symbol === "function" && o[Symbol.iterator]; - if (!m) return o; - var i = m.call(o), r, ar = [], e; - try { - while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value); - } - catch (error) { e = { error: error }; } - finally { - try { - if (r && !r.done && (m = i["return"])) m.call(i); - } - finally { if (e) throw e.error; } - } - return ar; -}; - -function __spread() { - for (var ar = [], i = 0; i < arguments.length; i++) - ar = ar.concat(__read(arguments[i])); - return ar; -}; - -function __await(v) { - return this instanceof __await ? (this.v = v, this) : new __await(v); -}; - -function __asyncGenerator(thisArg, _arguments, generator) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var g = generator.apply(thisArg, _arguments || []), i, q = []; - return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; - function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } - function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } - function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } - function fulfill(value) { resume("next", value); } - function reject(value) { resume("throw", value); } - function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } -}; - -function __asyncDelegator(o) { - var i, p; - return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i; - function verb(n, f) { if (o[n]) i[n] = function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === "return" } : f ? f(v) : v; }; } -}; - -function __asyncValues(o) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var m = o[Symbol.asyncIterator]; - return m ? m.call(o) : typeof __values === "function" ? __values(o) : o[Symbol.iterator](); -}; diff --git a/js/gulp/argv.js b/js/gulp/argv.js index 33553704eec25..253553c733c8a 100644 --- a/js/gulp/argv.js +++ b/js/gulp/argv.js @@ -22,9 +22,12 @@ const argv = require(`command-line-args`)([ { name: `target`, type: String, defaultValue: `` }, { name: `module`, type: String, defaultValue: `` }, { name: `coverage`, type: Boolean, defaultValue: false }, + { name: `integration`, alias: `i`, type: Boolean, defaultValue: false }, { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] }, - { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] } -]); + { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }, + { name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] }, + { name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] }, +], { partial: true }); const { targets, modules } = argv; diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js index d160ecb0e5de4..cc33ee14497b6 100644 --- a/js/gulp/arrow-task.js +++ b/js/gulp/arrow-task.js @@ -27,7 +27,8 @@ const { memoizeTask } = require('./memoize-task'); const { Observable, ReplaySubject } = require('rxjs'); const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, format) { - const out = targetDir(target), srcGlob = `src/**/*`; + const out = targetDir(target); + const srcGlob = `src/**/*.ts`; const es5Glob = `${targetDir(`es5`, `cjs`)}/**/*.js`; const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`; const es5UmdGlob = `${targetDir(`es5`, `umd`)}/**/*.js`; @@ -48,7 +49,7 @@ const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, forma }))({}); const arrowTSTask = ((cache) => memoizeTask(cache, function copyTS(target, format) { - return observableFromStreams(gulp.src(`src/**/*`), gulp.dest(targetDir(target, format))); + return observableFromStreams(gulp.src(`src/**/*.ts`), gulp.dest(targetDir(target, format))); }))({}); diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js index 950bf40e22a39..a1f0a9a698843 100644 --- a/js/gulp/closure-task.js +++ b/js/gulp/closure-task.js @@ -37,10 +37,14 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form const externs = path.join(src, `${mainExport}.externs`); return observableFromStreams( gulp.src([ -/* external libs first --> */ `closure-compiler-scripts/*.js`, -/* then sources glob --> */ `${src}/**/*.js`, -/* and exclusions last --> */ `!${src}/format/*.js`, - `!${src}/Arrow.externs.js`, +/* external libs first --> */ `node_modules/tslib/package.json`, + `node_modules/tslib/tslib.es6.js`, + `node_modules/flatbuffers/package.json`, + `node_modules/flatbuffers/js/flatbuffers.mjs`, + `node_modules/text-encoding-utf-8/package.json`, + `node_modules/text-encoding-utf-8/src/encoding.js`, +/* then sources globs --> */ `${src}/**/*.js`, +/* and exclusions last --> */ `!${src}/Arrow.externs.js`, ], { base: `./` }), sourcemaps.init(), closureCompiler(createClosureArgs(entry, externs)), @@ -53,12 +57,14 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form const createClosureArgs = (entry, externs) => ({ third_party: true, warning_level: `QUIET`, - dependency_mode: `LOOSE`, + dependency_mode: `STRICT`, rewrite_polyfills: false, externs: `${externs}.js`, entry_point: `${entry}.js`, + module_resolution: `NODE`, // formatting: `PRETTY_PRINT`, compilation_level: `ADVANCED`, + package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, language_in: gCCLanguageNames[`es2015`], diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js index b46b2bb14edcb..ae6b8071d2ab7 100644 --- a/js/gulp/test-task.js +++ b/js/gulp/test-task.js @@ -15,10 +15,19 @@ // specific language governing permissions and limitations // under the License. +const del = require('del'); const path = require('path'); -const child_process = require(`child_process`); const { argv } = require('./argv'); +const { promisify } = require('util'); +const glob = promisify(require('glob')); +const stat = promisify(require('fs').stat); +const mkdirp = promisify(require('mkdirp')); +const rimraf = promisify(require('rimraf')); +const child_process = require(`child_process`); const { memoizeTask } = require('./memoize-task'); +const readFile = promisify(require('fs').readFile); +const exec = promisify(require('child_process').exec); +const parseXML = promisify(require('xml2js').parseString); const jestArgv = []; argv.update && jestArgv.push(`-u`); @@ -29,16 +38,129 @@ const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`]; const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`)); const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) { - const opts = Object.assign({}, testOptions); + const opts = { ...testOptions }; const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv]; - opts.env = Object.assign({}, opts.env, { TEST_TARGET: target, TEST_MODULE: format }); + if (!argv.integration) { + args.push('test/vector-tests.ts'); + } + opts.env = { ...opts.env, + TEST_TARGET: target, + TEST_MODULE: format, + TEST_TS_SOURCE: !!argv.coverage, + TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]), + TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]), + }; return !debug ? child_process.spawn(jest, args, opts) : child_process.exec(`node --inspect-brk ${jest} ${args.join(` `)}`, opts); }))({}, jestArgv, { - env: Object.assign({}, process.env), + env: { ...process.env }, stdio: [`ignore`, `inherit`, `inherit`], }); module.exports = testTask; module.exports.testTask = testTask; +module.exports.cleanTestData = cleanTestData; +module.exports.createTestData = createTestData; + +// Pull C++ and Java paths from environment vars first, otherwise sane defaults +const ARROW_HOME = process.env.ARROW_HOME || path.resolve('../'); +const ARROW_JAVA_DIR = process.env.ARROW_JAVA_DIR || path.join(ARROW_HOME, 'java'); +const CPP_EXE_PATH = process.env.ARROW_CPP_EXE_PATH || path.join(ARROW_HOME, 'cpp/build/debug'); +const ARROW_INTEGRATION_DIR = process.env.ARROW_INTEGRATION_DIR || path.join(ARROW_HOME, 'integration'); +const CPP_JSON_TO_ARROW = path.join(CPP_EXE_PATH, 'json-integration-test'); +const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file'); +const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream'); + +const testFilesDir = path.join(ARROW_HOME, 'js/test/data'); +const cppFilesDir = path.join(testFilesDir, 'cpp'); +const javaFilesDir = path.join(testFilesDir, 'java'); +const jsonFilesDir = path.join(testFilesDir, 'json'); + +async function cleanTestData() { + return await del(`${testFilesDir}/**`); +} + +async function createTestJSON() { + await mkdirp(jsonFilesDir); + await exec(`shx cp ${ARROW_INTEGRATION_DIR}/data/*.json ${jsonFilesDir}`); + await exec(`python ${ARROW_INTEGRATION_DIR}/integration_test.py --write_generated_json ${jsonFilesDir}`); +} + +async function createTestData() { + + let JAVA_TOOLS_JAR = process.env.ARROW_JAVA_INTEGRATION_JAR; + if (!JAVA_TOOLS_JAR) { + const pom_version = await + readFile(path.join(ARROW_JAVA_DIR, 'pom.xml')) + .then((pom) => parseXML(pom.toString())) + .then((pomXML) => pomXML.project.version[0]); + JAVA_TOOLS_JAR = path.join(ARROW_JAVA_DIR, `/tools/target/arrow-tools-${pom_version}-jar-with-dependencies.jar`); + } + + await cleanTestData().then(createTestJSON); + await mkdirp(path.join(cppFilesDir, 'file')); + await mkdirp(path.join(javaFilesDir, 'file')); + await mkdirp(path.join(cppFilesDir, 'stream')); + await mkdirp(path.join(javaFilesDir, 'stream')); + + const errors = []; + const names = await glob(path.join(jsonFilesDir, '*.json')); + + for (let jsonPath of names) { + const name = path.parse(path.basename(jsonPath)).name; + const arrowCppFilePath = path.join(cppFilesDir, 'file', `${name}.arrow`); + const arrowJavaFilePath = path.join(javaFilesDir, 'file', `${name}.arrow`); + const arrowCppStreamPath = path.join(cppFilesDir, 'stream', `${name}.arrow`); + const arrowJavaStreamPath = path.join(javaFilesDir, 'stream', `${name}.arrow`); + try { + await generateCPPFile(path.resolve(jsonPath), arrowCppFilePath); + await generateCPPStream(arrowCppFilePath, arrowCppStreamPath); + } catch (e) { errors.push(`${e.stdout}\n${e.message}`); } + try { + await generateJavaFile(path.resolve(jsonPath), arrowJavaFilePath); + await generateJavaStream(arrowJavaFilePath, arrowJavaStreamPath); + } catch (e) { errors.push(`${e.stdout}\n${e.message}`); } + } + if (errors.length) { + console.error(errors.join(`\n`)); + process.exit(1); + } + + async function generateCPPFile(jsonPath, filePath) { + await rimraf(filePath); + return await exec( + `${CPP_JSON_TO_ARROW} ${ + `--integration --mode=JSON_TO_ARROW`} ${ + `--json=${jsonPath} --arrow=${filePath}`}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateCPPStream(filePath, streamPath) { + await rimraf(streamPath); + return await exec( + `${CPP_FILE_TO_STREAM} ${filePath} > ${streamPath}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateJavaFile(jsonPath, filePath) { + await rimraf(filePath); + return await exec( + `java -cp ${JAVA_TOOLS_JAR} ${ + `org.apache.arrow.tools.Integration -c JSON_TO_ARROW`} ${ + `-j ${path.resolve(jsonPath)} -a ${filePath}`}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateJavaStream(filePath, streamPath) { + await rimraf(streamPath); + return await exec( + `java -cp ${JAVA_TOOLS_JAR} ${ + `org.apache.arrow.tools.FileToStream`} ${filePath} ${streamPath}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } +} diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js index 2c66846667cd3..2fd9f1350a6f1 100644 --- a/js/gulp/typescript-task.js +++ b/js/gulp/typescript-task.js @@ -19,9 +19,11 @@ const { targetDir, tsconfigName, observableFromStreams } = require('./util'); +const del = require('del'); const gulp = require('gulp'); const path = require('path'); const ts = require(`gulp-typescript`); +const gulpRename = require(`gulp-rename`); const sourcemaps = require('gulp-sourcemaps'); const { memoizeTask } = require('./memoize-task'); const { Observable, ReplaySubject } = require('rxjs'); @@ -36,8 +38,26 @@ const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target ); const writeDTypes = observableFromStreams(dts, gulp.dest(out)); const writeJS = observableFromStreams(js, sourcemaps.write(), gulp.dest(out)); - return Observable.forkJoin(writeDTypes, writeJS).publish(new ReplaySubject()).refCount(); + return Observable + .forkJoin(writeDTypes, writeJS) + .concat(maybeCopyRawJSArrowFormatFiles(target, format)) + .publish(new ReplaySubject()).refCount(); }))({}); - + module.exports = typescriptTask; -module.exports.typescriptTask = typescriptTask; \ No newline at end of file +module.exports.typescriptTask = typescriptTask; + +function maybeCopyRawJSArrowFormatFiles(target, format) { + if (target !== `es5` || format !== `cls`) { + return Observable.empty(); + } + return Observable.defer(async () => { + const outFormatDir = path.join(targetDir(target, format), `format`); + await del(path.join(outFormatDir, '*.js')); + await observableFromStreams( + gulp.src(path.join(`src`, `format`, `*_generated.js`)), + gulpRename((p) => { p.basename = p.basename.replace(`_generated`, ``); }), + gulp.dest(outFormatDir) + ).toPromise(); + }); +} \ No newline at end of file diff --git a/js/gulp/uglify-task.js b/js/gulp/uglify-task.js index 804d450453644..5c605cb7882bd 100644 --- a/js/gulp/uglify-task.js +++ b/js/gulp/uglify-task.js @@ -39,7 +39,7 @@ const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJ const targetConfig = { ...commonConfig, output: { ...commonConfig.output, - path: path.resolve(`./${out}`) } }; + path: path.resolve(`./${out}`) } }; const webpackConfigs = [ [mainExport, PublicNames] diff --git a/js/gulp/util.js b/js/gulp/util.js index 21ffc3127339c..e23fc398562d4 100644 --- a/js/gulp/util.js +++ b/js/gulp/util.js @@ -27,7 +27,7 @@ const releasesRootDir = `targets`; const knownTargets = [`es5`, `es2015`, `esnext`]; const knownModules = [`cjs`, `esm`, `cls`, `umd`]; const moduleFormatsToSkipCombosOf = { cls: true }; -const metadataFiles = [`LICENSE`, `README.md`]; +const metadataFiles = [`../LICENSE.txt`, `../NOTICE.txt`, `README.md`]; const packageJSONFields = [ `version`, `license`, `description`, `author`, `homepage`, `repository`, diff --git a/js/gulpfile.js b/js/gulpfile.js index 4cf0342c3be78..cd0ac70029aba 100644 --- a/js/gulpfile.js +++ b/js/gulpfile.js @@ -19,11 +19,11 @@ const del = require('del'); const gulp = require('gulp'); const path = require('path'); const { Observable } = require('rxjs'); -const testsTask = require('./gulp/test-task'); const buildTask = require('./gulp/build-task'); const cleanTask = require('./gulp/clean-task'); const packageTask = require('./gulp/package-task'); const { targets, modules } = require('./gulp/argv'); +const { testTask, createTestData, cleanTestData } = require('./gulp/test-task'); const { targetDir, taskName, combinations, @@ -35,8 +35,8 @@ const { for (const [target, format] of combinations([`all`], [`all`])) { const task = taskName(target, format); gulp.task(`clean:${task}`, cleanTask(target, format)); - gulp.task( `test:${task}`, testsTask(target, format)); - gulp.task(`debug:${task}`, testsTask(target, format, true)); + gulp.task( `test:${task}`, testTask(target, format)); + gulp.task(`debug:${task}`, testTask(target, format, true)); gulp.task(`build:${task}`, gulp.series(`clean:${task}`, buildTask(target, format), packageTask(target, format))); @@ -50,7 +50,7 @@ knownTargets.forEach((target) => gulp.series( gulp.parallel( cleanTask(target, `umd`), - cleanTask(UMDSourceTargets[target], `cls`), + cleanTask(UMDSourceTargets[target], `cls`) ), buildTask(UMDSourceTargets[target], `cls`), buildTask(target, `umd`), packageTask(target, `umd`) @@ -86,6 +86,8 @@ const buildConcurrent = (tasks) => () => .merge(...knownTargets.map((target) => del(`${targetDir(target, `cls`)}/**`))))); +gulp.task(`test:cleanTestData`, cleanTestData); +gulp.task(`test:createTestData`, createTestData); gulp.task( `test`, gulp.series(getTasks(`test`))); gulp.task(`debug`, gulp.series(getTasks(`debug`))); gulp.task(`clean`, gulp.parallel(getTasks(`clean`))); diff --git a/js/lerna.json b/js/lerna.json index c8fb8c072c61e..0bf16fdfd57be 100644 --- a/js/lerna.json +++ b/js/lerna.json @@ -2,8 +2,10 @@ "lerna": "2.0.0", "version": "0.1.1", "packages": [ + "targets/ts", "targets/es5/*", "targets/es2015/*", - "targets/esnext/*" + "targets/esnext/*", + "targets/apache-arrow" ] } diff --git a/js/prepublish.sh b/js/npm-release.sh similarity index 85% rename from js/prepublish.sh rename to js/npm-release.sh index b40504ae808cb..0bf70d9798538 100644 --- a/js/prepublish.sh +++ b/js/npm-release.sh @@ -17,10 +17,4 @@ # specific language governing permissions and limitations # under the License. -npm run clean -npm run lint -npm run build -npm run test -npm --no-git-tag-version version patch &>/dev/null -npm run bundle -npm run lerna:publish \ No newline at end of file +lerna publish --yes --skip-git --force-publish=* \ No newline at end of file diff --git a/js/package.json b/js/package.json index ba93a34686288..24bc27f5b6325 100644 --- a/js/package.json +++ b/js/package.json @@ -15,17 +15,17 @@ "build": "gulp build", "clean": "gulp clean", "debug": "gulp debug", - "bundle": "gulp bundle", - "package": "gulp package", "perf": "node ./perf/index.js", - "test:coverage": "gulp test -t esnext -m esm --coverage", - "validate": "npm-run-all clean lint build test bundle", - "lerna:publish": "lerna exec --bail=false npm publish", - "prepublishOnly": "sh ./prepublish.sh", + "release": "./npm-release.sh", + "validate": "run-s --silent lint build clean", + "test:coverage": "gulp test -t ts --coverage", + "test:cleanTestData": "gulp test:cleanTestData", + "test:createTestData": "gulp test:createTestData", "doc": "shx rm -rf ./doc && esdoc", "lint": "npm-run-all -p lint:*", "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"", - "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"" + "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"", + "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1" }, "repository": { "type": "git", @@ -53,14 +53,15 @@ "command-line-usage": "4.0.1" }, "dependencies": { - "flatbuffers": "1.7.0", + "flatbuffers": "trxcllnt/flatbuffers-esm", "text-encoding": "0.6.4" }, "devDependencies": { - "@std/esm": "0.12.5", + "@std/esm": "0.13.0", "@types/flatbuffers": "1.6.5", - "@types/jest": "21.1.5", - "@types/node": "8.0.47", + "@types/glob": "5.0.33", + "@types/jest": "21.1.6", + "@types/node": "8.0.51", "@types/text-encoding": "0.0.32", "benchmark": "2.1.4", "command-line-args": "4.0.7", @@ -68,20 +69,22 @@ "del": "3.0.0", "esdoc": "1.0.3", "esdoc-standard-plugin": "1.0.0", - "google-closure-compiler": "20170910.0.0", + "glob": "7.1.2", + "google-closure-compiler": "20171112.0.0", "gulp": "github:gulpjs/gulp#4.0", "gulp-json-transform": "0.4.5", "gulp-rename": "1.2.2", "gulp-sourcemaps": "2.6.1", "gulp-typescript": "3.2.3", + "ix": "2.3.1", "jest": "21.2.1", "jest-environment-node-debug": "2.0.0", "json": "9.0.6", - "lerna": "2.5.0", - "lint-staged": "4.3.0", + "lerna": "2.5.1", + "lint-staged": "5.0.0", "merge2": "1.2.0", "mkdirp": "0.5.1", - "npm-run-all": "4.1.1", + "npm-run-all": "4.1.2", "pump": "1.0.2", "rimraf": "2.6.2", "rxjs": "5.5.2", @@ -89,12 +92,13 @@ "source-map-loader": "0.2.3", "text-encoding-utf-8": "1.0.1", "trash": "4.1.0", - "ts-jest": "21.1.4", + "ts-jest": "21.2.1", "tslib": "1.8.0", "tslint": "5.8.0", "typescript": "2.6.1", "uglifyjs-webpack-plugin": "1.0.1", - "webpack": "3.8.1" + "webpack": "3.8.1", + "xml2js": "0.4.19" }, "lint-staged": { "*.@(ts)": [ diff --git a/js/perf/arrows/file/dictionary.arrow b/js/perf/arrows/file/dictionary.arrow deleted file mode 100644 index 34d41db1f200159406c459b8c076d5ce0c3d30ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2522 zcmeHJ&1(};5T9+*G{mHqHnv(ta_B)s!H*ykC?w(`f?#L`@#bQlX&@hEla^FGYFyCl*NI8<1N7r49A1Uz3Rk-wEvR zLio|iYm>=D?S_ycE00v|-a0$whwl6poGAiD5*{f-vL43ThfM(<#?BTZKnj_r@FRGZ zuGRg%zeb+Aq?OUn8Q|rH>9$L?ZnM)b^@3)7y?FD+Ey%;#mt^rTwKFFovhl8Q1!_($ zlJWEtL)h|5{AXHOih zg1!KX%z6W?>@oA;tk{pQh?7R_wCw;VMkrXGRg>48IPE+nwBy|A(O=$|I45qzimpJ{h--txcvLUj$sb`iJStOf#2S4nX6BGyYdM<>mX!N+M#q1dO&)hQLnOoB4t^Hvy6ry{T@m@tC>`gegsGP9~nlJaXo%T6L zm4jK(9FSI3KM3}!CTKUi9kXe6O|aA3f(!MS#eC;+_re+Dyz_BTePX)8eVhsJQg~Ri zKQP7SunOaH-c&-=A9<}je!f5@ARhBlD4vSTCxQ_7b2}8B3OXsJ zo^z=DIUK$SRQ|LxuKaj7dh@+-=jFG-NTghDp@73?R(3W=%`Do3c>m}kswb~!;&Po? zTz=Hd;=TFTnQ30kku!_uaa=w=WBOGeN+}~Kud`5I=O^;6vby(vr24GAVc{EsZx~`3 zxCLa~*UpghB~(KqWflh5cJi|2!iAz&F6qxsUX1AZ4rPz@ZAwZT&gbMSdh!)@&(D8- MMaQ^*75ks=H#cC^v;Y7A diff --git a/js/perf/arrows/file/simple.arrow b/js/perf/arrows/file/simple.arrow deleted file mode 100644 index 838db6dc8eda5376aaaf977c5f96c50397582a5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1642 zcmeHIJxc>Y5S{x-E`~!=1UW$=g@vVpg@xdyvk-(J*x1}DwgF44l$Ms3{sezQ3QK>C zrQP}7+>FLVB_!H8VfN$A?7W@X-AjKkI62#uh2=aD>0os#tJ0K)v~-Ln2o)D;D;5K4 zL)k~rXXvHPPphiJg2plCThPpK`t2A-4oa*);8@d=0I^%(73C1=$r12CLggrn8+oi_ zM>%{Ly9?zWLYtGTmZnBxEL!idE=T=K1qD@_LLSm6wo7*Sa3Cn)RIfVw0Y zfD-loklcoj``mlSN9H$;kH5i@h)QmDkCPidn!?9Z{xtIr`2NlOKIa)y-0pK?n{1X4 z=ORDvi*w8ya(!OMAN^hqzczRB8t+;^>-8J_tfwpUUzw%(Go&cWvb=yS$iRD_;^pu~ prFaE=t<2^C?KYHW$}@hjgWmT8pS(n{|9`)h`+nV`u6_NFc>}#ma&iCw diff --git a/js/perf/arrows/file/struct.arrow b/js/perf/arrows/file/struct.arrow deleted file mode 100644 index 3d2c018e6c27c95057f2a18395a9dc6031e6b201..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2354 zcmeHJJ5L)y5MCPt7wZBk2y#J?B~nBw5RVv4cSZyvVqWKjR} z6s4k~Oa4I`{{(&lB_(a7NY3}|?Kp^s{6L5%V|}|jbMx4po7-{9b?24_Rns&dQtAP$ zZPlULRI9S|8bc_obC#`0A=(+is+wGdDvKpXyMleIejtO0puY>U<_|asV;be?rC_vtA32!F>+3PZ?aFWNwxU?K&|QVQPY-z;pc@f8Ls-qN37n%QY`%I zxY&rxo9w6m*goA-#&UDS5j1ee;DZ_$xWVP-wcK6z)5gufTCRbFbJ;&0~}qu0pfG@{&q z?#j*&Ubi*rclBPXsd}vfYg-^S&f4w+o&s!FH)vU~zCB8165A>F`SjF!_TAd@Xfl)D zbU%K`O?jhpD~ZxZ!h2O1*z^ltawxr(8Jo$yzOWVE-8OE2b_g3cIlB=%&N4;^kpA=4 j4F+65J`V#6Pz4u|?*qgFK+Fi1 ggAhzW>OT-53Bj4@;tp^bh#AP{z@!A!VJz}!0H3Q0ZvX%Q diff --git a/js/perf/arrows/multi/count/schema.arrow b/js/perf/arrows/multi/count/schema.arrow deleted file mode 100644 index dfd24e9e0018c471a053b8621cb7ea8b46a5bd9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmXYqF%E<<3OT-53Bj4@;to(5m^h3N(||6HEYAc~glsQJ4>}(t);5XP35JT0wDqM7?xI^BFD2^XWnihKA86z-kaHl z0XqO2Y!R?R?LCtf&I65e6{&_$Er>DqON^1rWzV@I_L`@aJfj#35`_jFSW8A2V8h-QC8fk$0e`;lU^$MH+=mDH!3Sj^M diff --git a/js/perf/arrows/multi/origins/schema.arrow b/js/perf/arrows/multi/origins/schema.arrow deleted file mode 100644 index 0d10fb0e2d1354e5a147f49b5283632d4fdde04e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1604 zcmeIxJ!lj`6bJCRGapA1Bua|d$Pxq#Q7lBQLUI>l5-+(k7firH-tE4-9d35!oY{?D z7p!cw6C1I!5^U^j#6|==vD3yTSSZ+OBjIR?4LO})pAhm>Is_B%h zG;7ATjOuK%OM^P8Bq%e5aoyO|FRRB=QN*enORC3)wY*)IY3iDpZO3(Mwq?XR$#X;)-z?RB2G9 z4(G`>D72I?0^sG85Q-X@_K`Ly>C#4~93G)FdR{fNn;Ba`lXP!B!4Z`*^O;Mf-jq6W z#$s*Qab;-}soOCnYvkfQr?Qcb%UP>M%9JyU0G%>Yk%_TfQM}FmMk+>SNi%a5W4oCP zDzTM_y+BHlTVTD-c7aoCWO_;AK3#qvRmvi^5iY6GH67BtWRQ^2OJo&tU|orEG$h(v z){!JlU8AI1c1m_v2)P=)U*&05$pzQAO|EJjZlleLT{UYqajGi5jpAS(%~cusbgz(h ict8K@)(@CUEJJFWRa|4yt$ZX2qYWXY)dL0a`6ZF zAN&!Xds6V=u_x(K5D$W9*YA7#rmU}Osk6+RH}l@id^7WQHYc(qB6V4miZrBZ_gG}k z+9jx2TN#!bm@)ug1UIoZAy?onxWzN-McG6@J_Z|K;*)_z#PvjifOrz0@p}F~NvL~OQvkp=nH7?6#=&VrtXpgYCH;GLZ zw}_5;f5%pXPHbh1Tl>RaC>8f=E8UOr*mcQKA@FP*IK5O?yIlwVft$FA7?tv=t*cS#x%Tk#_rEb89=F)82g$Le<8 diff --git a/js/perf/arrows/stream/simple.arrow b/js/perf/arrows/stream/simple.arrow deleted file mode 100644 index 2c68c0e44b0af2ed78a47edbd038ae8fa8ab574b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1188 zcmcIiJ5Iw;5M2La6IsYbgd7PZ7aV|so>x*(asiu&HWDROj*yaLaD)_;9EOt4%&d3$ zIDjGrHd;TwnSC>FH(o>{m57vbC=&J3d$Bh?>poGB!NdI5h?f00HmfkzUnN7vrV_(~{cQ4I?sKBIu0-%dG3D)Y~UDR>;`PleEZ%U^bsHi(7p#q z0^}Hj5>P*pUHCYN=-?Lw&sfa%1-yj}%;AoSJVi8G^r=%vp_8Ln_B2Murp|oyc?Jvu z6bC*IJpI^W#lihR{5bOdyq@O;Z5_l2DqH4%0^3S;zqVdo`&9G1QrX*-7z*I$W_<^A zeZ%Tx(qpT8*I6VbA?RVm^y_>9jy34>H`T~{;~1#*AIa*a;f-!)!=GrozxnztJg%(d zMDMraXg;fXPW<1@Ik~2^)1UihcOHlC?CCpxe*3k+RbuLuGkjW$*_VxKHzEJ^{3aLc z6ao4**o&qgvHWkcu$5mtntqv@nV6e8dQ*D8k;%BbDQ{diTF>4Nmnf+|e$v1;y^hdth(oOG^{&ro_cHKv$qq#D1c{F0n>w%$wGQ(GXHvhcBAxRy*QpJlH;%3Hj9}5!;<>ec0a0N6*h?f8W diff --git a/js/perf/config.js b/js/perf/config.js index 4fbcda3799587..cca1080154790 100644 --- a/js/perf/config.js +++ b/js/perf/config.js @@ -17,22 +17,14 @@ const fs = require('fs'); const path = require('path'); -const arrowFormats = ['file', 'stream']; -const arrowFileNames = ['simple', 'struct', 'dictionary']; -const multipartArrows = ['count', 'latlong', 'origins']; -let arrowTestConfigurations = []; +const glob = require('glob'); -arrowTestConfigurations = multipartArrows.reduce((configs, folder) => { - const schemaPath = path.resolve(__dirname, `./arrows/multi/${folder}/schema.arrow`); - const recordsPath = path.resolve(__dirname, `./arrows/multi/${folder}/records.arrow`); - return [...configs, [`multipart ${folder}`, fs.readFileSync(schemaPath), fs.readFileSync(recordsPath)]]; -}, arrowTestConfigurations); +const config = []; +const filenames = glob.sync(path.resolve(__dirname, `../test/data/cpp/stream`, `*.arrow`)); -arrowTestConfigurations = arrowFormats.reduce((configs, format) => { - return arrowFileNames.reduce((configs, name) => { - const arrowPath = path.resolve(__dirname, `./arrows/${format}/${name}.arrow`); - return [...configs, [`${name} ${format}`, fs.readFileSync(arrowPath)]]; - }, configs); -}, arrowTestConfigurations); +for (const filename of filenames) { + const { name } = path.parse(filename); + config.push({ name, buffers: [fs.readFileSync(filename)] }); +} -module.exports = arrowTestConfigurations; +module.exports = config; diff --git a/js/perf/index.js b/js/perf/index.js index 3a2ed96772330..9eac40e64ac71 100644 --- a/js/perf/index.js +++ b/js/perf/index.js @@ -16,24 +16,24 @@ // under the License. // Use the ES5 UMD target as perf baseline -// const { Table, readBuffers } = require('../targets/es5/umd'); -// const { Table, readBuffers } = require('../targets/es5/cjs'); -const { Table, readBuffers } = require('../targets/es2015/umd'); -// const { Table, readBuffers } = require('../targets/es2015/cjs'); +// const { Table, readVectors } = require('../targets/es5/umd'); +// const { Table, readVectors } = require('../targets/es5/cjs'); +const { Table, readVectors } = require('../targets/es2015/umd'); +// const { Table, readVectors } = require('../targets/es2015/cjs'); +const config = require('./config'); const Benchmark = require('benchmark'); -const arrowTestConfigurations = require('./config'); const suites = []; -for (let [name, ...buffers] of arrowTestConfigurations) { +for (let { name, buffers} of config) { const parseSuite = new Benchmark.Suite(`Parse ${name}`, { async: true }); const sliceSuite = new Benchmark.Suite(`Slice ${name} vectors`, { async: true }); const iterateSuite = new Benchmark.Suite(`Iterate ${name} vectors`, { async: true }); const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true }); parseSuite.add(createFromTableTest(name, buffers)); - parseSuite.add(createReadBuffersTest(name, buffers)); - for (const vector of Table.from(...buffers).columns) { + parseSuite.add(createReadVectorsTest(name, buffers)); + for (const vector of Table.from(buffers).columns) { sliceSuite.add(createSliceTest(vector)); iterateSuite.add(createIterateTest(vector)); getByIndexSuite.add(createGetByIndexTest(vector)); @@ -66,16 +66,16 @@ function createFromTableTest(name, buffers) { return { async: true, name: `Table.from`, - fn() { table = Table.from(...buffers); } + fn() { table = Table.from(buffers); } }; } -function createReadBuffersTest(name, buffers) { +function createReadVectorsTest(name, buffers) { let vectors; return { async: true, - name: `readBuffers`, - fn() { for (vectors of readBuffers(...buffers)) {} } + name: `readVectors`, + fn() { for (vectors of readVectors(buffers)) {} } }; } diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts index 3196550884dbf..e6527b34e10dd 100644 --- a/js/src/Arrow.ts +++ b/js/src/Arrow.ts @@ -15,45 +15,64 @@ // specific language governing permissions and limitations // under the License. -import { readBuffers } from './reader/arrow'; +import { Table } from './vector/table'; +import { Vector } from './vector/vector'; +import { Utf8Vector } from './vector/utf8'; +import { DictionaryVector } from './vector/dictionary'; +import { StructVector, StructRow } from './vector/struct'; +import { readVectors, readVectorsAsync } from './reader/arrow'; +import { ListVector, BinaryVector, FixedSizeListVector } from './vector/list'; -import { Vector } from './types/types'; -import { ListVector } from './types/list'; -import { Utf8Vector } from './types/utf8'; -import { BoolVector } from './types/vector/bool'; -import { DateVector } from './types/vector/date'; -import { RowVector } from './types/table/row'; -import { TableVector } from './types/table/table'; -import { StructVector } from './types/table/struct'; -import { DictionaryVector } from './types/dictionary'; -import { FixedSizeListVector } from './types/fixedsizelist'; -import { LongVector, Int64Vector, Uint64Vector, } from './types/vector/long'; import { - TypedVector, + BoolVector, Int8Vector, Int16Vector, Int32Vector, + Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, + Uint64Vector, + Float16Vector, Float32Vector, - Float64Vector -} from './types/vector/typed'; - -import './types/table/from'; + Float64Vector, + Date32Vector, + Date64Vector, + Time32Vector, + Time64Vector, + DecimalVector, + TimestampVector, +} from './vector/numeric'; +export { Table, Vector, StructRow }; +export { readVectors, readVectorsAsync }; +export { NumericVectorConstructor } from './vector/numeric'; +export { List, TypedArray, TypedArrayConstructor } from './vector/types'; export { - Vector, - readBuffers, + BoolVector, + ListVector, + Utf8Vector, + Int8Vector, + Int16Vector, + Int32Vector, + Int64Vector, + Uint8Vector, + Uint16Vector, + Uint32Vector, + Uint64Vector, + Date32Vector, + Date64Vector, + Time32Vector, + Time64Vector, + BinaryVector, + StructVector, + Float16Vector, + Float32Vector, + Float64Vector, + DecimalVector, + TimestampVector, DictionaryVector, - RowVector as Row, - TableVector as Table, - StructVector, Utf8Vector, - ListVector, FixedSizeListVector, - BoolVector, TypedVector, LongVector, - DateVector, Float32Vector, Float64Vector, - Int8Vector, Int16Vector, Int32Vector, Int64Vector, - Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, + FixedSizeListVector, }; /* These exports are needed for the closure umd targets */ @@ -61,18 +80,14 @@ try { const Arrow = eval('exports'); if (typeof Arrow === 'object') { // string indexers tell closure compiler not to rename these properties + Arrow['readVectors'] = readVectors; + Arrow['readVectorsAsync'] = readVectorsAsync; + Arrow['Table'] = Table; Arrow['Vector'] = Vector; - Arrow['Table'] = TableVector; - Arrow['readBuffers'] = readBuffers; + Arrow['StructRow'] = StructRow; Arrow['BoolVector'] = BoolVector; - Arrow['Utf8Vector'] = Utf8Vector; Arrow['ListVector'] = ListVector; - Arrow['StructVector'] = StructVector; - Arrow['DictionaryVector'] = DictionaryVector; - Arrow['FixedSizeListVector'] = FixedSizeListVector; - Arrow['LongVector'] = LongVector; - Arrow['TypedVector'] = TypedVector; - Arrow['DateVector'] = DateVector; + Arrow['Utf8Vector'] = Utf8Vector; Arrow['Int8Vector'] = Int8Vector; Arrow['Int16Vector'] = Int16Vector; Arrow['Int32Vector'] = Int32Vector; @@ -81,8 +96,19 @@ try { Arrow['Uint16Vector'] = Uint16Vector; Arrow['Uint32Vector'] = Uint32Vector; Arrow['Uint64Vector'] = Uint64Vector; + Arrow['Date32Vector'] = Date32Vector; + Arrow['Date64Vector'] = Date64Vector; + Arrow['Time32Vector'] = Time32Vector; + Arrow['Time64Vector'] = Time64Vector; + Arrow['BinaryVector'] = BinaryVector; + Arrow['StructVector'] = StructVector; + Arrow['Float16Vector'] = Float16Vector; Arrow['Float32Vector'] = Float32Vector; Arrow['Float64Vector'] = Float64Vector; + Arrow['DecimalVector'] = DecimalVector; + Arrow['TimestampVector'] = TimestampVector; + Arrow['DictionaryVector'] = DictionaryVector; + Arrow['FixedSizeListVector'] = FixedSizeListVector; } } catch (e) { /* not the UMD bundle */ } /* end closure exports */ diff --git a/js/src/format/File_generated.ts b/js/src/format/File.ts similarity index 99% rename from js/src/format/File_generated.ts rename to js/src/format/File.ts index d0b774ae34095..56f50ed20e936 100644 --- a/js/src/format/File_generated.ts +++ b/js/src/format/File.ts @@ -1,7 +1,7 @@ // automatically generated by the FlatBuffers compiler, do not modify import { flatbuffers } from 'flatbuffers'; -import * as NS16187549871986683199 from './Schema_generated'; +import * as NS16187549871986683199 from './Schema'; /** * ---------------------------------------------------------------------- * Arrow File metadata diff --git a/js/closure-compiler-scripts/File_generated.js b/js/src/format/File_generated.js similarity index 95% rename from js/closure-compiler-scripts/File_generated.js rename to js/src/format/File_generated.js index bb82cc4ccc6e4..12aae293ea4eb 100644 --- a/js/closure-compiler-scripts/File_generated.js +++ b/js/src/format/File_generated.js @@ -1,12 +1,5 @@ +import { org } from './Schema'; // automatically generated by the FlatBuffers compiler, do not modify -goog.module("module$targets$es5$cls$format$File_generated"); -goog.module.declareLegacyNamespace(); -var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated"); -/** - * @const - * @namespace - */ -var org = Schema_.org; /** * @const @@ -259,6 +252,5 @@ org.apache.arrow.flatbuf.Block.createBlock = function(builder, offset, metaDataL builder.writeInt64(offset); return builder.offset(); }; +export { org }; -// Exports for Node.js and RequireJS -exports.org = org; diff --git a/js/src/format/Message_generated.ts b/js/src/format/Message.ts similarity index 94% rename from js/src/format/Message_generated.ts rename to js/src/format/Message.ts index daa781f9b9290..4610fbef2e1c8 100644 --- a/js/src/format/Message_generated.ts +++ b/js/src/format/Message.ts @@ -1,7 +1,7 @@ // automatically generated by the FlatBuffers compiler, do not modify import { flatbuffers } from 'flatbuffers'; -import * as NS16187549871986683199 from './Schema_generated'; +import * as NS16187549871986683199 from './Schema'; export namespace org.apache.arrow.flatbuf { export import Schema = NS16187549871986683199.org.apache.arrow.flatbuf.Schema; } @@ -181,7 +181,7 @@ export namespace org.apache.arrow.flatbuf { */ buffers(index: number, obj?: NS16187549871986683199.org.apache.arrow.flatbuf.Buffer): NS16187549871986683199.org.apache.arrow.flatbuf.Buffer | null { let offset = this.bb.__offset(this.bb_pos, 8); - return offset ? (obj || new NS16187549871986683199.org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null; + return offset ? (obj || new NS16187549871986683199.org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null; } /** @@ -236,7 +236,7 @@ export namespace org.apache.arrow.flatbuf { * @param {number} numElems */ static startBuffersVector(builder: flatbuffers.Builder, numElems: number) { - builder.startVector(24, numElems, 8); + builder.startVector(16, numElems, 8); } /** @@ -251,12 +251,12 @@ export namespace org.apache.arrow.flatbuf { } } /** - * ---------------------------------------------------------------------- * For sending dictionary encoding information. Any Field can be * dictionary-encoded, but in this case none of its children may be * dictionary-encoded. - * There is one vector / column per dictionary - * + * There is one vector / column per dictionary, but that vector / column + * may be spread across multiple dictionary batches by using the isDelta + * flag * * @constructor */ @@ -308,11 +308,22 @@ export namespace org.apache.arrow.flatbuf { return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null; } + /** + * If isDelta is true the values in the dictionary are to be appended to a + * dictionary with the indicated id + * + * @returns {boolean} + */ + isDelta(): boolean { + let offset = this.bb.__offset(this.bb_pos, 8); + return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false; + } + /** * @param {flatbuffers.Builder} builder */ static startDictionaryBatch(builder: flatbuffers.Builder) { - builder.startObject(2); + builder.startObject(3); } /** @@ -331,6 +342,14 @@ export namespace org.apache.arrow.flatbuf { builder.addFieldOffset(1, dataOffset, 0); } + /** + * @param {flatbuffers.Builder} builder + * @param {boolean} isDelta + */ + static addIsDelta(builder: flatbuffers.Builder, isDelta: boolean) { + builder.addFieldInt8(2, +isDelta, +false); + } + /** * @param {flatbuffers.Builder} builder * @returns {flatbuffers.Offset} diff --git a/js/closure-compiler-scripts/Message_generated.js b/js/src/format/Message_generated.js similarity index 93% rename from js/closure-compiler-scripts/Message_generated.js rename to js/src/format/Message_generated.js index 0c1a1a99d74ca..ef46c98057c9f 100644 --- a/js/closure-compiler-scripts/Message_generated.js +++ b/js/src/format/Message_generated.js @@ -1,12 +1,5 @@ +import { org } from './Schema'; // automatically generated by the FlatBuffers compiler, do not modify -goog.module("module$targets$es5$cls$format$Message_generated"); -goog.module.declareLegacyNamespace(); -var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated"); -/** - * @const - * @namespace - */ -var org = Schema_.org; /** * @const @@ -200,7 +193,7 @@ org.apache.arrow.flatbuf.RecordBatch.prototype.nodesLength = function() { */ org.apache.arrow.flatbuf.RecordBatch.prototype.buffers = function(index, obj) { var offset = this.bb.__offset(this.bb_pos, 8); - return offset ? (obj || new org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 24, this.bb) : null; + return offset ? (obj || new org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset) + index * 16, this.bb) : null; }; /** @@ -255,7 +248,7 @@ org.apache.arrow.flatbuf.RecordBatch.addBuffers = function(builder, buffersOffse * @param {number} numElems */ org.apache.arrow.flatbuf.RecordBatch.startBuffersVector = function(builder, numElems) { - builder.startVector(24, numElems, 8); + builder.startVector(16, numElems, 8); }; /** @@ -268,12 +261,12 @@ org.apache.arrow.flatbuf.RecordBatch.endRecordBatch = function(builder) { }; /** - * ---------------------------------------------------------------------- * For sending dictionary encoding information. Any Field can be * dictionary-encoded, but in this case none of its children may be * dictionary-encoded. - * There is one vector / column per dictionary - * + * There is one vector / column per dictionary, but that vector / column + * may be spread across multiple dictionary batches by using the isDelta + * flag * * @constructor */ @@ -326,11 +319,22 @@ org.apache.arrow.flatbuf.DictionaryBatch.prototype.data = function(obj) { return offset ? (obj || new org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos + offset), this.bb) : null; }; +/** + * If isDelta is true the values in the dictionary are to be appended to a + * dictionary with the indicated id + * + * @returns {boolean} + */ +org.apache.arrow.flatbuf.DictionaryBatch.prototype.isDelta = function() { + var offset = this.bb.__offset(this.bb_pos, 8); + return offset ? !!this.bb.readInt8(this.bb_pos + offset) : false; +}; + /** * @param {flatbuffers.Builder} builder */ org.apache.arrow.flatbuf.DictionaryBatch.startDictionaryBatch = function(builder) { - builder.startObject(2); + builder.startObject(3); }; /** @@ -349,6 +353,14 @@ org.apache.arrow.flatbuf.DictionaryBatch.addData = function(builder, dataOffset) builder.addFieldOffset(1, dataOffset, 0); }; +/** + * @param {flatbuffers.Builder} builder + * @param {boolean} isDelta + */ +org.apache.arrow.flatbuf.DictionaryBatch.addIsDelta = function(builder, isDelta) { + builder.addFieldInt8(2, +isDelta, +false); +}; + /** * @param {flatbuffers.Builder} builder * @returns {flatbuffers.Offset} @@ -481,6 +493,5 @@ org.apache.arrow.flatbuf.Message.endMessage = function(builder) { org.apache.arrow.flatbuf.Message.finishMessageBuffer = function(builder, offset) { builder.finish(offset); }; +export { org }; -// Exports for Node.js and RequireJS -exports.org = org; diff --git a/js/src/format/Schema_generated.ts b/js/src/format/Schema.ts similarity index 99% rename from js/src/format/Schema_generated.ts rename to js/src/format/Schema.ts index c5b3e5011d790..d9b45ed20089c 100644 --- a/js/src/format/Schema_generated.ts +++ b/js/src/format/Schema.ts @@ -7,9 +7,25 @@ import { flatbuffers } from 'flatbuffers'; */ export namespace org.apache.arrow.flatbuf { export enum MetadataVersion { + /** + * 0.1.0 + */ V1 = 0, + + /** + * 0.2.0 + */ V2 = 1, - V3 = 2 + + /** + * 0.3.0 -> 0.7.1 + */ + V3 = 2, + + /** + * >= 0.8.0 + */ + V4 = 3 } } @@ -2049,7 +2065,6 @@ export namespace org.apache.arrow.flatbuf { /** * @param {flatbuffers.Builder} builder - * @param {number} page * @param {flatbuffers.Long} offset * @param {flatbuffers.Long} length * @returns {flatbuffers.Offset} diff --git a/js/closure-compiler-scripts/Schema_generated.js b/js/src/format/Schema_generated.js similarity index 98% rename from js/closure-compiler-scripts/Schema_generated.js rename to js/src/format/Schema_generated.js index 5b76443886543..f89cf2fa8fa0a 100644 --- a/js/closure-compiler-scripts/Schema_generated.js +++ b/js/src/format/Schema_generated.js @@ -1,6 +1,4 @@ // automatically generated by the FlatBuffers compiler, do not modify -goog.module("module$targets$es5$cls$format$Schema_generated"); -goog.module.declareLegacyNamespace(); /** * @const @@ -30,9 +28,25 @@ org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {}; * @enum */ org.apache.arrow.flatbuf.MetadataVersion = { + /** + * 0.1.0 + */ V1: 0, 0: 'V1', + + /** + * 0.2.0 + */ V2: 1, 1: 'V2', + + /** + * 0.3.0 -> 0.7.1 + */ V3: 2, 2: 'V3', + + /** + * >= 0.8.0 + */ + V4: 3, 3: 'V4' }; /** @@ -103,7 +117,7 @@ org.apache.arrow.flatbuf.Type = { Union: 14, 14: 'Union', FixedSizeBinary: 15, 15: 'FixedSizeBinary', FixedSizeList: 16, 16: 'FixedSizeList', - Map: 17, 17: 'Map', + Map: 17, 17: 'Map' }; /** @@ -131,7 +145,7 @@ org.apache.arrow.flatbuf.VectorType = { /** * Type vector used in Union type */ - TYPE: 3, 3: 'TYPE', + TYPE: 3, 3: 'TYPE' }; /** @@ -2005,16 +2019,6 @@ org.apache.arrow.flatbuf.Buffer.prototype.__init = function(i, bb) { return this; }; -/** - * The shared memory page id where this buffer is located. Currently this is - * not used - * - * @returns {number} - */ -org.apache.arrow.flatbuf.Buffer.prototype.page = function() { - return this.bb.readInt32(this.bb_pos); -}; - /** * The relative offset into the shared memory page where the bytes for this * buffer starts @@ -2022,7 +2026,7 @@ org.apache.arrow.flatbuf.Buffer.prototype.page = function() { * @returns {flatbuffers.Long} */ org.apache.arrow.flatbuf.Buffer.prototype.offset = function() { - return this.bb.readInt64(this.bb_pos + 8); + return this.bb.readInt64(this.bb_pos); }; /** @@ -2032,22 +2036,19 @@ org.apache.arrow.flatbuf.Buffer.prototype.offset = function() { * @returns {flatbuffers.Long} */ org.apache.arrow.flatbuf.Buffer.prototype.length = function() { - return this.bb.readInt64(this.bb_pos + 16); + return this.bb.readInt64(this.bb_pos + 8); }; /** * @param {flatbuffers.Builder} builder - * @param {number} page * @param {flatbuffers.Long} offset * @param {flatbuffers.Long} length * @returns {flatbuffers.Offset} */ -org.apache.arrow.flatbuf.Buffer.createBuffer = function(builder, page, offset, length) { - builder.prep(8, 24); +org.apache.arrow.flatbuf.Buffer.createBuffer = function(builder, offset, length) { + builder.prep(8, 16); builder.writeInt64(length); builder.writeInt64(offset); - builder.pad(4); - builder.writeInt32(page); return builder.offset(); }; @@ -2226,6 +2227,5 @@ org.apache.arrow.flatbuf.Schema.endSchema = function(builder) { org.apache.arrow.flatbuf.Schema.finishSchemaBuffer = function(builder, offset) { builder.finish(offset); }; +export { org }; -// Exports for Node.js and RequireJS -exports.org = org; diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts index 033bfecae61dd..8d3aafc722160 100644 --- a/js/src/reader/arrow.ts +++ b/js/src/reader/arrow.ts @@ -15,66 +15,143 @@ // specific language governing permissions and limitations // under the License. +import { Vector } from '../vector/vector'; import { flatbuffers } from 'flatbuffers'; -import * as Schema_ from '../format/Schema_generated'; -import * as Message_ from '../format/Message_generated'; -export import Schema = Schema_.org.apache.arrow.flatbuf.Schema; -export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; +import { readVector, readValueVector } from './vector'; +import { + readFileFooter, readFileMessages, + readStreamSchema, readStreamMessages +} from './format'; -import { readFile } from './file'; -import { readStream } from './stream'; -import { readVector } from './vector'; -import { readDictionary } from './dictionary'; -import { Vector, Column } from '../types/types'; +import * as File_ from '../format/File'; +import * as Schema_ from '../format/Schema'; +import * as Message_ from '../format/Message'; import ByteBuffer = flatbuffers.ByteBuffer; +import Footer = File_.org.apache.arrow.flatbuf.Footer; import Field = Schema_.org.apache.arrow.flatbuf.Field; -export type Dictionaries = { [k: string]: Vector } | null; -export type IteratorState = { nodeIndex: number; bufferIndex: number }; - -export function* readRecords(...bytes: ByteBuffer[]) { - try { - yield* readFile(...bytes); - } catch (e) { - try { - yield* readStream(...bytes); - } catch (e) { - throw new Error('Invalid Arrow buffer'); - } +import Schema = Schema_.org.apache.arrow.flatbuf.Schema; +import Message = Message_.org.apache.arrow.flatbuf.Message; +import ArrowBuffer = Schema_.org.apache.arrow.flatbuf.Buffer; +import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; +import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; +import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; +import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion; +import DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch; +import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding; + +export type ArrowReaderContext = { + schema?: Schema; + footer?: Footer | null; + dictionaries: Map; + dictionaryEncodedFields: Map; + readMessages: (bb: ByteBuffer, footer: Footer) => Iterable; +}; + +export interface VectorReaderContext { + offset: number; + bytes: Uint8Array; + batch: RecordBatch; + dictionaries: Map; + readNextNode(): FieldNode; + readNextBuffer(): ArrowBuffer; +} + +export function* readVectors(buffers: Iterable, context?: ArrowReaderContext) { + const context_ = context || {} as ArrowReaderContext; + for (const buffer of buffers) { + yield* readBuffer(toByteBuffer(buffer), context_); } } -export function* readBuffers(...bytes: Array) { - const dictionaries: Dictionaries = {}; - const byteBuffers = bytes.map(toByteBuffer); - for (let { schema, batch } of readRecords(...byteBuffers)) { - let vectors: Column[] = []; - let state = { nodeIndex: 0, bufferIndex: 0 }; - let fieldsLength = schema.fieldsLength(); - let index = -1, field: Field, vector: Vector; - if (batch.id) { - // A dictionary batch only contain a single vector. Traverse each - // field and its children until we find one that uses this dictionary - while (++index < fieldsLength) { - if (field = schema.fields(index)!) { - if (vector = readDictionary(field, batch, state, dictionaries)!) { - dictionaries[batch.id] = dictionaries[batch.id] && dictionaries[batch.id].concat(vector) || vector; - break; - } +export async function* readVectorsAsync(buffers: AsyncIterable, context?: ArrowReaderContext) { + const context_ = context || {} as ArrowReaderContext; + for await (const buffer of buffers) { + yield* readBuffer(toByteBuffer(buffer), context_); + } +} + +function* readBuffer(bb: ByteBuffer, readerContext: ArrowReaderContext) { + + let { schema, footer, readMessages, dictionaryEncodedFields, dictionaries } = readerContext; + + if (!schema) { + ({ schema, footer, readMessages, dictionaryEncodedFields } = readSchema(bb)); + readerContext.schema = schema; + readerContext.readMessages = readMessages; + readerContext.dictionaryEncodedFields = dictionaryEncodedFields; + readerContext.dictionaries = dictionaries = new Map(); + } + + const fieldsLength = schema.fieldsLength(); + const context = new BufferReaderContext(bb.bytes(), dictionaries); + + for (const message of readMessages(bb, footer!)) { + + let id: string; + let field: Field; + let vector: Vector; + let vectors: Array; + + context.message = message; + + if (message.headerType() === MessageHeader.DictionaryBatch) { + let batch: DictionaryBatch; + if (batch = message.header(new DictionaryBatch())!) { + context.batch = batch.data()!; + id = batch.id().toFloat64().toString(); + field = dictionaryEncodedFields.get(id)!; + vector = readValueVector(field, context); + if (batch.isDelta() && dictionaries.has(id)) { + vector = dictionaries.get(id)!.concat(vector); } + dictionaries.set(id, vector); } - } else { - while (++index < fieldsLength) { - if ((field = schema.fields(index)!) && - (vector = readVector(field, batch, state, dictionaries)!)) { - vectors[index] = vector as Column; - } + continue; + } + + vectors = new Array(fieldsLength); + context.batch = message.header(new RecordBatch())!; + + for (let i = -1; ++i < fieldsLength;) { + if ((field = schema.fields(i)!) || (vectors[i] = null as any)) { + vectors[i] = readVector(field, context); } - yield vectors; } + + yield vectors; } } +function readSchema(bb: ByteBuffer) { + let schema: Schema, readMessages, footer = readFileFooter(bb); + if (footer) { + schema = footer.schema()!; + readMessages = readFileMessages; + } else if (schema = readStreamSchema(bb)!) { + readMessages = readStreamMessages; + } else { + throw new Error('Invalid Arrow buffer'); + } + return { schema, footer, readMessages, dictionaryEncodedFields: readDictionaryEncodedFields(schema, new Map()) }; +} + +function readDictionaryEncodedFields(parent: Schema | Field, fields: Map) { + let field: Field, encoding: DictionaryEncoding, id: string; + let getField = parent instanceof Field ? parent.children : parent.fields; + let getFieldCount = parent instanceof Field ? parent.childrenLength : parent.fieldsLength; + for (let i = -1, n = getFieldCount.call(parent); ++i < n;) { + if (field = getField.call(parent, i)!) { + if ((encoding = field.dictionary()!) && + (id = encoding.id().toFloat64().toString())) { + !fields.has(id) && fields.set(id, field); + } + readDictionaryEncodedFields(field, fields); + } + } + return fields; +} + function toByteBuffer(bytes?: Uint8Array | Buffer | string) { let arr: Uint8Array = bytes as any || new Uint8Array(0); if (typeof bytes === 'string') { @@ -86,3 +163,33 @@ function toByteBuffer(bytes?: Uint8Array | Buffer | string) { } return new ByteBuffer(arr); } + +class BufferReaderContext implements VectorReaderContext { + public offset: number; + public batch: RecordBatch; + private nodeIndex: number; + private bufferIndex: number; + private metadataVersion: MetadataVersion; + constructor(public bytes: Uint8Array, + public dictionaries: Map) { + } + set message(m: Message) { + this.nodeIndex = 0; + this.bufferIndex = 0; + this.offset = m.bb.position(); + this.metadataVersion = m.version(); + } + public readNextNode() { + return this.batch.nodes(this.nodeIndex++)!; + } + public readNextBuffer() { + const buffer = this.batch.buffers(this.bufferIndex++)!; + // If this Arrow buffer was written before version 4, + // advance the buffer's bb_pos 8 bytes to skip past + // the now-removed page id field. + if (this.metadataVersion < MetadataVersion[`V4`]) { + buffer.bb_pos += (8 * this.bufferIndex); + } + return buffer; + } +} \ No newline at end of file diff --git a/js/src/reader/dictionary.ts b/js/src/reader/dictionary.ts deleted file mode 100644 index 0c58ace3b0dca..0000000000000 --- a/js/src/reader/dictionary.ts +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { readVector } from './vector'; -import { MessageBatch } from './message'; -import { DictionaryVector } from '../types/dictionary'; -import * as Schema_ from '../format/Schema_generated'; -import { IteratorState, Dictionaries } from './arrow'; -import Field = Schema_.org.apache.arrow.flatbuf.Field; - -export function readDictionary(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector | null { - let vector: DictionaryVector | null, id, encoding = field.dictionary(); - if (encoding && batch.id === (id = encoding.id().toFloat64().toString())) { - return readVector(field, batch, iterator, null) as DictionaryVector; - } - for (let i = -1, n = field.childrenLength() | 0; ++i < n;) { - if (vector = readDictionary(field.children(i)!, batch, iterator, dictionaries)) { - return vector; - } - } - return null; -} diff --git a/js/src/reader/file.ts b/js/src/reader/file.ts deleted file mode 100644 index bd60b476312a0..0000000000000 --- a/js/src/reader/file.ts +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { flatbuffers } from 'flatbuffers'; -import * as File_ from '../format/File_generated'; -import * as Schema_ from '../format/Schema_generated'; -import * as Message_ from '../format/Message_generated'; -import { PADDING, readMessageBatches } from './message'; - -import ByteBuffer = flatbuffers.ByteBuffer; -import Footer = File_.org.apache.arrow.flatbuf.Footer; -export import Schema = Schema_.org.apache.arrow.flatbuf.Schema; -export import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; - -const MAGIC_STR = 'ARROW1'; -const MAGIC = new Uint8Array(MAGIC_STR.length); -for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) { - MAGIC[i] = MAGIC_STR.charCodeAt(i); -} - -export function _checkMagic(buffer: Uint8Array, index = 0) { - for (let i = -1, n = MAGIC.length; ++i < n;) { - if (MAGIC[i] !== buffer[index + i]) { - return false; - } - } - return true; -} - -const magicLength = MAGIC.length; -const magicAndPadding = magicLength + PADDING; -const magicX2AndPadding = magicLength * 2 + PADDING; - -export function* readFile(...bbs: ByteBuffer[]) { - for (let bb of bbs) { - let fileLength = bb.capacity(); - let footerLength: number, footerOffset: number; - if ((fileLength < magicX2AndPadding /* Arrow buffer too small */) || - (!_checkMagic(bb.bytes(), 0) /* Missing magic start */) || - (!_checkMagic(bb.bytes(), fileLength - magicLength) /* Missing magic end */) || - (/* Invalid footer length */ - (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 && - (footerLength + magicX2AndPadding > fileLength))) { - throw new Error('Invalid file'); - } - bb.setPosition(footerOffset - footerLength); - let schema, footer = Footer.getRootAsFooter(bb); - if (!(schema = footer.schema()!)) { - return; - } - for (let i = -1, n = footer.dictionariesLength(); ++i < n;) { - let block = footer.dictionaries(i)!; - bb.setPosition(block.offset().low); - for (let batch of readMessageBatches(bb)) { - yield { schema, batch }; - break; - } - } - for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) { - const block = footer.recordBatches(i)!; - bb.setPosition(block.offset().low); - for (let batch of readMessageBatches(bb)) { - yield { schema, batch }; - break; - } - } - } -} diff --git a/js/src/reader/format.ts b/js/src/reader/format.ts new file mode 100644 index 0000000000000..fd8f1b40d918a --- /dev/null +++ b/js/src/reader/format.ts @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { flatbuffers } from 'flatbuffers'; +import * as File_ from '../format/File'; +import * as Schema_ from '../format/Schema'; +import * as Message_ from '../format/Message'; +import ByteBuffer = flatbuffers.ByteBuffer; +import Footer = File_.org.apache.arrow.flatbuf.Footer; +import Schema = Schema_.org.apache.arrow.flatbuf.Schema; +import Message = Message_.org.apache.arrow.flatbuf.Message; +import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; + +const PADDING = 4; +const MAGIC_STR = 'ARROW1'; +const MAGIC = new Uint8Array(MAGIC_STR.length); +for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) { + MAGIC[i] = MAGIC_STR.charCodeAt(i); +} + +export function _checkMagic(buffer: Uint8Array, index = 0) { + for (let i = -1, n = MAGIC.length; ++i < n;) { + if (MAGIC[i] !== buffer[index + i]) { + return false; + } + } + return true; +} + +const magicLength = MAGIC.length; +const magicAndPadding = magicLength + PADDING; +const magicX2AndPadding = magicLength * 2 + PADDING; + +export function readStreamSchema(bb: ByteBuffer) { + if (!_checkMagic(bb.bytes(), 0)) { + for (const message of readMessages(bb)) { + if (message.headerType() === MessageHeader.Schema) { + return message.header(new Schema()); + } + } + } + return null; +} + +export function readFileFooter(bb: ByteBuffer) { + let fileLength = bb.capacity(); + let footerLength: number, footerOffset: number; + if ((fileLength < magicX2AndPadding /* Arrow buffer too small */) || + (!_checkMagic(bb.bytes(), 0) /* Missing magic start */) || + (!_checkMagic(bb.bytes(), fileLength - magicLength) /* Missing magic end */) || + (/* Invalid footer length */ + (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 && + (footerLength + magicX2AndPadding > fileLength))) { + return null; + } + bb.setPosition(footerOffset - footerLength); + return Footer.getRootAsFooter(bb); +} + +export function* readFileMessages(bb: ByteBuffer, footer: Footer) { + for (let i = -1, n = footer.dictionariesLength(); ++i < n;) { + bb.setPosition(footer.dictionaries(i)!.offset().low); + yield readMessage(bb, bb.readInt32(bb.position())); + } + for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) { + bb.setPosition(footer.recordBatches(i)!.offset().low); + yield readMessage(bb, bb.readInt32(bb.position())); + } +} + +export function readMessage(bb: ByteBuffer, length: number) { + bb.setPosition(bb.position() + PADDING); + const message = Message.getRootAsMessage(bb); + bb.setPosition(bb.position() + length); + return message; +} + +export function* readMessages(bb: ByteBuffer) { + let length; + while (bb.position() < bb.capacity() && + (length = bb.readInt32(bb.position())) > 0) { + yield readMessage(bb, length); + } +} + +export function* readStreamMessages(bb: ByteBuffer) { + for (const message of readMessages(bb)) { + switch (message.headerType()) { + case MessageHeader.RecordBatch: + case MessageHeader.DictionaryBatch: + yield message; + break; + default: continue; + } + // position the buffer after the body to read the next message + bb.setPosition(bb.position() + message.bodyLength().low); + } +} diff --git a/js/src/reader/message.ts b/js/src/reader/message.ts deleted file mode 100644 index 6c8a969021f90..0000000000000 --- a/js/src/reader/message.ts +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { flatbuffers } from 'flatbuffers'; -import * as Message_ from '../format/Message_generated'; -import ByteBuffer = flatbuffers.ByteBuffer; -import Message = Message_.org.apache.arrow.flatbuf.Message; -import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; -import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; -import DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch; - -export const PADDING = 4; -export type MessageBatch = { - id?: string; - offset: number; - bytes: Uint8Array; - data: RecordBatch; -}; - -export function* readMessages(bb: ByteBuffer) { - let message, length; - while (bb.position() < bb.capacity() && - (length = bb.readInt32(bb.position())) > 0) { - bb.setPosition(bb.position() + PADDING); - message = Message.getRootAsMessage(bb); - bb.setPosition(bb.position() + length); - yield message; - } -} - -export function* readMessageBatches(bb: ByteBuffer) { - let bytes = bb.bytes(); - for (let message of readMessages(bb)) { - let type = message.headerType(); - let id: string | void, data: RecordBatch; - if (type === MessageHeader.RecordBatch) { - data = message.header(new RecordBatch())!; - } else if (type === MessageHeader.DictionaryBatch) { - let header = message.header(new DictionaryBatch())!; - id = header.id().toFloat64().toString(); - data = header.data()!; - } else { - continue; - } - yield { id, data, bytes, offset: bytes.byteOffset + bb.position() }; - // position the buffer after the body to read the next message - bb.setPosition(bb.position() + message.bodyLength().low); - } -} diff --git a/js/src/reader/stream.ts b/js/src/reader/stream.ts deleted file mode 100644 index 2062b1a8c4c10..0000000000000 --- a/js/src/reader/stream.ts +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { _checkMagic } from './file'; -import { flatbuffers } from 'flatbuffers'; -import * as Schema_ from '../format/Schema_generated'; -import * as Message_ from '../format/Message_generated'; -import { readMessages, readMessageBatches } from './message'; - -import ByteBuffer = flatbuffers.ByteBuffer; -import Schema = Schema_.org.apache.arrow.flatbuf.Schema; -import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; - -export function* readStream(...bbs: ByteBuffer[]) { - if (!bbs.length || _checkMagic(bbs[0].bytes(), 0)) { - throw new Error('Invalid Arrow Stream'); - } - for (const message of readMessages(bbs[0])) { - let schema: Schema; - if (message.headerType() === MessageHeader.Schema && (schema = message.header(new Schema())!)) { - for (const bb of bbs) { - for (const batch of readMessageBatches(bb)) { - yield { schema, batch }; - } - } - break; - } - } -} diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts index 4d3321833209d..0f95b769e141f 100644 --- a/js/src/reader/vector.ts +++ b/js/src/reader/vector.ts @@ -15,288 +15,257 @@ // specific language governing permissions and limitations // under the License. -import { flatbuffers } from 'flatbuffers'; -import { MessageBatch } from './message'; -import * as Schema_ from '../format/Schema_generated'; -import * as Message_ from '../format/Message_generated'; -import { IteratorState, Dictionaries } from './arrow'; +import { VectorReaderContext } from './arrow'; +import * as Schema_ from '../format/Schema'; +import * as Message_ from '../format/Message'; +import { TypedArray, TypedArrayConstructor } from '../vector/types'; import { - Vector, Column, - IntArray, FloatArray, - TypedArray, TypedArrayConstructor, -} from '../types/types'; - -import { - DictionaryVector, - Utf8Vector, StructVector, - ListVector, FixedSizeListVector, - DateVector, Float32Vector, Float64Vector, + Vector, BoolVector, BinaryVector, DictionaryVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, -} from '../types/arrow'; + Utf8Vector, ListVector, FixedSizeListVector, StructVector, + Float16Vector, Float32Vector, Float64Vector, DecimalVector, + Date32Vector, Date64Vector, Time32Vector, Time64Vector, TimestampVector, +} from '../vector/arrow'; import Int = Schema_.org.apache.arrow.flatbuf.Int; +import Date = Schema_.org.apache.arrow.flatbuf.Date; +import Time = Schema_.org.apache.arrow.flatbuf.Time; import Type = Schema_.org.apache.arrow.flatbuf.Type; import Field = Schema_.org.apache.arrow.flatbuf.Field; -import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; +import Buffer = Schema_.org.apache.arrow.flatbuf.Buffer; +import Decimal = Schema_.org.apache.arrow.flatbuf.Decimal; +import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit; +import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit; +// import Interval = Schema_.org.apache.arrow.flatbuf.Interval; +import Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp; +// import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit; import Precision = Schema_.org.apache.arrow.flatbuf.Precision; -import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType; -import VectorLayout = Schema_.org.apache.arrow.flatbuf.VectorLayout; +import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList; import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint; -import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding; +import FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary; -export function readVector(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries): Column | DictionaryVector | null { - return readDictionaryVector(field, batch, state, dictionaries) || - readTypedVector(field, batch, state, dictionaries); +export function readVector(field: Field, state: VectorReaderContext) { + return readDictionaryVector(field, state) || readValueVector(field, state); } -function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): Column | DictionaryVector | null { - let typeType = field.typeType(), readTyped = typedVectorReaders[typeType]; - if (!readTyped) { - throw new Error('Unrecognized vector name "' + Type[typeType] + '" type "' + typeType + '"'); - } - return readTyped(field, batch, iterator, dictionaries) as Column; -} +/* a dictionary index defaults to signed 32 bit int if unspecified */ +const defaultDictionaryIndexType = { bitWidth: () => 32, isSigned: () => true } as Int; +const intVectors = [ + [/* unsigned */ [Uint8Vector, Uint8Array ], /* signed */ [Int8Vector , Int8Array ]], + [/* unsigned */ [Uint16Vector, Uint16Array], /* signed */ [Int16Vector, Int16Array]], + [/* unsigned */ [Uint32Vector, Uint32Array], /* signed */ [Int32Vector, Int32Array]],, + [/* unsigned */ [Uint64Vector, Uint32Array], /* signed */ [Int64Vector, Int32Array]] +] as [any, TypedArrayConstructor][][]; -function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries): DictionaryVector | null { - let data: Vector, encoding: DictionaryEncoding; - if (dictionaries && - (encoding = field.dictionary()!) && - (data = dictionaries[encoding.id().toFloat64().toString()])) { - let indexType = encoding.indexType() || - /* a dictionary index defaults to signed 32 bit int if unspecified */ - { bitWidth: () => 32, isSigned: () => true }; - // workaround for https://issues.apache.org/jira/browse/ARROW-1363 - let indexField = createSyntheticDictionaryIndexField(field, indexType); - let keys = readIntVector(indexField, batch, iterator, null, indexType)!; - return new DictionaryVector({ data, keys: keys! }); +export function readDictionaryVector(field: Field, state: VectorReaderContext) { + const encoding = field.dictionary()!; + if (encoding) { + const type = encoding.indexType() || defaultDictionaryIndexType; + const data = state.dictionaries.get(encoding.id().toFloat64().toString())!; + const [IntVector, IntArray] = intVectors[type.bitWidth() >>> 4]![+type.isSigned()]; + const { fieldNode, validity, data: keys } = readNumericBuffers(field, state, IntArray); + return new DictionaryVector({ + validity, data, field, fieldNode, + keys: new IntVector({ field, fieldNode, data: keys }) + }); } return null; } -const IntViews = [Int8Array, Int16Array, Int32Array, Int32Array ]; -const Int32Views = [Int32Array, Int32Array, Int32Array, Int32Array ]; -const UintViews = [Uint8Array, Uint16Array, Uint32Array, Uint32Array ]; -const Uint8Views = [Uint8Array, Uint8Array, Uint8Array, Uint8Array ]; -const Uint32Views = [Uint32Array, Uint32Array, Uint32Array, Uint32Array ]; -const FloatViews = [Int8Array, Int16Array, Float32Array, Float64Array]; - -const createIntDataViews = createTypedArray.bind(null, IntViews, null); -const createUintDataViews = createTypedArray.bind(null, UintViews, null); -const createDateDataViews = createTypedArray.bind(null, Uint32Views, null); -const createFloatDataViews = createTypedArray.bind(null, FloatViews, null); -const createNestedDataViews = createTypedArray.bind(null, Uint32Views, null); -const createValidityDataViews = createTypedArray.bind(null, Uint8Views, null); -const createUtf8DataViews = createTypedArray.bind(null, Uint8Views, Int32Views); - -// Define as computed properties for closure-compiler -const floatVectors = { - [Precision.HALF]: Float32Vector, - [Precision.SINGLE]: Float32Vector, - [Precision.DOUBLE]: Float64Vector, -} as { [k: number]: any }; +export function readValueVector(field: Field, state: VectorReaderContext): Vector { + switch (field.typeType()) { + case Type.NONE: return readNullVector(); + case Type.Null: return readNullVector(); + // case Type.Map: return readMapVector(field, state); + case Type.Int: return readIntVector(field, state); + case Type.Bool: return readBoolVector(field, state); + case Type.Date: return readDateVector(field, state); + case Type.List: return readListVector(field, state); + case Type.Utf8: return readUtf8Vector(field, state); + case Type.Time: return readTimeVector(field, state); + // case Type.Union: return readUnionVector(field, state); + case Type.Binary: return readBinaryVector(field, state); + case Type.Decimal: return readDecimalVector(field, state); + case Type.Struct_: return readStructVector(field, state); + case Type.FloatingPoint: return readFloatVector(field, state); + case Type.Timestamp: return readTimestampVector(field, state); + case Type.FixedSizeList: return readFixedSizeListVector(field, state); + case Type.FixedSizeBinary: return readFixedSizeBinaryVector(field, state); + } + throw new Error(`Unrecognized Vector { name: ${Type[field.typeType()]}, type: ${field.typeType()} }`); +} -// and again as string-indexed keys for Uglify... -floatVectors[Precision['HALF']] = Float32Vector; -floatVectors[Precision['SINGLE']] = Float32Vector; -floatVectors[Precision['DOUBLE']] = Float64Vector; +export function readNullVector() { + return new Vector(); +} -const intVectors = [ - [/* unsigned */ Uint8Vector, /* signed */ Int8Vector ], - [/* unsigned */ Uint16Vector, /* signed */ Int16Vector], - [/* unsigned */ Uint32Vector, /* signed */ Int32Vector], - [/* unsigned */ Uint64Vector, /* signed */ Int64Vector] -] as any[][]; +export function readBoolVector(field: Field, state: VectorReaderContext) { + return new BoolVector(readNumericBuffers(field, state, Uint8Array)); +} -function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { - let type = (primitiveType || field.type(new Int())!); - return type.isSigned() ? - read_IntVector(field, batch, iterator, dictionaries, type) : - readUintVector(field, batch, iterator, dictionaries, type); +export function readDateVector(field: Field, state: VectorReaderContext) { + const type = field.type(new Date())!; + switch (type.unit()) { + case DateUnit.DAY: return new Date32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] }); + case DateUnit.MILLISECOND: return new Date64Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] }); + } + throw new Error(`Unrecognized Date { unit: ${type.unit()} }`); } -function read_IntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { - return readVectorLayout(createIntDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType); +export function readTimeVector(field: Field, state: VectorReaderContext) { + const type = field.type(new Time())!; + switch (type.bitWidth()) { + case 32: return new Time32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: TimeUnit[type.unit()] }); + case 64: return new Time64Vector({ ...readNumericBuffers(field, state, Uint32Array), unit: TimeUnit[type.unit()] }); + } + throw new Error(`Unrecognized Time { unit: ${type.unit()}, bitWidth: ${type.bitWidth()} }`); } -function readUintVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { - return readVectorLayout(createUintDataViews, createIntVector, field, batch, iterator, dictionaries, primitiveType); +export function readTimestampVector(field: Field, state: VectorReaderContext) { + const type = field.type(new Timestamp())!; + const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array); + return new TimestampVector({ + field, fieldNode, validity, data, + timezone: type.timezone()!, + unit: TimeUnit[type.unit()], + }); } -function createIntVector(argv: VectorFactoryArgv) { - let { field, fieldNode, data, validity, offsets, primitiveType } = argv; - let type = primitiveType || field.type(new Int())!, bitWidth = type.bitWidth(); - let IntVector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()]; - return new IntVector({ fieldNode, field, validity, data: data! || offsets! }); - // ---------------------------------------------------- 👆: - // Workaround for https://issues.apache.org/jira/browse/ARROW-1363 - // This bug causes dictionary encoded vector indicies' IntVector data - // buffers to be tagged as VectorType.OFFSET (0) in the field metadata - // instead of VectorType.DATA. The `readVectorLayout` routine strictly - // obeys the types in the field metadata, so if we're parsing an Arrow - // file written by a version of the library published before ARROW-1363 - // was fixed, the IntVector's data buffer will be null, and the offset - // buffer will be the actual data. If data is null, it's safe to assume - // the offset buffer is the data, because IntVectors don't have offsets. +export function readListVector(field: Field, state: VectorReaderContext) { + const { fieldNode, validity, offsets } = readListBuffers(field, state); + return new ListVector({ + field, fieldNode, validity, offsets, + values: readVector(field.children(0)!, state) + }); } -function bindVectorReader(createBufferView: BufferViewFactory, createVector: VectorFactory) { - return function readVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType) { - return readVectorLayout(createBufferView, createVector, field, batch, iterator, dictionaries, primitiveType); - }; +export function readStructVector(field: Field, state: VectorReaderContext) { + const n = field.childrenLength(); + const columns = new Array(n); + const fieldNode = state.readNextNode(); + const validity = readValidityBuffer(field, fieldNode, state); + for (let i = -1, child: Field; ++i < n;) { + if (child = field.children(i)!) { + columns[i] = readVector(child, state); + } + } + return new StructVector({ field, fieldNode, validity, columns }); } -const readFloatVector = bindVectorReader(createFloatDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv) => { - const type = field.type(new FloatingPoint())!; - const FloatVector = floatVectors[type.precision()]; - return new FloatVector({ field, fieldNode, validity, data: data! }); -}); +export function readBinaryVector(field: Field, state: VectorReaderContext) { + return new BinaryVector(readBinaryBuffers(field, state)); +} -const readDateVector = bindVectorReader(createDateDataViews, ({ field, fieldNode, data, validity }: VectorFactoryArgv) => { - return new DateVector({ field, fieldNode, validity, data: data! }); -}); +export function readDecimalVector(field: Field, state: VectorReaderContext) { + const type = field.type(new Decimal())!; + const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array); + return new DecimalVector({ + scale: type.scale(), + precision: type.precision(), + field, fieldNode, validity, data + }); +} -const readUtf8Vector = bindVectorReader(createUtf8DataViews, ({ field, fieldNode, data, offsets, validity }: VectorFactoryArgv) => { +export function readUtf8Vector(field: Field, state: VectorReaderContext) { + const { fieldNode, validity, offsets, data } = readBinaryBuffers(field, state); return new Utf8Vector({ field, fieldNode, - values: new ListVector({ - validity, - offsets: offsets as Int32Array, - values: new Uint8Vector({ data: data! }) - }) as any as Vector + values: new BinaryVector({ + validity, offsets, data + }) }); -}); +} -const readListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, offsets, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv) => { - return new ListVector({ +export function readFixedSizeListVector(field: Field, state: VectorReaderContext) { + const type = field.type(new FixedSizeList())!; + const fieldNode = state.readNextNode(); + const validity = readValidityBuffer(field, fieldNode, state); + return new FixedSizeListVector({ field, fieldNode, validity, - offsets: offsets! as Int32Array, - values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)! + size: type.listSize(), + values: readVector(field.children(0)!, state) }); -}); +} -const readFixedSizeListVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv) => { +export function readFixedSizeBinaryVector(field: Field, state: VectorReaderContext) { + const type = field.type(new FixedSizeBinary())!; + const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint8Array); return new FixedSizeListVector({ + size: type.byteWidth(), field, fieldNode, validity, - listSize: field.type(new FixedSizeList())!.listSize(), - values: readVector(field.children(0)!, messageBatch, iterator, dictionaries)! + values: new Uint8Vector({ data }) }); -}); +} -const readStructVector = bindVectorReader(createNestedDataViews, ({ field, fieldNode, validity, iterator, messageBatch, dictionaries }: VectorFactoryArgv>) => { - let columns: Column[] = []; - for (let i = -1, n = field.childrenLength(); ++i < n;) { - columns[i] = readVector(field.children(i)!, messageBatch, iterator, dictionaries) as Column; +export function readFloatVector(field: Field, state: VectorReaderContext) { + const type = field.type(new FloatingPoint())!; + switch (type.precision()) { + case Precision.HALF: return new Float16Vector(readNumericBuffers(field, state, Uint16Array)); + case Precision.SINGLE: return new Float32Vector(readNumericBuffers(field, state, Float32Array)); + case Precision.DOUBLE: return new Float64Vector(readNumericBuffers(field, state, Float64Array)); } - return new StructVector({ field, fieldNode, validity, columns }); -}); + throw new Error(`Unrecognized FloatingPoint { precision: ${type.precision()} }`); +} -// Define as computed properties for closure-compiler -const typedVectorReaders = { - [Type.Int]: readIntVector, - [Type.Date]: readDateVector, - [Type.List]: readListVector, - [Type.Utf8]: readUtf8Vector, - [Type.Struct_]: readStructVector, - [Type.FloatingPoint]: readFloatVector, - [Type.FixedSizeList]: readFixedSizeListVector, -} as { [k: number]: (...args: any[]) => Vector | null }; +export function readIntVector(field: Field, state: VectorReaderContext) { + const type = field.type(new Int())!; + if (type.isSigned()) { + switch (type.bitWidth()) { + case 8: return new Int8Vector(readNumericBuffers(field, state, Int8Array)); + case 16: return new Int16Vector(readNumericBuffers(field, state, Int16Array)); + case 32: return new Int32Vector(readNumericBuffers(field, state, Int32Array)); + case 64: return new Int64Vector(readNumericBuffers(field, state, Int32Array)); + } + } + switch (type.bitWidth()) { + case 8: return new Uint8Vector(readNumericBuffers(field, state, Uint8Array)); + case 16: return new Uint16Vector(readNumericBuffers(field, state, Uint16Array)); + case 32: return new Uint32Vector(readNumericBuffers(field, state, Uint32Array)); + case 64: return new Uint64Vector(readNumericBuffers(field, state, Uint32Array)); + } + throw new Error(`Unrecognized Int { isSigned: ${type.isSigned()}, bitWidth: ${type.bitWidth()} }`); +} -// and again as string-indexed keys for Uglify... -typedVectorReaders[Type['Int']] = readIntVector; -typedVectorReaders[Type['Date']] = readDateVector; -typedVectorReaders[Type['List']] = readListVector; -typedVectorReaders[Type['Utf8']] = readUtf8Vector; -typedVectorReaders[Type['Struct_']] = readStructVector; -typedVectorReaders[Type['FloatingPoint']] = readFloatVector; -typedVectorReaders[Type['FixedSizeList']] = readFixedSizeListVector; +function readListBuffers(field: Field, state: VectorReaderContext) { + const fieldNode = state.readNextNode(); + const validity = readValidityBuffer(field, fieldNode, state); + const offsets = readDataBuffer(Int32Array, state); + return { field, fieldNode, validity, offsets }; +} -type VectorFactory = (argv: VectorFactoryArgv) => V; -type PrimitiveType = { bitWidth(): number; isSigned(): boolean }; -type BufferViewFactory = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => T; +function readBinaryBuffers(field: Field, state: VectorReaderContext) { + const fieldNode = state.readNextNode(); + const validity = readValidityBuffer(field, fieldNode, state); + const offsets = readDataBuffer(Int32Array, state); + const data = readDataBuffer(Uint8Array, state); + return { field, fieldNode, validity, offsets, data }; +} -interface VectorFactoryArgv { - field: Field; - fieldNode: FieldNode; - iterator: IteratorState; - dictionaries: Dictionaries; - messageBatch: MessageBatch; - data?: T; - offsets?: TypedArray; - validity?: Uint8Array; - primitiveType?: PrimitiveType; +function readNumericBuffers(field: Field, state: VectorReaderContext, ArrayConstructor: TypedArrayConstructor) { + const fieldNode = state.readNextNode(); + const validity = readValidityBuffer(field, fieldNode, state); + const data = readDataBuffer(ArrayConstructor, state); + return { field, fieldNode, validity, data }; } -function readVectorLayout( - createBufferView: BufferViewFactory, createVector: VectorFactory, - field: Field, messageBatch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, primitiveType?: PrimitiveType -) { - let fieldNode: FieldNode, recordBatch = messageBatch.data; - if (!(fieldNode = recordBatch.nodes(iterator.nodeIndex)!)) { - return null; - } - iterator.nodeIndex += 1; - let type, bitWidth, layout, buffer, bufferLength; - let data: T | undefined, offsets: TypedArray | undefined, validity: Uint8Array | undefined; - for (let i = -1, n = field.layoutLength(); ++i < n;) { - if (!(layout = field.layout(i)!) || - !(buffer = recordBatch.buffers(iterator.bufferIndex)!)) { - continue; - } - iterator.bufferIndex += 1; - if ((type = layout.type()) === VectorType.TYPE || - (bufferLength = buffer.length().low) <= 0 || - (bitWidth = layout.bitWidth()) <= 0) { - continue; - } else if (type === VectorType.DATA) { - data = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength); - } else if (type === VectorType.OFFSET) { - offsets = createBufferView(messageBatch, type, bitWidth, buffer.offset().low, bufferLength); - } else if (fieldNode.nullCount().low > 0) { - validity = createValidityDataViews(messageBatch, type, bitWidth, buffer.offset().low, fieldNode.length().low); - } - } - return createVector({ data, offsets, validity, field, fieldNode, iterator, messageBatch, dictionaries, primitiveType }); +function readDataBuffer(ArrayConstructor: TypedArrayConstructor, state: VectorReaderContext) { + return createTypedArray(ArrayConstructor, state.bytes, state.offset, state.readNextBuffer()); } -function createTypedArray( - bufferViews: TypedArrayConstructor[], offsetViews: TypedArrayConstructor[] | null, - batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number -) { - const buffer = batch.bytes.buffer; - const byteLength = buffer.byteLength; - const byteOffset = batch.offset + offset; - const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || bufferViews); - const dataViewLength = ((byteOffset + length) <= byteLength - ? length - : byteLength - byteOffset - ) / DataViewType['BYTES_PER_ELEMENT']; - return new DataViewType(buffer, byteOffset, dataViewLength); +function readValidityBuffer(field: Field, fieldNode: FieldNode, state: VectorReaderContext) { + return createValidityArray(field, fieldNode, state.bytes, state.offset, state.readNextBuffer()); } -function valueForBitWidth(bitWidth: number, values: T[]) { - return values[bitWidth >> 4] || values[3]; +function createValidityArray(field: Field, fieldNode: FieldNode, bytes: Uint8Array, offset: number, buffer: Buffer) { + return field.nullable() && fieldNode.nullCount().low > 0 && createTypedArray(Uint8Array, bytes, offset, buffer) || null; } -function createSyntheticDictionaryIndexField(field: Field, type: PrimitiveType) { - let layouts = [] as VectorLayout[]; - let builder = new flatbuffers.Builder(); - if (field.nullable()) { - VectorLayout.startVectorLayout(builder); - VectorLayout.addBitWidth(builder, 8); - VectorLayout.addType(builder, VectorType.VALIDITY); - builder.finish(VectorLayout.endVectorLayout(builder)); - layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer())); - builder = new flatbuffers.Builder(); - } - VectorLayout.startVectorLayout(builder); - VectorLayout.addBitWidth(builder, type.bitWidth()); - VectorLayout.addType(builder, VectorType.DATA); - builder.finish(VectorLayout.endVectorLayout(builder)); - layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer())); - return Object.create(field, { - layout: { value(i: number) { return layouts[i]; } }, - layoutLength: { value() { return layouts.length; } } - }); +function createTypedArray(ArrayConstructor: TypedArrayConstructor, bytes: Uint8Array, offset: number, buffer: Buffer) { + return new ArrayConstructor( + bytes.buffer, + bytes.byteOffset + offset + buffer.offset().low, + buffer.length().low / ArrayConstructor.BYTES_PER_ELEMENT + ); } diff --git a/js/src/types/arrow.ts b/js/src/types/arrow.ts deleted file mode 100644 index e18f5da4f1fd6..0000000000000 --- a/js/src/types/arrow.ts +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import * as Schema_ from '../format/Schema_generated'; -import * as Message_ from '../format/Message_generated'; -import Field = Schema_.org.apache.arrow.flatbuf.Field; -import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; - -import { BoolVector } from './vector/bool'; -import { DictionaryVector } from './dictionary'; -import { nullableMixin, fieldMixin } from './vector/traits'; -import { ListVector as ListVectorBase } from './list'; -import { Utf8Vector as Utf8VectorBase } from './utf8'; -import { Vector, Column, TypedArray } from './types'; -import { DateVector as DateVectorBase } from './vector/date'; -import { TableVector as TableVectorBase } from './table/table'; -import { StructVector as StructVectorBase } from './table/struct'; -import { FixedSizeListVector as FixedSizeListVectorBase } from './fixedsizelist'; -import { - LongVector as LongVectorBase, - Int64Vector as Int64VectorBase, - Uint64Vector as Uint64VectorBase, -} from './vector/long'; -import { - TypedVector, - Int8Vector as Int8VectorBase, - Int16Vector as Int16VectorBase, - Int32Vector as Int32VectorBase, - Uint8Vector as Uint8VectorBase, - Uint16Vector as Uint16VectorBase, - Uint32Vector as Uint32VectorBase, - Float32Vector as Float32VectorBase, - Float64Vector as Float64VectorBase, -} from './vector/typed'; - -export { TypedArray, TypedVector }; -export { Column, BoolVector, DictionaryVector }; -export class ListVector extends MixinArrowTraits(ListVectorBase) {} -export class Utf8Vector extends MixinArrowTraits(Utf8VectorBase) {} -export class TableVector extends MixinArrowTraits(TableVectorBase) {} -export class StructVector extends MixinArrowTraits(StructVectorBase) {} -export class FixedSizeListVector extends MixinArrowTraits(FixedSizeListVectorBase) {} -export class DateVector extends MixinArrowTraits(DateVectorBase) {} -export class LongVector extends MixinArrowTraits(LongVectorBase) {} -export class Int8Vector extends MixinArrowTraits(Int8VectorBase) {} -export class Int16Vector extends MixinArrowTraits(Int16VectorBase) {} -export class Int32Vector extends MixinArrowTraits(Int32VectorBase) {} -export class Int64Vector extends MixinArrowTraits(Int64VectorBase) {} -export class Uint8Vector extends MixinArrowTraits(Uint8VectorBase) {} -export class Uint16Vector extends MixinArrowTraits(Uint16VectorBase) {} -export class Uint32Vector extends MixinArrowTraits(Uint32VectorBase) {} -export class Uint64Vector extends MixinArrowTraits(Uint64VectorBase) {} -export class Float32Vector extends MixinArrowTraits(Float32VectorBase) {} -export class Float64Vector extends MixinArrowTraits(Float64VectorBase) {} - -export function MixinArrowTraits, TArgv>(BaseVector: new (argv: TArgv) => T) { - const FieldVector = fieldMixin(BaseVector); - const NullableVector = nullableMixin(BaseVector); - const NullableFieldVector = nullableMixin(FieldVector); - return function(this: any, argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })) { - return new ((!isFieldArgv(argv) ? !isNullableArgv(argv) ? - BaseVector : NullableVector : !isNullableArgv(argv) ? - FieldVector : NullableFieldVector - ) as any)(argv); - } as any as { new (argv: TArgv & (object | { validity: Uint8Array } | { field: Field, fieldNode: FieldNode })): T }; -} - -function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } { - return x && x.field instanceof Field && x.fieldNode instanceof FieldNode; -} - -function isNullableArgv(x: any): x is { validity: Uint8Array } { - return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array; -} diff --git a/js/src/types/fixedsizelist.ts b/js/src/types/fixedsizelist.ts deleted file mode 100644 index 6311d891d5058..0000000000000 --- a/js/src/types/fixedsizelist.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { List, Vector } from './types'; -import { VirtualVector } from './vector/virtual'; - -export class FixedSizeListVector> extends Vector { - readonly listSize: number; - readonly values: Vector; - constructor(argv: { listSize: number, values: Vector }) { - super(); - this.values = argv.values; - this.listSize = argv.listSize; - } - get(index: number) { - return this.values.slice(this.listSize * index, this.listSize * (index + 1)); - } - concat(...vectors: Vector[]): Vector { - return new VirtualVector(Array, this, ...vectors); - } -} diff --git a/js/src/types/list.ts b/js/src/types/list.ts deleted file mode 100644 index ca9170b5908d5..0000000000000 --- a/js/src/types/list.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { List, Vector } from './types'; -import { VirtualVector } from './vector/virtual'; - -export class ListVector> extends Vector { - readonly offsets: Int32Array; - readonly values: Vector; - constructor(argv: { offsets: Int32Array, values: Vector }) { - super(); - this.values = argv.values; - this.offsets = argv.offsets; - } - get(index: number) { - return this.values.slice(this.offsets[index], this.offsets[index + 1]); - } - concat(...vectors: Vector[]): Vector { - return new VirtualVector(Array, this, ...vectors); - } -} \ No newline at end of file diff --git a/js/src/types/table/from.ts b/js/src/types/table/from.ts deleted file mode 100644 index ae0755961eb7d..0000000000000 --- a/js/src/types/table/from.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { Column } from '../types'; -import { TableVector } from './table'; -import { readBuffers } from '../../reader/arrow'; - -export function fromBuffers(...bytes: Array) { - let columns: Column[] = null as any; - for (let vectors of readBuffers(...bytes)) { - columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i]) as Column); - } - return new TableVector({ columns }); -} - -TableVector.from = fromBuffers; - -declare module './table' { - namespace TableVector { export let from: typeof fromBuffers; } -} \ No newline at end of file diff --git a/js/src/types/table/row.ts b/js/src/types/table/row.ts deleted file mode 100644 index 432cfd7364ba3..0000000000000 --- a/js/src/types/table/row.ts +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { Row, Vector, Struct } from '../types'; -import { VirtualVector } from '../vector/virtual'; - -export class RowVector extends Vector implements Row { - readonly row: number; - readonly length: number; - readonly table: Struct; - [Symbol.toStringTag]() { return 'Row'; } - constructor(table: Struct, row: number) { - super(); - this.row = row; - this.table = table; - this.length = table.columns.length; - } - get(index: number) { - const col = this.table.columns[index]; - return col ? col.get(this.row) as T : null; - } - col(key: string) { - const col = this.table.col(key); - return col ? col.get(this.row) as T : null; - } - *[Symbol.iterator]() { - const { row } = this; - for (const col of this.table.columns) { - yield col ? col.get(row) : null; - } - } - concat(...rows: Vector[]): Vector { - return new VirtualVector(Array, this, ...rows as any[]); - } - toArray() { return [...this]; } - toJSON() { return this.toArray(); } - toString() { return `Row [${this.length})` } - toObject(): Record { - const { row } = this, map = Object.create(null); - for (const col of this.table.columns) { - if (col && col.name) { - map[col.name] = col.get(row); - } - } - return map; - } -} diff --git a/js/src/types/table/struct.ts b/js/src/types/table/struct.ts deleted file mode 100644 index de6a3a05692aa..0000000000000 --- a/js/src/types/table/struct.ts +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { RowVector } from './row'; -import { toString } from './toString'; -import { VirtualVector } from '../vector/virtual'; -import { Row, Vector, Column, Struct } from '../types'; - -export interface StructVector { - toString(): string; - toString(index: boolean): string; - toString(options: { index: boolean }): string; -} - -export class StructVector extends Vector> implements Struct { - readonly length: number; - readonly columns: Column[]; - constructor(argv: { columns: Column[] }) { - super(); - this.columns = argv.columns || []; - if (!this.length) { - this.length = Math.max(...this.columns.map((col) => col.length)) | 0; - } - } - get(index: number): StructRow { - return new StructRow(this, index); - } - col(name: string) { - return this.columns.find((col) => col.name === name) || null; - } - key(index: number) { - return this.columns[index] ? this.columns[index].name : null; - } - select(...columns: string[]) { - return new StructVector({ columns: columns.map((name) => this.col(name)!) }); - } - concat(...structs: Vector>[]): Vector> { - return new VirtualVector(Array, this, ...structs as any[]); - } - toString(x?: any) { - return toString(this, x); - } -} - -export class StructRow extends RowVector { - toString() { - return JSON.stringify(this); - } -} \ No newline at end of file diff --git a/js/src/types/table/table.ts b/js/src/types/table/table.ts deleted file mode 100644 index d9074dec2d382..0000000000000 --- a/js/src/types/table/table.ts +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { StructVector, StructRow } from './struct'; - -export class TableVector extends StructVector { - get(index: number): TableRow { - return new TableRow(this, index); - } -} - -export class TableRow extends StructRow { - toString() { - return this.toArray().map((x) => JSON.stringify(x)).join(', '); - } -} \ No newline at end of file diff --git a/js/src/types/table/toString.ts b/js/src/types/table/toString.ts deleted file mode 100644 index 85b23ca246058..0000000000000 --- a/js/src/types/table/toString.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Struct } from '../types'; - -export function toString(source: Struct, options?: any) { - const index = typeof options === 'object' ? options && !!options.index - : typeof options === 'boolean' ? !!options - : false; - const { length } = source; - if (length <= 0) { return ''; } - const rows = new Array(length + 1); - const maxColumnWidths = [] as number[]; - rows[0] = source.columns.map((_, i) => source.key(i)); - index && rows[0].unshift('Index'); - for (let i = -1, n = rows.length - 1; ++i < n;) { - rows[i + 1] = [...source.get(i)!]; - index && rows[i + 1].unshift(i); - } - // Pass one to convert to strings and count max column widths - for (let i = -1, n = rows.length; ++i < n;) { - const row = rows[i]; - for (let j = -1, k = row.length; ++j < k;) { - const val = row[j] = `${row[j]}`; - maxColumnWidths[j] = !maxColumnWidths[j] - ? val.length - : Math.max(maxColumnWidths[j], val.length); - } - } - // Pass two to pad each one to max column width - for (let i = -1, n = rows.length; ++i < n;) { - const row = rows[i]; - for (let j = -1, k = row.length; ++j < k;) { - row[j] = leftPad(row[j], ' ', maxColumnWidths[j]); - } - rows[i] = row.join(', '); - } - return rows.join('\n'); -} - -function leftPad(str: string, fill: string, n: number) { - return (new Array(n + 1).join(fill) + str).slice(-1 * n); -} diff --git a/js/src/types/vector/bool.ts b/js/src/types/vector/bool.ts deleted file mode 100644 index b2eea81f87f05..0000000000000 --- a/js/src/types/vector/bool.ts +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { TypedVector } from './typed'; - -export interface BoolVector extends TypedVector { - set(index: number, value: boolean): void; -} - -export class BoolVector extends TypedVector { - static pack = pack; - get(index: number) { - return (this.data[index >> 3] & 1 << index % 8) !== 0; - } - set(index: number, value: boolean) { - if (index > -1 === false) { - return; - } else if (value) { - this.data[index >> 3] |= (1 << (index % 8)); - } else { - this.data[index >> 3] &= ~(1 << (index % 8)); - } - } -} - -export function pack(values: Iterable) { - let xs = [], n, i = 0; - let bit = 0, byte = 0; - for (const value of values) { - value && (byte |= 1 << bit); - if (++bit === 8) { - xs[i++] = byte; - byte = bit = 0; - } - } - if (i === 0 || bit > 0) { xs[i++] = byte; } - if (i % 8 && (n = i + 8 - i % 8)) { - do { xs[i] = 0; } while (++i < n); - } - return new Uint8Array(xs); -} diff --git a/js/src/types/vector/date.ts b/js/src/types/vector/date.ts deleted file mode 100644 index 82dc82e64021f..0000000000000 --- a/js/src/types/vector/date.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { TypedVector } from './typed'; - -export class DateVector extends TypedVector { - get(index: number): Date { - return new Date(4294967296 * /* 2^32 */ - (super.get(index * 2 + 1) as any) + /* high */ - (super.get(index * 2) as any) /* low */ - ); - } -} - -(DateVector.prototype as any).stride = 2; diff --git a/js/src/types/vector/long.ts b/js/src/types/vector/long.ts deleted file mode 100644 index de8eb0c13710b..0000000000000 --- a/js/src/types/vector/long.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { TypedVector } from './typed'; -import { TypedArray } from '../types'; -import { flatbuffers } from 'flatbuffers'; -import Long = flatbuffers.Long; - -export class LongVector extends TypedVector { - get(index: number) { - return new Long( - super.get(index * 2) as any, /* low */ - super.get(index * 2 + 1) as any /* high */ - ); - } -} - -(LongVector.prototype as any).stride = 2; - -export class Int64Vector extends LongVector {} -export class Uint64Vector extends LongVector {} \ No newline at end of file diff --git a/js/src/types/vector/typed.ts b/js/src/types/vector/typed.ts deleted file mode 100644 index fc093f2cb5f2a..0000000000000 --- a/js/src/types/vector/typed.ts +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { Vector } from '../types'; -import { VirtualVector } from './virtual'; -import { TypedArray, TypedArrayConstructor } from '../types'; - -export interface TypedVector { - slice(start?: number, end?: number): TArray; -} - -export class TypedVector extends Vector { - readonly data: TArray; - readonly stride: number; - readonly length: number; - constructor(argv: { data: TArray } | TArray) { - super(); - const data = ArrayBuffer.isView(argv) ? argv : argv.data; - this.length = ((this.data = data).length / this.stride) | 0; - } - get(index: number): T | null { - return this.data[index] as any; - } - concat(...vectors: Vector[]): Vector { - return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors); - } - slice(start?: number, end?: number) { - const { data, stride } = this, from = start! | 0; - const to = end === undefined ? data.length : Math.max(end | 0, from); - return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0); - } -} - -(TypedVector.prototype as any).stride = 1; - -export class Int8Vector extends TypedVector {} -export class Int16Vector extends TypedVector {} -export class Int32Vector extends TypedVector {} -export class Uint8Vector extends TypedVector {} -export class Uint16Vector extends TypedVector {} -export class Uint32Vector extends TypedVector {} -export class Float32Vector extends TypedVector {} -export class Float64Vector extends TypedVector {} diff --git a/js/src/types/vector/virtual.ts b/js/src/types/vector/virtual.ts deleted file mode 100644 index 7f56012dc7d4b..0000000000000 --- a/js/src/types/vector/virtual.ts +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { TypedVector } from './typed'; -import { Vector, Column, TypedArray, TypedArrayConstructor } from '../types'; - -export class VirtualVector implements Column { - readonly name: string; - readonly type: string; - readonly length: number; - readonly vectors: Vector[]; - readonly offsets: Uint32Array; - readonly ArrayType: ArrayConstructor | TypedArrayConstructor; - constructor(ArrayType: ArrayConstructor | TypedArrayConstructor, ...vectors: Vector[]) { - this.vectors = vectors; - this.ArrayType = ArrayType; - this.name = (vectors[0] as any).name; - this.type = (vectors[0] as any).type; - this.length = vectors.reduce((sum, vec) => sum + vec.length, 0); - this.offsets = Uint32Array.from(vectors.reduce((sums, vector, index) => [...sums, vector.length + sums[index]], [0])); - } - *[Symbol.iterator]() { - for (const vector of this.vectors) { - yield* vector; - } - } - get nullable() { - return (this.vectors as Column[]).some((vec) => vec.nullable); - } - get nullCount() { - return (this.vectors as Column[]).reduce((sum, v) => sum + v.nullCount | 0, 0); - } - get metadata() { - return new Map( - (this.vectors as Column[]).reduce((entries, v) => [ - ...entries, ...v.metadata.entries() - ], [] as [string, string][]) - ); - } - get(index: number) { - return findIndex(this.offsets, index) ? this.vectors[_vector].get(_offset) : null; - } - concat(...vectors: Vector[]) { - return new VirtualVector(this.ArrayType, ...this.vectors, ...vectors); - } - slice(begin?: number, end?: number) { - const ArrayType = this.ArrayType as any; - // clamp begin and end values between the virtual length - clampRange(this.length, begin!, end); - const from = _from, total = _total; - // find the start vector index and adjusted value index offset - if (!findIndex(this.offsets, from)) { return new ArrayType(0); } - const set = ArrayType === Array ? arraySet : typedArraySet as any; - let index = _vector, vectors = this.vectors as TypedVector[]; - let vector = vectors[index], source = vector.slice(_offset, _offset + total), target = source; - // Perf optimization: if the first slice contains all the values we're looking for, - // we don't have to copy values to a target Array. If we're slicing a TypedArray, - // this is a significant improvement as we avoid the memcpy 🎉 - if ((source.length / vector.stride | 0) < total) { - let vectorsLength = vectors.length; - let count = 0, length = 0, sources = []; - do { - sources.push(source); - length += source.length; - count += (source.length / vector.stride | 0); - } while ( - (count < total) && - (vector = vectors[index = (++index % vectorsLength)]) && - (source = vector.slice(0, Math.min(vector.length, total - count))) - ); - target = new ArrayType(length); - for (let i = -1, j = 0, n = sources.length; ++i < n;) { - j = set(sources[i], target, j); - } - } - return target; - } -} - -let _from = -1, _total = -1; -function clampRange(length: number, start: number, end?: number) { - let total = length, from = start || 0; - let to = end === end && typeof end == 'number' ? end : total; - if (to < 0) { to = total + to; } - if (from < 0) { from = total - (from * -1) % total; } - if (to < from) { from = to; to = start; } - _from = from; - _total = !isFinite(total = (to - from)) || total < 0 ? 0 : total; -} - -let _offset = -1, _vector = -1; -function findIndex(offsets: Uint32Array, index: number) { - let offset = 0, left = 0, middle = 0, right = offsets.length - 1; - while (index < offsets[right] && index >= (offset = offsets[left])) { - if (left + 1 === right) { - _vector = left; - _offset = index - offset; - return true; - } - middle = left + ((right - left) / 2) | 0; - index >= offsets[middle] ? (left = middle) : (right = middle); - } - return false; -} - -function arraySet(source: T[], target: T[], index: number) { - for (let i = 0, n = source.length; i < n;) { - target[index++] = source[i++]; - } - return index; -} - -function typedArraySet(source: TypedArray, target: TypedArray, index: number) { - return target.set(source, index) || index + source.length; -} diff --git a/js/src/vector/arrow.ts b/js/src/vector/arrow.ts new file mode 100644 index 0000000000000..88fec7e44542b --- /dev/null +++ b/js/src/vector/arrow.ts @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Schema_ from '../format/Schema'; +import * as Message_ from '../format/Message'; +import Field = Schema_.org.apache.arrow.flatbuf.Field; +import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; + +import { Vector } from './vector'; +import { Utf8Vector as Utf8VectorBase } from './utf8'; +import { StructVector as StructVectorBase } from './struct'; +import { DictionaryVector as DictionaryVectorBase } from './dictionary'; +import { + ListVector as ListVectorBase, + BinaryVector as BinaryVectorBase, + FixedSizeListVector as FixedSizeListVectorBase +} from './list'; + +import { + BoolVector as BoolVectorBase, + Int8Vector as Int8VectorBase, + Int16Vector as Int16VectorBase, + Int32Vector as Int32VectorBase, + Int64Vector as Int64VectorBase, + Uint8Vector as Uint8VectorBase, + Uint16Vector as Uint16VectorBase, + Uint32Vector as Uint32VectorBase, + Uint64Vector as Uint64VectorBase, + Float16Vector as Float16VectorBase, + Float32Vector as Float32VectorBase, + Float64Vector as Float64VectorBase, + Date32Vector as Date32VectorBase, + Date64Vector as Date64VectorBase, + Time32Vector as Time32VectorBase, + Time64Vector as Time64VectorBase, + DecimalVector as DecimalVectorBase, + TimestampVector as TimestampVectorBase, +} from './numeric'; + +import { nullableMixin, fieldMixin } from './traits'; + +function MixinArrowTraits, TArgv>( + Base: new (argv: TArgv) => T, + Field: new (argv: TArgv & { field: Field, fieldNode: FieldNode }) => T, + Nullable: new (argv: TArgv & { validity: Uint8Array }) => T, + NullableField: new (argv: TArgv & { validity: Uint8Array, field: Field, fieldNode: FieldNode }) => T, +) { + return function(argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })) { + return new (!isFieldArgv(argv) + ? !isNullableArgv(argv) ? Base : Nullable + : !isNullableArgv(argv) ? Field : NullableField + )(argv as any); + } as any as { new (argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })): T }; +} + +function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } { + return x && x.field instanceof Field && x.fieldNode instanceof FieldNode; +} + +function isNullableArgv(x: any): x is { validity: Uint8Array } { + return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array; +} + +export { Vector }; +export class ListVector extends MixinArrowTraits( + ListVectorBase, + class ListVector extends fieldMixin(ListVectorBase) {} as any, + class ListVector extends nullableMixin(ListVectorBase) {} as any, + class ListVector extends nullableMixin(fieldMixin(ListVectorBase)) {} as any +) {} + +export class BinaryVector extends MixinArrowTraits( + BinaryVectorBase, + class BinaryVector extends fieldMixin(BinaryVectorBase) {} as any, + class BinaryVector extends nullableMixin(BinaryVectorBase) {} as any, + class BinaryVector extends nullableMixin(fieldMixin(BinaryVectorBase)) {} as any +) {} + +export class Utf8Vector extends MixinArrowTraits( + Utf8VectorBase, + class Utf8Vector extends fieldMixin(Utf8VectorBase) {} as any, + class Utf8Vector extends nullableMixin(Utf8VectorBase) {} as any, + class Utf8Vector extends nullableMixin(fieldMixin(Utf8VectorBase)) {} as any +) {} + +export class BoolVector extends MixinArrowTraits( + BoolVectorBase, + class BoolVector extends fieldMixin(BoolVectorBase) {} as any, + class BoolVector extends nullableMixin(BoolVectorBase) {} as any, + class BoolVector extends nullableMixin(fieldMixin(BoolVectorBase)) {} as any +) {} + +export class Int8Vector extends MixinArrowTraits( + Int8VectorBase, + class Int8Vector extends fieldMixin(Int8VectorBase) {} as any, + class Int8Vector extends nullableMixin(Int8VectorBase) {} as any, + class Int8Vector extends nullableMixin(fieldMixin(Int8VectorBase)) {} as any +) {} + +export class Int16Vector extends MixinArrowTraits( + Int16VectorBase, + class Int16Vector extends fieldMixin(Int16VectorBase) {} as any, + class Int16Vector extends nullableMixin(Int16VectorBase) {} as any, + class Int16Vector extends nullableMixin(fieldMixin(Int16VectorBase)) {} as any +) {} + +export class Int32Vector extends MixinArrowTraits( + Int32VectorBase, + class Int32Vector extends fieldMixin(Int32VectorBase) {} as any, + class Int32Vector extends nullableMixin(Int32VectorBase) {} as any, + class Int32Vector extends nullableMixin(fieldMixin(Int32VectorBase)) {} as any +) {} + +export class Int64Vector extends MixinArrowTraits( + Int64VectorBase, + class Int64Vector extends fieldMixin(Int64VectorBase) {} as any, + class Int64Vector extends nullableMixin(Int64VectorBase) {} as any, + class Int64Vector extends nullableMixin(fieldMixin(Int64VectorBase)) {} as any +) {} + +export class Uint8Vector extends MixinArrowTraits( + Uint8VectorBase, + class Uint8Vector extends fieldMixin(Uint8VectorBase) {} as any, + class Uint8Vector extends nullableMixin(Uint8VectorBase) {} as any, + class Uint8Vector extends nullableMixin(fieldMixin(Uint8VectorBase)) {} as any +) {} + +export class Uint16Vector extends MixinArrowTraits( + Uint16VectorBase, + class Uint16Vector extends fieldMixin(Uint16VectorBase) {} as any, + class Uint16Vector extends nullableMixin(Uint16VectorBase) {} as any, + class Uint16Vector extends nullableMixin(fieldMixin(Uint16VectorBase)) {} as any +) {} + +export class Uint32Vector extends MixinArrowTraits( + Uint32VectorBase, + class Uint32Vector extends fieldMixin(Uint32VectorBase) {} as any, + class Uint32Vector extends nullableMixin(Uint32VectorBase) {} as any, + class Uint32Vector extends nullableMixin(fieldMixin(Uint32VectorBase)) {} as any +) {} + +export class Uint64Vector extends MixinArrowTraits( + Uint64VectorBase, + class Uint64Vector extends fieldMixin(Uint64VectorBase) {} as any, + class Uint64Vector extends nullableMixin(Uint64VectorBase) {} as any, + class Uint64Vector extends nullableMixin(fieldMixin(Uint64VectorBase)) {} as any +) {} + +export class Date32Vector extends MixinArrowTraits( + Date32VectorBase, + class Date32Vector extends fieldMixin(Date32VectorBase) {} as any, + class Date32Vector extends nullableMixin(Date32VectorBase) {} as any, + class Date32Vector extends nullableMixin(fieldMixin(Date32VectorBase)) {} as any +) {} + +export class Date64Vector extends MixinArrowTraits( + Date64VectorBase, + class Date64Vector extends fieldMixin(Date64VectorBase) {} as any, + class Date64Vector extends nullableMixin(Date64VectorBase) {} as any, + class Date64Vector extends nullableMixin(fieldMixin(Date64VectorBase)) {} as any +) {} + +export class Time32Vector extends MixinArrowTraits( + Time32VectorBase, + class Time32Vector extends fieldMixin(Time32VectorBase) {} as any, + class Time32Vector extends nullableMixin(Time32VectorBase) {} as any, + class Time32Vector extends nullableMixin(fieldMixin(Time32VectorBase)) {} as any +) {} + +export class Time64Vector extends MixinArrowTraits( + Time64VectorBase, + class Time64Vector extends fieldMixin(Time64VectorBase) {} as any, + class Time64Vector extends nullableMixin(Time64VectorBase) {} as any, + class Time64Vector extends nullableMixin(fieldMixin(Time64VectorBase)) {} as any +) {} + +export class Float16Vector extends MixinArrowTraits( + Float16VectorBase, + class Float16Vector extends fieldMixin(Float16VectorBase) {} as any, + class Float16Vector extends nullableMixin(Float16VectorBase) {} as any, + class Float16Vector extends nullableMixin(fieldMixin(Float16VectorBase)) {} as any +) {} + +export class Float32Vector extends MixinArrowTraits( + Float32VectorBase, + class Float32Vector extends fieldMixin(Float32VectorBase) {} as any, + class Float32Vector extends nullableMixin(Float32VectorBase) {} as any, + class Float32Vector extends nullableMixin(fieldMixin(Float32VectorBase)) {} as any +) {} + +export class Float64Vector extends MixinArrowTraits( + Float64VectorBase, + class Float64Vector extends fieldMixin(Float64VectorBase) {} as any, + class Float64Vector extends nullableMixin(Float64VectorBase) {} as any, + class Float64Vector extends nullableMixin(fieldMixin(Float64VectorBase)) {} as any +) {} + +export class StructVector extends MixinArrowTraits( + StructVectorBase, + class StructVector extends fieldMixin(StructVectorBase) {} as any, + class StructVector extends nullableMixin(StructVectorBase) {} as any, + class StructVector extends nullableMixin(fieldMixin(StructVectorBase)) {} as any +) {} + +export class DecimalVector extends MixinArrowTraits( + DecimalVectorBase, + class DecimalVector extends fieldMixin(DecimalVectorBase) {} as any, + class DecimalVector extends nullableMixin(DecimalVectorBase) {} as any, + class DecimalVector extends nullableMixin(fieldMixin(DecimalVectorBase)) {} as any +) {} + +export class TimestampVector extends MixinArrowTraits( + TimestampVectorBase, + class TimestampVector extends fieldMixin(TimestampVectorBase) {} as any, + class TimestampVector extends nullableMixin(TimestampVectorBase) {} as any, + class TimestampVector extends nullableMixin(fieldMixin(TimestampVectorBase)) {} as any +) {} + +export class DictionaryVector extends MixinArrowTraits( + DictionaryVectorBase, + class DictionaryVector extends fieldMixin(DictionaryVectorBase) {} as any, + class DictionaryVector extends nullableMixin(DictionaryVectorBase) {} as any, + class DictionaryVector extends nullableMixin(fieldMixin(DictionaryVectorBase)) {} as any +) {} + +export class FixedSizeListVector extends MixinArrowTraits( + FixedSizeListVectorBase, + class FixedSizeListVector extends fieldMixin(FixedSizeListVectorBase) {} as any, + class FixedSizeListVector extends nullableMixin(FixedSizeListVectorBase) {} as any, + class FixedSizeListVector extends nullableMixin(fieldMixin(FixedSizeListVectorBase)) {} as any +) {} diff --git a/js/src/types/dictionary.ts b/js/src/vector/dictionary.ts similarity index 62% rename from js/src/types/dictionary.ts rename to js/src/vector/dictionary.ts index cafa753311d3b..b7375c0904276 100644 --- a/js/src/types/dictionary.ts +++ b/js/src/vector/dictionary.ts @@ -15,28 +15,19 @@ // specific language governing permissions and limitations // under the License. -import { Vector, Column } from './types'; -import { VirtualVector } from './vector/virtual'; +import { Vector } from './vector'; +import { VirtualVector } from './virtual'; -export interface DictionaryVector extends Vector { - getValue(key: number): T; - getKey(index: number): number; -} - -export class DictionaryVector extends Vector implements Column, DictionaryVector { +export class DictionaryVector extends Vector { + readonly length: number; readonly data: Vector; - readonly keys: Column; + readonly keys: Vector; constructor(argv: { data: Vector, keys: Vector }) { super(); this.data = argv.data; - this.keys = argv.keys as Column; + this.keys = argv.keys; + this.length = this.keys.length; } - get name () { return this.keys.name; } - get type () { return this.keys.type; } - get length () { return this.keys.length; } - get metadata () { return this.keys.metadata; } - get nullable () { return this.keys.nullable; } - get nullCount () { return this.keys.nullCount; } get(index: number) { return this.getValue(this.getKey(index)!); } @@ -50,9 +41,8 @@ export class DictionaryVector extends Vector implements Column, Diction return new VirtualVector(Array, this, ...vectors); } *[Symbol.iterator]() { - const { data, keys } = this; - for (let i = -1, n = keys.length; ++i < n;) { - yield data.get(keys.get(i)!); + for (let i = -1, n = this.length; ++i < n;) { + yield this.get(i); } } } diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts new file mode 100644 index 0000000000000..97913f8d8878c --- /dev/null +++ b/js/src/vector/list.ts @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { List } from './types'; +import { Vector } from './vector'; +import { VirtualVector } from './virtual'; + +export class BinaryVector extends Vector { + readonly data: Uint8Array; + readonly offsets: Int32Array; + constructor(argv: { offsets: Int32Array, data: Uint8Array }) { + super(); + this.data = argv.data; + this.offsets = argv.offsets; + } + get(index: number) { + return this.data.subarray(this.offsets[index], this.offsets[index + 1]); + } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(Array, this, ...vectors); + } +} + +export class ListVector extends Vector { + readonly offsets: Int32Array; + readonly values: Vector; + constructor(argv: { offsets: Int32Array, values: Vector }) { + super(); + this.values = argv.values; + this.offsets = argv.offsets; + } + get(index: number) { + const { offsets, values } = this; + const from = offsets[index]; + const xs = new Array(offsets[index + 1] - from); + for (let i = -1, n = xs.length; ++i < n;) { + xs[i] = values.get(i + from); + } + return xs; + } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(Array, this, ...vectors); + } +} + +export class FixedSizeListVector> extends Vector { + readonly size: number; + readonly values: Vector; + constructor(argv: { size: number, values: Vector }) { + super(); + this.size = argv.size; + this.values = argv.values; + } + get(index: number) { + return this.values.slice(this.size * index, this.size * (index + 1)); + } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(Array, this, ...vectors); + } +} diff --git a/js/src/vector/numeric.ts b/js/src/vector/numeric.ts new file mode 100644 index 0000000000000..fe4767809f465 --- /dev/null +++ b/js/src/vector/numeric.ts @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { VirtualVector } from './virtual'; +import { TypedArray, TypedArrayConstructor } from './types'; + +export class NumericVector extends Vector { + readonly data: TArray; + readonly length: number; + readonly stride: number; + constructor(argv: { data: TArray }) { + super(); + const data = (ArrayBuffer.isView(argv) ? argv : argv.data) as TArray; + this.length = ((this.data = data).length / this.stride) | 0; + } + get(index: number) { + return this.data[index] as any; + } + concat(...vectors: Vector[]): Vector { + return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors); + } + slice(start?: number, end?: number) { + const { data, stride } = this, from = start! | 0; + const to = end === undefined ? data.length : Math.max(end | 0, from); + return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0); + } +} + +export class FixedWidthNumericVector extends NumericVector { + get(index: number) { + return this.data.slice(this.stride * index, this.stride * (index + 1)) as TArray; + } +} + +export class BoolVector extends NumericVector { + static pack(values: Iterable) { + let xs = [], n, i = 0; + let bit = 0, byte = 0; + for (const value of values) { + value && (byte |= 1 << bit); + if (++bit === 8) { + xs[i++] = byte; + byte = bit = 0; + } + } + if (i === 0 || bit > 0) { xs[i++] = byte; } + if (i % 8 && (n = i + 8 - i % 8)) { + do { xs[i] = 0; } while (++i < n); + } + return new Uint8Array(xs); + } + get(index: number) { + return (this.data[index >> 3] & 1 << index % 8) !== 0; + } + set(index: number, value: boolean) { + if (index > -1 === false) { + return; + } else if (value) { + this.data[index >> 3] |= (1 << (index % 8)); + } else { + this.data[index >> 3] &= ~(1 << (index % 8)); + } + } +} + +export class Int8Vector extends NumericVector {} +export class Int16Vector extends NumericVector {} +export class Int32Vector extends NumericVector {} +export class Int64Vector extends FixedWidthNumericVector {} + +export class Uint8Vector extends NumericVector {} +export class Uint16Vector extends NumericVector {} +export class Uint32Vector extends NumericVector {} +export class Uint64Vector extends FixedWidthNumericVector {} + +export class Float16Vector extends NumericVector { + get(index: number) { + return Math.min((super.get(index)! - 32767) / 32767, 1); + } +} + +export class Float32Vector extends NumericVector {} +export class Float64Vector extends NumericVector {} + +export class Date32Vector extends NumericVector { + public readonly unit: string; + constructor(argv: { data: Int32Array, unit: string }) { + super(argv); + this.unit = argv.unit; + } + get(index: number): Date { + return new Date(86400000 * (super.get(index) as any)); + } +} + +export class Date64Vector extends NumericVector { + public readonly unit: string; + constructor(argv: { unit: string, data: Int32Array }) { + super(argv); + this.unit = argv.unit; + } + get(index: number): Date { + return new Date(4294967296 * /* 2^32 */ + (super.get(index * 2 + 1) as any) + /* high */ + (super.get(index * 2) as any) /* low */ + ); + } +} + +export class Time32Vector extends NumericVector { + public readonly unit: string; + constructor(argv: { data: Int32Array, unit: string }) { + super(argv); + this.unit = argv.unit; + } +} + +export class Time64Vector extends FixedWidthNumericVector { + public readonly unit: string; + constructor(argv: { unit: string, data: Uint32Array }) { + super(argv); + this.unit = argv.unit; + } +} + +export class DecimalVector extends FixedWidthNumericVector { + readonly scale: number; + readonly precision: number; + constructor(argv: { precision: number, scale: number, data: Uint32Array }) { + super(argv); + this.scale = argv.scale; + this.precision = argv.precision; + } +} + +export class TimestampVector extends FixedWidthNumericVector { + readonly unit: string; + readonly timezone: string; + constructor(argv: { unit: string, timezone: string, data: Uint32Array }) { + super(argv); + this.unit = argv.unit; + this.timezone = argv.timezone; + } +} + +export interface NumericVectorConstructor { + readonly prototype: NumericVector; + new (argv: { data: TArray }): NumericVector; +} + +(DecimalVector.prototype as any).stride = 4; +(NumericVector.prototype as any).stride = 1; +(FixedWidthNumericVector.prototype as any).stride = 2; diff --git a/js/src/vector/struct.ts b/js/src/vector/struct.ts new file mode 100644 index 0000000000000..c43f6efc48fbe --- /dev/null +++ b/js/src/vector/struct.ts @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { VirtualVector } from './virtual'; + +export class StructVector extends Vector> { + readonly length: number; + readonly columns: Vector[]; + constructor(argv: { columns: Vector[] }) { + super(); + this.columns = argv.columns || []; + } + get(index: number): StructRow { + return new StructRow(this, index); + } + col(name: string) { + return this.columns.find((col) => col.name === name) || null; + } + key(index: number) { + return this.columns[index] ? this.columns[index].name : null; + } + select(...columns: string[]) { + return new StructVector({ columns: columns.map((name) => this.col(name)!) }); + } + concat(...structs: Vector>[]): Vector> { + return new VirtualVector(Array, this, ...structs as any[]); + } + toString(options?: any) { + const index = typeof options === 'object' ? options && !!options.index + : typeof options === 'boolean' ? !!options + : false; + const { length } = this; + if (length <= 0) { return ''; } + const rows = new Array(length + 1); + const maxColumnWidths = [] as number[]; + rows[0] = this.columns.map((_, i) => this.key(i)); + index && rows[0].unshift('Index'); + for (let i = -1, n = rows.length - 1; ++i < n;) { + rows[i + 1] = [...this.get(i)!]; + index && rows[i + 1].unshift(i); + } + // Pass one to convert to strings and count max column widths + for (let i = -1, n = rows.length; ++i < n;) { + const row = rows[i]; + for (let j = -1, k = row.length; ++j < k;) { + const val = row[j] = stringify(row[j]); + maxColumnWidths[j] = !maxColumnWidths[j] + ? val.length + : Math.max(maxColumnWidths[j], val.length); + } + } + // Pass two to pad each one to max column width + for (let i = -1, n = rows.length; ++i < n;) { + const row = rows[i]; + for (let j = -1, k = row.length; ++j < k;) { + row[j] = leftPad(row[j], ' ', maxColumnWidths[j]); + } + rows[i] = row.join(', '); + } + return rows.join('\n'); + } +} + +export class StructRow extends Vector { + readonly row: number; + readonly length: number; + readonly table: StructVector; + [Symbol.toStringTag]() { return 'Row'; } + constructor(table: StructVector, row: number) { + super(); + this.row = row; + this.table = table; + this.length = table.columns.length; + } + get(index: number) { + const col = this.table.columns[index]; + return col ? col.get(this.row) as T : null; + } + col(key: string) { + const col = this.table.col(key); + return col ? col.get(this.row) as T : null; + } + *[Symbol.iterator]() { + const { row } = this; + for (const col of this.table.columns) { + yield col ? col.get(row) : null; + } + } + concat(...rows: Vector[]): Vector { + return new VirtualVector(Array, this, ...rows as any[]); + } + toArray() { return [...this]; } + toJSON() { return this.toArray(); } + toString() { return JSON.stringify(this); } + toObject(): Record { + const { row } = this, map = Object.create(null); + for (const col of this.table.columns) { + if (col && col.name) { + map[col.name] = col.get(row); + } + } + return map; + } +} + +function leftPad(str: string, fill: string, n: number) { + return (new Array(n + 1).join(fill) + str).slice(-1 * n); +} + +function stringify(x: any) { + return Array.isArray(x) ? JSON.stringify(x) : ArrayBuffer.isView(x) ? `[${x}]` : `${x}`; +} diff --git a/js/src/vector/table.ts b/js/src/vector/table.ts new file mode 100644 index 0000000000000..b15092a72df24 --- /dev/null +++ b/js/src/vector/table.ts @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { StructVector, StructRow } from './struct'; +import { readVectors, readVectorsAsync } from '../reader/arrow'; + +export class Table extends StructVector { + static from(buffers?: Iterable) { + let columns: Vector[] = []; + if (buffers) { + for (let vectors of readVectors(buffers)) { + columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) => + vec && col && col.concat(vec) || col || vec + ) as Vector[]; + } + } + return new Table({ columns }); + } + static async fromAsync(buffers?: AsyncIterable) { + let columns: Vector[] = []; + if (buffers) { + for await (let vectors of readVectorsAsync(buffers)) { + columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) => + vec && col && col.concat(vec) || col || vec + ) as Vector[]; + } + } + return new Table({ columns }); + } + readonly length: number; + constructor(argv: { columns: Vector[] }) { + super(argv); + this.length = Math.max(...this.columns.map((col) => col.length)) | 0; + } + get(index: number): TableRow { + return new TableRow(this, index); + } +} + +export class TableRow extends StructRow { + toString() { + return this.toArray().map((x) => JSON.stringify(x)).join(', '); + } +} diff --git a/js/src/types/vector/traits.ts b/js/src/vector/traits.ts similarity index 88% rename from js/src/types/vector/traits.ts rename to js/src/vector/traits.ts index 872c40b64d559..ca933f1605957 100644 --- a/js/src/types/vector/traits.ts +++ b/js/src/vector/traits.ts @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -import { BoolVector } from './bool'; -import { Vector, Column } from '../types'; -import * as Schema_ from '../../format/Schema_generated'; -import * as Message_ from '../../format/Message_generated'; +import { Vector } from './vector'; +import { BoolVector } from './numeric'; +import * as Schema_ from '../format/Schema'; +import * as Message_ from '../format/Message'; import Type = Schema_.org.apache.arrow.flatbuf.Type; import Field = Schema_.org.apache.arrow.flatbuf.Field; import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; @@ -38,7 +38,7 @@ export const nullableMixin = (superclass: new (argv: TA }; export const fieldMixin = (superclass: new (argv: TArgv) => T) => - class extends (superclass as Ctor) implements Column { + class extends (superclass as Ctor) implements Vector { readonly field: Field; readonly type: string; readonly length: number; @@ -53,8 +53,8 @@ export const fieldMixin = (superclass: new (argv: TArgv this.fieldNode = fieldNode; this.nullable = field.nullable(); this.type = Type[field.typeType()]; + this.length = fieldNode.length().low | 0; this.nullCount = fieldNode.nullCount().low; - this.length = (fieldNode.length().low / this.stride) | 0; } get name() { return this.field.name()!; } get metadata() { diff --git a/js/src/vector/types.ts b/js/src/vector/types.ts new file mode 100644 index 0000000000000..363fcf2265c30 --- /dev/null +++ b/js/src/vector/types.ts @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export interface TypedArrayConstructor { + readonly prototype: T; + readonly BYTES_PER_ELEMENT: number; + new (length: number): T; + new (elements: Iterable): T; + new (arrayOrArrayBuffer: ArrayLike | ArrayBufferLike): T; + new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T; +} + +export interface TypedArray extends Iterable { + [index: number]: number; + readonly length: number; + readonly byteLength: number; + readonly byteOffset: number; + readonly buffer: ArrayBufferLike; + readonly BYTES_PER_ELEMENT: number; + [Symbol.iterator](): IterableIterator; + slice(start?: number, end?: number): TypedArray; + subarray(begin: number, end?: number): TypedArray; + set(array: ArrayLike, offset?: number): void; +} + +export type List = T[] | TypedArray; +export type FloatArray = Float32Array | Float64Array; +export type IntArray = Int8Array | Int16Array | Int32Array; +export type UintArray = Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array; diff --git a/js/src/types/utf8.ts b/js/src/vector/utf8.ts similarity index 94% rename from js/src/types/utf8.ts rename to js/src/vector/utf8.ts index 178704f6161f5..ba875cf333fe7 100644 --- a/js/src/types/utf8.ts +++ b/js/src/vector/utf8.ts @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -import { Vector } from './types'; +import { Vector } from './vector'; +import { VirtualVector } from './virtual'; import { TextDecoder } from 'text-encoding-utf-8'; -import { VirtualVector } from './vector/virtual'; const decoder = new TextDecoder('utf-8'); diff --git a/js/src/types/types.ts b/js/src/vector/vector.ts similarity index 56% rename from js/src/types/types.ts rename to js/src/vector/vector.ts index f732bc0971f35..8047c89923223 100644 --- a/js/src/types/types.ts +++ b/js/src/vector/vector.ts @@ -15,64 +15,20 @@ // specific language governing permissions and limitations // under the License. -import * as Schema_ from '../format/Schema_generated'; +import * as Schema_ from '../format/Schema'; import Type = Schema_.org.apache.arrow.flatbuf.Type; -export interface TypedArrayConstructor { - readonly prototype: T; - readonly BYTES_PER_ELEMENT: number; - new (length: number): T; - new (elements: Iterable): T; - new (arrayOrArrayBuffer: ArrayLike | ArrayBufferLike): T; - new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T; -} - -export interface TypedArray extends Iterable { - [index: number]: number; - readonly length: number; - readonly byteLength: number; - readonly byteOffset: number; - readonly buffer: ArrayBufferLike; - readonly BYTES_PER_ELEMENT: number; - [Symbol.iterator](): IterableIterator; - slice(start?: number, end?: number): TypedArray; - subarray(begin: number, end?: number): TypedArray; - set(array: ArrayLike, offset?: number): void; -} - -export type FloatArray = Float32Array | Float64Array; -export type IntArray = Int8Array | Int16Array | Int32Array; -export type UintArray = Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array; - -export type List = T[] | TypedArray; - export interface Vector extends Iterable { - readonly length: number; - get(index: number): T | null; - concat(...vectors: Vector[]): Vector; - slice(start?: number, end?: number): R; -} - -export interface Row extends Vector { - col(key: string): T | null; -} - -export interface Column extends Vector { readonly name: string; readonly type: string; + readonly length: number; readonly nullable: boolean; readonly nullCount: number; readonly metadata: Map; + get(index: number): T | null; + concat(...vectors: Vector[]): Vector; + slice(start?: number, end?: number): R; } - -export interface Struct extends Vector> { - readonly columns: Column[]; - key(key: number): string | null; - col(key: string): Column | null; - select(...columns: string[]): Struct; - concat(...structs: Vector>[]): Vector>; -} - export class Vector implements Vector { slice(start?: number, end?: number): R { let { length } = this, from = start! | 0; @@ -91,8 +47,8 @@ export class Vector implements Vector { } (Vector.prototype as any).name = ''; -(Vector.prototype as any).type = Type[0]; (Vector.prototype as any).stride = 1; (Vector.prototype as any).nullable = !1; (Vector.prototype as any).nullCount = 0; (Vector.prototype as any).metadata = new Map(); +(Vector.prototype as any).type = Type[Type.NONE]; diff --git a/js/src/vector/virtual.ts b/js/src/vector/virtual.ts new file mode 100644 index 0000000000000..6ec3a8eef9f4d --- /dev/null +++ b/js/src/vector/virtual.ts @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { NumericVector } from './numeric'; +import { TypedArray, TypedArrayConstructor } from './types'; + +export class VirtualVector implements Vector { + readonly name: string; + readonly type: string; + readonly length: number; + readonly vectors: Vector[]; + readonly offsets: Uint32Array; + readonly ArrayType: ArrayConstructor | TypedArrayConstructor; + constructor(ArrayType: ArrayConstructor | TypedArrayConstructor, ...vectors: Vector[]) { + this.vectors = vectors; + this.ArrayType = ArrayType; + this.name = (vectors[0] as any).name; + this.type = (vectors[0] as any).type; + this.length = vectors.reduce((sum, vec) => sum + vec.length, 0); + this.offsets = Uint32Array.from(vectors.reduce((sums, vector, index) => [...sums, vector.length + sums[index]], [0])); + } + *[Symbol.iterator]() { + for (const vector of this.vectors) { + yield* vector; + } + } + get nullable() { + return (this.vectors as Vector[]).some((vec) => vec.nullable); + } + get nullCount() { + return (this.vectors as Vector[]).reduce((sum, v) => sum + v.nullCount | 0, 0); + } + get metadata() { + return new Map( + (this.vectors as Vector[]).reduce((entries, v) => [ + ...entries, ...v.metadata.entries() + ], [] as [string, string][]) + ); + } + get(index: number) { + // find the vector index and adjusted value offset (inlined) + let offsets = this.offsets, offset = 0; + let left = 0, middle = 0, right = offsets.length - 1; + while (index < offsets[right] && index >= (offset = offsets[left])) { + if (left + 1 === right) { + return this.vectors[left].get(index - offset); + } + middle = left + ((right - left) / 2) | 0; + index >= offsets[middle] ? (left = middle) : (right = middle); + } + return null; + } + concat(...vectors: Vector[]) { + return new VirtualVector(this.ArrayType, ...this.vectors, ...vectors); + } + slice(begin?: number, end?: number) { + + // clamp begin and end values between the virtual length (inlined) + // let [from, total] = clampRange(this.length, begin!, end); + let total = this.length, from = begin! | 0; + let to = end === end && typeof end == 'number' ? end : total; + if (to < 0) { to = total + to; } + if (from < 0) { from = total - (from * -1) % total; } + if (to < from) { from = to; to = begin! | 0; } + total = !isFinite(total = (to - from)) || total < 0 ? 0 : total; + + // find the vector index and adjusted value offset (inlined) + let offsets = this.offsets, ArrayType = this.ArrayType as any; + let offset = 0, index = 0, middle = 0, right = offsets.length - 1; + while (from < offsets[right] && from >= (offset = offsets[index])) { + if (index + 1 === right) { + from -= offset; + let set = ArrayType === Array ? arraySet : typedArraySet as any; + let vectors = this.vectors as any as NumericVector[]; + let vector = vectors[index], source = vector.slice(from, from + total), target = source; + // Perf optimization: if the first slice contains all the values we're looking for, + // we don't have to copy values to a target Array. If we're slicing a TypedArray, + // this is a significant improvement as we avoid the memcpy 🎉 + if ((source.length / vector.stride | 0) < total) { + let vectorsLength = vectors.length; + let count = 0, length = 0, sources = []; + do { + sources.push(source); + length += source.length; + count += (source.length / vector.stride | 0); + } while ( + (count < total) && + (vector = vectors[index = (++index % vectorsLength)]) && + (source = vector.slice(0, Math.min(vector.length, total - count))) + ); + target = new ArrayType(length); + for (let i = -1, j = 0, n = sources.length; ++i < n;) { + j = set(sources[i], target, j); + } + } + return target; + } + middle = index + ((right - index) / 2) | 0; + from >= offsets[middle] ? (index = middle) : (right = middle); + } + return new ArrayType(0); + } +} + +function arraySet(source: T[], target: T[], index: number) { + for (let i = 0, n = source.length; i < n;) { + target[index++] = source[i++]; + } + return index; +} + +function typedArraySet(source: TypedArray, target: TypedArray, index: number) { + return target.set(source, index) || index + source.length; +} diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts index 722781db658dc..66a5c21ac2dff 100644 --- a/js/test/Arrow.ts +++ b/js/test/Arrow.ts @@ -21,6 +21,7 @@ const path = require('path'); const target = process.env.TEST_TARGET!; const format = process.env.TEST_MODULE!; +const useSrc = process.env.TEST_TS_SOURCE === `true`; // these are duplicated in the gulpfile :< const targets = [`es5`, `es2015`, `esnext`]; @@ -32,55 +33,17 @@ function throwInvalidImportError(name: string, value: string, values: string[]) let modulePath = ``; -if (target === `ts` || target === `apache-arrow`) modulePath = target; +if (useSrc) modulePath = '../src'; +else if (target === `ts` || target === `apache-arrow`) modulePath = target; else if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets); else if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats); else modulePath = path.join(target, format); -let Arrow: any = require(path.resolve(`./targets`, modulePath, `Arrow`)); +export { List } from '../src/Arrow'; +export { TypedArray } from '../src/Arrow'; +export { TypedArrayConstructor } from '../src/Arrow'; +export { NumericVectorConstructor } from '../src/Arrow'; -import { - Table as Table_, - Vector as Vector_, - readBuffers as readBuffers_, - BoolVector as BoolVector_, - TypedVector as TypedVector_, - ListVector as ListVector_, - Utf8Vector as Utf8Vector_, - DateVector as DateVector_, - Int8Vector as Int8Vector_, - Int16Vector as Int16Vector_, - Int32Vector as Int32Vector_, - Int64Vector as Int64Vector_, - Uint8Vector as Uint8Vector_, - Uint16Vector as Uint16Vector_, - Uint32Vector as Uint32Vector_, - Uint64Vector as Uint64Vector_, - Float32Vector as Float32Vector_, - Float64Vector as Float64Vector_, - StructVector as StructVector_, - DictionaryVector as DictionaryVector_, - FixedSizeListVector as FixedSizeListVector_, -} from '../src/Arrow'; - -export let Table = Arrow.Table as typeof Table_; -export let Vector = Arrow.Vector as typeof Vector_; -export let readBuffers = Arrow.readBuffers as typeof readBuffers_; -export let BoolVector = Arrow.BoolVector as typeof BoolVector_; -export let TypedVector = Arrow.TypedVector as typeof TypedVector_; -export let ListVector = Arrow.ListVector as typeof ListVector_; -export let Utf8Vector = Arrow.Utf8Vector as typeof Utf8Vector_; -export let DateVector = Arrow.DateVector as typeof DateVector_; -export let Int8Vector = Arrow.Int8Vector as typeof Int8Vector_; -export let Int16Vector = Arrow.Int16Vector as typeof Int16Vector_; -export let Int32Vector = Arrow.Int32Vector as typeof Int32Vector_; -export let Int64Vector = Arrow.Int64Vector as typeof Int64Vector_; -export let Uint8Vector = Arrow.Uint8Vector as typeof Uint8Vector_; -export let Uint16Vector = Arrow.Uint16Vector as typeof Uint16Vector_; -export let Uint32Vector = Arrow.Uint32Vector as typeof Uint32Vector_; -export let Uint64Vector = Arrow.Uint64Vector as typeof Uint64Vector_; -export let Float32Vector = Arrow.Float32Vector as typeof Float32Vector_; -export let Float64Vector = Arrow.Float64Vector as typeof Float64Vector_; -export let StructVector = Arrow.StructVector as typeof StructVector_; -export let DictionaryVector = Arrow.DictionaryVector as typeof DictionaryVector_; -export let FixedSizeListVector = Arrow.FixedSizeListVector as typeof FixedSizeListVector_; +import * as Arrow_ from '../src/Arrow'; +export let Arrow: typeof Arrow_ = require(path.resolve(`./targets`, modulePath, `Arrow`)); +export default Arrow; \ No newline at end of file diff --git a/js/test/__snapshots__/reader-tests.ts.snap b/js/test/__snapshots__/reader-tests.ts.snap deleted file mode 100644 index 961ce87861c3f..0000000000000 --- a/js/test/__snapshots__/reader-tests.ts.snap +++ /dev/null @@ -1,497 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` -Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], -] -`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` -Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], -] -`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`; - -exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` -Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], -] -`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` -Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], -] -`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` -Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], -] -`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`; - -exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` -Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], -] -`; - -exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`; - -exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; - -exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; - -exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` -Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], -] -`; - -exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` -Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], -] -`; - -exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`; - -exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; - -exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; - -exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` -Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "12345", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], -] -`; - -exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` -Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "67890", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], -] -`; - -exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"row_count"`; - -exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`; - -exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `1`; - -exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `10000`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_lat"`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"FloatingPoint"`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `35.393089294433594`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `35.393089294433594`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `35.393089294433594`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `29.533695220947266`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `29.533695220947266`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"origin_lon"`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `-97.6007308959961`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `-97.6007308959961`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `-97.6007308959961`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `-98.46977996826172`; - -exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `-98.46977996826172`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_city"`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Utf8"`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `"Oklahoma City"`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `"Oklahoma City"`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"Oklahoma City"`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"San Antonio"`; - -exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `"San Antonio"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`; - -exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`; - -exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` -Array [ - null, - "MhRNxD4", -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = ` -Array [ - 137773603, - "3F9HBxK", -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = ` -Array [ - 410361374, - "aVd88fp", -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` -Array [ - null, - "3loZrRf", -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = ` -Array [ - null, - null, -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = ` -Array [ - null, - null, -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = ` -Array [ - null, - "78SLiRw", -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = ` -Array [ - null, - "0ilsf82", -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = ` -Array [ - null, - "LjS9MbU", -] -`; - -exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = ` -Array [ - null, - null, -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` -Array [ - null, - "MhRNxD4", -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = ` -Array [ - 137773603, - "3F9HBxK", -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = ` -Array [ - 410361374, - "aVd88fp", -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` -Array [ - null, - "3loZrRf", -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = ` -Array [ - null, - null, -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = ` -Array [ - null, - null, -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = ` -Array [ - null, - "78SLiRw", -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = ` -Array [ - null, - "0ilsf82", -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = ` -Array [ - null, - "LjS9MbU", -] -`; - -exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = ` -Array [ - null, - null, -] -`; diff --git a/js/test/__snapshots__/table-tests.ts.snap b/js/test/__snapshots__/table-tests.ts.snap deleted file mode 100644 index 401b992d9d7b5..0000000000000 --- a/js/test/__snapshots__/table-tests.ts.snap +++ /dev/null @@ -1,1815 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`; - -exports[`dictionary file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`; - -exports[`dictionary file Arrow Table creates a Table from Arrow buffers 3`] = `3`; - -exports[`dictionary file Arrow Table creates a Table from Arrow buffers 4`] = ` -Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], -] -`; - -exports[`dictionary file Arrow Table creates a Table from Arrow buffers 5`] = ` -Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], -] -`; - -exports[`dictionary file Arrow Table creates a Table from Arrow buffers 6`] = ` -Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], -] -`; - -exports[`dictionary file Arrow Table enumerates Table rows 1`] = ` -Object { - "example-csv": Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], - ], -} -`; - -exports[`dictionary file Arrow Table enumerates Table rows 2`] = ` -Object { - "example-csv": Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], - ], -} -`; - -exports[`dictionary file Arrow Table enumerates Table rows 3`] = ` -Object { - "example-csv": Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], - ], -} -`; - -exports[`dictionary file Arrow Table enumerates Table rows compact 1`] = ` -Array [ - Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], - ], -] -`; - -exports[`dictionary file Arrow Table enumerates Table rows compact 2`] = ` -Array [ - Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], - ], -] -`; - -exports[`dictionary file Arrow Table enumerates Table rows compact 3`] = ` -Array [ - Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], - ], -] -`; - -exports[`dictionary file Arrow Table toString() prints a pretty Table 1`] = ` -" example-csv -[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] - [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] - [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" -`; - -exports[`dictionary file Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, example-csv - 0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] - 1, [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] - 2, [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" -`; - -exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 1`] = `"example-csv"`; - -exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`; - -exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 3`] = `3`; - -exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 4`] = ` -Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], -] -`; - -exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 5`] = ` -Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], -] -`; - -exports[`dictionary stream Arrow Table creates a Table from Arrow buffers 6`] = ` -Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], -] -`; - -exports[`dictionary stream Arrow Table enumerates Table rows 1`] = ` -Object { - "example-csv": Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], - ], -} -`; - -exports[`dictionary stream Arrow Table enumerates Table rows 2`] = ` -Object { - "example-csv": Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], - ], -} -`; - -exports[`dictionary stream Arrow Table enumerates Table rows 3`] = ` -Object { - "example-csv": Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], - ], -} -`; - -exports[`dictionary stream Arrow Table enumerates Table rows compact 1`] = ` -Array [ - Array [ - "Hermione", - 25, - Float32Array [ - -53.235599517822266, - 40.231998443603516, - ], - ], -] -`; - -exports[`dictionary stream Arrow Table enumerates Table rows compact 2`] = ` -Array [ - Array [ - "Severus", - 30, - Float32Array [ - -62.22999954223633, - 3, - ], - ], -] -`; - -exports[`dictionary stream Arrow Table enumerates Table rows compact 3`] = ` -Array [ - Array [ - "Harry", - 20, - Float32Array [ - 23, - -100.23652648925781, - ], - ], -] -`; - -exports[`dictionary stream Arrow Table toString() prints a pretty Table 1`] = ` -" example-csv -[\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] - [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] - [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" -`; - -exports[`dictionary stream Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`dictionary stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, example-csv - 0, [\\"Hermione\\",25,{\\"0\\":-53.235599517822266,\\"1\\":40.231998443603516}] - 1, [\\"Severus\\",30,{\\"0\\":-62.22999954223633,\\"1\\":3}] - 2, [\\"Harry\\",20,{\\"0\\":23,\\"1\\":-100.23652648925781}]" -`; - -exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`; - -exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`; - -exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 3`] = `2`; - -exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 4`] = ` -Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], -] -`; - -exports[`dictionary2 file Arrow Table creates a Table from Arrow buffers 5`] = ` -Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], -] -`; - -exports[`dictionary2 file Arrow Table enumerates Table rows 1`] = ` -Object { - "struct": Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], - ], -} -`; - -exports[`dictionary2 file Arrow Table enumerates Table rows 2`] = ` -Object { - "struct": Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], - ], -} -`; - -exports[`dictionary2 file Arrow Table enumerates Table rows compact 1`] = ` -Array [ - Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], - ], -] -`; - -exports[`dictionary2 file Arrow Table enumerates Table rows compact 2`] = ` -Array [ - Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], - ], -] -`; - -exports[`dictionary2 file Arrow Table toString() prints a pretty Table 1`] = ` -" struct - [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] -[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" -`; - -exports[`dictionary2 file Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`dictionary2 file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct - 0, [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] - 1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" -`; - -exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 1`] = `"struct"`; - -exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`; - -exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 3`] = `2`; - -exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 4`] = ` -Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "12345", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], -] -`; - -exports[`multi_dictionary file Arrow Table creates a Table from Arrow buffers 5`] = ` -Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "67890", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], -] -`; - -exports[`multi_dictionary file Arrow Table enumerates Table rows 1`] = ` -Object { - "struct": Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "12345", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], - ], -} -`; - -exports[`multi_dictionary file Arrow Table enumerates Table rows 2`] = ` -Object { - "struct": Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "67890", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], - ], -} -`; - -exports[`multi_dictionary file Arrow Table enumerates Table rows compact 1`] = ` -Array [ - Array [ - "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", - "12345", - "Airbus", - 1502880750, - Float32Array [ - 32.45663833618164, - 1.8712350130081177, - ], - ], -] -`; - -exports[`multi_dictionary file Arrow Table enumerates Table rows compact 2`] = ` -Array [ - Array [ - "50fb46f4-fefa-42c1-919c-0121974cdd00", - "67890", - "Boeing", - 1502880750, - Float32Array [ - 38.766666412353516, - -4.181231498718262, - ], - ], -] -`; - -exports[`multi_dictionary file Arrow Table toString() prints a pretty Table 1`] = ` -" struct - [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] -[\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" -`; - -exports[`multi_dictionary file Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`multi_dictionary file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct - 0, [\\"a0fb47f9-f8fb-4403-a64a-786d7611f8ef\\",\\"12345\\",\\"Airbus\\",1502880750,{\\"0\\":32.45663833618164,\\"1\\":1.8712350130081177}] - 1, [\\"50fb46f4-fefa-42c1-919c-0121974cdd00\\",\\"67890\\",\\"Boeing\\",1502880750,{\\"0\\":38.766666412353516,\\"1\\":-4.181231498718262}]" -`; - -exports[`multipart count Arrow Table creates a Table from Arrow buffers 1`] = `"row_count"`; - -exports[`multipart count Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`; - -exports[`multipart count Arrow Table creates a Table from Arrow buffers 3`] = `1`; - -exports[`multipart count Arrow Table creates a Table from Arrow buffers 4`] = `10000`; - -exports[`multipart count Arrow Table enumerates Table rows 1`] = ` -Object { - "row_count": 10000, -} -`; - -exports[`multipart count Arrow Table enumerates Table rows compact 1`] = ` -Array [ - 10000, -] -`; - -exports[`multipart count Arrow Table toString() prints a pretty Table 1`] = ` -"row_count - 10000" -`; - -exports[`multipart count Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`multipart count Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, row_count - 0, 10000" -`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 1`] = `"origin_lat"`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 2`] = `"FloatingPoint"`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 3`] = `5`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 4`] = `35.393089294433594`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 5`] = `35.393089294433594`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 6`] = `35.393089294433594`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 7`] = `29.533695220947266`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 8`] = `29.533695220947266`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 9`] = `"origin_lon"`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 11`] = `5`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 12`] = `-97.6007308959961`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 13`] = `-97.6007308959961`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 14`] = `-97.6007308959961`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 15`] = `-98.46977996826172`; - -exports[`multipart latlong Arrow Table creates a Table from Arrow buffers 16`] = `-98.46977996826172`; - -exports[`multipart latlong Arrow Table enumerates Table rows 1`] = ` -Object { - "origin_lat": 35.393089294433594, - "origin_lon": -97.6007308959961, -} -`; - -exports[`multipart latlong Arrow Table enumerates Table rows 2`] = ` -Object { - "origin_lat": 35.393089294433594, - "origin_lon": -97.6007308959961, -} -`; - -exports[`multipart latlong Arrow Table enumerates Table rows 3`] = ` -Object { - "origin_lat": 35.393089294433594, - "origin_lon": -97.6007308959961, -} -`; - -exports[`multipart latlong Arrow Table enumerates Table rows 4`] = ` -Object { - "origin_lat": 29.533695220947266, - "origin_lon": -98.46977996826172, -} -`; - -exports[`multipart latlong Arrow Table enumerates Table rows 5`] = ` -Object { - "origin_lat": 29.533695220947266, - "origin_lon": -98.46977996826172, -} -`; - -exports[`multipart latlong Arrow Table enumerates Table rows compact 1`] = ` -Array [ - 35.393089294433594, - -97.6007308959961, -] -`; - -exports[`multipart latlong Arrow Table enumerates Table rows compact 2`] = ` -Array [ - 35.393089294433594, - -97.6007308959961, -] -`; - -exports[`multipart latlong Arrow Table enumerates Table rows compact 3`] = ` -Array [ - 35.393089294433594, - -97.6007308959961, -] -`; - -exports[`multipart latlong Arrow Table enumerates Table rows compact 4`] = ` -Array [ - 29.533695220947266, - -98.46977996826172, -] -`; - -exports[`multipart latlong Arrow Table enumerates Table rows compact 5`] = ` -Array [ - 29.533695220947266, - -98.46977996826172, -] -`; - -exports[`multipart latlong Arrow Table toString() prints a pretty Table 1`] = ` -" origin_lat, origin_lon -35.393089294433594, -97.6007308959961 -35.393089294433594, -97.6007308959961 -35.393089294433594, -97.6007308959961 -29.533695220947266, -98.46977996826172 -29.533695220947266, -98.46977996826172" -`; - -exports[`multipart latlong Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`multipart latlong Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, origin_lat, origin_lon - 0, 35.393089294433594, -97.6007308959961 - 1, 35.393089294433594, -97.6007308959961 - 2, 35.393089294433594, -97.6007308959961 - 3, 29.533695220947266, -98.46977996826172 - 4, 29.533695220947266, -98.46977996826172" -`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 1`] = `"origin_city"`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 2`] = `"Utf8"`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 3`] = `5`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 4`] = `"Oklahoma City"`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 5`] = `"Oklahoma City"`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 6`] = `"Oklahoma City"`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 7`] = `"San Antonio"`; - -exports[`multipart origins Arrow Table creates a Table from Arrow buffers 8`] = `"San Antonio"`; - -exports[`multipart origins Arrow Table enumerates Table rows 1`] = ` -Object { - "origin_city": "Oklahoma City", -} -`; - -exports[`multipart origins Arrow Table enumerates Table rows 2`] = ` -Object { - "origin_city": "Oklahoma City", -} -`; - -exports[`multipart origins Arrow Table enumerates Table rows 3`] = ` -Object { - "origin_city": "Oklahoma City", -} -`; - -exports[`multipart origins Arrow Table enumerates Table rows 4`] = ` -Object { - "origin_city": "San Antonio", -} -`; - -exports[`multipart origins Arrow Table enumerates Table rows 5`] = ` -Object { - "origin_city": "San Antonio", -} -`; - -exports[`multipart origins Arrow Table enumerates Table rows compact 1`] = ` -Array [ - "Oklahoma City", -] -`; - -exports[`multipart origins Arrow Table enumerates Table rows compact 2`] = ` -Array [ - "Oklahoma City", -] -`; - -exports[`multipart origins Arrow Table enumerates Table rows compact 3`] = ` -Array [ - "Oklahoma City", -] -`; - -exports[`multipart origins Arrow Table enumerates Table rows compact 4`] = ` -Array [ - "San Antonio", -] -`; - -exports[`multipart origins Arrow Table enumerates Table rows compact 5`] = ` -Array [ - "San Antonio", -] -`; - -exports[`multipart origins Arrow Table toString() prints a pretty Table 1`] = ` -" origin_city -Oklahoma City -Oklahoma City -Oklahoma City - San Antonio - San Antonio" -`; - -exports[`multipart origins Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`multipart origins Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, origin_city - 0, Oklahoma City - 1, Oklahoma City - 2, Oklahoma City - 3, San Antonio - 4, San Antonio" -`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 1`] = `"foo"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 3`] = `5`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 4`] = `1`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 5`] = `null`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 6`] = `3`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 7`] = `4`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 8`] = `5`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 9`] = `"bar"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 11`] = `5`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 12`] = `1`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 13`] = `null`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 14`] = `null`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 15`] = `4`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 16`] = `5`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 17`] = `"baz"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 18`] = `"Utf8"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 19`] = `5`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 20`] = `"aa"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 21`] = `null`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 22`] = `null`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 23`] = `"bbb"`; - -exports[`simple file Arrow Table creates a Table from Arrow buffers 24`] = `"cccc"`; - -exports[`simple file Arrow Table enumerates Table rows 1`] = ` -Object { - "bar": 1, - "baz": "aa", - "foo": 1, -} -`; - -exports[`simple file Arrow Table enumerates Table rows 2`] = ` -Object { - "bar": null, - "baz": null, - "foo": null, -} -`; - -exports[`simple file Arrow Table enumerates Table rows 3`] = ` -Object { - "bar": null, - "baz": null, - "foo": 3, -} -`; - -exports[`simple file Arrow Table enumerates Table rows 4`] = ` -Object { - "bar": 4, - "baz": "bbb", - "foo": 4, -} -`; - -exports[`simple file Arrow Table enumerates Table rows 5`] = ` -Object { - "bar": 5, - "baz": "cccc", - "foo": 5, -} -`; - -exports[`simple file Arrow Table enumerates Table rows compact 1`] = ` -Array [ - 1, - 1, - "aa", -] -`; - -exports[`simple file Arrow Table enumerates Table rows compact 2`] = ` -Array [ - null, - null, - null, -] -`; - -exports[`simple file Arrow Table enumerates Table rows compact 3`] = ` -Array [ - 3, - null, - null, -] -`; - -exports[`simple file Arrow Table enumerates Table rows compact 4`] = ` -Array [ - 4, - 4, - "bbb", -] -`; - -exports[`simple file Arrow Table enumerates Table rows compact 5`] = ` -Array [ - 5, - 5, - "cccc", -] -`; - -exports[`simple file Arrow Table toString() prints a pretty Table 1`] = ` -" foo, bar, baz - 1, 1, aa -null, null, null - 3, null, null - 4, 4, bbb - 5, 5, cccc" -`; - -exports[`simple file Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`simple file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, foo, bar, baz - 0, 1, 1, aa - 1, null, null, null - 2, 3, null, null - 3, 4, 4, bbb - 4, 5, 5, cccc" -`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 1`] = `"foo"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 2`] = `"Int"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 3`] = `5`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 4`] = `1`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 5`] = `null`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 6`] = `3`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 7`] = `4`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 8`] = `5`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 9`] = `"bar"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 10`] = `"FloatingPoint"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 11`] = `5`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 12`] = `1`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 13`] = `null`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 14`] = `null`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 15`] = `4`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 16`] = `5`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 17`] = `"baz"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 18`] = `"Utf8"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 19`] = `5`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 20`] = `"aa"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 21`] = `null`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 22`] = `null`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 23`] = `"bbb"`; - -exports[`simple stream Arrow Table creates a Table from Arrow buffers 24`] = `"cccc"`; - -exports[`simple stream Arrow Table enumerates Table rows 1`] = ` -Object { - "bar": 1, - "baz": "aa", - "foo": 1, -} -`; - -exports[`simple stream Arrow Table enumerates Table rows 2`] = ` -Object { - "bar": null, - "baz": null, - "foo": null, -} -`; - -exports[`simple stream Arrow Table enumerates Table rows 3`] = ` -Object { - "bar": null, - "baz": null, - "foo": 3, -} -`; - -exports[`simple stream Arrow Table enumerates Table rows 4`] = ` -Object { - "bar": 4, - "baz": "bbb", - "foo": 4, -} -`; - -exports[`simple stream Arrow Table enumerates Table rows 5`] = ` -Object { - "bar": 5, - "baz": "cccc", - "foo": 5, -} -`; - -exports[`simple stream Arrow Table enumerates Table rows compact 1`] = ` -Array [ - 1, - 1, - "aa", -] -`; - -exports[`simple stream Arrow Table enumerates Table rows compact 2`] = ` -Array [ - null, - null, - null, -] -`; - -exports[`simple stream Arrow Table enumerates Table rows compact 3`] = ` -Array [ - 3, - null, - null, -] -`; - -exports[`simple stream Arrow Table enumerates Table rows compact 4`] = ` -Array [ - 4, - 4, - "bbb", -] -`; - -exports[`simple stream Arrow Table enumerates Table rows compact 5`] = ` -Array [ - 5, - 5, - "cccc", -] -`; - -exports[`simple stream Arrow Table toString() prints a pretty Table 1`] = ` -" foo, bar, baz - 1, 1, aa -null, null, null - 3, null, null - 4, 4, bbb - 5, 5, cccc" -`; - -exports[`simple stream Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`simple stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, foo, bar, baz - 0, 1, 1, aa - 1, null, null, null - 2, 3, null, null - 3, 4, 4, bbb - 4, 5, 5, cccc" -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 3`] = `17`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 4`] = `null`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 5`] = ` -Array [ - null, - "MhRNxD4", -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 6`] = ` -Array [ - 137773603, - "3F9HBxK", -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 7`] = ` -Array [ - 410361374, - "aVd88fp", -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 8`] = `null`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 9`] = ` -Array [ - null, - "3loZrRf", -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 10`] = `null`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 11`] = `null`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 12`] = ` -Array [ - null, - null, -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 13`] = ` -Array [ - null, - null, -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 14`] = `null`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 15`] = ` -Array [ - null, - "78SLiRw", -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 16`] = `null`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 17`] = `null`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 18`] = ` -Array [ - null, - "0ilsf82", -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 19`] = ` -Array [ - null, - "LjS9MbU", -] -`; - -exports[`struct file Arrow Table creates a Table from Arrow buffers 20`] = ` -Array [ - null, - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows 1`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct file Arrow Table enumerates Table rows 2`] = ` -Object { - "struct_nullable": Array [ - null, - "MhRNxD4", - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 3`] = ` -Object { - "struct_nullable": Array [ - 137773603, - "3F9HBxK", - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 4`] = ` -Object { - "struct_nullable": Array [ - 410361374, - "aVd88fp", - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 5`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct file Arrow Table enumerates Table rows 6`] = ` -Object { - "struct_nullable": Array [ - null, - "3loZrRf", - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 7`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct file Arrow Table enumerates Table rows 8`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct file Arrow Table enumerates Table rows 9`] = ` -Object { - "struct_nullable": Array [ - null, - null, - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 10`] = ` -Object { - "struct_nullable": Array [ - null, - null, - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 11`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct file Arrow Table enumerates Table rows 12`] = ` -Object { - "struct_nullable": Array [ - null, - "78SLiRw", - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 13`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct file Arrow Table enumerates Table rows 14`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct file Arrow Table enumerates Table rows 15`] = ` -Object { - "struct_nullable": Array [ - null, - "0ilsf82", - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 16`] = ` -Object { - "struct_nullable": Array [ - null, - "LjS9MbU", - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows 17`] = ` -Object { - "struct_nullable": Array [ - null, - null, - ], -} -`; - -exports[`struct file Arrow Table enumerates Table rows compact 1`] = ` -Array [ - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 2`] = ` -Array [ - Array [ - null, - "MhRNxD4", - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 3`] = ` -Array [ - Array [ - 137773603, - "3F9HBxK", - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 4`] = ` -Array [ - Array [ - 410361374, - "aVd88fp", - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 5`] = ` -Array [ - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 6`] = ` -Array [ - Array [ - null, - "3loZrRf", - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 7`] = ` -Array [ - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 8`] = ` -Array [ - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 9`] = ` -Array [ - Array [ - null, - null, - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 10`] = ` -Array [ - Array [ - null, - null, - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 11`] = ` -Array [ - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 12`] = ` -Array [ - Array [ - null, - "78SLiRw", - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 13`] = ` -Array [ - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 14`] = ` -Array [ - null, -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 15`] = ` -Array [ - Array [ - null, - "0ilsf82", - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 16`] = ` -Array [ - Array [ - null, - "LjS9MbU", - ], -] -`; - -exports[`struct file Arrow Table enumerates Table rows compact 17`] = ` -Array [ - Array [ - null, - null, - ], -] -`; - -exports[`struct file Arrow Table toString() prints a pretty Table 1`] = ` -" struct_nullable - null - [null,\\"MhRNxD4\\"] -[137773603,\\"3F9HBxK\\"] -[410361374,\\"aVd88fp\\"] - null - [null,\\"3loZrRf\\"] - null - null - [null,null] - [null,null] - null - [null,\\"78SLiRw\\"] - null - null - [null,\\"0ilsf82\\"] - [null,\\"LjS9MbU\\"] - [null,null]" -`; - -exports[`struct file Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`struct file Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct_nullable - 0, null - 1, [null,\\"MhRNxD4\\"] - 2, [137773603,\\"3F9HBxK\\"] - 3, [410361374,\\"aVd88fp\\"] - 4, null - 5, [null,\\"3loZrRf\\"] - 6, null - 7, null - 8, [null,null] - 9, [null,null] - 10, null - 11, [null,\\"78SLiRw\\"] - 12, null - 13, null - 14, [null,\\"0ilsf82\\"] - 15, [null,\\"LjS9MbU\\"] - 16, [null,null]" -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 1`] = `"struct_nullable"`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 2`] = `"Struct_"`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 3`] = `17`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 4`] = `null`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 5`] = ` -Array [ - null, - "MhRNxD4", -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 6`] = ` -Array [ - 137773603, - "3F9HBxK", -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 7`] = ` -Array [ - 410361374, - "aVd88fp", -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 8`] = `null`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 9`] = ` -Array [ - null, - "3loZrRf", -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 10`] = `null`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 11`] = `null`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 12`] = ` -Array [ - null, - null, -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 13`] = ` -Array [ - null, - null, -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 14`] = `null`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 15`] = ` -Array [ - null, - "78SLiRw", -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 16`] = `null`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 17`] = `null`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 18`] = ` -Array [ - null, - "0ilsf82", -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 19`] = ` -Array [ - null, - "LjS9MbU", -] -`; - -exports[`struct stream Arrow Table creates a Table from Arrow buffers 20`] = ` -Array [ - null, - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows 1`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 2`] = ` -Object { - "struct_nullable": Array [ - null, - "MhRNxD4", - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 3`] = ` -Object { - "struct_nullable": Array [ - 137773603, - "3F9HBxK", - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 4`] = ` -Object { - "struct_nullable": Array [ - 410361374, - "aVd88fp", - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 5`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 6`] = ` -Object { - "struct_nullable": Array [ - null, - "3loZrRf", - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 7`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 8`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 9`] = ` -Object { - "struct_nullable": Array [ - null, - null, - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 10`] = ` -Object { - "struct_nullable": Array [ - null, - null, - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 11`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 12`] = ` -Object { - "struct_nullable": Array [ - null, - "78SLiRw", - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 13`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 14`] = ` -Object { - "struct_nullable": null, -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 15`] = ` -Object { - "struct_nullable": Array [ - null, - "0ilsf82", - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 16`] = ` -Object { - "struct_nullable": Array [ - null, - "LjS9MbU", - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows 17`] = ` -Object { - "struct_nullable": Array [ - null, - null, - ], -} -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 1`] = ` -Array [ - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 2`] = ` -Array [ - Array [ - null, - "MhRNxD4", - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 3`] = ` -Array [ - Array [ - 137773603, - "3F9HBxK", - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 4`] = ` -Array [ - Array [ - 410361374, - "aVd88fp", - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 5`] = ` -Array [ - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 6`] = ` -Array [ - Array [ - null, - "3loZrRf", - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 7`] = ` -Array [ - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 8`] = ` -Array [ - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 9`] = ` -Array [ - Array [ - null, - null, - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 10`] = ` -Array [ - Array [ - null, - null, - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 11`] = ` -Array [ - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 12`] = ` -Array [ - Array [ - null, - "78SLiRw", - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 13`] = ` -Array [ - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 14`] = ` -Array [ - null, -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 15`] = ` -Array [ - Array [ - null, - "0ilsf82", - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 16`] = ` -Array [ - Array [ - null, - "LjS9MbU", - ], -] -`; - -exports[`struct stream Arrow Table enumerates Table rows compact 17`] = ` -Array [ - Array [ - null, - null, - ], -] -`; - -exports[`struct stream Arrow Table toString() prints a pretty Table 1`] = ` -" struct_nullable - null - [null,\\"MhRNxD4\\"] -[137773603,\\"3F9HBxK\\"] -[410361374,\\"aVd88fp\\"] - null - [null,\\"3loZrRf\\"] - null - null - [null,null] - [null,null] - null - [null,\\"78SLiRw\\"] - null - null - [null,\\"0ilsf82\\"] - [null,\\"LjS9MbU\\"] - [null,null]" -`; - -exports[`struct stream Arrow Table toString() prints an empty Table 1`] = `""`; - -exports[`struct stream Arrow Table toString({ index: true }) prints a pretty Table with an Index column 1`] = ` -"Index, struct_nullable - 0, null - 1, [null,\\"MhRNxD4\\"] - 2, [137773603,\\"3F9HBxK\\"] - 3, [410361374,\\"aVd88fp\\"] - 4, null - 5, [null,\\"3loZrRf\\"] - 6, null - 7, null - 8, [null,null] - 9, [null,null] - 10, null - 11, [null,\\"78SLiRw\\"] - 12, null - 13, null - 14, [null,\\"0ilsf82\\"] - 15, [null,\\"LjS9MbU\\"] - 16, [null,null]" -`; diff --git a/js/test/arrows/file/dictionary.arrow b/js/test/arrows/file/dictionary.arrow deleted file mode 100644 index 34d41db1f200159406c459b8c076d5ce0c3d30ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2522 zcmeHJ&1(};5T9+*G{mHqHnv(ta_B)s!H*ykC?w(`f?#L`@#bQlX&@hEla^FGYFyCl*NI8<1N7r49A1Uz3Rk-wEvR zLio|iYm>=D?S_ycE00v|-a0$whwl6poGAiD5*{f-vL43ThfM(<#?BTZKnj_r@FRGZ zuGRg%zeb+Aq?OUn8Q|rH>9$L?ZnM)b^@3)7y?FD+Ey%;#mt^rTwKFFovhl8Q1!_($ zlJWEtL)h|5{AXHOih zg1!KX%z6W?>@oA;tk{pQh?7R_wCw;VMkrXGRg>48IPE+nwBy|A(O=$|I45qzimpJ{h--txcvLUj$sb`iJStOf#2S4nX6BGyYdM<>mX!N+M#q1dO&)hQLnOoB4t^Hvy6ry{T@m@tC>`gegsGP9~nlJaXo%T6L zm4jK(9FSI3KM3}!CTKUi9kXe6O|aA3f(!MS#eC;+_re+Dyz_BTePX)8eVhsJQg~Ri zKQP7SunOaH-c&-=A9<}je!f5@ARhBlD4vSTCxQ_7b2}8B3OXsJ zo^z=DIUK$SRQ|LxuKaj7dh@+-=jFG-NTghDp@73?R(3W=%`Do3c>m}kswb~!;&Po? zTz=Hd;=TFTnQ30kku!_uaa=w=WBOGeN+}~Kud`5I=O^;6vby(vr24GAVc{EsZx~`3 zxCLa~*UpghB~(KqWflh5cJi|2!iAz&F6qxsUX1AZ4rPz@ZAwZT&gbMSdh!)@&(D8- MMaQ^*75ks=H#cC^v;Y7A diff --git a/js/test/arrows/file/dictionary2.arrow b/js/test/arrows/file/dictionary2.arrow deleted file mode 100644 index 1537f54db79eeeb2243e91846994737099a8ea96..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2762 zcmeHJJ!n%=6h5zwHMFU=lGb7~Jg`F%14;YL9~5cn(4kus7;==KdlyX+rSubWy7m(7FMd2t)S4T zSKK*gb|wQmdAeSsT(8uEYQ63MLgzQ}Vu)vWr{I$VMEb2z3z&idKz^U$YlF!b9RnuF z0D3Wi55-b?$}I&YHbJdLl#Ov1gLJvFtMlH*xTb+@R)DeLw%iJ9WvENQVkz*7l})d} zLBaL>()RseE~syN$LsCny^mbv{T!OTr%WR@lx5?K{WP&L#C@x-eN^3% zJ*=@;D*$;m0g*G-D}0kP?@Uzfj;NT78wZAf;)9x3xEa(dTN}sDhF2s3m_9G6x-Ge@ zINDN#HoRGB9MRcJo-z4+|N7?X+nu3V40ZmqDnF(cIRxu;K1$iO=y{m1LG z;oq5`cdlD{)z@uTIpA!#nmD!|EE$8=_q6VgQ`7cNUNr6W*#A)5P2crY9{1S4*Ta6G z?WPV++52uDC!O;icZ)f1qq#l>n4XnV{<@R(^GQGFuO}TRHJdDDoKiBI%edK0I_>8? z{~GsXd?)GS5tnpk%IRc2oi8U->6vss>y%wLmHHQ7H{HyCT;9J!nE824@Yx4z;pOhn z>!YKS3$NclI&9xHb}9U=@Vtp>U>RWOJzeCH&eMd=h+5Z)K4~L;LpgnCvUR8L%<$Q~ Yf?fnbcz;gcnJ0Z`9^!X5ul~dS1zMm*_5c6? diff --git a/js/test/arrows/file/multi_dictionary.arrow b/js/test/arrows/file/multi_dictionary.arrow deleted file mode 100644 index 113d30da7856560455dc0b63229671825d1a9fdd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3482 zcmeHK%}Z2K6hALb2R}+fEwaUcE+ib78K3h-l8Ot1Br%}^7mhRU>3}-J_ymI%Eec!| zxNuR>!XUzBv}zNyEN~I%qCX&_MO2HH)%5$ld$02t5n4!_JaXQ-_uO;O$M2r|<~cGo zb?rt>#I6@<6Ok4^f*Wn?p z){ZYljNKyaEciW%)nTm7oDkW>;t$N}=d<=Rjqxn_Eg0EZl*Y%4UzG~uhrylTi}Ox# zDKl9tEPI)iav?ut55yDDGuOPck}DTVUa?%=|6S)`{LR3h$vuiaDX>VLr6f^M_%OeI z*oKjb4aqtbH2{wW{E;kQ_U5x*mOx-e+_*+-`e3>cvZv$T#JHpxR+fjc$?eH+RV!Vc z2ajbvXS%TDWH`v=9XGqW=nZ-0Rp(&6!Ms1}yuSqUUW0Fj3gbaMjc*KPvQnwsfg~>D z;6?BQu*gYk9Z!?rCD?Iq2e=tLeY@mju6yOe^85k(*$NlH>^0)R_#t@O@bd`EI;6og z4!^Yf^6BNn*K5rK7)l>>5EkQY!j?W54+8PS@ZAlJI^eqv{TlLf+Q1rXm;@8oZ7^$i zh9_%KeNF3nAA?{m6BF03{B12`J`R@5_&J7U8q(Cllj-mys54?d3#oomMkXn%68Q@R!iD{p&H4ptBL34*-Ww2&m z)$?sap6VQUZ|QdyY;G8aem#^y;u!`>)uhffsAnt^b2G%fLH|Zi+zhd1vuC7Go?BfR zEjny&&{5Q1?Sp?)Ke09RH}&&9ZT7_C1NLAdnMy~MhkAflJ^wEJdUT>Y)%m|NL4Yv( zS?UhfH;Dv-__yxwsSXZ1-&(yN1KZ4584qDS7&i(n7qqPPQ__0&dyv-jq}Kkf)(7jL zYF+DV?KiaE^r@c8qeJYs4$*IGy{VhiMsRH&M?J5_g>lV&F_OAo@HhvcY}B2zlWscV zrrf!RZAS+p*@T^qBvXleG7*cpDaRe8O~SP!E`HJ?c03o0q+{t^BpQpy(n&j)&qt$G z{|eRpxdz$HzxHh2BAfYXm4-atwML)b`*yj#y?gZe&iyyl-X)6a-NXJN_kkC{bUnV)q2o9g(Rir?qFufgxb|LvP<1D9dM H|DWs+-cqT+ diff --git a/js/test/arrows/file/simple.arrow b/js/test/arrows/file/simple.arrow deleted file mode 100644 index 838db6dc8eda5376aaaf977c5f96c50397582a5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1642 zcmeHIJxc>Y5S{x-E`~!=1UW$=g@vVpg@xdyvk-(J*x1}DwgF44l$Ms3{sezQ3QK>C zrQP}7+>FLVB_!H8VfN$A?7W@X-AjKkI62#uh2=aD>0os#tJ0K)v~-Ln2o)D;D;5K4 zL)k~rXXvHPPphiJg2plCThPpK`t2A-4oa*);8@d=0I^%(73C1=$r12CLggrn8+oi_ zM>%{Ly9?zWLYtGTmZnBxEL!idE=T=K1qD@_LLSm6wo7*Sa3Cn)RIfVw0Y zfD-loklcoj``mlSN9H$;kH5i@h)QmDkCPidn!?9Z{xtIr`2NlOKIa)y-0pK?n{1X4 z=ORDvi*w8ya(!OMAN^hqzczRB8t+;^>-8J_tfwpUUzw%(Go&cWvb=yS$iRD_;^pu~ prFaE=t<2^C?KYHW$}@hjgWmT8pS(n{|9`)h`+nV`u6_NFc>}#ma&iCw diff --git a/js/test/arrows/file/struct.arrow b/js/test/arrows/file/struct.arrow deleted file mode 100644 index 3d2c018e6c27c95057f2a18395a9dc6031e6b201..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2354 zcmeHJJ5L)y5MCPt7wZBk2y#J?B~nBw5RVv4cSZyvVqWKjR} z6s4k~Oa4I`{{(&lB_(a7NY3}|?Kp^s{6L5%V|}|jbMx4po7-{9b?24_Rns&dQtAP$ zZPlULRI9S|8bc_obC#`0A=(+is+wGdDvKpXyMleIejtO0puY>U<_|asV;be?rC_vtA32!F>+3PZ?aFWNwxU?K&|QVQPY-z;pc@f8Ls-qN37n%QY`%I zxY&rxo9w6m*goA-#&UDS5j1ee;DZ_$xWVP-wcK6z)5gufTCRbFbJ;&0~}qu0pfG@{&q z?#j*&Ubi*rclBPXsd}vfYg-^S&f4w+o&s!FH)vU~zCB8165A>F`SjF!_TAd@Xfl)D zbU%K`O?jhpD~ZxZ!h2O1*z^ltawxr(8Jo$yzOWVE-8OE2b_g3cIlB=%&N4;^kpA=4 j4F+65J`V#6Pz4u|?*qgFK+Fi1 ggAhzW>OT-53Bj4@;tp^bh#AP{z@!A!VJz}!0H3Q0ZvX%Q diff --git a/js/test/arrows/multi/count/schema.arrow b/js/test/arrows/multi/count/schema.arrow deleted file mode 100644 index dfd24e9e0018c471a053b8621cb7ea8b46a5bd9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 184 zcmXYqF%E<<3OT-53Bj4@;to(5m^h3N(||6HEYAc~glsQJ4>}(t);5XP35JT0wDqM7?xI^BFD2^XWnihKA86z-kaHl z0XqO2Y!R?R?LCtf&I65e6{&_$Er>DqON^1rWzV@I_L`@aJfj#35`_jFSW8A2V8h-QC8fk$0e`;lU^$MH+=mDH!3Sj^M diff --git a/js/test/arrows/multi/origins/schema.arrow b/js/test/arrows/multi/origins/schema.arrow deleted file mode 100644 index 0d10fb0e2d1354e5a147f49b5283632d4fdde04e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1604 zcmeIxJ!lj`6bJCRGapA1Bua|d$Pxq#Q7lBQLUI>l5-+(k7firH-tE4-9d35!oY{?D z7p!cw6C1I!5^U^j#6|==vD3yTSSZ+OBjIR?4LO})pAhm>Is_B%h zG;7ATjOuK%OM^P8Bq%e5aoyO|FRRB=QN*enORC3)wY*)IY3iDpZO3(Mwq?XR$#X;)-z?RB2G9 z4(G`>D72I?0^sG85Q-X@_K`Ly>C#4~93G)FdR{fNn;Ba`lXP!B!4Z`*^O;Mf-jq6W z#$s*Qab;-}soOCnYvkfQr?Qcb%UP>M%9JyU0G%>Yk%_TfQM}FmMk+>SNi%a5W4oCP zDzTM_y+BHlTVTD-c7aoCWO_;AK3#qvRmvi^5iY6GH67BtWRQ^2OJo&tU|orEG$h(v z){!JlU8AI1c1m_v2)P=)U*&05$pzQAO|EJjZlleLT{UYqajGi5jpAS(%~cusbgz(h ict8K@)(@CUEJJFWRa|4yt$ZX2qYWXY)dL0a`6ZF zAN&!Xds6V=u_x(K5D$W9*YA7#rmU}Osk6+RH}l@id^7WQHYc(qB6V4miZrBZ_gG}k z+9jx2TN#!bm@)ug1UIoZAy?onxWzN-McG6@J_Z|K;*)_z#PvjifOrz0@p}F~NvL~OQvkp=nH7?6#=&VrtXpgYCH;GLZ zw}_5;f5%pXPHbh1Tl>RaC>8f=E8UOr*mcQKA@FP*IK5O?yIlwVft$FA7?tv=t*cS#x%Tk#_rEb89=F)82g$Le<8 diff --git a/js/test/arrows/stream/simple.arrow b/js/test/arrows/stream/simple.arrow deleted file mode 100644 index 2c68c0e44b0af2ed78a47edbd038ae8fa8ab574b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1188 zcmcIiJ5Iw;5M2La6IsYbgd7PZ7aV|so>x*(asiu&HWDROj*yaLaD)_;9EOt4%&d3$ zIDjGrHd;TwnSC>FH(o>{m57vbC=&J3d$Bh?>poGB!NdI5h?f00HmfkzUnN7vrV_(~{cQ4I?sKBIu0-%dG3D)Y~UDR>;`PleEZ%U^bsHi(7p#q z0^}Hj5>P*pUHCYN=-?Lw&sfa%1-yj}%;AoSJVi8G^r=%vp_8Ln_B2Murp|oyc?Jvu z6bC*IJpI^W#lihR{5bOdyq@O;Z5_l2DqH4%0^3S;zqVdo`&9G1QrX*-7z*I$W_<^A zeZ%Tx(qpT8*I6VbA?RVm^y_>9jy34>H`T~{;~1#*AIa*a;f-!)!=GrozxnztJg%(d zMDMraXg;fXPW<1@Ik~2^)1UihcOHlC?CCpxe*3k+RbuLuGkjW$*_VxKHzEJ^{3aLc z6ao4**o&qgvHWkcu$5mtntqv@nV6e8dQ*D8k;%BbDQ{diTF>4Nmnf+|e$v1;y^hdth(oOG^{&ro_cHKv$qq#D1c{F0n>w%$wGQ(GXHvhcBAxRy*QpJlH;%3Hj9}5!;<>ec0a0N6*h?f8W diff --git a/js/test/integration-tests.ts b/js/test/integration-tests.ts new file mode 100644 index 0000000000000..4147e862b19a4 --- /dev/null +++ b/js/test/integration-tests.ts @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Arrow from './Arrow'; +import { zip } from 'ix/iterable/zip'; +import { config, formats } from './test-config'; + +const { Table, readVectors } = Arrow; + +expect.extend({ + toEqualVector(v1: any, v2: any) { + + const format = (x: any, y: any, msg= ' ') => `${ + this.utils.printExpected(x)}${ + msg}${ + this.utils.printReceived(y) + }`; + + let getFailures = new Array(); + let propsFailures = new Array(); + let iteratorFailures = new Array(); + let allFailures = [ + { title: 'get', failures: getFailures }, + { title: 'props', failures: propsFailures }, + { title: 'iterator', failures: iteratorFailures } + ]; + + let props = ['name', 'type', 'length', 'nullable', 'nullCount', 'metadata']; + for (let i = -1, n = props.length; ++i < n;) { + const prop = props[i]; + if (this.utils.stringify(v1[prop]) !== this.utils.stringify(v2[prop])) { + propsFailures.push(`${prop}: ${format(v1[prop], v2[prop], ' !== ')}`); + } + } + + for (let i = -1, n = v1.length; ++i < n;) { + let x1 = v1.get(i), x2 = v2.get(i); + if (this.utils.stringify(x1) !== this.utils.stringify(x2)) { + getFailures.push(`${i}: ${format(x1, x2, ' !== ')}`); + } + } + + let i = -1; + for (let [x1, x2] of zip(v1, v2)) { + ++i; + if (this.utils.stringify(x1) !== this.utils.stringify(x2)) { + iteratorFailures.push(`${i}: ${format(x1, x2, ' !== ')}`); + } + } + + return { + pass: allFailures.every(({ failures }) => failures.length === 0), + message: () => [ + `${v1.name}: (${format('cpp', 'java', ' !== ')})\n`, + ...allFailures.map(({ failures, title }) => + !failures.length ? `` : [`${title}:`, ...failures].join(`\n`)) + ].join('\n') + }; + } +}); + +describe(`Integration`, () => { + for (const format of formats) { + describe(format, () => { + for (const [cppArrow, javaArrow] of zip(config.cpp[format], config.java[format])) { + describe(`${cppArrow.name}`, () => { + testReaderIntegration(cppArrow.buffers, javaArrow.buffers); + testTableFromBuffersIntegration(cppArrow.buffers, javaArrow.buffers); + }); + } + }); + } +}); + +function testReaderIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) { + test(`cpp and java vectors report the same values`, () => { + expect.hasAssertions(); + for (const [cppVectors, javaVectors] of zip(readVectors(cppBuffers), readVectors(javaBuffers))) { + expect(cppVectors.length).toEqual(javaVectors.length); + for (let i = -1, n = cppVectors.length; ++i < n;) { + (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]); + } + } + }); +} + +function testTableFromBuffersIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) { + test(`cpp and java tables report the same values`, () => { + expect.hasAssertions(); + const cppTable = Table.from(cppBuffers); + const javaTable = Table.from(javaBuffers); + const cppVectors = cppTable.columns; + const javaVectors = javaTable.columns; + expect(cppTable.length).toEqual(javaTable.length); + expect(cppVectors.length).toEqual(javaVectors.length); + for (let i = -1, n = cppVectors.length; ++i < n;) { + (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]); + } + }); +} diff --git a/js/test/reader-tests.ts b/js/test/reader-tests.ts index a7f9f4110237f..309cec9e23b14 100644 --- a/js/test/reader-tests.ts +++ b/js/test/reader-tests.ts @@ -15,36 +15,55 @@ // specific language governing permissions and limitations // under the License. -import { readBuffers } from './Arrow'; -import arrowTestConfigurations from './test-config'; +import Arrow from './Arrow'; +const { readVectors } = Arrow; +import { config, sources, formats } from './test-config'; -for (let [name, ...buffers] of arrowTestConfigurations) { - describe(`${name} readBuffers`, () => { - test(`enumerates each batch as an Array of Vectors`, () => { - expect.hasAssertions(); - for (let vectors of readBuffers(...buffers)) { - for (let vector of vectors) { - expect(vector.name).toMatchSnapshot(); - expect(vector.type).toMatchSnapshot(); - expect(vector.length).toMatchSnapshot(); - for (let i = -1, n = vector.length; ++i < n;) { - expect(vector.get(i)).toMatchSnapshot(); +describe(`readBuffers`, () => { + for (const source of sources) { + describe(source, () => { + for (const format of formats) { + describe(format, () => { + for (const { name, buffers } of config[source][format]) { + describe(name, () => { + testReaderIterator(buffers); + testVectorIterator(buffers); + }); } - } + }); } }); - test(`vector iterators report the same values as get`, () => { - expect.hasAssertions(); - for (let vectors of readBuffers(...buffers)) { - for (let vector of vectors) { - let i = -1, n = vector.length; - for (let v of vector) { - expect(++i).toBeLessThan(n); - expect(v).toEqual(vector.get(i)); - } - expect(++i).toEqual(n); + } +}); + +function testReaderIterator(buffers: Uint8Array[]) { + test(`reads each batch as an Array of Vectors`, () => { + expect.hasAssertions(); + for (const vectors of readVectors(buffers)) { + for (const vector of vectors) { + expect(vector.name).toMatchSnapshot(); + expect(vector.type).toMatchSnapshot(); + expect(vector.length).toMatchSnapshot(); + for (let i = -1, n = vector.length; ++i < n;) { + expect(vector.get(i)).toMatchSnapshot(); } } - }); + } + }); +} + +function testVectorIterator(buffers: Uint8Array[]) { + test(`vector iterators report the same values as get`, () => { + expect.hasAssertions(); + for (const vectors of readVectors(buffers)) { + for (const vector of vectors) { + let i = -1, n = vector.length; + for (let v of vector) { + expect(++i).toBeLessThan(n); + expect(v).toEqual(vector.get(i)); + } + expect(++i).toEqual(n); + } + } }); } diff --git a/js/test/table-tests.ts b/js/test/table-tests.ts index d0d70059e1561..5ec04a72ff33b 100644 --- a/js/test/table-tests.ts +++ b/js/test/table-tests.ts @@ -15,75 +15,122 @@ // specific language governing permissions and limitations // under the License. -import { Table, readBuffers } from './Arrow'; -import arrowTestConfigurations from './test-config'; +import Arrow from './Arrow'; +const { Table, readVectors } = Arrow; +import { config, sources, formats } from './test-config'; -for (let [name, ...buffers] of arrowTestConfigurations) { - describe(`${name} Table`, () => { - test(`creates a Table from Arrow buffers`, () => { - expect.hasAssertions(); - const table = Table.from(...buffers); - for (const vector of table.columns) { - expect(vector.name).toMatchSnapshot(); - expect(vector.type).toMatchSnapshot(); - expect(vector.length).toMatchSnapshot(); - for (let i = -1, n = vector.length; ++i < n;) { - expect(vector.get(i)).toMatchSnapshot(); - } - } - }); - test(`vector iterators report the same values as get`, () => { - expect.hasAssertions(); - const table = Table.from(...buffers); - for (const vector of table.columns) { - let i = -1, n = vector.length; - for (let v of vector) { - expect(++i).toBeLessThan(n); - expect(v).toEqual(vector.get(i)); - } - expect(++i).toEqual(n); - } - }); - test(`batch and Table Vectors report the same values`, () => { - expect.hasAssertions(); - let rowsTotal = 0; - let table = Table.from(...buffers); - for (let vectors of readBuffers(...buffers)) { - let rowsNow = Math.max(...vectors.map((v) => v.length)); - for (let vi = -1, vn = vectors.length; ++vi < vn;) { - let v1 = vectors[vi]; - let v2 = table.columns[vi]; - expect(v1.name).toEqual(v2.name); - expect(v1.type).toEqual(v2.type); - for (let i = -1, n = v1.length; ++i < n;) { - expect(v1.get(i)).toEqual(v2.get(i + rowsTotal)); +describe(`Table`, () => { + for (const source of sources) { + describe(source, () => { + for (const format of formats) { + describe(format, () => { + for (const { name, buffers } of config[source][format]) { + describe(name, () => { + testTableFromBuffers(buffers); + testColumnIterators(buffers); + testReaderVectorsAndTableColumns(buffers); + testTableRowIterator(buffers); + testTableRowIteratorCompact(buffers); + testEmptyTableToString(); + testTableToStringPretty(buffers); + testTableToStringPrettyWithIndex(buffers); + }); } - } - rowsTotal += rowsNow; + }); } }); - test(`enumerates Table rows`, () => { - expect.hasAssertions(); - const table = Table.from(...buffers); - for (const row of table) { - expect(row!.toObject()).toMatchSnapshot(); + } +}); + +function testTableFromBuffers(buffers: Uint8Array[]) { + test(`creates a Table from Arrow buffers`, () => { + expect.hasAssertions(); + const table = Table.from(buffers); + for (const vector of table.columns) { + expect(vector.name).toMatchSnapshot(); + expect(vector.type).toMatchSnapshot(); + expect(vector.length).toMatchSnapshot(); + for (let i = -1, n = vector.length; ++i < n;) { + expect(vector.get(i)).toMatchSnapshot(); } - }); - test(`enumerates Table rows compact`, () => { - expect.hasAssertions(); - const table = Table.from(...buffers); - for (const row of table) { - expect(row!.toArray()).toMatchSnapshot(); + } + }); +} + +function testColumnIterators(buffers: Uint8Array[]) { + test(`vector iterators report the same values as get`, () => { + expect.hasAssertions(); + const table = Table.from(buffers); + for (const vector of table.columns) { + let i = -1, n = vector.length; + for (let v of vector) { + expect(++i).toBeLessThan(n); + expect(v).toEqual(vector.get(i)); } - }); - test(`toString() prints an empty Table`, () => { - expect(Table.from().toString()).toMatchSnapshot(); - }); - test(`toString() prints a pretty Table`, () => { - expect(Table.from(...buffers).toString()).toMatchSnapshot(); - }); - test(`toString({ index: true }) prints a pretty Table with an Index column`, () => { - expect(Table.from(...buffers).toString({ index: true })).toMatchSnapshot(); - }); + expect(++i).toEqual(n); + } }); } + +function testReaderVectorsAndTableColumns(buffers: Uint8Array[]) { + test(`batch and Table Vectors report the same values`, () => { + expect.hasAssertions(); + let rowsTotal = 0; + let table = Table.from(buffers); + for (let vectors of readVectors(buffers)) { + let rowsNow = Math.max(...vectors.map((v) => v.length)); + for (let vi = -1, vn = vectors.length; ++vi < vn;) { + let v1 = vectors[vi]; + let v2 = table.columns[vi]; + expect(v1.name).toEqual(v2.name); + expect(v1.type).toEqual(v2.type); + for (let i = -1, n = v1.length; ++i < n;) { + expect(v1.get(i)).toEqual(v2.get(i + rowsTotal)); + } + } + rowsTotal += rowsNow; + } + }); +} + +function testTableRowIterator(buffers: Uint8Array[]) { + test(`enumerates Table rows`, () => { + expect.hasAssertions(); + const table = Table.from(buffers); + expect(table.length).toMatchSnapshot(); + expect(table.columns.length).toMatchSnapshot(); + for (const row of table) { + expect(row!.toObject()).toMatchSnapshot(); + } + }); +} + +function testTableRowIteratorCompact(buffers: Uint8Array[]) { + test(`enumerates Table rows compact`, () => { + expect.hasAssertions(); + const table = Table.from(buffers); + expect(table.length).toMatchSnapshot(); + expect(table.columns.length).toMatchSnapshot(); + for (const row of table) { + expect(row!.toArray()).toMatchSnapshot(); + } + }); +} + +function testEmptyTableToString() { + test(`toString() prints an empty Table`, () => { + expect(Table.from().toString()).toMatchSnapshot(); + }); +} + +function testTableToStringPretty(buffers: Uint8Array[]) { + test(`toString() prints a pretty Table`, () => { + expect(Table.from(buffers).toString()).toMatchSnapshot(); + }); +} + +function testTableToStringPrettyWithIndex(buffers: Uint8Array[]) { + test(`toString({ index: true }) prints a pretty Table with an Index column`, () => { + expect(Table.from(buffers).toString({ index: true })).toMatchSnapshot(); + }); +} \ No newline at end of file diff --git a/js/test/test-config.ts b/js/test/test-config.ts index 89de1cc6c70c9..d185ecc922c47 100644 --- a/js/test/test-config.ts +++ b/js/test/test-config.ts @@ -17,26 +17,36 @@ import * as fs from 'fs'; import * as path from 'path'; -const arrowFormats = ['file', 'stream']; -const arrowFileNames = ['simple', 'struct', 'dictionary', 'dictionary2', 'multi_dictionary']; -const multipartArrows = ['count', 'latlong', 'origins']; -export let arrowTestConfigurations = [] as (string | Buffer)[][]; +import * as glob from 'glob'; -arrowTestConfigurations = arrowFormats.reduce((configs, format) => { - return arrowFileNames.reduce((configs, name) => { - const arrowPath = path.resolve(__dirname, `./arrows/${format}/${name}.arrow`); - try { - const arrowFile = fs.readFileSync(arrowPath); - return [...configs, [`${name} ${format} Arrow`, arrowFile]]; - } catch (e) {} - return configs; - }, configs); -}, arrowTestConfigurations); +export const sources = (process.env.TEST_SOURCES + ? JSON.parse(process.env.TEST_SOURCES + '') + : [`cpp`, `java`]) as ['cpp' | 'java']; -arrowTestConfigurations = multipartArrows.reduce((configs, folder) => { - const schemaPath = path.resolve(__dirname, `./arrows/multi/${folder}/schema.arrow`); - const recordsPath = path.resolve(__dirname, `./arrows/multi/${folder}/records.arrow`); - return [...configs, [`multipart ${folder} Arrow`, fs.readFileSync(schemaPath), fs.readFileSync(recordsPath)]]; -}, arrowTestConfigurations); +export const formats = (process.env.TEST_FORMATS + ? JSON.parse(process.env.TEST_FORMATS + '') + : [`file`, `stream`]) as ['file' | 'stream']; -export default arrowTestConfigurations; +export const config = sources.reduce((sources, source) => ({ + ...sources, + [source]: formats.reduce((formats, format) => ({ + ...formats, + [format]: loadArrows(source, format) + }), {}) +}), {}) as { + [k in 'cpp' | 'java']: { + [k in 'file' | 'stream']: Arrows + } +}; + +export type Arrows = { name: string, buffers: Uint8Array[] }[]; + +function loadArrows(source: string, format: string) { + const arrows = []; + const filenames = glob.sync(path.resolve(__dirname, `data/${source}/${format}`, `*.arrow`)); + for (const filename of filenames) { + const { name } = path.parse(filename); + arrows.push({ name, buffers: [fs.readFileSync(filename)] }); + } + return arrows as Arrows; +} diff --git a/js/test/tsconfig.json b/js/test/tsconfig.json index c1ae204212b8f..838bb1d7041e3 100644 --- a/js/test/tsconfig.json +++ b/js/test/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../tsconfig.json", "include": ["./**/*.ts"], "compilerOptions": { - "target": "ESNEXT", + "target": "es2015", "module": "commonjs", "allowJs": true, "importHelpers": false, diff --git a/js/test/vector-tests.ts b/js/test/vector-tests.ts index 0eca2327bff43..0aaba19ee9144 100644 --- a/js/test/vector-tests.ts +++ b/js/test/vector-tests.ts @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. -import { flatbuffers } from 'flatbuffers'; -import Long = flatbuffers.Long; +import Arrow from './Arrow'; import { + TypedArray, + TypedArrayConstructor, + NumericVectorConstructor, +} from './Arrow'; + +const { BoolVector, - TypedVector, Int64Vector, Uint64Vector, Int8Vector, @@ -28,28 +32,29 @@ import { Uint8Vector, Uint16Vector, Uint32Vector, + Float16Vector, Float32Vector, Float64Vector, -} from './Arrow'; +} = Arrow; -const LongVectors = { - Int64Vector: [Int64Vector, Int32Array], - Uint64Vector: [Uint64Vector, Uint32Array] +const FixedSizeVectors = { + Int64Vector: [Int64Vector, Int32Array] as [NumericVectorConstructor, any], + Uint64Vector: [Uint64Vector, Uint32Array] as [NumericVectorConstructor, any] }; -const TypedVectors = { - Int8Vector: [Int8Vector, Int8Array], - Int16Vector: [Int16Vector, Int16Array], - Int32Vector: [Int32Vector, Int32Array], - Uint8Vector: [Uint8Vector, Uint8Array], - Uint16Vector: [Uint16Vector, Uint16Array], - Uint32Vector: [Uint32Vector, Uint32Array], - Float32Vector: [Float32Vector, Float32Array], - Float64Vector: [Float64Vector, Float64Array] +const FixedWidthVectors = { + Int8Vector: [Int8Vector, Int8Array] as [NumericVectorConstructor, any], + Int16Vector: [Int16Vector, Int16Array] as [NumericVectorConstructor, any], + Int32Vector: [Int32Vector, Int32Array] as [NumericVectorConstructor, any], + Uint8Vector: [Uint8Vector, Uint8Array] as [NumericVectorConstructor, any], + Uint16Vector: [Uint16Vector, Uint16Array] as [NumericVectorConstructor, any], + Uint32Vector: [Uint32Vector, Uint32Array] as [NumericVectorConstructor, any], + Float32Vector: [Float32Vector, Float32Array] as [NumericVectorConstructor, any], + Float64Vector: [Float64Vector, Float64Array] as [NumericVectorConstructor, any] }; -const longVectors = toMap<[typeof TypedVector, any]>(LongVectors, Object.keys(LongVectors)); -const byteVectors = toMap<[typeof TypedVector, any]>(TypedVectors, Object.keys(TypedVectors)); +const fixedSizeVectors = toMap(FixedSizeVectors, Object.keys(FixedSizeVectors)); +const fixedWidthVectors = toMap(FixedWidthVectors, Object.keys(FixedWidthVectors)); const bytes = Array.from( { length: 5 }, () => Uint8Array.from( @@ -127,30 +132,66 @@ describe(`BoolVector`, () => { }); }); -for (const [VectorName, [VectorType, ArrayType]] of longVectors) { +describe('Float16Vector', () => { + const values = concatTyped(Uint16Array, ...bytes); + const vector = bytes + .map((b) => new Float16Vector({ data: new Uint16Array(b.buffer) })) + .reduce((v: any, v2) => v.concat(v2)); + const n = values.length; + const clamp = (x: number) => Math.min((x - 32767) / 32767, 1); + test(`gets expected values`, () => { + let i = -1; + while (++i < n) { + expect(vector.get(i)).toEqual(clamp(values[i])); + } + }); + test(`iterates expected values`, () => { + expect.hasAssertions(); + let i = -1; + for (let v of vector) { + expect(++i).toBeLessThan(n); + expect(v).toEqual(clamp(values[i])); + } + }); + test(`slices the entire array`, () => { + expect(vector.slice()).toEqual(values); + }); + test(`slice returns a TypedArray`, () => { + expect(vector.slice()).toBeInstanceOf(Uint16Array); + }); + test(`slices from -20 to length`, () => { + expect(vector.slice(-20)).toEqual(values.slice(-20)); + }); + test(`slices from 0 to -20`, () => { + expect(vector.slice(0, -20)).toEqual(values.slice(0, -20)); + }); + test(`slices the array from 0 to length - 20`, () => { + expect(vector.slice(0, n - 20)).toEqual(values.slice(0, n - 20)); + }); + test(`slices the array from 0 to length + 20`, () => { + expect(vector.slice(0, n + 20)).toEqual( + concatTyped(Uint16Array, values, values.slice(0, 20))); + }); +}); + +for (const [VectorName, [VectorType, ArrayType]] of fixedSizeVectors) { describe(`${VectorName}`, () => { const values = concatTyped(ArrayType, ...bytes); const vector = bytes - .map((b) => new VectorType({ - data: new ArrayType(b.buffer) - })) + .map((b) => new VectorType({ data: new ArrayType(b.buffer) })) .reduce((v: any, v2) => v.concat(v2)); const n = values.length * 0.5; test(`gets expected values`, () => { let i = -1; while (++i < n) { - expect(vector.get(i)).toEqual(new Long( - values[i * 2], values[i * 2 + 1] - )); + expect(vector.get(i)).toEqual(values.slice(2 * i, 2 * (i + 1))); } }); test(`iterates expected values`, () => { let i = -1; for (let v of vector) { expect(++i).toBeLessThan(n); - expect(v).toEqual(new Long( - values[i * 2], values[i * 2 + 1] - )); + expect(v).toEqual(values.slice(2 * i, 2 * (i + 1))); } }); test(`slices the entire array`, () => { @@ -175,13 +216,11 @@ for (const [VectorName, [VectorType, ArrayType]] of longVectors) { }); } -for (const [VectorName, [VectorType, ArrayType]] of byteVectors) { +for (const [VectorName, [VectorType, ArrayType]] of fixedWidthVectors) { describe(`${VectorName}`, () => { const values = concatTyped(ArrayType, ...bytes); const vector = bytes - .map((b) => new VectorType({ - data: new ArrayType(b.buffer) - })) + .map((b) => new VectorType({ data: new ArrayType(b.buffer) })) .reduce((v: any, v2) => v.concat(v2)); const n = values.length; @@ -221,14 +260,14 @@ for (const [VectorName, [VectorType, ArrayType]] of byteVectors) { }); } -function toMap(entries: any, keys: string[]) { +function toMap(entries: Record, keys: string[]) { return keys.reduce((map, key) => { map.set(key, entries[key] as T); return map; }, new Map()); } -function concatTyped(ArrayType: any, ...bytes: any[]) { +function concatTyped(ArrayType: TypedArrayConstructor, ...bytes: any[]) { const BPE = ArrayType.BYTES_PER_ELEMENT; return bytes.reduce((v, bytes) => { const l = bytes.byteLength / BPE; @@ -237,5 +276,5 @@ function concatTyped(ArrayType: any, ...bytes: any[]) { a.set(v); a.set(b, v.length); return a; - }, new ArrayType(0)) as Array; + }, new ArrayType(0)) as T; } \ No newline at end of file diff --git a/js/tsconfig/tsconfig.es5.cls.json b/js/tsconfig/tsconfig.es5.cls.json index 6e6f213b4d6e5..502432da04348 100644 --- a/js/tsconfig/tsconfig.es5.cls.json +++ b/js/tsconfig/tsconfig.es5.cls.json @@ -4,8 +4,6 @@ "compilerOptions": { "target": "ES5", "module": "es2015", - "declaration": false, - "noEmitHelpers": true, - "importHelpers": false + "declaration": false } } From cb5da9c1bc71fafd76f5cafc60ef237385a36b2b Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 20 Nov 2017 09:27:02 -0500 Subject: [PATCH 063/177] ARROW-1778: [Python] Link parquet-cpp statically, privately in manylinux1 wheels Author: Uwe L. Korn Closes #1331 from xhochy/ARROW-1778 and squashes the following commits: 65cf4c6a [Uwe L. Korn] ARROW-1778: [Python] Link parquet-cpp statically, privately in manylinux1 wheels --- cpp/cmake_modules/FindThrift.cmake | 102 ++++++++++++++++++++++++++++ python/CMakeLists.txt | 46 +++++++++---- python/manylinux1/Dockerfile-x86_64 | 2 +- python/manylinux1/build_arrow.sh | 6 +- python/setup.py | 7 +- 5 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 cpp/cmake_modules/FindThrift.cmake diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake new file mode 100644 index 0000000000000..25f50825c3735 --- /dev/null +++ b/cpp/cmake_modules/FindThrift.cmake @@ -0,0 +1,102 @@ +# Copyright 2012 Cloudera Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# - Find Thrift (a cross platform RPC lib/tool) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Thrift_HOME - When set, this path is inspected instead of standard library +# locations as the root of the Thrift installation. +# The environment variable THRIFT_HOME overrides this variable. +# +# This module defines +# THRIFT_VERSION, version string of ant if found +# THRIFT_INCLUDE_DIR, where to find THRIFT headers +# THRIFT_CONTRIB_DIR, where contrib thrift files (e.g. fb303.thrift) are installed +# THRIFT_STATIC_LIB, THRIFT static library +# THRIFT_FOUND, If false, do not try to use ant + +# prefer the thrift version supplied in THRIFT_HOME +if( NOT "${THRIFT_HOME}" STREQUAL "") + file( TO_CMAKE_PATH "${THRIFT_HOME}" _native_path ) + list( APPEND _thrift_roots ${_native_path} ) +elseif ( Thrift_HOME ) + list( APPEND _thrift_roots ${Thrift_HOME} ) +endif() + +message(STATUS "THRIFT_HOME: ${THRIFT_HOME}") +find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h HINTS + ${_thrift_roots} + NO_DEFAULT_PATH + PATH_SUFFIXES "include" +) + +find_path(THRIFT_CONTRIB_DIR share/fb303/if/fb303.thrift HINTS + ${_thrift_roots} + NO_DEFAULT_PATH +) + +if (MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX) + set(THRIFT_MSVC_STATIC_LIB_SUFFIX md) +endif() + +find_library(THRIFT_STATIC_LIB NAMES + ${CMAKE_STATIC_LIBRARY_PREFIX}thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} + HINTS ${_thrift_roots} + NO_DEFAULT_PATH + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" +) + +find_program(THRIFT_COMPILER thrift HINTS + ${_thrift_roots} + NO_DEFAULT_PATH + PATH_SUFFIXES "bin" +) + +if (THRIFT_STATIC_LIB) + set(THRIFT_FOUND TRUE) + exec_program(${THRIFT_COMPILER} + ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN) +else () + set(THRIFT_FOUND FALSE) +endif () + +if (THRIFT_FOUND) + if (NOT Thrift_FIND_QUIETLY) + message(STATUS "Thrift version: ${THRIFT_VERSION}") + endif () +else () + if (NOT Thrift_FIND_QUIETLY) + set(THRIFT_ERR_MSG "Thrift compiler/libraries NOT found: ${THRIFT_RETURN}") + set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} (${THRIFT_INCLUDE_DIR}, ${THRIFT_STATIC_LIB}).") + if ( _thrift_roots ) + set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in ${_thrift_roots}.") + else () + set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in system search paths.") + endif () + if ( Thrift_FIND_REQUIRED ) + message(FATAL_ERROR "${THRIFT_ERR_MSG}") + else () + message(STATUS "${THRIFT_ERR_MSG}") + endif () + endif () +endif () + + +mark_as_advanced( + THRIFT_STATIC_LIB + THRIFT_COMPILER + THRIFT_INCLUDE_DIR +) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 8c7348298db5f..00a4ee68737c6 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -61,6 +61,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") option(PYARROW_BUILD_PARQUET "Build the PyArrow Parquet integration" OFF) + option(PYARROW_PARQUET_USE_SHARED + "Rely on parquet shared libraries where relevant" + ON) option(PYARROW_BUILD_PLASMA "Build the PyArrow Plasma integration" OFF) @@ -282,24 +285,41 @@ if (PYARROW_BUILD_PARQUET) endif() include_directories(SYSTEM ${PARQUET_INCLUDE_DIR}) - if (PYARROW_BUNDLE_ARROW_CPP) - bundle_arrow_lib(PARQUET_SHARED_LIB - ABI_VERSION ${PARQUET_ABI_VERSION} - SO_VERSION ${PARQUET_SO_VERSION}) + if (PYARROW_PARQUET_USE_SHARED) + if (PYARROW_BUNDLE_ARROW_CPP) + bundle_arrow_lib(PARQUET_SHARED_LIB + ABI_VERSION ${PARQUET_ABI_VERSION} + SO_VERSION ${PARQUET_SO_VERSION}) + if (MSVC) + bundle_arrow_implib(PARQUET_SHARED_IMP_LIB) + endif() + endif() if (MSVC) - bundle_arrow_implib(PARQUET_SHARED_IMP_LIB) + ADD_THIRDPARTY_LIB(parquet + SHARED_LIB ${PARQUET_SHARED_IMP_LIB}) + else() + ADD_THIRDPARTY_LIB(parquet + SHARED_LIB ${PARQUET_SHARED_LIB}) endif() - endif() - if (MSVC) - ADD_THIRDPARTY_LIB(parquet - SHARED_LIB ${PARQUET_SHARED_IMP_LIB}) + set(LINK_LIBS + ${LINK_LIBS} + parquet_shared) else() + find_package(Thrift) + set(Boost_USE_STATIC_LIBS ON) + find_package(Boost COMPONENTS regex REQUIRED) + ADD_THIRDPARTY_LIB(boost_regex + STATIC_LIB ${Boost_REGEX_LIBRARY_RELEASE}) ADD_THIRDPARTY_LIB(parquet - SHARED_LIB ${PARQUET_SHARED_LIB}) + STATIC_LIB ${PARQUET_STATIC_LIB}) + ADD_THIRDPARTY_LIB(thrift + STATIC_LIB ${THRIFT_STATIC_LIB}) + set(LINK_LIBS + ${LINK_LIBS} + parquet_static + thrift_static + boost_regex_static) endif() - set(LINK_LIBS - ${LINK_LIBS} - parquet_shared) set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _parquet) diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 index 69d8d3dbf8cef..919a32be715b0 100644 --- a/python/manylinux1/Dockerfile-x86_64 +++ b/python/manylinux1/Dockerfile-x86_64 @@ -29,5 +29,5 @@ RUN /check_arrow_visibility.sh WORKDIR / RUN git clone https://github.com/apache/parquet-cpp.git WORKDIR /parquet-cpp -RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BOOST_USE_SHARED=OFF -GNinja . +RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BUILD_SHARED=OFF -DPARQUET_BUILD_STATIC=ON -DPARQUET_BOOST_USE_SHARED=OFF -GNinja . RUN ninja install diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 074bd0056a948..ced4556e92ad5 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -40,11 +40,11 @@ cd /arrow/python # PyArrow build configuration export PYARROW_BUILD_TYPE='release' export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_STATIC_PARQUET=1 export PYARROW_WITH_PLASMA=1 export PYARROW_BUNDLE_ARROW_CPP=1 -# Need as otherwise arrow_io is sometimes not linked -export LDFLAGS="-Wl,--no-as-needed" export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig +export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr' # Ensure the target directory exists mkdir -p /io/dist @@ -65,7 +65,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do # Clear output directory rm -rf dist/ echo "=== (${PYTHON}) Building wheel ===" - PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp + PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py build_ext --inplace --with-parquet --with-static-parquet --bundle-arrow-cpp PATH="$PATH:$(cpython_path $PYTHON)/bin" $PYTHON_INTERPRETER setup.py bdist_wheel echo "=== (${PYTHON}) Test the existence of optional modules ===" diff --git a/python/setup.py b/python/setup.py index ccab8fb6581f2..2270cf79c2a8b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -82,6 +82,7 @@ def run(self): user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'), ('build-type=', None, 'build type (debug or release)'), ('with-parquet', None, 'build the Parquet extension'), + ('with-static-parquet', None, 'link parquet statically'), ('with-plasma', None, 'build the Plasma extension'), ('bundle-arrow-cpp', None, 'bundle the Arrow C++ libraries')] + @@ -102,6 +103,8 @@ def initialize_options(self): self.with_parquet = strtobool( os.environ.get('PYARROW_WITH_PARQUET', '0')) + self.with_static_parquet = strtobool( + os.environ.get('PYARROW_WITH_STATIC_PARQUET', '0')) self.with_plasma = strtobool( os.environ.get('PYARROW_WITH_PLASMA', '0')) self.bundle_arrow_cpp = strtobool( @@ -144,6 +147,8 @@ def _run_cmake(self): if self.with_parquet: cmake_options.append('-DPYARROW_BUILD_PARQUET=on') + if self.with_static_parquet: + cmake_options.append('-DPYARROW_PARQUET_USE_SHARED=off') if self.with_plasma: cmake_options.append('-DPYARROW_BUILD_PLASMA=on') @@ -225,7 +230,7 @@ def _run_cmake(self): move_shared_libs(build_prefix, build_lib, "arrow_python") if self.with_plasma: move_shared_libs(build_prefix, build_lib, "plasma") - if self.with_parquet: + if self.with_parquet and not self.with_static_parquet: move_shared_libs(build_prefix, build_lib, "parquet") print('Bundling includes: ' + pjoin(build_prefix, 'include')) From 284e6c9fbf88e77809e4b7376d9439265ad0d20d Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 20 Nov 2017 14:21:40 -0500 Subject: [PATCH 064/177] ARROW-1826: [JAVA] Avoid branching in copyFrom for fixed width scalars cc @jacques-n , @BryanCutler , @icexelloss Note: there was no performance regression or different behavior introduced as part of refactor patches. This is something I had noticed in the old code during refactoring but lost track of it. copyFrom function for fixed width scalars does a double check if the cell in the source vector is NULL or not. The old code used to do the same. There is a straightforward way to avoid double check but the new changes avoid branch all-together; since it is fine to copy garbage from source vector as long as the validity bit is copied correctly to indicate NULL or non-NULL value in the target cell. Will add unit tests. Author: siddharth Closes #1325 from siddharthteotia/ARROW-1826 and squashes the following commits: 8fc5b3f1 [siddharth] added unit tests 84de0cda [siddharth] ARROW-1826: [JAVA] Avoid branching in copyFrom --- .../arrow/vector/NullableBigIntVector.java | 8 +- .../arrow/vector/NullableBitVector.java | 7 +- .../arrow/vector/NullableDateDayVector.java | 8 +- .../arrow/vector/NullableDateMilliVector.java | 8 +- .../arrow/vector/NullableDecimalVector.java | 10 +- .../arrow/vector/NullableFloat4Vector.java | 8 +- .../arrow/vector/NullableFloat8Vector.java | 8 +- .../arrow/vector/NullableIntVector.java | 8 +- .../vector/NullableIntervalDayVector.java | 8 +- .../vector/NullableIntervalYearVector.java | 8 +- .../arrow/vector/NullableSmallIntVector.java | 8 +- .../arrow/vector/NullableTimeMicroVector.java | 8 +- .../arrow/vector/NullableTimeMilliVector.java | 8 +- .../arrow/vector/NullableTimeNanoVector.java | 8 +- .../arrow/vector/NullableTimeSecVector.java | 8 +- .../arrow/vector/NullableTimeStampVector.java | 8 +- .../arrow/vector/NullableTinyIntVector.java | 8 +- .../arrow/vector/NullableUInt1Vector.java | 8 +- .../arrow/vector/NullableUInt2Vector.java | 8 +- .../arrow/vector/NullableUInt4Vector.java | 8 +- .../arrow/vector/NullableUInt8Vector.java | 8 +- .../org/apache/arrow/vector/TestCopyFrom.java | 1037 +++++++++++++++++ 22 files changed, 1098 insertions(+), 108 deletions(-) create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java index eca6592c54ad2..8d44c92e46664 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -137,11 +137,9 @@ public Long getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); + valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java index bd363f5b7f1f3..086331e13d452 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java @@ -286,11 +286,8 @@ public Boolean getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + BitVectorHelper.setValidityBit(valueBuffer, thisIndex, from.getBit(fromIndex)); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java index c5ddc9e87be6a..bb14d1ac0d41c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -139,11 +139,9 @@ public Integer getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); + valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java index 7e37055dab188..97205c318317c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -144,11 +144,9 @@ public LocalDateTime getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); + valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index dcc551094ae28..5d2782a4c8460 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -151,13 +151,9 @@ public BigDecimal getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { - if (from.isSet(fromIndex) != 0) { - from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, - thisIndex * TYPE_WIDTH, TYPE_WIDTH); - BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java index 834f16c211cdb..5d7128d2bdfa3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -138,11 +138,9 @@ public Float getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final float value = from.valueBuffer.getFloat(fromIndex * TYPE_WIDTH); + valueBuffer.setFloat(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java index b054d9b05c06a..7aa546150b319 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -138,11 +138,9 @@ public Double getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final double value = from.valueBuffer.getDouble(fromIndex * TYPE_WIDTH); + valueBuffer.setDouble(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index e5dfbcedcf90e..7c6f29ad7c559 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -143,11 +143,9 @@ public Integer getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); + valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java index 8d2fb5661d9eb..862f6429cb8cd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java @@ -186,13 +186,9 @@ private StringBuilder getAsStringBuilderHelper(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { - if (from.isSet(fromIndex) != 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); - from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, thisIndex * TYPE_WIDTH, TYPE_WIDTH); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java index ac121d2f078ff..3fba5bfeeac00 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java @@ -170,11 +170,9 @@ private StringBuilder getAsStringBuilderHelper(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); + valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java index 5dd1492a3e095..d5c66b115acde 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -138,11 +138,9 @@ public Short getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final short value = from.valueBuffer.getShort(fromIndex * TYPE_WIDTH); + valueBuffer.setShort(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java index 25da4f68d506c..ca8e1aacf6e0e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -139,11 +139,9 @@ public Long getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); + valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java index 7d2d5d1f10fe2..a28ccfe00f372 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -140,11 +140,9 @@ public LocalDateTime getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); + valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java index e93a63fe2a1f9..e6903c011e676 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -138,11 +138,9 @@ public Long getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); + valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java index 8329fa7813987..85426a36948db 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -138,11 +138,9 @@ public Integer getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); + valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java index d65cb83e15123..e1b719f66a784 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -70,11 +70,9 @@ public long get(int index) throws IllegalStateException { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); + valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java index 106359ddd9666..cadfe608538cc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -138,11 +138,9 @@ public Byte getObject(int index) { * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH); + valueBuffer.setByte(thisIndex * TYPE_WIDTH, value); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java index 52f6c151bdff9..ebd9b56996cf6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java @@ -108,11 +108,9 @@ public Byte getObject(int index) { } public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH); + valueBuffer.setByte(thisIndex * TYPE_WIDTH, value); } public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java index 42ba6e139bf71..bedb3f35e6bac 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java @@ -109,11 +109,9 @@ public Character getObject(int index) { } public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final char value = from.valueBuffer.getChar(fromIndex * TYPE_WIDTH); + valueBuffer.setChar(thisIndex * TYPE_WIDTH, value); } public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java index 1ec21490d8447..99b547ca4fdef 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java @@ -109,11 +109,9 @@ public Integer getObject(int index) { } public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); + valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java index 408a810cfa0af..e6664a09265d7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java @@ -109,11 +109,9 @@ public Long getObject(int index) { } public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); + valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java new file mode 100644 index 0000000000000..7cdfbdf452d84 --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java @@ -0,0 +1,1037 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.types.Types; +import org.joda.time.Period; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.math.BigDecimal; + +import static org.apache.arrow.vector.TestUtils.newVector; +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Tested field types: + * + * NullableInt + * NullableBigInt + * NullableFloat4 + * NullableFloat8 + * NullableBit + * NullableDecimal + * NullableIntervalDay + * NullableIntervalYear + * NullableSmallInt + * NullableTinyInt + * NullableVarChar + * NullableTimeMicro + * NullableTimeMilli + * NullableTimeStamp* + */ + +public class TestCopyFrom { + + private final static String EMPTY_SCHEMA_PATH = ""; + + private BufferAllocator allocator; + + @Before + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void terminate() throws Exception { + allocator.close(); + } + + @Test /* NullableVarChar */ + public void testCopyFromWithNulls() { + try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator); + final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) { + + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + continue; + } + byte[] b = Integer.toString(i).getBytes(); + vector.setSafe(i, b, 0, b.length); + } + + /* NO reAlloc() should have happened in setSafe() */ + capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + vector.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + } + } + + vector2.allocateNew(); + capacity = vector2.getValueCapacity(); + assertEquals(4095, capacity); + + for (int i = 0; i < 4095; i++) { + vector2.copyFromSafe(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + + /* NO reAlloc() should have happened in copyFrom */ + capacity = vector2.getValueCapacity(); + assertEquals(4095, capacity); + + vector2.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + } + } + + @Test /* NullableVarChar */ + public void testCopyFromWithNulls1() { + try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator); + final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) { + + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + continue; + } + byte[] b = Integer.toString(i).getBytes(); + vector.setSafe(i, b, 0, b.length); + } + + /* NO reAlloc() should have happened in setSafe() */ + capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + vector.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024 * 10, 1024); + + capacity = vector2.getValueCapacity(); + assertEquals(1024, capacity); + + for (int i = 0; i < 4095; i++) { + vector2.copyFromSafe(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + + /* 2 reAllocs should have happened in copyFromSafe() */ + capacity = vector2.getValueCapacity(); + assertEquals(4096, capacity); + + vector2.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + } + } + + @Test /* NullableIntVector */ + public void testCopyFromWithNulls2() { + try (final NullableIntVector vector1 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator); + final NullableIntVector vector2 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, 1000 + i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, 1000 + i, vector1.get(i)); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, 1000 + i, vector2.get(i)); + } + } + } + } + + @Test /* NullableBigIntVector */ + public void testCopyFromWithNulls3() { + try (final NullableBigIntVector vector1 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator); + final NullableBigIntVector vector2 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, 10000000000L + (long)i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + 10000000000L + (long)i, vector1.get(i)); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + 10000000000L + (long)i, vector2.get(i)); + } + } + } + } + + @Test /* NullableBitVector */ + public void testCopyFromWithNulls4() { + try (final NullableBitVector vector1 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator); + final NullableBitVector vector2 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + int counter = 0; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + if ((counter&1) == 0) { + vector1.setSafe(i, 1); + } else { + vector1.setSafe(i, 0); + } + counter++; + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + counter = 0; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + if ((counter&1) == 0) { + assertTrue(vector1.getObject(i)); + } else { + assertFalse(vector1.getObject(i)); + } + counter++; + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + counter = 0; + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + if ((counter&1) == 0) { + assertTrue(vector2.getObject(i)); + } else { + assertFalse(vector2.getObject(i)); + } + counter++; + } + } + } + } + + @Test /* NullableFloat4Vector */ + public void testCopyFromWithNulls5() { + try (final NullableFloat4Vector vector1 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator); + final NullableFloat4Vector vector2 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, 100.25f + (float)i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + 100.25f + (float)i, vector1.get(i), 0); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + 100.25f + i*1.0f, vector2.get(i), 0); + } + } + } + } + + @Test /* NullableFloat8Vector */ + public void testCopyFromWithNulls6() { + try (final NullableFloat8Vector vector1 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator); + final NullableFloat8Vector vector2 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, 123456.7865 + (double) i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + 123456.7865 + (double) i, vector1.get(i), 0); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + 123456.7865 + (double) i, vector2.get(i), 0); + } + } + } + } + + @Test /* NullableIntervalDayVector */ + public void testCopyFromWithNulls7() { + try (final NullableIntervalDayVector vector1 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator); + final NullableIntervalDayVector vector2 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + final int days = 10; + final int milliseconds = 10000; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, days + i, milliseconds + i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + final Period p = vector1.getObject(i); + assertEquals(days + i, p.getDays()); + assertEquals(milliseconds + i, p.getMillis()); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + final Period p = vector2.getObject(i); + assertEquals(days + i, p.getDays()); + assertEquals(milliseconds + i, p.getMillis()); + } + } + } + } + + @Test /* NullableIntervalYearVector */ + public void testCopyFromWithNulls8() { + try (final NullableIntervalYearVector vector1 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator); + final NullableIntervalYearVector vector2 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + final int interval = 30; /* 2 years 6 months */ + final Period[] periods = new Period[4096]; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, interval + i); + final Period p = new Period(); + final int years = (interval + i) / org.apache.arrow.vector.util.DateUtility.yearsToMonths; + final int months = (interval + i) % org.apache.arrow.vector.util.DateUtility.yearsToMonths; + periods[i] = p.plusYears(years).plusMonths(months);; + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + final Period p = vector1.getObject(i); + assertEquals(interval + i, vector1.get(i)); + assertEquals(periods[i], p); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + final Period p = vector2.getObject(i); + assertEquals(periods[i], p); + } + } + } + } + + @Test /* NullableSmallIntVector */ + public void testCopyFromWithNulls9() { + try (final NullableSmallIntVector vector1 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator); + final NullableSmallIntVector vector2 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + final short val = 1000; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, val + (short)i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + (short)i, vector1.get(i)); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + (short)i, vector2.get(i)); + } + } + } + } + + @Test /* NullableTimeMicroVector */ + public void testCopyFromWithNulls10() { + try (final NullableTimeMicroVector vector1 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator); + final NullableTimeMicroVector vector2 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + final long val = 100485765432L; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, val + (long)i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + (long)i, vector1.get(i)); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + (long) i, vector2.get(i)); + } + } + } + } + + @Test /* NullableTimeMilliVector */ + public void testCopyFromWithNulls11() { + try (final NullableTimeMilliVector vector1 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator); + final NullableTimeMilliVector vector2 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + final int val = 1000; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, val + i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + i, vector1.get(i)); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + i, vector2.get(i)); + } + } + } + } + + @Test /* NullableTinyIntVector */ + public void testCopyFromWithNulls12() { + try (final NullableTinyIntVector vector1 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator); + final NullableTinyIntVector vector2 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + byte val = -128; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, val); + val++; + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + val = -128; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, val, vector1.get(i)); + val++; + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + val = -128; + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, val, vector2.get(i)); + val++; + } + } + } + } + + @Test /* NullableDecimalVector */ + public void testCopyFromWithNulls13() { + try (final NullableDecimalVector vector1 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16); + final NullableDecimalVector vector2 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + final double baseValue = 104567897654.876543654; + final BigDecimal[] decimals = new BigDecimal[4096]; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + BigDecimal decimal = new BigDecimal(baseValue + (double)i); + vector1.setSafe(i, decimal); + decimals[i] = decimal; + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + final BigDecimal decimal = vector1.getObject(i); + assertEquals(decimals[i], decimal); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + final BigDecimal decimal = vector2.getObject(i); + assertEquals(decimals[i], decimal); + } + } + } + } + + @Test /* NullableTimeStampVector */ + public void testCopyFromWithNulls14() { + try (final NullableTimeStampVector vector1 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator); + final NullableTimeStampVector vector2 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) { + + vector1.allocateNew(); + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(0, vector1.getValueCount()); + + final long val = 20145678912L; + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + continue; + } + vector1.setSafe(i, val + (long)i); + } + + vector1.setValueCount(4096); + + /* No realloc should have happened in setSafe or + * setValueCount + */ + assertEquals(4096, vector1.getValueCapacity()); + assertEquals(4096, vector1.getValueCount()); + + for (int i = 0; i < 4096; i++) { + if ((i&1) == 0) { + assertNull(vector1.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + (long)i, vector1.get(i)); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024); + assertEquals(1024, vector2.getValueCapacity()); + + for (int i = 0; i < 4096; i++) { + vector2.copyFromSafe(i, i, vector1); + } + + /* 2 realloc should have happened in copyFromSafe() */ + assertEquals(4096, vector2.getValueCapacity()); + vector2.setValueCount(8192); + /* setValueCount() should have done another realloc */ + assertEquals(8192, vector2.getValueCount()); + assertEquals(8192, vector2.getValueCapacity()); + + /* check vector data after copy and realloc */ + for (int i = 0; i < 8192; i++) { + if (((i&1) == 0) || (i >= 4096)) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, + val + (long) i, vector2.get(i)); + } + } + } + } +} From d887d9122a17530126a40a60944e84207d40d412 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 21 Nov 2017 14:35:22 +0100 Subject: [PATCH 065/177] ARROW-1830: [Python] Relax restriction that Parquet files in a dataset end in .parq or .parquet It seems that some setups may not use these file extensions, and so our assumption is too rigid. cc @dbtsai Author: Wes McKinney Closes #1340 from wesm/ARROW-1830 and squashes the following commits: d123ae1 [Wes McKinney] Relax restriction that Parquet files in a dataset end in .parq or .parquet --- python/pyarrow/parquet.py | 12 ++++-------- python/pyarrow/tests/test_parquet.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index 3023e1771e404..37da662806366 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -421,10 +421,6 @@ def read(self, columns=None, nthreads=1, partitions=None, return table -def _is_parquet_file(path): - return path.endswith('parq') or path.endswith('parquet') - - class PartitionSet(object): """A data structure for cataloguing the observed Parquet partitions at a particular level. So if we have @@ -556,14 +552,14 @@ def _visit_level(self, level, base_path, part_keys): filtered_files = [] for path in files: full_path = self.pathsep.join((base_path, path)) - if _is_parquet_file(path): - filtered_files.append(full_path) - elif path.endswith('_common_metadata'): + if path.endswith('_common_metadata'): self.common_metadata_path = full_path elif path.endswith('_metadata'): self.metadata_path = full_path - elif not self._should_silently_exclude(path): + elif self._should_silently_exclude(path): print('Ignoring path: {0}'.format(full_path)) + else: + filtered_files.append(full_path) # ARROW-1079: Filter out "private" directories starting with underscore filtered_directories = [self.pathsep.join((base_path, x)) diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 522815fce16c8..274ff458fd6cc 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1020,7 +1020,7 @@ def _visit_level(base_dir, level, part_keys): if level == DEPTH - 1: # Generate example data - file_path = pjoin(level_dir, 'data.parq') + file_path = pjoin(level_dir, guid()) filtered_df = _filter_partition(df, this_part_keys) part_table = pa.Table.from_pandas(filtered_df) From e98adc3b54aab1139239c5b475dc36a809412d99 Mon Sep 17 00:00:00 2001 From: ksdevlife <33046664+ksdevlife@users.noreply.github.com> Date: Tue, 21 Nov 2017 09:56:48 -0500 Subject: [PATCH 066/177] =?UTF-8?q?ARROW-1840:=20[Website]=20The=20install?= =?UTF-8?q?ation=20command=20failed=20on=20Windows10=20anaconda=20envir?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …onment Author: ksdevlife <33046664+ksdevlife@users.noreply.github.com> Closes #1342 from ksdevlife/1840__ and squashes the following commits: bc0f964e [ksdevlife] [Website] The installation command failed on Windows10 anaconda environment --- site/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/install.md b/site/install.md index 0ef2008db9061..67b26983b46c4 100644 --- a/site/install.md +++ b/site/install.md @@ -53,7 +53,7 @@ Install them with: ```shell conda install arrow-cpp=0.7.* -c conda-forge -conda install pyarrow==0.7.* -c conda-forge +conda install pyarrow=0.7.* -c conda-forge ``` ### Python Wheels on PyPI (Unofficial) From c436376b0ab880202c4e2f2735aff74b4667d98b Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 21 Nov 2017 09:57:36 -0500 Subject: [PATCH 067/177] ARROW-1838: [C++] Conform kernel API to use Datum for input and output We could also add helper methods that do automatic boxing into a `Datum` at a later time if it helps with code complexity. Author: Wes McKinney Closes #1339 from wesm/ARROW-1838 and squashes the following commits: f83a866e [Wes McKinney] Conform kernel API to use Datum for input and output --- cpp/src/arrow/compute/compute-test.cc | 2 +- cpp/src/arrow/compute/kernel.h | 2 +- cpp/src/arrow/compute/kernels/cast.cc | 11 +++++++---- cpp/src/arrow/compute/kernels/hash.cc | 5 +++-- cpp/src/arrow/compute/kernels/util-internal.cc | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index 58a991c60f24b..fa408ae40cd85 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -697,7 +697,7 @@ TEST_F(TestCast, PreallocatedMemory) { out_data->buffers.push_back(out_values); Datum out(out_data); - ASSERT_OK(kernel->Call(&this->ctx_, *arr->data(), &out)); + ASSERT_OK(kernel->Call(&this->ctx_, Datum(arr), &out)); // Buffer address unchanged ASSERT_EQ(out_values.get(), out_data->buffers[1].get()); diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h index 0037245d610b4..7ff506ca0cf2b 100644 --- a/cpp/src/arrow/compute/kernel.h +++ b/cpp/src/arrow/compute/kernel.h @@ -131,7 +131,7 @@ struct ARROW_EXPORT Datum { /// \brief An array-valued function of a single input argument class ARROW_EXPORT UnaryKernel : public OpKernel { public: - virtual Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) = 0; + virtual Status Call(FunctionContext* ctx, const Datum& input, Datum* out) = 0; }; } // namespace compute diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc index c866054eadd8a..d595d2ea507c1 100644 --- a/cpp/src/arrow/compute/kernels/cast.cc +++ b/cpp/src/arrow/compute/kernels/cast.cc @@ -740,20 +740,23 @@ class CastKernel : public UnaryKernel { can_pre_allocate_values_(can_pre_allocate_values), out_type_(out_type) {} - Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override { + Status Call(FunctionContext* ctx, const Datum& input, Datum* out) override { + DCHECK_EQ(Datum::ARRAY, input.kind()); + + const ArrayData& in_data = *input.array(); ArrayData* result; if (out->kind() == Datum::NONE) { - out->value = std::make_shared(out_type_, input.length); + out->value = std::make_shared(out_type_, in_data.length); } result = out->array().get(); if (!is_zero_copy_) { RETURN_NOT_OK( - AllocateIfNotPreallocated(ctx, input, can_pre_allocate_values_, result)); + AllocateIfNotPreallocated(ctx, in_data, can_pre_allocate_values_, result)); } - func_(ctx, options_, input, result); + func_(ctx, options_, in_data, result); RETURN_IF_ERROR(ctx); return Status::OK(); diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc index 3af41609fef28..95f039932152c 100644 --- a/cpp/src/arrow/compute/kernels/hash.cc +++ b/cpp/src/arrow/compute/kernels/hash.cc @@ -658,8 +658,9 @@ class HashKernelImpl : public HashKernel { explicit HashKernelImpl(std::unique_ptr hasher) : hasher_(std::move(hasher)) {} - Status Call(FunctionContext* ctx, const ArrayData& input, Datum* out) override { - RETURN_NOT_OK(Append(ctx, input)); + Status Call(FunctionContext* ctx, const Datum& input, Datum* out) override { + DCHECK_EQ(Datum::ARRAY, input.kind()); + RETURN_NOT_OK(Append(ctx, *input.array())); return Flush(out); } diff --git a/cpp/src/arrow/compute/kernels/util-internal.cc b/cpp/src/arrow/compute/kernels/util-internal.cc index df68637e088f7..28428bfcba6c6 100644 --- a/cpp/src/arrow/compute/kernels/util-internal.cc +++ b/cpp/src/arrow/compute/kernels/util-internal.cc @@ -34,13 +34,13 @@ Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel, const Datum& value, std::vector* outputs) { if (value.kind() == Datum::ARRAY) { Datum output; - RETURN_NOT_OK(kernel->Call(ctx, *value.array(), &output)); + RETURN_NOT_OK(kernel->Call(ctx, value, &output)); outputs->push_back(output); } else if (value.kind() == Datum::CHUNKED_ARRAY) { const ChunkedArray& array = *value.chunked_array(); for (int i = 0; i < array.num_chunks(); i++) { Datum output; - RETURN_NOT_OK(kernel->Call(ctx, *(array.chunk(i)->data()), &output)); + RETURN_NOT_OK(kernel->Call(ctx, Datum(array.chunk(i)), &output)); outputs->push_back(output); } } else { From cac09122e79b2b8faa063a087b93f63d86a6f74e Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Tue, 21 Nov 2017 09:59:24 -0500 Subject: [PATCH 068/177] ARROW-1841: [JS] Update text-encoding-utf-8 and tslib for node ESModules support Updates the `text-encoding-utf-8` dependency to version 1.0.2, [which now supports](https://github.com/arv/text-encoding-utf-8/pull/2) ESModules in node >= v8.6.0 via the `--experimental-modules` flag. We currently support ESModules in node in the main `apache-arrow` package, but need our dependencies to also expose ESModule forms as well. I have also issued a [PR to flatbuffers](https://github.com/google/flatbuffers/pull/4504) to add ESModules support, and are using a temporary [fork of flatbuffers](https://github.com/trxcllnt/flatbuffers-esm) in my github until that PR is merged. This PR enables the following workflow: ```js // file - index.mjs // run via `node --experimental-modules index.mjs` import util from 'util'; import * as fs from 'fs'; import { Table } from 'apache-arrow'; (async () => { const buffer = await util.promisify(fs.readFile)('simple.arrow'); console.log(Table.from([buffer]).toString()); /* foo, bar, baz 1, 1, aa null, null, null 3, null, null 4, 4, bbb 5, 5, cccc */ })(); ``` Author: Paul Taylor Closes #1338 from trxcllnt/update-text-encoding-utf8 and squashes the following commits: fbecde59 [Paul Taylor] synthesize an mjs file for tslib on postinstall 4b050c95 [Paul Taylor] update text-encoding-utf-8 dependency with node ESModules support --- js/gulp/package-task.js | 6 ++++++ js/package.json | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js index 7b4b15a33e6ef..ad56d172eaec7 100644 --- a/js/gulp/package-task.js +++ b/js/gulp/package-task.js @@ -49,6 +49,12 @@ const createMainPackageJson = (target, format) => (orig) => ({ browser: `${mainExport}.es5.min.js`, [`browser:es2015`]: `${mainExport}.es2015.min.js`, [`@std/esm`]: { esm: `mjs` }, + // Temporary workaround until https://github.com/Microsoft/tslib/pull/44 is merged + scripts: { + postinstall: `npm i shx && npm run tslib_mjs && npm run tslib_pkg && npm r shx`, + tslib_mjs: `shx cp $(node -e \"console.log(require.resolve('tslib/tslib.es6.js'))\") $(node -e \"var r=require,p=r('path');console.log(p.join(p.dirname(r.resolve('tslib')),'tslib.mjs'))\")`, + tslib_pkg: `node -e \"var r=require,p=r('path'),f=r('fs'),k=p.join(p.dirname(r.resolve('tslib')),'package.json'),x=JSON.parse(f.readFileSync(k));x.main='tslib';f.writeFileSync(k,JSON.stringify(x))\"` + } }); const createTypeScriptPackageJson = (target, format) => (orig) => ({ diff --git a/js/package.json b/js/package.json index 24bc27f5b6325..1a110b2d5de89 100644 --- a/js/package.json +++ b/js/package.json @@ -25,7 +25,8 @@ "lint": "npm-run-all -p lint:*", "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"", "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"", - "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1" + "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1", + "postinstall": "shx cp node_modules/tslib/tslib.es6.js node_modules/tslib/tslib.mjs" }, "repository": { "type": "git", @@ -54,7 +55,8 @@ }, "dependencies": { "flatbuffers": "trxcllnt/flatbuffers-esm", - "text-encoding": "0.6.4" + "text-encoding-utf-8": "^1.0.2", + "tslib": "^1.8.0" }, "devDependencies": { "@std/esm": "0.13.0", @@ -90,10 +92,8 @@ "rxjs": "5.5.2", "shx": "0.2.2", "source-map-loader": "0.2.3", - "text-encoding-utf-8": "1.0.1", "trash": "4.1.0", "ts-jest": "21.2.1", - "tslib": "1.8.0", "tslint": "5.8.0", "typescript": "2.6.1", "uglifyjs-webpack-plugin": "1.0.1", From 15ed08054b07a1c3fda6d712c947b8bacc5afc7e Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 21 Nov 2017 16:22:49 -0500 Subject: [PATCH 069/177] ARROW-1703: [C++] Vendor exact version of jemalloc we depend on Author: Uwe L. Korn Closes #1334 from xhochy/ARROW-1703 and squashes the following commits: 7282583f [Uwe L. Korn] ARROW-1703: [C++] Vendor exact version of jemalloc we depend on --- LICENSE.txt | 31 ++++++++++++++++++ cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 ++-- ...7976c60b0e6e4f4a365c751027244dada7a.tar.gz | Bin 0 -> 454719 bytes cpp/thirdparty/jemalloc/README.md | 22 +++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz create mode 100644 cpp/thirdparty/jemalloc/README.md diff --git a/LICENSE.txt b/LICENSE.txt index 84e6a4e2a2a97..30966d36f37a1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -552,3 +552,34 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 42d7eddc9c9d9..411cf7584f7f1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -24,7 +24,7 @@ set(GFLAGS_VERSION "2.2.0") set(GTEST_VERSION "1.8.0") set(GBENCHMARK_VERSION "1.1.0") set(FLATBUFFERS_VERSION "1.7.1") -set(JEMALLOC_VERSION "4.4.0") +set(JEMALLOC_VERSION "17c897976c60b0e6e4f4a365c751027244dada7a") set(SNAPPY_VERSION "1.1.3") set(BROTLI_VERSION "v0.6.0") set(LZ4_VERSION "1.7.5") @@ -471,8 +471,8 @@ if (ARROW_JEMALLOC) set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}") set(JEMALLOC_VENDORED 1) ExternalProject_Add(jemalloc_ep - URL https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 - CONFIGURE_COMMAND ./configure "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" + URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz + CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" && touch doc/jemalloc.html && touch doc/jemalloc.3 ${EP_LOG_OPTIONS} BUILD_IN_SOURCE 1 BUILD_COMMAND ${MAKE} diff --git a/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz b/cpp/thirdparty/jemalloc/17c897976c60b0e6e4f4a365c751027244dada7a.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..29d9266a12ded2452f3bd4f89421c76f50bdcc84 GIT binary patch literal 454719 zcmV(%K;pk2iwFP!000001MFP;ciKp@o}c+E`ua>xY+r*7#&*WL*)s&=ggLhH1)NOo zWRACy8t7OLRxM$ZIrG2Yuc{>w55Ka`WcE(98xNAYtE=nzRY@?CH`Cu(sHCf+ z@1FkH!Lh%;&;P}-_+Q+teplWt@9phX5BB!UxL&UARrbFdd;wmTj>N<=!je%K#&?ao z@BZTE{zHzvif}^TS3&%;eBiwJ;oygZeP@5Cx1;t|wO^IH`+Ls8UU{c-P^ng3>B<9H z_}V;wJD%?NsC?;pp;LUQ_O~Cu2KmFm!QSoukI(b|U)d?|?|!%U^xIuN{^I`sc>R~; zXf#%nFe*)a??G$W_+PzE|K)>fc~SqB{auXz!h7fAFRuTwii{3}L-G8#UZOqs_veLz z3dTAL14;^q1tEMHNBV|76~$@ieW%+vJrt+Sv)AwTtGFmc{ruuk+`QPoOwUg{Z|jFb z`YxX@PTH-^Ch7eb`?)DMeDUsr~nNiz<*kPZeF&O9r2_ zokvr=h0mwp>rNU49l#en&uZGJSe)0oZ(cfwAAB=0`_&K6(iNV4=;`2Zv+Ie8_{uXH z|BPqrzHn7v2g+qx2nTe<8m62_;UJQ}IM$vrVjcV2_;9M7C^TU|7H_plq$fIY;_6V8 ztEHU}Z|fh%&R9kU6iPiAuZd?warH7I>G(3h#)8lZgT9V@QEVGW_&OL|7q9oL#b*%6 zv*N&U1pYCNRdYJYWND5suT-JHPI76SNRMJGsGc-KQFO$bOk$?6Hh;Oa&145jq{R1m z1=w-C%x;Cri{i&!zSxI!gr|GkS$HnFp0{$ih?R-Wid$F087k-MKCt#V@gMv2tFbCM z{f8m+d*c62d2e?q{#TLzzs3JAfArKq2SrZ?gG3J0p;)V>R$RQ+@i0u{qI6&$(kuGa zwStTW=8#>b=t$TXQic2A$zOY`-;Nc>XZZg>$1(;a=p~3f4_V_r|KF)B`Ty?DUiq8< z{{hD)639*bl4810q4(13fAiD7((!rzuLnUGsfVs{kN@xPmY3@PN_lVRoB#g-$M$Av z;DqDSW}&bR3_k^&MGIQzFP6+OyD*BvKBm}FCCJbpP0ze=kiQcrrZB_6nB<1e_AvIn z(v9)#836bQ2*S*+Oqaaz7q8INg-g4jIz1UVI3w_k|C|B~6PU(F9XMX%s_8gfh*cCw zZ+hi2yHXlHYWO8;5OepOQKZK*R+oYFl|jX!9yUA)9y%fEdD?j}c{|!n{9fppQhXCX zVDy7|G5V9bbUbN{S}m2CChxd;yHKYid`WfVePe=wx0>OFF_#}zeD^rme0<+}cFg^> zB1O`?drUkTsZ6PiS=@q-^ z+eW+7Y@HPfo28N4!(Zd}@i!cykn5!%>?N-6ICNrA65?-SR759UP(tvfz^)9=V~YH~ z89UqWbg*05PET(3tJz2R#7E`C*gF1qM7^fXmABf;Q_`r_H|L1Y-QktKb{_ptJw5~f z#gQCq^Wg6Pz4&i`XJ>B-|5fOD(HGr$di=%l->i8vP=Sgxu+yyHpQeUG?j09JXby#^ zgXD(Kg;2rxP}J)$she7Fot`&O8g1L!e7Uxk_L?u+jp?Nvnb14|kyX4nw4tXUw2Ggi zzgxMLln~N&!@wM-XE?Vl;_Lu&x+0L9zeXHjrJ|pBu?A*+8cEML_CIH*cM&D%|70{> zZI~#f6Z7O$W6olC8{TJG%){*WPJ8(`9pQEz{~IK4)y5OhT-L?I{r9Uo-h++gajy4x zvJY{-M^Jr~`+Wx8hbjFLzOzccUn}z7$Kp&^e>x6V=>1R2W8Q}Tv`JQcne@1@#FxpI zw=DAczI&fjeCeGmtntJfnPYv$GKE4<^}|Tby(N&n9Ka`YuLS%yzv$tAr#1_g*1oUT z>79Z6{WbA>I^g>S-t0>N`PT>*UJ@we8RC2P6@hQX^I!LPeEnZCIvB zlQdweF_wW5fB#4)>PJTgVQ>vbOHSzjqg1GQp5VDgM9QdWtlZKv!2=>#5vSoN?Rioh zhe_aa@3|zz!zlbV@IemS_gfZIcUaH80k{6$x|#6{J$9D?J^^XaWI8zS$D= zri}$k%bKT!Fke)-FE8Lm4k<2^q=NG|BKJUm;JFOJbfQ z7)n25#R**ugZp3uu}rN*_QEmU%7hSvF|@yKHnu3+#zSpj4wwQm`2_=afg3D}YD4=f zDitcrTwt}F6f!QbJvRYoUxty)5!}#8=+#)3FcHI~Z76_Uh+dXek(QpB37a*TE~UcM!Wt7r?pqjlVhE>lY`rwm83N zpSL;{^tUNa8n%BPHV5RpY>%i6!Kz# z=<5Jdr^IIKSadF4b-K;&MYkbdw^~OXBHU@T-!|)w&fmmIt3whJ7oElyR_NAf;Q~lN zJh+XquP!=G7F6@B+i15h&b!Ul*#*T$DCn8zEGZM9v5myZ=1w+IvD!yG*jDPk=5U%;x0XXr2pD!^8fAe z8TfA~gMm5;AO7vUJMmw+3aGdA{!jU!`tALnKjJu^W;)9H4(nD7kup6CPNXrRqh*}s zv9J$tf&7Ma%MWV}#dv=dMzIXy0yPl&`bJp{LGS!irbN0UIElmu?BElev2Ti?^u~IQ zqA_+xm;x1KQcA*6p%y7VN^r zVKg}`aG^hpVs-o}t&(jySK{KiG0uqmpgz0!_YKUTdcA%IbNu>}}cKG+~KUBV6c zKVt+PG3sD35bEuzK=r{uKMDzwfZbjc(pPE;VZ=H}!o-N&>kFVqbvb0eMSv{0GDPJ^ zR{}P~gV1w(o{VE`{f&h1p%v*`;F8V)t$ri{DH1297pv%bofvrlnj|{l4~@T>nzjTu zyABf%!j*)KWJ0irR0Ie_`|(3;2S`+dP0)(KCMpAlMWj;T&WT+{))(&HYv6?p~cbZ-JT-BLOYX@rmwq9wrJ zLX`$3QGBPO2z&TIgznt6*SpJQICrQOobk}}?4@17D7StFSxynx;d z6Z~G7NT?6{$tne&I7;HVb^A%+(7E*B`@G1?<+1cmx-^HB>wj%bH;AoVp)4ZrLqJgU z^`T5^12^$UOAKRDF;ac$kkQyyo-mWZ8A6gNtB(}R&I+**Yh%eiCVuL)&WcpONRD~w zi&xxLyoB=x@%jdoJ}{4$$|{A>%H&5yOdrq}rXH_uP^Mk+((}aM4OMWz$5Q099a>}q zL~-gCT$2a!yF@(PFmX!BF)b(^2RbCBrCo(9Bo5_0H^NCGUMPfRNsfjnd=W*#?VY&(mYJmswQpYmf4u$+PnS<;@!;f<(3Q0)fa6EnDXi z2@|4ARYSUT_N!14l$*sNX7hyF=ULE9dOH5JVN++7Qs2A#wgr_I{V?XYa9!bPU)u;}(uqVJp>s*K#7cfEN2;fRQ(m84Y+oHm zO1Q0#q4H+a?=qR7}cO0Cf{Fh{9_l-0+%ubncQ|+gW7xr3-kc zAp803H{HT1%GAr_cB4VNww6r6j^xHkN7aQ+t(vAZ@yMSJ2?qea}v6vzTzOw-zng`R0Dxm9a}Z=qkSk= zywd@K2wBKncreov3x>hg&R)$0R|fp)EANaXl2NW=D~LG=GuBenMx(d%0~lhRq}b)x zVMV<;ON}gD6c;~l+)*U0{j?_52?K6SWe#pup}>_{rXI-&JG=G>TU!b|G7tS%Czowf zpLFHOdKchFkfrus7{+i-Bo*!nU>}imVW@}{3USU=X*gp{G{eRB1t=@>Txy}RM*&|9 zE!eZ=Y*r>}pYYZ!rVW)G*`RVg)R;o0ze#$AdJZ-)M~XZD3tof>9oeTSbJ^#SQ6vj) z{>M?Zyx>+Gd`VX&&IzXUcm0o25D$^_kP!*KCS*v1)kO+%W{@X5s{W8E<;2!v86aJe zj;_>v?UqK&jSL7=6s1VzabCCL!UhrX;W(p^(l#V!i%9TDLrn&QE!#*7I$nucDRvi0 zE&1Ys+6Orkf#CXVCuUA&;Xo!qnin2SmPL9Qi}jJ0%yptz;u`|xzD(n;0!Rb>KB3*)YU!M|#9QeNASrh)BA`T@cRLz?r0Mr6!fUZ}vnRzz0{R5nNy-eT zqr|!#3@jsh%bNUC5D+K7x&a5J%rHeSk4zDj)RMW3 zp6)qJ#Qk^;Eda~e<_kSKp5kIy1V0U9l=&Ndsbq^xYs>LMqkg9WS$hNFW^p&x(Zh(a zRm9yr`)Ca-M<|_@d7eUd`>P_?6s!oI;typhA`43l2wBAon1&g@qJVcLvH5pc&FB?U zpx8~nEFzIsY(pthTp=)Um}?{P(B&@DEw;`zotwocUj{&X8BIAxsRJCUTz(`^Kg*`$ zZG;8>kaO^S3n+~g%dV~>P$$X@G0~T!`DmC{2qsZn+@uNg&WP9T93i2_*ov7kz8+EG zWROu-wUW}l;aKusc}+si-IESS!F1Jyq`lH5(hNx>IdnuiYY?zf7Pc@qJB%FNqXO5K ziO5QyCdCZbWh|_-2W3My(}u<7#xjnn5Yqc8fpxsN8|BR>)l}A$ZRfXZK~ALpmFQ?L zSz2A&wy1FzvaeXTeZyJL)iWFSRLsQGfY6Sw!oDjf>l-5dAt>_rz|f7$g!`!}{FsYw zG<_Ni^NnJJGl%&GK~gK1NS9V&sh{3PZ2#MLwo!HAfo8H$m)Wjc!5Cd5+ES}y@K)IZ@@olgREG3}tTrJ1VG++(- zqU$V@tOS}Jj#aMde zISeM#G)Ya9p z{IMYr8BkptGxo4GG5w&?)nyty3CUo~42s0x1rox+vc#Y*WnmjY1X;C7e4idI&ULqq zq&k2ydKC6#PcL_VHo2owwn1fgv1$ekg&3yw@)QV2wq6xa$Z+TxpT|FKMo-2#4^d$p zCIfy$3A9k}bMOTz5C!Or>0uV{e80-wfGoS-B7CwXGIxohlB{Qft%_mr1-o`&&;{ zty_0X;3PBq?DKAZGcokKuT`s7-Rr!6Iu*4dqzSkSQ9Q(eFg-$GC5d?Gow8s!T z=*!%lP)7+ry#V=9fxsUb*JvewP#)FK{7SVX2E{yfYeb)I*TO)aisG|wdodndS{U9h{Q!QG% zqQhXPc!hnTL}uCa`iju@^(-4C?j0goeqnLO`IFwdFv_pT&_~SeojIpBwwupu( zI~@$@!0^~#;65g&c) z>Q(3M=1*#)#A}FhKUj?O0Yvtz)BD-36FNt%an+V%$mLR77j#Wc$u4pdmGa{$tV2CQSjsB1DTo{L_?eeHwk5)lv7-B) zs#6SY1_Y|04tHf@aZfa|Y5|t?RQ043xF^rIw5%~%Xqd+dN{`H!aY7YFhXN**p%OVg zfDYJ5fIxy^jX-IowRX2ehI^EZ;|ZG)u;4wufSDz8!3K4ON*A7WhB;4WBd(}uE8uC1 za$*qER(ORoYyzb=MRSdd0qU~On~0A8KEsGs)nsN_v6emlOm$O4?9f-k)NUe=Kkb?S zFcxvtly;YBUt-V3`qs{uA{o>RIOt8{VLTPHKiL*}Bgde@Cj5dEnkefcxD2}^>;Vaoyan;x0ZFslSzEdmngVEv_3`w%q&?}$uKx_UlcKZ78-5@ zf;jy=qdN;61J=`M>_Xb58rCqOm)TJXzhlSDXNKQrM1Og?t26_-DL@1w4eD=r5Wg&X*QsohW zwc01#Bx5eQzPlc)IaR?0n>LS9rAfi55)8`v{Kr>cJa+&>d;S2rmRlW*!q#IIgO|m?5mvmFX2i2Z?+6EF-ShrM*<-q$j+UoZINO~~29}n?fkNs& zvsQ+VSP5|@cO;g0U^(R~@w2R|mlEt+s*e7(mt~E?S%*pW4lE@aj*xAPRVp{;;xE?z zI`8M+%@cLZxURGwMzsUFps?gv6H|^ybA?js8ZZ)l-jyZh0<5#2s=m)tEKq-9wa%PF z8nJT%%MmYdap^@LP0!VgO}H7Ab;QwGn)V!LH%|L3U19@y$3u#R8|#qbzR3BiJ=RPn zRck_Tz)h%K;j@ItqzgE|eIR1U-cF%WpgvZd;oV$2)4X)vGuv&M zQ-2WE=pCMl1<#G=uGTchbcaBb8F>c)K)pFx6Yt(v?`7R?X6$d_$u&eEbN7&CUB7QX zzG6nj*)T!iO{5|wE;vSF-TuMbFiD4Pek(lI#Wzxpo$Zr8K=Kr@GQ?@uXkR&r*O&ea>BQ>|e6X*^9X zSwt7vIkGUvhs=2}*;f$g@LsbxKao8iK^PmcWA=445O;)xqddewJeCI=a|uDyvKIqPtRtd@K3jP( zC^yz;h;r7Ku&S1yr2q;H)T;jgJHvK2dTysb~5xV zumcQ)8P;bsYt>+<1wD!|6rkZI9y4OX02`nVxq$oDS^=PRy0wjEAbwQ2;!2y>@VEHGNNg*p$x-n zj0f_Hu`sPOElh+h0x%U}!&$n|Dyl5E6z+?WV=I)U1h-gjmK;}R*j6$3FnYByTa~iQ z3ro=y^+UQAVqK8vNjwp$whlmU zY>0Bc01)@f{hWFS0YQl2&X@qScFp?k$nLVEB&^SzZpraOOjdW8MCd3pr1XoN8S!u( zTszs~b7qxusJ-3uO-O)(P4EoI?7%ai1knkoC=ccYi;QXInf52hmelEF#BLQuQboMW zeC0T3hWmm59540e{p%gbM7Q=fkB(d~0(UG4rn62zvr&R@`Jv1tJD;dAVPHCwlS=db z9rQfnC6H{OctmfrN87l^Tty8clugaCjS?2}T3f3kZ!VK}EIDwLQE@`~q@sD{u-$fp zmWCZTdaOFVf*3ntFS4r9T)6$pp$BZeVHZn^1<8SBnf0@CydvOgZIB*%2}Yurm#a!U zGF3rfBoq^qP-AGqIKB+w`IF_IfsUA`I~` zW~8hfm?+5^yJ~k?a>l37@|NFM*znL(IYp4N{@64XvDhdPOHkX0Ia;Xrvm(p-QJFa&?|CT|Ycdsk~cGmG7Ae2Nm~yC}jNf>W>d<~d zhXbLsXx(YPx^@+v**D#2Ahf&u+^sSK$j`V#dh0g=#9*7R3a`MJ{c%}xIE=+4UAUEW z4n?cevEp*RY+{Sx4;8LVj1K6G`19!`NwmLSM$%a~LMv(Yiu0NlXVr_FlD z@6puJxDA%C(2FU$;z5s}PiF%WS+kLbu%amKIb^ku|J+0u#=mN}!O678P8$!6)Y0s}Ru-Omt zn`CDEQthmcvqk8lj|F!+>{NU4S5SY~#06as`EAngj~{kAG8PL3(cj+d&VRh$+&_MA zaUR>HaX?3rguyV4puW#|00Qc)V54T)oRz4yIXi@gUj9QPD9lEFs50jaqu;RN&4HSy@oa?9&vMhy zr-`0^^J-g7u;9RkxN=i>-Pv$9kV~nY6gW7i)tY06u@Alk&ZE8*1{1U+R9yC3u~8^3SlXaQTm(V6jEw*QJ|i-_6!V6zyJb@dQfaYyZVx5@et2~dCsvA z^Z6}(#w*C;Pbf5Jr>9IeLh((1KcboSOP62~9zS2<>a3Ia3H-ycl_lHETP&0>-A%%{ zhGyiqjmeenXL*bBuJ*|lxF-7i;xoE|C!cA8&1Lz1gWvyTT%dpF8^p`#aP@e3qGq)7iIdzS|XM zPsf&9)_gRH(6Nbw(*q7u_s{r z{k~6P!hIs=C@>Zxh?FmbGo|AA9JpIpfsB|T4)Ns0jo^p|2!6jN%cTO#!cSfAW#F~Sdd>6Jsb>s99 z1WINct&tfMuM2LF={_WT;&;kU+%p3-HWO1rjLI`6UNH3jA`8^QEr?l8sj}Jxb}|En zE$&XEWMRldl{|p4cTOY*z4PTQC^NA)EhBO6;pEG!jD+q}=!xc5>uEK1 z@EeK*#*8q=-Eom*`X-~98bZ+&1zU$*8%4ecfgtf`&@*@@NWl*0vVTV6b!_(1MlS1* zBy1Jq7`8$`SJdO2g=0@~+LP;b4#h-jou@^l&f;#*yoHVQz5TR8dIuF&3_eNJBctz= zy$X(|p?N?GKx!(qh9J~>{!=l(u0uT~Z-+NJ6u3aL)w~~{2awSW4v|9h8TY;M`E{lO zdDDgGxjS~oy%y!V;T`HRR09j(7g%cRmdT=Zxq+pK#YVgf`J(yz71o@$Rs0mQ8&+YP zN{yta=^6YNInpCq$!jN^8qp-}^z_vFMNd+Kv7wq%1%RIQ43%XDvl!c(BwMW7i_T%ky-Q zm>^P&1{o#HyR3*iM2to*XwrsSJ@mN=K>7FMs*~|j5pD^gL%H5T%GBLx9vgVYD4c=h& zyX&!dRc62znPT+-`Khyd7E4UPM8i|21`<^-w3RzJ^b&Ba4Q00{0$>_q2EI)tyt}jb zwyb#|Bmmz^^RF>`4!IXy94t&sXzOnNgZP@&z#2U)+Isii+a#!-!Z7m8IT)iaTa*+d z?wqioBxT0kv&LZcpE~>i-h@c!JO^w;;GY^B&b!HLKs8L!P0#MFNO|;5WTJp|k||?c zc$2Y}>>k0vq%Z9)Pg2zk;0GxBc!UZKiX4sST(}ifi_ZvihAIQ&wtsB5||& z;yqyf>#cgs3hlEDvP7MFJQXx->qCJpCJ4mCC4h8z0ghD};&6kTWuDCsHLl*FtLRGv zYexK)TbX#G*SsJ;?B^^_n_?new{n~A=O&2ShRW7It8&}D3ZZ0HBUo;0rYf0EM@m|j$_A8d@x0&^66S)YH`@`2=6rGaXLU+wfZvldy1G@toC88 zOyWg-Q#%`tz7yJk$dD2oo@)i3MC@uWOoLenD2n;wDSh2oS2gB*UervitQ)?2DqCw9 z@6u*dI@@?QdhUt1v&h)RK(i~FzmpmCwmt1qJlsa?LnflNyzv2D0(KbUj%sHRI3w;I z-3*?ctbSXiWlmUi+NV2F^-d;y236lSMaZq%gSsam7va7NSy>S&Cg>c06jsoI-*~X` zR-!P#yJhjAoGHs?#7_hhFm5ER05Cm1#-r&^`Xk}!i#(8XFzKJKt3&f)>b;mnUT z8@Z>B@5_Y-6tZofI%xv3=UE3UAt~51f)&DiQAIrB!m9D3Y(uLRmMKs=PnZ}=5c6zq zAUMpiwq>U_ggt>(xy_l164xqqa#PXo({xIBq%ajNh4|5*No8(ijzUCLwlp#^dTSxe zBwr;~uf};0;grZDOtaeuihY3G3eJ)Ofgsp^MkC>QW&;533uhIx$u6JqXlniVVD`mE z3My&u3(Q+z4YAooz`GTD^9)zhGnTUz@hYB9(K+M)*ir>i**pfu4 zb&wf?F+3Y7wO%UUFFfr+%mqskAqUX*8po7U!z9|w$NpxPIYisdGYwgv&JBT0d33I^% zYK%TmYD%gYvsx2q-uQLch_ve9K~x!0MoAWKir5zrk4&@Xx2zCN4Aw4uI}j=fA!Z+l z3&_I-#xAiV{56SRnLAWQvXP`|3QT8<*k`FD@Qz9h zaUKdTJ_jaffI(PD$cqMe%)6QTWBZj(u8sMy*{bI0h?xChe@aX$aSlx*?jWeA-U5Gy zJgC-8=mta6*l?quEf5bCn4&J60vq6;BqsiH&x;tNK4M1u;?GxjXcKQ-77l-S5OCf; zdnoU;s1441#k~g_IbPSlG%y&kLDua_4?%pLCCZg)kYa06-9sqS6j2CA{Zn+HhB8-A z9(Nu;wAP1)p(X&!xdmji6JpG#Dnh!5J4mSojf#Meor{CeO1$nk6AQkhS8tEi zY_C#goMnOq7Ht$<#s$gPp_&5GfOA}hsYD*@$|H+7aKhW3B;s5$8?hhx%mxe0yT2*2 zc?1t47r^P!JbdFtZ?axzVm@cy0^SI_+8jpNvZzk&G26R?!EBiFBZhOZ*$>=K=aqty z(4#i-&-};+faa0X+t&WMe`*n~7buy*_dtM(mV9)yvlg)QZ1c$^%v7vt#GQ_?>fg zQtR{XY{-+-iRlHQ*ZAM!k=&29dH3Tq?and@M4XRDraU01cb7GdDPfIwR*9Tqj7211 ziV2=BfM4>+Ay^+HI+Li5?mq3#g24R+hTUWBWB49{tu=P(AS#L5?`sGUD-z)S0<$mL zMb}>#E+}J9ci7-bUWYzvW)xFQ+YsRFYflxwuT@iJIAWTW}&n#D8`q3WB5Ui%dp6koku(6}*3dJs@fVk6g1Uj}U2fZL- z7c7-XiJ>AAmQIO#oAdt?ipmd8ztJ2LC6h^_qN~RbeJs)_SB@J~V=umSZjP{VL8KGz zY{uanj&M)(@`(p9+u)Cqvw`OF&Iq{LCOJr#^)GDr>IWm(psl^@QH*pWh_r2( z1!~swEhcl{6|eVL5G!(j@t+n^P@ZlxR<_3d5Idjr+UQJ01K8n)@7g1ey9;bGXPZ36 z-HJy6M?3omZ+G4X&&kg2{tugbc*8`15T#wjLN6g4%mQ76EH+G4?w}uB2TP1eTognA z$=w-yQ&o>>SPD;|QJz46)m4nHIUWE!OsF?u$)Os{IcB4hI`JTazi_xL($95n43;gq zz+DQ2aI{a`ufqw7(ws8cSr@8!l*O}eRot7^SvFf@-_&44{tF27qKT7ihI)%DX7#~1l2K=j!28tK`bC~L{)OpPnE(^osWH>+72~m z1cxJMrpH0$Ro`er0fbhpmJ}6}PxdjlO(h}U+!idC7|up)V~|;uazXoHg)_<-Hq;o9os*IarJ+F&6sfWT4Z%k_m}-ehQE{K> zP|w6T(cWS&XiWLm1j4zxM~!Q~I_oR8yT7-)zth>Wo2Us#`Xh;v9#B6-mV9O>aC!cu zCObQeau!!CZ!hSaELnOKi3g}4Je6VL`Wh!~59I~cK0;XId|VCD&*LpJLAo6Daxh4a zNvM3BwN0I*)7db-vSS&cY+Eb=4d0!pF$}j4<6&t;W)K>zJb{1PWJ0H9 z-zSq$EnxFEc`d*qoKO)jwGmW zpO1!@@Z5R5XagS4ZBN5SekL*j!or8GCoG5+z+BPXENdQ62D~Qjc9z+=FtLf`8L$yH za)}E!$eSKRuna+%G8Y=pC3C_>5$l-2P7>#ors1ATZV2v&m%=0n&@M7#;^a~wE_<0> z9qZ+6cRg}l(H7Y_QHGjue&b?bpeF%>)fmr+vvszB8}_*?-X`4`{L_bcA9hf3CJ!We z0VMK|=N*a=A&jA{DZX0|e5aW1H_45Lbq@KD?%N1n9IlB)cQW9MVZD#kK;I~^tjXnb}gR<}n(z#01~B^V1K z+!R<#XKg}&6lZ^fHUm-oJ=vr;^R}0^)(>tL;JD+xBP*mUym>yfFjrB+d1$myC0+Ql zkSo^2LHlJ`83=FncL_)h;>)}#gz39**n;fdB9&~GfeLY2K~^nXrhW&0I0WnC8ZZIM zO}I8a6Cs(cgqbX7eOYl`M zIjT*Xt~V8$n}mt|(6PkoBKjAZutvV|O_-F*QzVyHGN=Ta{H8vfKa{2SWsYVHXGS^Y(77r1B`ZK+sZ= zz5IL_nx$(rIW^R)MBJz;?wGk)$2o8H$|5se-=q=-YB`@K8%YfEgh!aigQ)gkZ5;`b zqbKXDbK{lKo6|;w@i{tQ7JaHrOaOo-Hrzs2VYPB%Z(;=aIXyDd>+ZlU6_ew?LW8{< zpE~6Xg6Gbsjq`>px$Xs)INyz78&@BDuvCH!X4AB(mLi7ynPcY%Ae&{PATV6UNK!4R zK4T;~NHUwn%vzsr82iI@Laa%M1m@gk7mja$#F0U)YKTfX;y^gCapLyxyBQ^raylVr z^lM|%=zTF#x7^G)14@>KetPnu+@skyB+C4x%El!(@Ex?U>HxHCYf4Vqy1A*nXqE#w zm02=c@}k^`>VG_rZ@8Ss)LPkLFS1|%bUt_#Y_Iinm;*pSd-Fak-=o0|K~m7@s%QzV zO1CHJSS;&(3z*QZ-l=&FojVUVQH0M^Tv&6CA8@Erq$`( zW=P<<6md<`bDT?nxT2g|FlJj-g4A(qqRQ9^@a7V;eJrDvXb zh7+UMsI;mEH;KRe;6^Uv6{9GI3oAAh1w;>zCZdUCen1mr4wiAKb;NI-q;z1{c%x2d z1CMF7{_JHbyi{f5&jKcZc&6$F+blw*$%qzo9%mAkje|ucjN@Z5#-hGn#ogotV0|X! zJ%at3mLPhB&Eq_Q@Z`x%v=dn^BK1t5V5goCuM~<)UcuJEyT9%3zpmeymdjFKN~Ja+ z&IhHK8OJrlcYM7&t&(K=*W0)&xu*Aw1eCt@N3Va>-);Khk#n!PPNu3cH+l!peMb;g z{ibR8avVFgx4Z9*_wLb~hx*$wEJxzf{v;rl2=M}HePo_B>mCo!!;H-)z4S)7SgAhe z@x*?1K_g5cYe@&|SS>r4JF=XrG^(8p03KVbtNFL}CR9^+h<%rzpw@edtt*a9HNJSx zg$P#a^)l9^X*Ds6Ig`M`WN}rMhxopg>_ei?yygvc&r~T}6_|LLlhYwMG!Z+P88frM zFKywYz9W{&qJ_65mn}|$T`l58Ip!DAk!t zb26BtP;jto$y>JJ83m6DAIp5Dr*henslfFpmI}T)(@(eoKk~qU0^9-k1c{q07~Kz> zog43RLJxhDWeV)%y-4)wTf|t&!aNx@lghM~riEeHiO>V*^iu`fYL68VV-lP69QehA z&^DzOn=mTPOlQ~kBEzS<0Q#lWCMhZQ{r!-mFSnOH}tOO*D^2mj-C>k5(-U5C$ ztfz2EI01OaXjEv+Vshw2iT?H(5O&2@?xvL8vn2?sD0A`>Z!aT&_tFTpS8--^OnO^S zw2sK~j0fbcte{vF0hi53p4faEMm~eRBV~w<=i5GH_`R1S)PJ&g9t0owq-HM0aRy>-&6T23qOg;)X&U}!m@fhF6crxg7(mCd!XWm%_LD7RYFAGlx33;Q3=N}8oYvGg-`ou*3gh$rqva+(WC66b5@S(|Hhr>tF-Y(k!up-5hl znvvb~Ib>59<1A)N2aOfMi2-i#k}^>$a?zIO%f!-@9^IysjIFl5t6TXUmgD0Rdsjmb z_d$Br?`kIYAg;Z+0esI1a&S)bnK_lS7|f^U)SO^I1o=GUs?h>e`RZ2B`@LQ=s;W$k z2ex4{guXUOxsi;Lq4&D!VyTuj>jX4oaRq<|NnOYSVi3_<~3BeTg-I>{y$R(3guCbWMSy=)@_VL_1=*8gtH^HOq!ogX{ zG5K$bM3)ZAwnDJ_Fn*W_dn9oTIR-I@=oUjb@o$}(aD*QIGsddPh!t9z1&Q9&OqhlO2jf+c zH%s*WY&eUDSlo&vpKamL6jiQ*<`PtS6O|oD-SJoug?}AtUfxC{r}?#)G`?89u!~b0 z8UGGUjiHOS5R+I7D3dwj9$mAkrN(+1Ly>t#q6yD*kL~)_>{GCKc{k5p=UlwEHvTKe zHWb_;`XW1vRcAaFkhW#}tVdSxFpp|BkCB0fT<8-uhS+=E+1%TGy^rXplT_2<h=DTGEh8NPwv zWij1y=IKvt6bQw+!A_RMYY``VqXtnq3N84ajo=ej&*{59^pZ~{P_t|z&7~s{W@6*c zb}h;frL##_1I$ejiJ1UsJMF$Cc3oa$NkxENLDC>pu|lgi$8YzvvjwKwVhv%@s7XCm z#7u^8W4vUXCzT2?qBwRQ3M6wObK}F0*gocjN+zZfZA~rEhD5~v8v1P|@Q?Yc2Uzp{ zV=VWEqs!+7%nJssWuMJ?Btn_fqIO7LzQAwJzxuwrlwe$jaLX!b$W)aRmbkQ<%v$!i z@^KQlQ?QeW?hvpxE51FLrV}r@e_nKe`JVOSkcKOv{K`IY ztkgLqNDarEhq-A3+M1)#g8(zhZf`UiD9jxeUYTd(#7&#bCOG~_@ixcRbpQQ70b&z4k3*Qe{&g# zJp)$NB20qt)ng4=p*V4<=TTlcnRcxUH%GX%{)h+fj&~3CkJ|RbyTgN5^q-wqyFUpf zmG9JEdUFh_kYD;JIvUeir78ef2<%nbLo@ou#QF=ksBG9o)8i0dqRW&4zTvUp@niZ zG6qO12iMj$&_sU`{};7VNRuF(48{^4FBlrh{Hfo3G^5ZR+aUv4(18hYHYb9i3jaBA zkGas6?K0$5nZaqfNo6j7T6R6^o=?(|BSreAV}4Luy0sTZYPM^HCq}B^qCiMAY$l!_vi--} z6QTQZLSWz8#VW3ZT@JLTd83;gxuGjM!6pX+K=y5u;DQq`K&C2XKPkJuw0OtF8|2>2Tp30NsTNqw9 z^s!b+ID}ksb)4UwbH&i68m*l}wy`Cl^){_pE7HWdpt|BY{yMX*YmVmiKjtY#Ae7QIsI^VC4>c|J%+XLQB9h+Za6p&v|jacBjVWag!O%n=Y` zJ+)w=hmr#gr$SbA+EDKLR7}rh&Jaz{#&{p-{SM015CmG*ec7*)GC4*y2%FhT!zvX^R{!d2l%H;G&v7$Mo*4GLWDMeD81GMixb zOHEnyXf5>?CcIa*h!y=nViII>?;PKZZnNt_)g=`DJSg^YBGK}|vRt*#a$ z<56M~dy08~I6S8aopr77aaB_*%f7DW*;iwm@3F5gt+HiH3yi4po|VYyZV)KN=RH}A zI`zl!%>9WNM5+@T&zqk#Vw-x21gM)Qgz~gwvuO%|Q#=&HUY>4sLZ*h7SB0}qW=fK~ z^#x0>7#(1cIQ#Z|iS@4HGqdv4|a7Je+RIH_lul~;~g;K*$~n;VKY%kxzC z_EY*jl04kkD&*tdHC`;@J`sl_kyJT*3zX|PDX97p6A=Dm37_wh0>iCwOz)^L z0X8R*agsV^JD$Wk7rNOAhg=dsuWSeAcwG1ykvJ#Q$+byA!+g8Nwd%sb-hE|T#U!ed z;!x%^X;wq=!gqZz%O@Emi;f7M`=>#Z7*Qx#XwZz^$BmZTCeEI4fv{3h1k$ z;qdc0jJljs>`X;K5{3#-4+qqkUY9WrZ`muX7(ZIoZ<;GWV%uuUcr{p?aFQw|4S2}f z=F#3EM{oBx$UP(0sVZ+7BZjUX4ra}qL_}ZV6y^TZOwLk_QV2Tf%PS@05pBK(<c-YtUOu!@ZFP7 zv85W4@UD({;6E_hwHd0$WF5X#%mq_x$O6*=NnH}D*x+$e*URHHskB+ciks{ z{qC>dJ?=hUIax^_Cl5~_#t$Ap>VEfVb!F|lwTBOTaWDQZZtw0N9dGXKeZ?3n^z-=f zL;jb3a{oSDefa3{|5|;p`smT4hu=MVyh`8KR@c@Z|F4yQ+N!-Dx?b^whMc77bWy#V z%JYl+Z}Rif1PhTznl?i?qL?!lvah-!EeVa-P=iJQml;$OXBC8LGNHXvJenLN=)6C$ zA>Xiv8jYC47-%{%Q5+)k+i1(1g#Bq$cZcrlqoHJ{WsD8)1t9LKv zJv9W5kp@md;7u))*kdn;EUidR1}i`DY!HBcVx8a^F)Oy6^&44jLDO} ziKGxM+Z8Q?3X%xV9xTubGMloxx zUbfJ&tQo4!1Lj%N8iA~^oJ_;n$Oh_aT?usyew1giFS6jyBnqz=h0rERURnvo(qOCp zN<6Y4?cm=Kwd|NX<{y{BMK)iDMeP?{rytscdel`rwUc?6Lp6^^6tt^smJ3d)|CDnj{y` zn`h6?`aRi%_bTUgv4xFNSy`4K;os!CFZnvG%>p=tTL07-h{fvQ4tf&1S&x)wvB5Jj z*bh60N4p36BDq0p%^*{EOPN5czVWP^4j|&zl({|mEHmF3p0JG5aVSRG#tM0-ouSVG zem?DMd~ItbE9?_1aU@JOEzw41Bc!Jm6SOHid#{w~cd?^#PRZE)u}tx;I!j1?Qn6O_ z#6tizuI$IjX9Vn4Y~Y&vgBkIwDvw{QF$d70(MQ1DC5-o#0HxR_+w(R+u8ASFg(7ooa(K@~SFMyv;)F$~K4g zG>pB2!CJH*&vgMEQx;T}Y*yt}xlC&pf}Nr^e+-(mye7U1&<-GG1s$)f;`G<0LHz;X zE(_w+i)H~yk_NMoE9%=_q>~1)g+xe{UaVrKn~wV;k+U5WRYaxnC}9<{-nExr&QcHP zXr@&<7-x@bwCm0wxB5%csY*ObF68d)V7PmO_&N(nA1y=V3}KaJH%0{h(=tIHwF z`BYxv=t}ous7~Q?2a}1n+LIB3TqSPgfECCINhljvw^@T;hGP? zrg)VxSkC8P3}?`bWnlH;Da!u-A!NP^2PGI*Ax;i~5L(b z#iqePrH3~ppt}tsqRCApDYq;`!I#0l6S*BVG?SROZ#8XZzG$fA74YL{OD#q)+;>-6E37+b1%5;Ox?-LI#-6V+Ll5KKg+?z$A-oxL@Qbc7- z=yhd`X$#C6xPt$fGcpI_!h3EJpHF%O-zE27KD@3WqQpuhGeM$Q0VL0ueNcSNt4X*` zNGicI1!PMH2x`R-NEG7NWOUJ=q~c#}`pQ2uP(T-x1h8n&sc@;%+M~zt@hiUYZl6pP z%JtPJ*vDe3Zow@o8J2K<9O+(4C0VcH6q|6#1hsDEt(mwpt!<)v#@d7G8<{Y69Bk=b!cWpm;g|a)j&*$1ZP6$ zIcWon_LptPMlY40@=Q5gbIDl26WlLPNp{9897}_I#vZnhi5H8+Qxa8$8&Ln8Ku~jP zoyj#snGbIU27OS=T@dpY(MlxJse09`?+qOW3iu1iQ>1p-S-govmatmuV)_<_A zObwZf|BI{&Dtr&qPtox17_nTx(|{fvYVOwn6dB(V<=4 z`8ZtKBpwlag47b?xx{7^wc|JM>VB+r;So(PS@9cuBWCdSsiqIIZel2iO6X5re!HF! zql`!sO`~;cjGdkNhTuds46;U9ijX-uxLP?%P>g(1u*{ilfvgu}745+$rT`tsKI&o< z(g4>w^`LD(0R7`=IGver(< z)|-u{vSiqATHC+qhqoN$={)6#gRsvP$syjofhTM*Fdl3&4A?>>bc8uQHv|{ zV%i67IdQ!cM4*f9ctUv15FSGU;>_*zCCgGy5{}Q~l6LvrR$i>p?jIca#L0~9Pkr)4 zzPMDyzQKti{akKmqhFH%wQ|Wg+AbvU#;O4zX;>15RhuS7>F2sXt*c>y*7G;k`YrW` z!=%@TtOcQFjd_w<6hrbvkDF)SXi^8q%xo+g(QLw~w!lb<13k?nD+Weldwmn6eZ=zjoZVJN2wq+|Fd&e6@ymg3#H@#9 zCGN{mESH?`2_z+6)NX#n)XTLkw#X7I)yDF$%IExGLp4lREUY=-G)I8}CBwl0T8LDc z4ib})4HjD-jV&~qf_tfjRhY4`62yinqX}5&#lHFSZtfk`bHV0ug0SvX9~dWL@!A_% zh*REnI%-*^xwXYO8|GBd$rwI1C$MGo+ZSLr(M615XRsKi7@FzHpwFyjC_(^;hh)nEJTPjo{FMJ%9-EClW89z zKs^IX0dH)LNoo{ry?d|z{F0r@&^vF^OVkrlS8-EdDJK*L3-DCp-%VBLJ4)0$QjDFG z;zn5)MVFNu0mivm0@ZtlS#S+S#tMEG%!93MttN3PR5OI_t){XI05|ErKh% zPelJpH+9qiI5s%)WP@|dehI>xDFde-J8@|kA)y|Hy=Ij_mJmw}c{N+o8xJHsB|8_# zE?i{;Xv{aBIi>57>itBm!9kg5JauP}emdHt%Xv*V>pX$bCDaU4NKfB zeokPtoO!{|0uQrracqN`*(J)K#f9c|z;5B0Rdt%-m`T@nSC{V#9)|ICURA@ZZ`#$D z*$w+zQ=O~vmdb3{e|8t(Ym?bTQzqFHVe>_Yef2RFc*^qf={)eYJ->Sn z-_g*R$CD>Imz9ThLM!2{9zG0a1%o6Gb4p0Ge1hnLHJdjnSI|bVzBxG~vmeuQ zG2zRwmYcl(;g;adaqZFLhF@7)ZWf+0A^swl1X5j(%47yE5Oz3Is7xNIrF==qhfB#s zooQwkgBweCDx%lVT3mEFi5l*-UV06~o2b9BUjGSPF=^)ITTe$N)^bc3jyweFUqRx? zM$_4lZ=U{s8>H-l+ZKtKmPbD!1Cf0>5v(fL+R!kfFXRU$1mUg7_`X^)$Qbg~{ys~m zCORU=%@H3YjSv*|Ch_UiOn8evn4anuoUof z!o$SR@&~KNk>>gXEYCPGk;v4H&Xs)?tZ#UD#j$2_T4qAoONn7ged8>m&cP1vn>2bH zrcCf|e40o~o6M40Or%*RP{Zx^lF2Acz+oijePeV$l+)-loBCVt2;&kR)i`u+-53~| zP&l||5Q3$B#A!UW_tAj9Nac7!^QOSs$x)BvAgw-b!p+$o6L(nid)#f)&{%A+{7T?)q7GW&+!>;l?Rbi5d|XV()Cd-4=V` zOTw-!_dc1OaVU&-MvrI*A)&fw{qOs|jkT4>5q18alS0a@bBqzkl8|dE_sia{Ky22H zFNhCWx9tQKB;E6o5gwi=2>I1N^+*u}wr>vmOxrQ)RjN5hvFHwa+Y8o2PFy4#hCYr5 zptP^WxLN3DCGGl|Cz%>)XY8i2U(5~kZ|5Q!rrU{_Z_bGl(? zYIrw5PzGk%JnUb-N_y*6VH=ndH#{K}UK{hOI^R2|;V(R}06T?X%l6d}f#~6i+-Q1A zCH}8bo-ZKXe5S%XLw;DyA7AsbZ~_D`_e4kmQ(H&fpl_grX^F637a+= zxg=9wAJrhg=)r@HiIfvuP&Scr-qG|1dC^!IyO;zBqBiYnHq?lv^5djuJn<4^KzrBJ~%$vt%PJ+hgUU;4n^iu{0 z79fgnz3eS6f5Xpit-!${Sq%0!=)N`72uOGX@~HpuUsqRF9*DcBGB;vJRBw3{fn&ZW zF$;c#`IO1HXhb&NPn_kXu&^FeW!H&H7mL)n!;L**=R6A&XP+$nNSZu4xepIG3SA?( ztDjopKH&(Z}auhM)iBK2Y44tK5p=z$iHE1=eF!U z{+GW#+Qy224ujMJn%SArUn~6j&DR zNw;hl1uG6>M@$k_d+n`yJ?HaZ*?RZxwon9M5F^8M666k)%`1}RULN$8MF7KBJ?9#t)+;KJ^aG&pZQrnYyl^#Dp!(sU-eld_W*FM) zvq{WeAiEu2tg?@UjMzI5 z=GJm(d*|rb@8g{iNs)fXfx9IWHswh-qV3J&P5uR_O{U>lp6B&Qo-Z&Xzh3{8jZvrU zf+$=eg3Ec%mHpT{isE74DlU@UPU+;5JY9HeTGDUvJ1K`h_-gu#$r6su&SENkE=7)4 zCbJM`P#++5l-~Bt`hO?-^)8Px4Wc`>>9XK@FrD--I2*M3)lpdy_vMO`e7^G5 zqZe8>%vIB{)R|x2P;=j`5^jN?lVWDW?Py)EH^VCV3d)KZsb}Mk>$L=j2XAOw->L`y4@RT;`uH!1%7)9<>W^+Y#trdb9kzRJ;-sFNy(2Mrk5}+suSd1CI!d(5keK-oJXa`_s-*M7QkUu{lPdL#^Sv{8Av3(^OzmTLO7{z5*!@My(c1y}Gw$-rC% z`S_xuWLBAq9RqOhP00{g&qDixk`9A%}sz$ z)IvyX8hGai{xnH0Uf9??nzDzJ3NrvWB#Bz?~f1I!4{v* zA1^8$UQ}dOr;Cno1pQO`&uYD~R0EY3NzDd6(FS)<&(3?3jJRVtuejG!rp5tQNwHvQ=Oh-LVdfUd5omR_APY zUM>7>3yl%KU%`B{m0+0PaSB_nYFGmF;(Vl0&Bm|a5*T{n2ciIn)47B{MTgZq7m0#c z{xe~uN$tBhY>RNn1wncGVr-CuNdk2eor?;PW^{z~Az?LL~{UsR3> zHPQ&3w+GuhDDnTMRsK7!5q;)g2x<@U$L1k^vjgj|H0oC8?OPrbztD(p<@UY>y63=l zDAZpGLH19Rzav{|_dIP7P|>v^NR|nBvRm}A{WqG@D{A#}bL-!8l3j-->B?3s$skL# zXX2QN-n^k}(+U2qua|XJ)~o`%r&IkeFo!vRxhocx)C~2b-WLI{Y=DS%3#ss zsQA{*0Q%cNap@cnBw>odO4|Cn>dtw{6=T5VAP8}Q_ZkcyLFFyV2M(#!* zD}M>gI96(D0Vf45D}k!M^m~6gjK^gq>?t<4014>l7Y^t3`T4x`^K?-5Ik`d-)v}WF zI4kW49(!fqhbN_9;fGOHK!?;HmUbkMcTVb=x5k|B-tMf`0BV?x%xv zq|z6#-q*Cu{5jB*b(=Z!8&;_QBRs z=cl)OU(@PpIFNRnD^OS_?Z$f%@1ov@|2HH@b!q7)daITXq?y+OL)Xn8=nZ!WPcQ9M z$pgfx;rV%(nn!4UVP_?e6$&+0%R3knhdsCLaC$YT)!*l~IePW>xV#a5ncHl7-kZ~k zo`H!)Z1drNtMmSNcdxU%f4uYha1$w(%%%zi&o7VWygX&7S0Nh5cGP*lzk7`K#T6kU zOFJ$<6e;6$-3aJdz9mwrl$3Dg<%EMPFL~-+VK3>%*Yk_z;qS2d4@}w?A z1v4t6g#1#I*7L?XFyOkO(it*_-B720i#j*l1+JrR!-$?XzvkZ;9OT~%#Fke!mmG9; zCIuqaD=Q-yz2e#E7Pm>U=bTcQE_T#(aY3yi{e7NIlLcE|O2p=hCCbtKO6An6D=#PE zz4k>K*AgSleWazp-MN;o`>XlUvSK?`5FO&tyPbYrWlTTVrv0)>pD z^(l(~;GK)pib$HB{g)7HW|t-B%bwz>iF zxKuk}iUxenS2Pp;Mw#LWquGr9$$T`iETI*TMI?t-Eb+of}u)n=|__t`?u3DFSC)y4&N3<%` z|GH-uSwWBMzR_dIZ3}nI9fzNZ75306zPWG*t*dO1Yk*$x8ED(6kLpn~jh0@_sTnOj zD^BLYAD;ye;!PBd8TLl`GSIRNJKb94Mcn_*S zqF3IX+nv?MhcnZ+2zWBLAe3xsabvcfN@6ragk!N{D4^lOf=_~PDXZz{o~RXCs` zlF-lHe8WGsZB~X~)MHo#7Ui#gu(<8Ly`7)X)*wFW(xGj<#fsS3w)@S_=Jw8EXM6YX zDDqj8f>x75A|QMmGd=T^pNsUOrUxBU#=E5Vs0&Z@r5XYVbNHA3gqG}03_Cy{>+O53 z^TIw@BpdEZ;nAo^g#*@k)NeR?vk?Bntfs>v=JUei^2g?+P*(ZN3o)Q=!cOSLb{$*3 zwZ$E%Sx^&OZ4RCb#0sGf`oMpu|MEhbHs8JtS~TAtgU|rEQG4OPm*~JjPMJdKu4s!Q znXf9Jic=npqGkp|n`f`6(-&^5ZlTL!3X83Dyw|c3>}|vDdD-A*zdWN4{K@7%#@fd- zda9$)SaQ(s_xVrT44OM-V7;Z&pHuOZf(%Z!cTM+Ba+l-)&*^}SRUEfyiIe+JT>!UxslTLNJ zcw4pO=3s&^8hq1k_pSF^Wn%NT+7!E8E{8Cm2Kc{jPj5M!KL*EnC&hI^iAVKH-cu1J zKHhK@)(ftVMK{NaCz4pleyL>|y)$^)#ZnnN%dybmZowaq>Q50afJh2)4)xm&qwgL) zik6m6T>I)%p*|K?LY*FB>Q{22k&<~i1L?Yb*>#fVGD)%cS~$(e4tl)A?bhe_w0ISM z&C4%Yjv-?n_Br9lw{1CTIkGVB-D+$`=k*sq^RbXCtG}ZHSVvd+Cr*8uj^?9SPwa}k8N^v7#-?3hD=H*+Kvrk21 zu}UpYAN@`zv*h-ZzZKLk8mi%0&#a%ugY3^AT%PTc&A~a{ejgW|c7y-hfepucbM;@a zedR;1{7Xmv(+4Z;`&S*#pE?zTnte1HIm~wZ@ovKDx@sT&6_;4HXq-UFclPqg-}%5p zN-lqp5YHbySelu%sVyt>lmyW)$u#6&zoNZS3q-u?d6Jc{>yEq9{_^tsN~J92-+FNu zzj6=8{ccV`_I2(3lb$bA`8^w(?|H5)=W9FnI}Qh#qyO;s-rOxE0sggn{indI;)A&+ zBwyR5k6xm3iwo0Sq4mGfuKjZ%{7*z3|I`cfPXL=ms4o1p`TnPW)iwWppkLN^ur(@r zVOt$yD<;FJc^WyDf%m}!are=KTlX*ijac=ewNhQNs}|Xnea>+xsMa2+~M#(>1l+F z4XsK6uyYq6FvAS*%L@oc^x6Bb4qfy=KYzl{*T?@l+}YfIyYn?;ERO%RvbOT*aX$Xn z>dM2%{}cb~U-0ASCyPzkbd%mu(~H&~)bp|FIaF6N9G}OWo2_z6gtV6;5lK2Zfe1qW zA>p3p5FJUXsqVFx$HLA)vbJ~QVNxN&TY}wdBvg~Ql~c#K*f)d-*H%_E`;^UX$5;euMa#9UO%_02Lk*0o2di}0sjWsduIbSHUDy>vp1{g17 zXL5SVVJ12CY@$hzjMB%ZyK(WWB{yYOF%j6$lX#5Cp{JUw&m`>ixeyJJ3CCW9w2g)- z@mVRti?*l4a*fU^$i9ofif<)zkDaNe2}d|_I7muT+LOd~&e`|6pJkVHyBifvvNTCx zLl_ael*PzqIYpia#C@agCVdXc$w`CGQVA?v+!W%cM!m{3y@W|3Jz1z{=N8j4B+C3r zl6h%@B|B0qI~q+8t&|_J1mPO7SpN`pYmgia3I1qDhBUJ?{x!Nx;?JYRWqzR(O-y== zcu+yi+==E6l0gxS9Pzvdy#P`)zUbD$yT9%3zh(n|-cZ?!bbLMOpPfs*aB71?;%s{o zlTu>bqrHhV6t?&30S=MVkIuqU4h^vm(ex7POkyc$ofwC7-Am7MU1JaX2@+ooIi(@( zqKW=$vjZxGP*3j<_trUK#(LY4=%ZxX{+}oR|GA&9fB#)z;MYJ$L_CzI*WCe?9u=EIIu7|M~s@ednK(?%fay z`X>JGy9cZJ^MCkY^}GMU|Nnv?5C3!S_gX#rwelAtccW*|b`D-u-s9ssp-K}Kxe)5{ zndBmSuIV~5RBkpo1JZ9%^E`^yqj!ixBDv*TTnbJCWNze2EjOTLQgM=8^tGm`s798O zM+1zA){s=i(oCo3_wU{inQvx;%uCdLM@dIEb4AcW5JleQ4lp?`(28~vHe!A&q#K`H z&uJ{Jdhs;YW~LQBEo`HFC7QYU1@?ud&1lo|6Eyp{s2THHVY49O|AMC7Ze)JkNoeY_ zMF-H8(E6G*T?by%WUsP7%?DW;8)hcqadv~v1`3qk{)trA?_^DOVgt5n%sUZ_EF0etz%$=ZMU=%mB0S{y(CtzEZgVYmXlP5B>kI__^~<)ShJ% zKGkG&5sj0{K$52zVRo%sk5>Qs*C)+2;^G_8>oh$ZBnWKNqH3FiLB!RNuq#U@7s%3b zhph}?Au;Kn%xrcm&Xqkf`MNmYl@P}eduNjFHX`Cob81p2!?f2wRoQz(asVUYEV4Vf zw91l#I+xLDB&pvye+)WGrt3T{{M^e;j-%V_tYDHXI`;{(FH6?+_$0kZoOqwQ8KuaJ z-H_aGoRAd~b#GMBn|M>A0lR~k1L4k{2J+~*9Wm3Oh4yBm*^-%1d#7snBQv3nbcJDF zYHHJ_bC7(#!+4rZ`tcxhJI)71g`DrlEx;{oKj~|xxFkZPPt3`K8=AB$iiT(r6Ww`k|5cO0eg;4s>K`)+TSx}*LNH}{Ws zca9pGp>cQrbt9rKqWyzoBwpBkyL(I(j}IC=aIs-@@G5${bGY?}{@i@IySID%Hy-iT z?(sfGdv$P#D;mAqJUrgrdcU`M7`=Oc`0n6nC&J>kcaOIAHh16dY`16zG-|Z-!_NM3 zbo2(gsbH;2)*TTGm`w|-B-tnWI!yk6Hc8;D#dk06nY44AAFcokVJUDeidqx%L_m}UFc6s+;M0$Am{+&dF zd~@(4?GDXzlNxXHHXrO`!3;Ay2Zw({$Jh%#heq_{n;rUoh@Isf+r(BJ(avoh`_*W~ zW5krXV<+u#3rSbpnK=T?HH7x~(_5@tmxy*=*8(-$^P5a7O-({@(Nz19%}%a4 zl^$Y{O4+2q3bHoZ3h8+vZ%^|d&I}AinlH) z>!jJ7CRfwBeZ98RM2@xra^%9wfJGfW*qfDhd3JG&4&xC$E_|{Gx6q`@<8w)c-Ot8@ z_&VcUiq*4=>fAw^%~N_=vW>UM5i)nk!vv2=cPi>`dVo0xojmMF5j(ixG@Cf(ETx+dRkzi#QW=(`5)m*xKe$ge< zYtiL-pVq@Ua$&W~nQ&uj$%xeDh&K3M6Y~WzG!!haT4b@Cvv`*US?74-S*3-*bdbr$WUfe$+wK&?5tg<_EXUJ+#gW+(`m}R zrPxi~I{vlcRik&gPaur*p-ZP*3sa_pqv)qf{VAOx5pX-}nqC_qgbO6eKTTR?dRoDi zXi{E&&_v+gmxR`ru^#S=?E~bAt5tjHv|4}a&zG+Dh2*-9PqK7CIJu$bf&HmrENBd7 zd;x_6?(g*v+#S7Vb-n{Hc7E6e35Ji!1)A$H;8bR?3>#!@Lw7aqoibyROwQvm-RSNF zA@_RR$-i%HZoS!geMs~$_w<>AqO)b68Ry;6+Pm#nTsqFysIw~XmAh!n1aTOQYhsz0 zhhg-War5_wd!1K1$6KH_(MNYS!MRkM8UMF=8uth2v2b^hc$wqh3fg1#o#>KLA}p7+ zRGc&L(wm*lcb#`cF9JWb$0%!q!ZRoYRQ29YzJ@{FPkASq$u%Q&&-zWyL>g7U&k}I4 z*~Xo!boTcBy#9tW?Xvd$`p1vox9-)yr_SefUAYO|i{S6(@@;ZK(7kc>Egf|~?QM*B zZm$myems7YpV!$Wy_}xwUVsGvZ*#DpuLi=F`XoH4RevPzD_`l7aCERbnpNEGCS#=L zQ+4K`|5-R{iDkuo(S8vVv$H*C>!fDVE7?Dp_0Im$`=dFHVcs~){;6AUPrwI<#vkcs zN|>Z0Z#i{^hxrfnc0Eq08%9S(L0!ao{gE9CYJ2j z+@05ln{PY&n{SI7KV-C6_8RAIw5CJ{Q1qfyL_LaF4fVwmJQghtIEp9;l+1XYoDq^_ zN~YcNZ!{u|R%`D1{}x>J-|~0$w`$bN?{d|@G{5Dq$8YWuls7Je>)%SiwJw`vvQ4XW z4g10CdB6Yf`v*VncV6!9BfvUVH5+{%rI(|+-Q7tdrtx&+FJd;j_9A%ep3;+dM!Qs_ z?K&quqkA5Mts`6t?zi7cT8(JbKf#k+XL57|QUD&N4Hlr$TR!V&=XlajlIdlVjG_l? z&67S8j?}~Bhx~`Zy`VkQD49&m@L7CH|LOmOUOBT;Lf_g7D_G~%>X;p!?MAfvSn0xE za?u&h2oL~d^hXE*1|Ldkx_3{sQ5|kd@1DGHM>b zqUFwMf7D-oYD;v-sc42!!I&U^qRq(}i(9#W96NumRI=HLSjlvhL;aODZITeJI1+Q5 zr^X&fIi(MZ{69s{ls9_LzwstL$GvfX2^eSNn8+YqB%9Ib#;cr#MY64u{$-T}WW8oM zKv{u|mG7I&;Fm^sidmjOvV+c8D4f7Z1NT%nmT31e*qR8(bD0qI#F_1rEoEwW`-tVU zvMBV$h^yJ0#FzE5^=JpdCh5E<3Cov3QPW*J5*T3|o1S2*Dr%0-(`>p9Ozz|U3Od}u z>KOC{1%t4U+5qnMG&K~EL1#^1ml*tFir>7=LMuDShxRj2X#H;T{FAn_92AgBq^0QQ zESb8FAQeYXU|3xr=5cUnyb+RYji^9iEqHo;eBLO~JGL(nGK+dCqHyN$%IOt;7+$d- zP&S3T(Yo}khpiqA!)qaITlDg+KUs0>MdjU7e@p1}Q&BH91hMLlDpqC06q2>I?s(>$ zb{wEJn~GZ8#l&!LnLQA!kO-a=7?*3u9!9LbfHaVJmT3vC+r|@O5Zv{jjRl|iH=w*F`hNU>sb>`QkrJCQAV)E7?cJD>~`B!blRqI`OXsf z(b{KM^@b{$L=^0dma*OB^*SJDVi3(O5KJ)EYEiEJFPZHnG0Wb7B?|fep`SuBR-Z zXoj*80nLs~u%i>?yebvnA`(VvvtbV0k1!#K&7Eur_)MxJ*Yw6S{P(#4UqGP0sXjY_ z40{^sS4Kh(cb$lrmP%)(6HmmfogB{Cf!Hqquz`4SU#d8QYIMfLAbzh$AM_G@l8y~M z{%mAswJ zC-}icN9IOIpB!mZj8~rZ`V;DP3zTok(`POc<6vn*6h`GR0lWajUQc2oR^my2dcAHZ ztl(^Zq&9?k=FuO(4bIGEX?+3AIY@pYHMDhEu> zL||ofajb6$evE)M;-U$6r!Y9X{Z7aDtg_!8>*9T;$T zKWY~Y*T=H~0k8jVD1N6cmV02KQ)iP-=j{MQXbcRwvl(a6ye2psIhuT%-s#*nbTGNv z*~QtSW68rxe{SEecME+J$v3aIBUTNSAOz$rOHOA4d2|f1K8DeyL z_AVCfI$H!2&# zH3~){O5?I4v(1enBf~(&b>Ew$_96(hic|Sy>18ytso^?O+&R^^&tmlGp{jjjup^5tTeTYtw}jk8kM_ z)Ep+~EyX-Ztc3NE>MMK!=?JHDR?MqnPF82pqG_WL!f6jYnnO0!60mGm#8FVFie59( z_lsVZHu=V;i8lU9vemP}!2(&-@@Ti;kiLj+!e*sv)|-d5lpF zG9>k^fO6xr;dIkURu`HXcKRp6F~V#gY^uNLL$PK{M*{+&atyT;C?8hafbKo=dbc#%r z@grwwlPvd`fu422rp@LP0}=eN$;S|E!I|!n=Ypg^Kj;_F55H2`@sy4~sM1juXS9=V z-)-+6nm*ncO4qr;m?VwN`g~k^bMSV@#I^EzfsQkBz3_JB_9F8(qv8xt49m$ToK$I$ zwM^$_Kl6=GLW7fAb|>WHT<6YyZb&tDj-4kn)1%u{|S8h3pt+H{Y$CDYGk4DmACA331 zOQvx7y5%Ufcf@An`0L)L=E!A0v-Dc4SMrt~zhk&}TmE1^T^)>638g`-?q!+U>e_>c z?LT@;!tQ?GCE@wzIh4gG5^~?L3*bSwW5Xa^}Zf&b!cFrCx#9gB4lhu!R%O zTl8&ls4)F`rg|UJZar_CC zJ#a>=t=6BKY3|Y^&ECamLFChH98HJgrqwzyu${x^BVpZ(WI{$P*>Q{8{0KXBk=m&! zlOXF_)ReG$0zu7$$)$+hiHYKN3F1H0>LTw%r?WBrWrZk2g0sMi4=O&LiVyJ;!W*rx z(+n{|=y+H;Mt|$$y|mP#kGzq&y5p=+)vq?9OVZK0RB?rij&@)Fy1cu8y!?eeef%~f zfR&wiTH!Tp!DvL}T}EQr1}0Ii1`M!sU~pr}#}1pE7#*1{B<`!jl#-c*`)r6KE(wbu z#Vj_!&4dPQIrrTO%8G47DpvgNL|-LkOk9=qWu$%B`)YpDjF^h8oQSHdM^-CVtGVy; zV3m~^p-r(bdd!O-RghII0C`jV3Mo$U>$q$oJU_=`S$98yWv7XnvjdKXcP3*UWx0_t`4;$xmu8@%r6_ zQ&OTvufFE^d)J!Jp3zB%BP_VquNVOe{UnOE<77DAr}qGZ-_@6hK8a-5;(S`q^*Hgh z@t3l=ULq0<$Ry-x7DX9s0I7x8q%yJB9nb9VxFYrsv2=6w?Af^m>7X5SDK{h%(qU&f zo?i1vEMg;)TP<*^nDy&qsv@--tE9fEI@)vNIdRkBOP13tSM?dQt#VDYINyqn+8-z1 zii-0sSXGx*=e0m_zKPMF^{b^#v^d|&D9W<(R#cR4;iP>KsDrka2`uZJMnzRCa%xAS%{wL8$9$kij#6DQ8HsWh_a>Fb0{g4V=x7L7`)LsiQ>W!aH}(2 zJv1?em0atx&8ov5H|WT30J7D2y9MYsyTrejZ4wJ9OIGTnl%}wjzY)pbiKULdk5-~} zth=n9UZ-2sv~o;ZaIt16)-3s|a?O$w2sTSTL$X=&d0cwzkZqQHh8VMKhW*mo5O0=V zJtwm&0+yWH?coabbC9G*?*S4%H(ASa@cx)xY3{%#UH%qtgz5CU!I^(9V8E8Q+g-d0 zi@&}^$skK~&@$1usW<0cz>{X?d^j7urv(^-S6f1j4=bPQ_55}grFQ=AOGii608zVyK<`$4qD{x_mhC#fPdT49 z_7uJB4+fk=Dw(Q+*BVaey-i91Aipm6{+7?qEJ9Uwqvfgi6q|GShD+g}52SPgb5=UW zWwn!D203+u#7KXZjr~BrjeJ$u|Q=7x4VuJ_s0>#}iyjX{+MrfMBq zv}C&ZRhxZr_ek}e!2$91w_4)T$cb?#0&+fqxZRg~WRQw~NQzMmK+UIv?s#Uz*!|4T zyea+;$Qd>pMWbPa;A+XVTelV=*2}vHU#wWZ)s#6>hnsB@KAnG{yFavXf-2szU!Pyx zw!Tyr5mgv;*who?b4mq~8Pnkg`M&X=(Xd@>-K&3mQTwpc{OhOt_1bSA+w@_n9XSx= z_dkY(A+GR}y~FrEnhiTz2DG-c<`=SW0)2d7J_4v~DCsjPihrMlec;>u`)+Ik@R4qJ z9*Z`R+@WP=#)FxyHlUn)Ko)ex8&aSDoaC^HeN9ILcunMRZNB>uM~Y^@3dc4{9%W9{ zbD8qJ`6A6KA*%cf{$*H17 zqSu)f7Svb>A5!y%uX}Il1h$|Yh9!{YOBN@=@6JaS)+cas$T`kt<3WE)=(n-lDA%a_ z?<>(1QzD1?d2Rd8hsQxRJP(opqt}00OutA8U%`bjg@~WA|S^sIMsgnQ#yE|SodZH54OUf4L; z7RBmbBXPfovdtuVld{oW*1%%A6CS_E2aDh4ue|O7orz-UJ-+M#;6}7%|0t<1uOIG*p5iy{Bxr3$t$c6Kl*R&&p^S|ElXK zdm>P($pv&*c=1TAV~bW63ZzbSDj99{oYEG(pF42Xig3D6RA&z^f zvHWnzfxCc{g|B8&^q`o z0Yy#x8@hR5e}-%td~U-eTWi$x^fYJ-A9G?31_1hNG-Z=hbwjWb2&~04OFmjb5BsnR z9t`NgxlEROOqUWK;YFrWKdPCa&X>W3y2H4F2qL=T5s^@Xo}eAq0le*}Q+V})!E9Jl zSapea{pfs=9+jHJ#un<$p|CYOBk+bY|?L6U=Yx*V6knXxy4@pxO(~Eo$1FG{a zZH}5Xi-ogiSk_GkgQPnR)?tWcDwt-lYI1ob)XEEy8dFXS=d;tfq z;I2d-=n<04KCwHQAI7Dkz6skBHNd!1*;TXUH_Nil@g|xvjmbBgfR+lj3@N2tr^JH% z^^2tz0|t&M_a8pE^tQHsPkcO(tiFIdre6 zVMo$k|6&ehvlql^FvM#E&=p@0rwHkofi@$zDccrb=IV=G#v-jMZ{y2VFu}D&Cn8;0 zbl~kjEh5j!>5A99Q}J5zxu8{-wu%1oYj-oR=GW}*TDZLyS;&)==E10x1;n`S0dvU6xq+_=(hGgaNL-FnZoX|QER`6h_h;E3S< zXHP40|C8iP#xYo`318Cyw>w5iXCT@8yD31s( zdXh#y67m_D&&3E|HwV_8%BCJon+nnh0GPu^B`W$eb7$SJhfRN(UZ{qF`%p)?$ zR>UEUWB6+k%K-%zPhG;nSw?%@C4yy`oTrzq{QfA*Z+A!^V!KgBxcH4_Q}VW2mTV5? zL-XJHKxb#WRs1J?610`!D)mFCWeB(OYqS(c&!Cq$q4R{&p?3Gzd^cY_qCOa|u!`v0 zZ#{yjSb;UW_m*?U5ymrO!^*mR;>^vtQ+$lf6o1t!9>tg6$*_+Fg0>P;3hi;9pD62!-5`lG2_5>?F9ID5xU z?NsqSo*--sxB(Mjta3*r)F#Kmh&cwPq*4uBHHkH|N-Pw~DRCYBL}Eg7NN4tuHl<|!)`ooWoc{8z7y)Exx!jRS4m}Gye>1YCvs0e zvzR%+VcOGbKsqy%Y8@9(`Jwk*7x|TP&Ws9scZipSi)1zWwkk(#hMq`dD+G2p2tu&2^gpUXPnBn9kfwV~7A0s%a&`PKehbYG` zCX%oUEF2I`vKhRFzHWi9W;SSQ}(`Xupo$O?@lw!J?G7%K&+-CF_J3--w{5RHcc z0Y0y-R#ZdXpQ&PV{)UUuYu?ON0#$qSsWucK>OO1O0zi~KkO#ClZ@x`2;nP!bA(EL@ z;Hde1L{Z}NlNkrjt|jv}v&>JN;YR&w7p>c_vSj7DP=9sa4G}TRKlhfD&0B(>7iZ^Y z_G5A~e_EmU@LsAB>oqD`F z8cT@4Z)`GT z6&xz4nT<|nOtT9`Q(|1k8q}1CpoP>Oi^ZH4HQb~hlL6-ep~pk~;lGhspX{+Gu7At; zeYs`u07qkH!ex(RJt9Hea^{g=F^4n73ea<=DMsLVn59nK5Q$~o?!HGme^VfbYCdUg z=lBNUDdbb_M59SGq_N}(|q=l)R;Zv&?y`fnXvNd3~7O)VY^M(E{U@~2T}>e z8}9h>0a~g)4P)FBr1`M@^T+HS7TPZGc7UIZ9ZLRI=!I_x`{b8Rex}`M;YO7T!*;%r(R)_Wdiq}A2{HOMV7|h7u zHB;O$-vNni?dTokCS9xA(u2C!s}ujcLs+WJ*$5fPo_7PPRX3KK)YqE8EHHw(of*vU zFoe0i>4A4ja{%8#gN07O!H1utkE2iJz~F=N>g_h+COatqjJ#bj5raIFQznm=R@EKE zqcgm^H|X8%{6qW6Qp1Rb>$tvn0wFD5v&|Zy&m*VkFPYQ*`(_?D98nF%2HyiC@Lj#{ zjt9Gi(CxU37UQmhK%}e1lZ8Wj)fll9@yqHzh@MoHR{eL6Hf=j_9tBlz64ec2@um6($!3{C)xkN$L$Rf^;~WQirU%5Z*<*% zQv;|454$5H77!Bif_$kqQqT*s50<%U@d$Z$B5h-m6PC5JQ<&41(OFqTnc!-GQu+8` z6+92(C8EMtV6^{2c(*Lj+@Jh%{<~d@e}duRLbJoKF*=OC^*l^&Wpwz}CWki-kuy0| zv%%X~8y0L>7g8gOk<3z_BeQiv{_eem#WK9_I?JJbxtZSqWQO2q!z*DWqY@&*MDh#d zqyA;dmpQF`@Y#gr3qIPYd-}EV$L@O0Udo@P)ixM&X3v;8XN4?F&e&aZ0yRtskoJw zR}D8QXnJpTWu>)JkJmPR=g)KhXz1 z`A^XY4LZK+eqH;d3*yuZ95cO7JYIC(r|#k3g+DZ-4mLb@*IQBaqv(va8)U$+E|(1m z?k+QyA4!&U8|yc0Z=BxP>QH-tIF|59OCR)9@ zNVFf<>1^U_!;BL<_M6d%cJvxS!!P>DWhvXuKQnM%YQ z<8X2?qPdS|R}FSGXVbKsm6Jw^+H##h&D-#YeZ@hgBckrpZeQGVb1L47CdT`qhoVV- zZvn!^tWH*`cFC|Iku)XSMK$ccs)zg4L%SRVe{g*6tq;Q{UppE_f75j8-2sf5X}`QY zV{CB(uyF1H-IQ)=Hby{mn724aigy<3>kpj3ZvLXE&f3atYCZVw##;7lFtE%xo%C2u zd2$^wt!!?CM5%*t-y@FsqoY_Q^`5bjhmmZHGe5>ADNrA`E6ISSH~@IGOam`BJnsoi zJrH(CB`N7wNXD(sbSHMQYVpx+TflY=44Qj?2K3*%y z^M5ByrJ^@89g{>W^HaMqN~XOWr7o4C@o;NIUb1@@`?L0z$K;4sP07ompif_+p}<-G zyiH|0Ba@`7QhCK;_vN7yo3NRl;1j(G1wL4?1^2Jf11>ufopsP6~+5XluL@}(iXOY8u8^DaH#6mC8*}oC=di!$*ep0dI>$3DR_xHA$dem+XRSV+Oia7;EZB z8If)DVwPB7otg2K1ED(yDn>h|POk@v7=hLPkxbHhRFfJ_oLrM`b3g6X zwYnwuG5y-|-aUG;R``Dp%rEr6Rr|Y$g?zKE&k_s=cInnv!&ikEw#$?@pZ~h_`HSVw zULZhQ=C1R{5w7!RPs6ixN2)vzEWae>{8%EIXpQ?WpwS&xeV(Q%F-B|m?mbwmo4oRD z)j3H9oP{g8IkPx1l==d#TI+ zHd?uQr9TbP%1VKmos|X^dyrL@^dJRZ586c#aAr0ba11;c*I|?Lru$sB;Z=NUi?!@O zqpZDhwQ{fBX45ZlLt}WhB8ePRdMiY$1DFz*dk*I|;9R~n?ggG%tB*sU&K!c#fR-ln zt&5ql?oG}bD98_FW$y1U!NLaVWk(kWZT(;3E-&j|zOx1KfNaQ9iPT9W*_|1BW`S>kM;9|BMEkE@Kq61_PBl*Zd%-B%PWi+?BgZR~Hs9TlfR}+S{ z5ASAn#^aC<(1Ev*g)q4me*v3>y2f@)$z({Yn<5$+F*wbp?K0CIh{8VU0H8;m(D2R9=DR-uD5Lx=6ex@OfWmwvr-hr*>%)T|kKg>) z&dJ4RcJtag*gxLcKPHm?pFcMkaNfCblH=L3ST-(!;n>~V`R7io1jB??=b)H59{Qfk zPyf_K{tJ>qH^8Eei4^J0hGU+r?Z#JWh>f*eP&gaZjt;kIGSB7>ZM0R2%brp7=YF z;>wf3HKvx;i;4aF(~R@ddO@_<;1-!>AfIg}Bc|nK6B&8N9BxO&#@J*NBa(0*Osluv zy*CJ5#CAaTo>yALuR;`}6$UylU?IyuNWi7WpMH~Nc2 z4zSlx;G*QVRAuOpu`yZ7u#nq6-d1j8iJv=CF-A6>rg9n{%mj=E>KAr6(0T-P1X~vh z&^gip)@?UY3#xn$z6Gl0pd_$Zbfm=m*l6h9gbVnQpEZBMaFso|{@%le9s)b1aW~=e|y+>{wb`Vn6gc`|5@tV-7A!tbdn|3~sz0{i;8P ze{xd{Netv%bDgS+{j;yr4=M!8cT+7`F?6|~Jj!(sV-OYunGFrX6PqOIlWBtGR zy(D0jQ+wt^QoTzi-2}YXskLjESHs)yd3EhL-MPJV=e5OV1U8cbuEt`(crVoas|F4l zW{3*e^k?2ERdKrL zfEq|ox#!u)tO5C1i5a3^mp+4_ra3IZUnka}ZS=8Cf6uU#aoksJl|En^gbDImhAq)4 zSc&#ipr5fs-o`KRETV3>JIqwp*J1>_W#_Z?+LLqDFslYni5*7A&KrP5qP;6Ya%0>s zs>A}7JN!m-LQ~ptLQp^~4PeTPzT}FAAi77a>G=#Ut-c>PWa*i4dNQp01w68zc{IOD z!>X9w28s<8ZzwuT8qo~vrLCmMZ|BGjjqSq0m1WV;0f^j2)hv9FaIsgFF4fX_53(m- z7eJ|0477FM%fVo`3_D1Tp3a+NgCNLazy(bnszR^{-o*?RPbt}tgFF8Ge} zY1H_#{Yvz$JVYfgEq-iZ4Stf$vn+r7_FH)ygLKhm?UP32kl*KwB2#JCKK#7?>3)5^ z_OXZme)u^;RA(A3%9XV4*X!R~FTB#94@(`w1*`P``NK+Fas{U{F|5cQQH*PVUu?`A z-bA=e|E@_xY$AOmla89==wAadU^Zp?Sa=Vb6eU5kBxsh{X0$4k_kykeu^sgP2lW3( z^#8~7|L-DaF_{m0gDa_l61?ZEVYn~jb>sb5^Z;J?zg59|inD^JfO{}Jt91KhX!y22z=g>u@S@DVnho=@FY*u9Ui`{4`Vz=S zMNB*8$ByXm(;+t$q?84*K=M zCzq_j&dsRog=2ep%`}#w4*dD~+zrniZuv&?AJhcm)NY04dFnMwM!nlKa=Z}oJ0u#m zVIt{tVJQ$;t`Q}vZM~6%NTYF}G4MCWOrhZ~!f6mZt&g^%e`;RUhS_l2q&!i8OrV$| zAfN*s^kE0k{g-;E2g(6>(tD-XH~SpJUAcO)^0@o(K@S0E+NWvSK8Yu-ZkPUCeL#;cvO%Iq z>pfhleqwax9i*?%KFi}Cd!<9`aR1VUDt>pbLPV9G{KC?(4gQsr0Rq|oTGfK8jd$4{ z!r8(Pnqe9vy3Q8I58~xE2PT$K6Pzn#Z%x|0lEBvX2T_SI_oi;?#>$nqn421=W51nQ zYE`vV3^7b8AH4jx?e}lrAp{T>^IMf=OPav~pVp(LZ_GOc-iWxe;uYIJ!o;4Kis}EY~}1{~X=toGo$l6w}MQdF3)2 z%KVy5R+~3f4sJBxv?I`WM}l69Nc&B4wZkpM<A%$0SDD6=yKr6J1O`7;sjP zQWXXPWV6a0NkIaIzsogP<8OoQE9key7cY6D>O$sCQCz*V>?m6q@-*$ftooSNNh<;?_~i5_Owp&6Ob z&Ycp=B)0L}vmpy*@3IJ?S~)D8NJt0>TL}9|^Fm(KOfpyjOO3ng`CQZJ&}i8oF{h4n z8KhY=lHIQ$Up%( zoIcg^=_isdkO5t}fL|R}Il_z#-6@`Xah*j&B_${Izz|T3;+w|y{zfhj6`8?tmrkA1 zL*Kfh&g_r!dv@e}frcx{5mtg3H=>XCL~3UT^~eV!U}Oh*d)k{swX)kq8eLBM(;y8B zh*~d*sa5FOZf|gKrSU*Iyy0dJUSI^JQ5>A4J0t>@L@M#3rON7GkwF>n6*#@W9rqa&wBj}dhCcW*!Wl-Bxi&KK31RK&7Pf1p8Jl7&)QV+ zd0B21bN}5E6>b7)Pnw*_NPQ? zy<-CxeH)*(sw&nqG_5Um4ATN20gC|3*fkyt+Ry#*TNsDFZ_mW}{Hjc>n2#TLKu z{9bJkc)+{VMgb+R0;Af;>|XujQ9g2qF}+wV znqz8>rVD`2dES|$64qZRGZlYiVn7bQ0(qMnzG${q=L8;lbD%y*el2Z(}I)Mwa$SEY6w#k^wyX|^amQQc~CAJxqnRY;ZA4#*f@%T73 zoG06cvH`6M$+SOSgNSd8brMsJ{=#fpHO|dsBhEvC3n`A7nZ^MgI9XywlzRB5-C|J$ zW~%?KsK(g%$7L*pG{;bHRqjxYO*Kz;6eq*oWQGjdn6u;tr4E+7*=Kf4k^6%SX4=Py zhUDw&C+9+oP4M>D-(9IpoHqlf3N~-7|KO9a&y;TuKZ!0OoW`S1lF`%#Gq;nf0q#x` zFV73_HzwW_>nOWj($(X9SN@de^|n^h8}28uEXV11k8tDw(UteP7c1;yHpG#n;r@pP zSR#k|@JV^60y!;?6Mp+IA!RKRAc!=wFS(K6MbZDn^s$YN5{hF){uNw|{M7x_Le?|( z>!-==3>!)nCipa1M}k!(%+=KAl&pR7xIn`nB_8+WnWB0^A7(P#n`&7R7(nKNO5I^D z@Cz&(*VxdJ{u09;b=z&-Q`Ld*w;P#>i^!kt==z$%F|4xoNu>X*Yj9v#!Urst4?#9- zxJ@T{6$JYAN;88+{*Y|wYcy|VLZh6_$QKX$DnUra0<2au!Hd}w&+Tbemo8rfwKsxC zOntZ<(3diU40PoeztF?SU^ruN{swOTtLq3IG8-l~DW)2+>chu!@npotARJ{vUO4W+ zw*s+G2m2COkwZ>3XEQv`yS4s2!z(Ku-CnU`69V4Qg*H%il{9ej3C9}yZsRVVwCQM0 zSZP15!zuU*e%y5Jb7D@rjbAHnJZG%Qrs3-a#P60QB4zF+k<1&a#1VOwu_i{f>oMXD zA=gWM%4uD?%pJU2wP#fkV=hYov85cr?7KBh+VO6UNR=6Pl6Jvm(MvuV^f{`Ioreb~ z5<7%xfL=<}0@71RnkOW4)^xHL{D8q=Y&NBvBanm~J=1IJ{Xo#c7f3doWzNYNUlq*~ zV)BXO2$d3D`F(I@JP{Uw3Xe3W&e|u3#9^+V{M`%aE75TW(Q`Lb-`l{O{n8+(-%v1l z;DJf*YKEj;)f$>weFW1{>1oAy?r1x>^t9~VoY*O>{<#;E9-eS7ybJ0EU9_de8|wDu z;d&&>CGiR1-e~TZa1bI?)gQ27yw4aRo`_f|Y9jkQ=Y-h!@g-d1-3-kGFz5wmrN3ip zZWBZ<54S#cWpjqV^F{MorBpg^BA|W=KybxDBu}#AufP~NI6DT+#`Bc>^UN3ky+$da z#``=GnuEtJA|}77Bl1Fa&#>EDNZVap(=&DM(C(iXk|>w%gI3QALgx*_>351!(l5oS zTQFEKC%qA-tT1c4o)*XR<6wAS?yteuPe+n>2H6|Z3-OU;&j(04{OQ zZB;l02f=UE7bSQ%!B2xyH1V#y4%TklK}pyw&#!J(5Txto{V3*5hA6=forR6o7BtEa zXPUCUD>oY3fU`QaI26}kn^%9%D0giX?^^w--)i3G_$+hiZ)kg}R-fjUSZtJi?vI`0 za<$oPdLGGvz9C<=Y87enVDSTOq4+62Jo9liBP{O~l4<_B#2z!IH$w;-+5(H|cm(D0 z^3!~&+65IX%sSI_);&*p9lhF3;lYk=63#PN$Mi=9$!a@JkKHDE(?(~)3dQ2 zrqk0KvnyPxQ{_+k>_yx!nmsWSwHiJz-8J{O-Q?I~Zt>S(Dt|tdKXd9NpVp!7@0a*{ z#?CN=2E0Gl;WM@h9Y=Kd&|yu#!KE3`NGUMBksqfYr+1%Zta@Hxx(J~zbFRWo(L7f3 zUWyBAuPvypI&9sfnq%=~OBb|6@|aN&#XS*qvI~$ypfDaMA`Q$5m%IpvHd3T<-!BD- z&|!|YwN4N56TEV=$%x3c94$vzQSHe})QtR^bm}%CuAYb!luH$#Za6g{^(a&vo6nC! z|JqZ;^D5rWK<=S4v~ch&8*{uz*dtio)$_aA`rY2MHp-sg01rBMJ_HC+#E|2(UWa18 zB$DcWHAr z=AW@G$>OymcE`lAG{0xPfA*|>CN2LAES)|r6uWo8@N!Y%O@u8Q{kovoC96_Q=g1AX zw|9-hr==(Ot6J_FQX#0RKW>4neCH4x*$+?vl>0RdNXBmNa?u zIUT$D?0l8+IREo;^lUsqoWCuEE#CN8EgBWnKUQH&XA!=}{5SMZjah}Z&jH-c{7CN} zaxAeAHK}3hCDrq|)Cx{b-OzY%bkKRL*|QXFH!-q>BzmtC7#H_(&}v za7Sxp_11!VrO(KnXopqM5z{=($w8%_`c6QDfyC5iiYUp}y~YvZzKh*dv&k);JjSbI zY_+tG$BjyqeP$9yX?bJVk>N)hk5VxQw~^8ft8bcY4ze^#cvT}8pjv1;(MlA8teeJ; zzw=o4r$Mn5I|MJW)#or9qiDbM#ntvcjrkEqO@voDQV$cORIX%Z*;~vq4d%8w+(E>l ziwRWjGa$fFaxQU-pYNlp9$Ix_y>)bp`MP+>dXQ{EMd#w>;+cy!@snFFJBe4H-+g4L z=_uqNa@7f?Qe_i;$S@s+@Iy<8_u}u1Us&LwM8}1UfJSfB_&1XfzeF?V3 zF6v=#y%^xSZgyT@gGJW%0Zi z*LpQDHS<+SM%^OcOzHsUTiec0&d)x~nqSNW@?zGOifNgc2kaSBRgW_%L{`h`-}mTY z5qqRAYa&D4t#}R?h(i^PfL-@##MPCR=$QuBQH%Rx!+=YXB)$?Vhsq==bYByGOw%+v z>6;WIC3CPYtCqdH!-4yl>yek!dgfUdk~xq54SJ=VPHFyzh`|r9ji)HG`7z>5@D228SF|S|>x7s|35xL5@lYk>1Q5OBKzBqFf zTKQqDzNthPqWjHR$&HDm)if9=x?!i5Y+i29<0ME+UeeCoLBjzTq!k{`;8p^_ISc{mMc+P_Y8lYvDrUjgm`GFpqa1VT0Lfgw7L|+x>y>GOfm!97@TX zsSnHH0s>KN=#v^$wUDl*{~2Y!{rs`j{_Qt$Ge4g6hc=5rwbiaV1rZ-O5iElgCYxq4 zhNtsYKH@|L*8I_vK(}JuWLvEkm?D!s;8Z=IA3Z8iv<9@BB|jz`WzK6$cOPNS);T~H zY{kSoEo#Sn#*(!HZc&nY`Poa804BYAf93 z+aR%Z4NZs70PgT>iH5pUP*K}2fsWdYp;~Guiq=uW?{?de6AqU5Vy5riAR*adqHng+ z&wZBL8MNfP@zkflYGpSn5cKmMpKGb!i|vETc0>KT z7W)TP`(gfC3oLfOqNYuS9$HI#+x+{1!efUV^Hd8kL_xIV_@dkv+F%h;_4`1(;O8JI zaLrzTiJw1a_uK7xH{QNYkC=y^6OouXWgpb%L=@(F9^@B-2M-D#`lA9)DGU{194NqqUocmt@P2;XcHWiM{pLKPN^6@bf_aM%)CU5;su~4?UJ?5PKcr3OJe>K7^ z`))}aw$=3Co!WwRafljj_=$JM1BQAqYP0F9Xk@5o&MGvN&+Mn}q_||hvjF$z0Pwkn zf_E_WJU7Ux8KqX;o}^>W;UI%C)A4LW0&hHx&_(bO+Yssl%dm(yn>neT3k~dK%VKui zi4Mj@kOGIOI8x<7GiLCx#=YU6Be+&j(Je0Q|=Zu^xJi@F8&UE zTVYq5h0*8Edu)XaHw-hshu-NeyC|J$b8g4T3ykaJzGN3?P9*YUVw1XzQ@Z10WlF&f z9{upTK~$Xy1XPUv6UBqe_}XJP{?KXByl!<`RPQT5>a2gOjiqX?i=^1#d1?G}x|0nX zk;D(^D4$$RN5%rj2%%xcAdC-c)m~ph(L>GjR1nw4YV>OfiV}fb?xV#(0a;MMRz6Oe z4|x)mPZv!b)X)aTX76-r2uXnE_8ueYJR7%xuB2TI;$F+*P`vT6`lPYC`q7K_m8%78 z8-2L!_on9?E3F6LJ!+gMaHOI?)*gL&>g}KqV-gi4-^-rF3i*(o)3I)xq*o1u4B;89 zJ#5I=@gRmk_-fiHnhaJ4@kufeb>GK9*?&G(c`~;;7i-LC3@?N?u&>FygDaez?dTQq zuX2j;k?l*n@M)^~9q?P3^omcVzKU~A3Ku<#HxE~Mgo`{+9ybZWo6d^i8i2*TIRfOp-C;y)HaKOv0_4NrOuXwAA(G zAX?4B*T;{DPXvkZBG_G!smJ{UIeUUV%nLyz`cZDO4woX0VfQ@ge$IKfpN6L?+3!9l zkYIjpc4N8MaxWR@Ui2kxS}pGN=+$*$ecUB*Q)%#T8>dJT*=yh*4)C)<@5;3?uy8Nq zsZX<=!@@`QGlRc9F3=t!$(Dw_%%e7RwOYS3``aV z8}qXxAzLMc#VdldS6b^hJu-FP#$y}zv$3F`LLL||8TBN-{Ng2I6aje(PH>Q3Mt4DX z@X8dg0c#=7cP=|M?dB*G`|6G8*!*#Xr02Jqp_hRzP`Io*g^L>^AA65+#RTublB?uP zSlQA`IQgJL(@QTP-h;Tb=n7WpJw#h(P3A?~?-40BIBoJ)eD7f!lK@U2nVNoKi?HxB z^%RExGh&<$Uol)zH2t`;vWmJnbF*(uXK3-O*RBv!uu#Dl@dpOp#hMULkDlfbLr&W$ zAYL0=U7E&7rl+BobP2pN$?yp9*SHjS;iY;uNs`EgE#7x2s*q<4ZS?Z=dPwV;Oc))4 z8#;|uId_-vUD%P|4;7xJH}{iMT$>C>vKc2|BwvyIMNjt^l_%kYpJ(s|=g-nK>~v&E z#BPMbazS)=zu3omm#C_KjI7d;E`lcgIT=sg4k%9dmjUCZ1UpSEpnDVr(654MFbdpP zR@)cAjoms0P)gJ1E{`bj2*2DaokMI+BAm&dvrvtcZGp*`a_E|30ccAR3#_vp{1<>z-4+&{w9T z=g}%drq}aoXk2~CY8WRQUkf_ABtmUff@|C)%?Zi5qaR+|h|L7C)6Cn2j5{5t>BKiX z=9#QIOzVI&g<5ZzAlqX`&&?UfCS){#^nSXRm&~oJ0S#wiOj8N;0`ZaUjIH4$VX|p~ z`chL;EP9)Ro~4>O61q+I?nNuD?}Bae>OF_01^v_w2DRGEq!K>oeh?z*}1Uv{#^>0y65E;?>* z)>zz_vwe8;wzPk3rLj`4{%I>Mthyo!)GJw8L)Y{)>VXW=eMet%#t`q+zR&9ZUVQku z`su#g4EeLxy7ztE9-Dk?->`#At8E3E-4WdydSQ6G8j;x{TK#iLzOKQbp7C-N0=*rE ze&OQm-|XLxk+ogH)rI%+qdkC+{(gSU9f7zPJ`HxlKa7vPYCAUzJJ`~ohkxT0BrB(X%?D^7Kd*K{--`jb}GCP|?a{*FoHW?FGosOS!p7Y7r zCxb^X=tS;BKXOyZq}X9O&Q!&#VyA zxEQ!P=?-n+>Sy14*xKGa-uzH$w=v|qc!b~1r_=FzyM1|i+3H>sk^7HUIyq~f4AQgq z+RDmb+pBBsl_xl=aXf0qO$P+n)h2!XN16_MllZJT;9M=&ZS;!dRs(uGdI(g(&j0Il zMz!f=2I3sIxULKULi(hzqN1)woPJa!Zo~8fk##GT_Pu)*A{BMYk&c3p==fY_PD|fW z^qT24cq5GZf2VPonPSFHqGhYOd@pj~)KD7>)Z$2Z_td9A#$^9TI7Q>ch`-J9?t1Z9 zc%?r3?8&^F(c2XM5W2qL8}<$jYa)M~it9--sB5i*lV1>1hM@KAOxmL0-$zI%-p{gG zFwFlpor!IPj#mc*a1;~o-4}1cDe^Ia#MK_OgG1VtICCA+{x+!HGe`-(`5=s~pl-TH&Su0MFNV66A!9t*m#W!OcIf5*3bHfA&^;ZiUz@2J~_ z_SX3%MX*#InmXSc^@weV_LB5;lJt7I7~tWF?21;FobDKiLl$(C5g07Eb4+rO@bYNe zR4^O5m5#3`On>8EHL=^&&yML_ND6 zf%c?-GPALxIX(TzSSaGl38Cf*`-}`h|Kfa7FtEFdSQ_+8{4Mdn*k(z~&iifR zi8`oHN4>sSqH;K}WNLvjqW#UyF7xbDc`JuA-e*pNhK}Xk=-S;HrBm9-hKUoAN$|sa zN*ViuYmgi_Ga9!$h>4S%wA_r=iZh~-z1=Z0qBZws-Q;$Yax-FQkXud4cq#SL?rdnI z)=`T#{Ce2-c}PG$>Boc2^>2?t=<%D~qv+`1)$xy;hdUAd{_gPLhu!U+ z?dav-ieUGD{?FzSRa#z-Hutyb|9^{ietLJfb959P97elu-|g-0Y@=7|>~M4ccz5Th z5$*18?Y-aL-KRJ2<@@7kpV*MS-M72PG{o^igC?+JT6)c*gICepox`m+^ylWw-M!u8 zzhOx9{c87kA0xh^AvdFUn}^4{TkrQa52JVQ58oXe?L=6}_U_Tv-sbMxo$VG)fCi0r ze%RSRj*i}J?(G%Vb#TZx?_mG-K2pKi*S4Dv!1|-)_F91=eoZ7dnTn_lG-gF>Bh4 zqxUb5j(3mWAMZr34-U4EwC|&x!yk6Hc8;D#dk06nYwwSC8WG-a43A!Dzo?jgfBF7s zmv?S=|9I!{@cp~v-Glvl^yc74+MB~@Ym=I8^T8hMqi?L4PUqn8Z|E1B!RJ62_{|P| zKg7oJmTh7$j%e$)j{RyhB<%;_w~;H)($Ehprapmk9O*GY`ZkS*XR`^ z|F}uRzUQ?AL}+gEs|T(PKGbOU70-72hh5BH>k}j#?JD5#_HDhd})pw_-%}?RG+sG82*f0x(yRj@3$N zWaU*cN0z4S7f zY!NL|tG75y{K08$`7$~A+@CJj#Q>I8pb2JZKX;H?1$j`CQ8KA5^QxEtt zB-cv)DV8SWQCo{Nijo?Y0Oi6?|V_ z@AhgUD0*7?%UMfA*~HW`C(WPEjTqB>`(lBV*iho`nBrlE0l9;sRH$fvFI9+(H4r{X zCq(KK^k=OQm@yqx;r4v&fLhEM5`2Jh3!1iRV#cWP;f-6+N>TMzmoD;X`qNW&+QAL>=C)6CUJ1GTDM7~Q3;?(f0&Y&a(TY8K1}n4oQh zMM_R*12Mfzh{dSqb^y7t-^vQh7XKkCtUD^KGs>^)<0y&>+Y=SGBg!uy?TE3U|E)%! zs@{=jg(HtGe!S^`JxprP*|i3`Wwm{uQ`oxibbHIrVgIN$nhofB_WfQy^gjhBA{bpc zDd1VTJGsf_@4US-gw6o^VWuaZQQ@mxA$*su`T^<@Iyp@PrANu=LNAnNv!W*?N}_i3;4zdYbFrY`t_PN7Ky8|Tl_qb~e-LXO>jeG* zB@BUrOa$fN&uKN?DN>QGwEo&Aw)bXCXk`<1voxTfB7$Q4LnY&uz~QmEWVJK&3FLr> z;{tI3*etg47c5h?hH!&Lz>LIPJ%r#;JS1m7PBUg{WDcJTXi%?Sz2&5fgbo`tpX+*y zyRQAiFKlvY6UA2h6d5#j)`+}W4+%Q@rw7VMe)=G6$6_y@q)k`Mo-P<(5+ zpcyK`M-(4XARGe;1Wg2)*$_JFzbpWr0?g>fa^HV(hhp~IZgP0qQPZa^_4IQeuW>Mc z`C{_gdDF(RxOw9Me#gY;0=rp7KnLFjgm1Sf2ZE(DaK+yEwxXG#>k$_j+f)SME&>JK z1ih9_{%-@lMl$U6IgQV34nn2}OF-IkIe>Y$iHsb|4x(hAzv?#p#cBo$Kx(JLaO};+ zW@@xc`VWeoD}ylFgeSNx7c?GB_$AHUP;yobeyC{*?Kz{E;l14f*uxxyvD` zIUhML1w?8>d1_#MVJzE*GB9;4B0wHJS5CZOJUr>2&1mm9Z4akAWT_>D z?>}>C}(?JI&M2j`o6k}xS*W~OuvgVi!okpx16H12tJ6Q~hQ zFH=AFoX#NQfK6ETG_7bx8BD}Z3F&_{P9av#S$|B1L=#9xWh8f;p@!xX;Jt`7sX9%- zq%yMgV7$!&SlN0EQ-OWA87vD_%1nh-4{>Z7U>MlspzHi!nkUHgv@= z%Oj*<94vdgZ)K||*XC&T^fdcUJfHc$0j)w`_I!ItLTAp$$ ztv)LNO}a;+F*yi=(whSC>4~z9Wqg0Z$H?I635iv-E~s{S@^zo?JZT&XMgE{yY&U+!atahU+fN+`KD|jWvZ| z>Q8Tmo<(Px&nfA44fkZ_UhTur&p+L(n@i%~5>6ki=sTX8ymS4j1IybjLG0#D@y8cS ztDa$>gA_mHWUTRfOxhW2Y#(WqfZ-L-PR>NQO|9Mv>6~yTyUeM*_?a>l=`%+JaxpDY z>LONdxwh^NG>x;MwJX=Cg&NZmYuxRYpg7qz9| zbKwTKogb^xG_FVY&Alz{mSdFkjF+hiB8V@%<<<>^C26o=39g1%G`JGIuDsyTgXB+p z!#)S5d@#(zbTZGJf7G^~C@7X_mDX!#UMw)~rI-bfgVJ$X5CQDybAPOMxNbKjogzWc zwQmD_o?aq`9^IEyAQ@yl2AC4@a`@)2_Wqg>TX8*l9zEfJ#Bp>sjjG?BhCf~|br+7u zw+44+Od8HVG}!cGW2is-WlX>-xk){u%tUe}C^2@i9Q{uAGYGF&n23+fa~W>E^tRvl z`E7Y#{@Y}me^`nUDikAr3P$0{gZ$+DkpzYb0XB}nL89hP8po4qzdJ)fSI#)C&1-C) z3gpPtm}VBY!9VjUts_PQth(anU0A_LHf%o^w$5B40mOsPZ25&8gbE=QM>%*dS8|6` zFQ}?U+p9mwM8gMIK>j6?)W$k4vKs%wX@U8d=0@}&bh85S9KMT7zckmv5Oe)OV#1sS z)pC*O=+c(;79{7n{hNlaLVqYb*W0xXUk<$wv=`A@S*KO`MGemDWd`NjDYF2>1Q%W0 zJG<$GZon8O8Zfy^876ST>+6TJk;MV=)OG^*P;NLn6+5l&b$1}3f=~){OUHxwn)#$` zn#4V3k+SO%0f_aY=45%s63udYtUNrOFzb;_6g*n~;$^8YtT)qa-Q$UdNuk*Xn_B}< z5=vz6+wUFbql1U%#?#^^QeSrv^3T0*HSMZi*w7q)vY6AKN{D2tgau_N3zOB7)Ji^j z8-@M3ZAkteEr0+0GUHYY;PP>OB^UL*Fbg+oX$Ho8{drUNvNM)U-tyZE7w~7!lsmrd z>=4SbBsph(C;7n%KL5cf+k{}I)2Ch`O4zynWAFZw3p!V})OqVn=7OL1#%Y#K4=afE ztzlo`x?e)LR?t9dvKv6PK@XXbEOqGpHl9QO=M~A&C{wd)rOC8F>9W_*o`WnKi^H^0 zbINn4lrcD!Si_rf6gI7ENHIxFm5++-f8rCyw{JX$KckAYA?!GsEZ%2ysL@+Bcxy3N zi?*)G=o^A(s-u`~HEQx%u}}3rZ7SPQH1UfwX!h4n{Y|f%JQr~~9OBW|Wc66Jy3Wj7 zwFl&yXu1OnXog!vCs_9Ar$U9ZNqRXwx8R+-F#QoB{*IUrWH~a7Fl5>_W$tgT=6K)5|~elO^y6I2Lk8phXUBHqfgZxB7P!`f=NI%>W?{GO*#?- zY%ULQG6MX-yo>F=*=2=FcQwd8#9^>YKa&`0!uEWK8z+r!qxT6O6w#S>WZO3&jaM1X zN#ZyE5PuTkISpy1XaHLq=btz$4-m2UoRAvYI21hEkegbXkD#$Y5}{BdVuoCDaF^E3 z(H%^8uY79gpn>nElhc&b1YSU)f5ibbq7NtdPZ$5`$$4zqDg0l4D*4D(LYXZ*ny)#N zU9r@ZTi3j%W@kw@joxiVsmd97`=R&a$beS%*wFqmR;tBBZFY%6Wc$aq`+O%_ z#@97)6wCT(W>FPfRh3Em+2}JnZ<&<1*;B|0da0P0LmQzgXIxS$W;0uMQ8dAgHLwEr zxvt$+&ATu0LPB|+%<_wW;Y@0Rl4@SE0?QrZwW@ya$t_I>vM}%D{TWK7R#aW*ENkUs zt$fN$u1ybm|MX;_nwO}NaIb+ZzvUmCNcV3zuX(pe)CvZ1vGZIF1+^u<+Z~LMSeyZS z1zfAUw!i#EQkd|slGl+JG_RMyAJObrDGd5?o3w8P_m(~!ZsR31f*x~7ZeBnq?X)zS zlQ?O8#mtuapDagVc2@L5(pB?$Fhxr=6&Hekb1R7Q8>YSbNZf-X?#W*8SA7uMuO@6a z@H^lnM=`3q8jD6)9OVV{z2YFQ-N`bI6$bwMbwK>JG?~PpFTIZq;@8NJ&d2ExC&^iIh40Jz2X8lzx85xO*;y-`)8y1HDHZ>)w#JI7ZU*7FrD$m4Ha%G zr~O-8K{n)Sxk0e-OCo>G?d&V+UVTN~-dEJkzNYfD+qnr|P-XFxe-99&M3KDSl&wp$#n7RsS8fYcZ+^*u1B49h>tlZWO1uOZ?4=+ zZ#?7mb+1wc)Cu1V#!Ag*bOvQ6;bmssIkW{zdeJDmMSq=i(&>)JHz9-V^oT`Bz0H9k z#^+bg;Ix`Zm+R}Xv{-OP9WxjCFJh@ceSW*9Co>}B`lA~G!PQ&{Qm)!Xax-*=wHJ1r zjv;+P@aNms`tWIPJqc0IRu4FIqIn~_xas7&8Qb$Q89>=8<=`m#X)zk^{39JDH$tLc z_a^k_*S!fF_;qjjZs>$>#8-apg_x2z6|O+>s-JIki&5W*jm zL9}$#PDc`jo~7y8;O2cr^>q9-wb8jQ?L19C-`KTZ*B7}EHTCOK&KT5uuXdPCccqae zIVG~F28gOlz?(SgKwWVv-kFHhb3Hxh&@_=DX-)iMp&+B0v+IoTiuNCe+pUq4K`>7++I^Jf*KZpYJj?)gR9>S%T{>`&j$rpeXUH#-{l zN4Fo42U%Z#KcXF&4_LW+M}q~AoUMDvX;E+2GbX&g{-5^u1@+DPZZT9-ovps20+>DS~Jg|Wk;8GyrcemcXHxhidPfuKwTs492jg!$C z?4Myt6b7ZRVT+$eeS|86iL`0U_Rw@q#Lq$X1%qg9)N78r^uBayF8ypMF}hj>AXgao zKx}=T*iR-{%+H;OrbN$Mzst$lN_RZy01NHt9-yb&bg(s^X&;k(2lH{{tHaH=JDr{V z?M`P=wJ{NM1i~FUom+GY9+`INMto~w#D$mJPX!^O{#Z}m2#NtVnPJs$4_0u684OBQ z7s%a0jJ)WQI48UKF^Sef+cP*F9UmU-zux;>l+Di2RO8+DSpx;PipjJOe`WyEFp0C- zM4ZyKkA5p@HB>)xDx!liqih3p9&}G~cc7(xyn#6=+tUeohIT_!SOZ~1?;Yo9c(;XF z;lRXyi&rFlN&-ikU4yR%itRF0fy1_*v@rONa3#3x^+y-!0AaQynS-XB%PU;wC~ay+ z5K_-ea{_Un`K;D>%gZ-N*X+o7DVwiODXkyM+>B%@NMXk>HXzaKjyID%()b`*hMl#% ztfIFSSelhC5?X4V#ez%xpn&KBso02C8;+d0QD`PoO;`Qv&4y#~Nn|zbB)RCPv&^8ROSo|Y<2qkS7I^}3Ow(d_!gHn8N*6!*ot7V8 z66_f)nwvLYCVYFL@7I7XgQSB;UnAn{a@2hB=^W88EP^;Ho&kQ^BYI9oFwIA-BZ;p= z2p+MERFI7Nc`Jh33Xe-~(SLcqmfv+>|HSkxCAXX9=eQ2;(-Qvvb$QwHa}2>psa%AF zZpAgb95Dr;#76;B(2M8I+;==VPNSHUvh>o~2`!h`&k+u13>K55JDX&zie$mQx*0Y3 z>e^mJu+fL>nfRm)@P=PAXUO;4Lr>qxik6ZuxwlU~`b=%^f+acZ{m;E~DY42}EpETJO)I)VobXS)aS2^xV>2bfm^`+-Cfg&1 zh}`CnVV6@L_P>Wc6Y%eNU@pv1p+W-+dfs1ci#poI=-xZe8rb;+gpT%eZJ?0(Vlhqt5*2tbkF1PHrlBigz@1_^!c=KnHTdy~t*j`4_{7 z?8vPEM5=~tiJSiFO#op~yf#^A!j}iL8_X&-MZ7Ve92w$8++4jJ@wYZ|T=^P6FwO3l z!NJ^V)3T#YyrVC9n)GLWlAHPIy)4)^9a^Zp6LDrmV&wSAr%l+Lzl6;YiZ1}cn?CHi zGda}bxUg?uaio{?M|xQ@(#zY9WcKY)x9lz4{4(>X!hD{$jz1qMBKTdKck?za9m=G0 zJ7sdm+n4xdy-a9az9C_+!LH;1UUr}yYP!F3(Z3gByh*SAL69ivDu3ojvlF&Xi(ZPT zVpE7F{rZQl%-3IzsOe?FeVMnRy+9(;oVggXk#t7M8Jk*r2Y=x({+Ao?fkB)LJ0*f0 z?(0VK%Mnd7-6m%!&NMiCRjZ%-tzk7tSf~3jhg{GVtRUgMf_0fEQpss^j!)(o_^^#V zUG$G+l5*-g7w?DRz?{ZSi2ZYvBc07c0(@*yH&Ju=($ zRa0i6e0~kzEM7$T*}xliJy)VxDeVHhco8NTLfL90(h4pJpR;4D*plU(gwO_LF!aGPe5=7Sv? zcRkJ%Tqh0IK&jbI2l#epYTz>?}+X53$qyrk1#=Il-@OwZBU93nWw%x z?6H9?%<{^;L&8|InX?IOa9zRs!#!tUhbY!KC8}_q3A;So$WFIZu_XO{y6m_wDBR(WS^uqa#=lS9oZ zTg*(`KaXa^j#9kq1S++Uz58|NgU^*$b7?hooFgQLhF(W=F%;_VbFWis28G6Szw*Ev1c?0j!9HxuvK*=dNLr>1^{Ct)r2a0_@Q|4a?2CG`XJ9 z8NBC+2%I94^C5+eoT4B^h9$Y*-ogLy)^ju{&Yo~MRmdiShHzo1=FoS|&mMgQEMPAi zCeXC(69+3wM@Zb_wY27VSvJR@PY;Uh?b1W; z2wix6puqISH&Q;b8-D9vyKO7Ous{$99OC>k^K_hHcfb6H&Z|a4)9`gj@l< zazI2*w_|ag4gcgMV2Zm8Tm*yBcMQh%(;In6vMN(P9qQV;P#Bj<)5nsakj}f!*E?0` zJAfI#6Rj!dHmv`db8slyO@lz=_2K5*&OY?iI+*}pcNDAM7HMz2-QL^X-|?`v^iYbr zxyZc`!=6Fm+Kb?D`#nJ8e+YqV|2%Xl&NHfOiJ#)5I)m{oI?(CWOXhmDveLRwFfPIp zwo#%3xj>{hJZJ3tDlvM3hTnV~ee=zk-)ip_*RmcbQrS|f^vri@ddpeV+#___G{$0li)$HZ44% zF(bgdx?de888qZx?N{l9rx;QnIr8k5FkEdm&y$#)`~M7RR{gyNWAhsPSyYI;NbR4E zhtzt6M2DUA68F%SYd_0*yq8+2z>=AI`@HWba3kem|gVln@Yz^sTP5xUpFxl<| zRf$rR80)1~y=#_Wfs)h5vw8!(4ckAjTp^gI z#GNG~4h%r+esH%nl-cSROPhHROFcB3O|QBWf%|o8AL8aeK7IIk6J92H9 z;4nuD`45f{v_=hJwX|kMD6HWtsp&*9S^HWS2 zpltZPH79#_WDm*A!0w9Mf-jYldN@t4Xg8)1a4I)v)E>$<~W;9j11I51T^+0$!&-3$!!L;sz(X>WxIC1#2tM)uT z&ED9DF#Dc6Ut)Gj9=>nIHUUjtw?&eH-~2?f)wy@X1?h&IbgZ_YL$Nu{Y#e06h)D5n-AdrMu-uv(rY_6EoScm)0s2wQbFa7hMSf`NpG>%p;+y9tkADLwh6Gfm}c{94tqsz%*Z zF40zQve@?x3}%$O_a0STyCUE?2n-%Y%5zbVUCAs>C`eSlDdOVh!DWzhZVC&;)rn0-IXVU!4-$ZA+k;x5J)p7{hqP%4X5(*ntD;|^f@#=vN$8Z3iimP zT@D@|=}Fhq2+GCu&b$Wnn9`nIFwtf?1+Zmy%h`HHjO|6w6M5f2QvJMg&k)mhq8*$b z1Jy9THo>kM|-O81UM$1aJ7!N)>ciuGkfS~d`E$?JWNk&{l{(%ml!d+z(;!b(*;OM#9e`{w7? zz4kZX_+m|Y9aH78ya8=XJLKE=vk`LVN3Sgjqt=lX<@7T~xb&4m^q><%Yw0+CQV}zm zU=qG)%_Fc|{J*j%xX^RMS#ES7EozA3q#mjYwnaaiH$CxYGRGRRw2Wz2F6ppLCs&*h z%Q2e}=>$9Eej-VJayl|!^_SbjIR*_K^PB`X?~fjq@>*r@-8rXVju8eQW)*is6+^o$pU-`(rOgCCFI+`2;z z!L+c;t%LpJo&Dq8gZ*1~>v`wrMP-HiPY)fAXlMV2U#tJNgA3O=e*12F_pqw&YAAVg z@OI}5c%1ex2^v;CRQ65O>;>InFA9`w`Qaz-63-)IybegZ(bn zqor>`I}z37496X(@~`?$+AS(<1@Pq|D1_gb+cMwC;jR3e6of2u#>}qoJPP)dOgZgT zEyN@_7T?XhJ1iJ2Srrd+tIU=@FS4cHUKJjtTcG0KmbP~GkKP}J=-KL;0(aWiF$i@$ zV!j{0)m!roF2R#706MaWFa7Foqj5!eAFS}n{KMZ3)b7Y6K#ZGR~*A#xJhC% zKBA`UJY4sWlXM#RItKmQ_30^Uhz2-?`L4nFd*Gz?rHw=uz%tw|DYK6h3q@nZM@t6t z2u@D}N1MbJlNw(zZ-U}%^pcB&1XrNkhn5v??+^8(?CgrEdVtH)I^22Nb>=H_YTw_;1BeR!?*v++1M(OPA+$k(r^&o*yXpLrZb75as zVz0n3zT!3I*{0HuCdGY$gW^KY(1%IaJPT^qVkH8_U_U&2rhptp$CGQl$KJ8av>G~- zhU-cjUB>-s;U4(ot^T>=bq7g2nvIR?aB|h3)>i5zs3=B;th0edikd1gH{_pu09yU| zM&#BKHEG3x{jJ-K`8V2K)!%a;%?O`@D^f7nU3kaxXK{Zue;R()24bo?<5-ZW;hm%@ zwuv;eDLpbns@LTBC&RQilh;a(P!MxVk{#kYIj)(R7i3O8;tp5@e?A+1hS>yUywv}o z&lp!sFj_&b?2!aYI7J`^y}d}nx10^q>2kwZiwv%17Gm&{5CV@!%@Nz`X`an4Phm4c zJIEc;$%D1#Nq-u#Ae(=?OeaW`UF#)Ka$rgtf4@IMViNP3{|p?D-nPN|b7 zXJfl#6C;{kAkN8{ki=8Z8udxUNkP_mU&b?sT1}p8Ce-Y_H37hbsCJnw(*)vfHyP7~ z8EL#Tc-=3HdUU1Us=VBJy}Qo z%7G>WWZe2y_*4Dac&?3#vMxw4ylN{YRbcnrOKg=#t81JqYVDzfsT@M`dIF#vnuM(g zFx8)8*V|7@`lEfz+>jqWm5ezXVRJeR{>Zdd4I@nL^I4ws&zf@DjpzsdGU!jIgQVEq zIGGGO{jBr&Az+F+Ty6N>4aM{m*u=mFovT2g+;rLC6Sg(wJ%;B7j>Cxh_Y%pHGv#`VOcADLq^ zW#B_j=I9cEDK0s#O9U%cqy2e;b5NX6dqPE5aWCoihw$k=j}gr$T7B%!Dyq?6#Y!3p zFgN%H2l}5t1X=5zPw3UVdQwOBGU{VJdgAej`Gy;`!ay)bca?go$?6|BkhPL})lS0{ zmbzaaeB*qVe^`Dh(Xs)*3e-k^V?`vvFb{vvF#9shOM7wV$qLukjk646nf2(LgZf1b zI!IA^@80TT$2>Vn;tLBvGIp(WD#WATs|vW3tF}3k8KF~Hc$mYryj~X`PuTddLX#MM zaCx>1-1^Xlg!MX%w@pTzNuq9pu>62w=jkBzQh@d)vnMA{=64Uu&Ow?9QA$w8l0w|2 zVf@+CQQ8i4?p{*z3B9oLTwSKH1Mj8?g-cJ(buw*4Co`Fw?Ev9dCbC!#qRDq87LyqX z=we)%PW!A(f3U`P+!srp$64p^&asrpJZesFge~Bcs{RM;6jrNz2j~i{K8Sb^i;MXtyNCqx$#ug=4iT3)T591I0v0+HC+@r&|U@`={zvcf}mL24{hjE?XT*AsQ zqmPO3I*~6*j1WAfr>%d&)m&LL?SIA9e6Ut_HO1bJWuISnHyM+2dR&4**fuh$rs{g` zkuWeVns*&qV4Md(fW`^5Rlp9l6;>hjClDLl*I!pZ1q`pt+cP*7`nbX;kut4IH=iw> z*r6FR{4Vp8JV5g~QKhw?>Q1SRbHC=;ei}u&r8E|S&xG!LKxkpW0%5h_T{PeDo{8J+ z9j$-g1k~k7NGZE<@}wDa5%L+tT@yw3mal7$I;h|HP?;mdY3W-EgByFa>>kn=(}N_= zuDNtwVFXxL=zeV~p9TN*hkbaOXT{_FgUJ6!*kOuHP^Y0-4TQS*Lom`M+<8Lq=Ha>* zClA&-YESs}aLye(e&{-Ik=^EmPjbVV5K3zylHzEs-K;+V5~{D`pr*RvvDs6g&xRVC zfGmO;y4MB$bUDB)LR8PjY)Gqsq*=Vuu zJ)_rYJBbHkw}U696?SvjKRch&jk;vr8DI2XKXH}#4h>Rh$@^lb?siFJTj7WMt1f+z zP5V_mFho1uUsiy=<5B+}y`LNRybo?WEj9M$7!Lf3ZlDg3(0QLYn;Ij~+=RZF^U{6$ zEzd;qt&T*!n8k0u;XF-wJ1nPVg?97yE!f9xagBM?|e%_vKU2@3%}dHuwvpy*5+3A?IIKABjyA~#S8jxcoc%==eS z-sC0ALjERIerpI_C{dxA#EjW8Y9#UwgwgbRVF+SL3^+m;rd_6qNhpDVK-0OQEd72YyOQ*HtR@DtEArG~RR??e%s0(Tc|Mt)H3&^%=1grccX*Oac|NJu&r@K0 ze$Vqyv3-e4NHFvAI~+eR(d&{D6(4wzW;jSLwyt5vnSWv61w-2~;7ObK;gdnv!z3Od z66x{zIao!(|5Rg?o3B0l5oTlRj z>-7=Z+~hc51}6;QVe(Y6#ih8XvMHm|ebPaLeW&zLU@1)yk)huT~^;Oz_l9wgEK>TXSRd{y~$d7v$OIS-O!{vn}X{Ck?374>O}7o zJnjbnAA4`w*hH2jiq2Q`E3$>FEvc;43&!B8GRAQgQVMETm3@C@H2xQBKyV4Q)y_%BoOi z>hUy+q4;ovVfd4XcR8su1cn$2C5*ftfYh%rFf?m{vP6Xdure+2p8}}lzMvl>2AsMB#i5Ym-u~TT1udmYI)y>xGhI0*ABQ*k;7#Q$GeNG00NhdUT=*)Ym*cS0I?0)S6w4_Y(QqA zt!V%)H+HeeP{x4?S&&*?kIhk5s4`Abvbu=h`nbS1!yp&}E@7EHZIDQ)6!(J5F>2-T zjQ()z;;F7W9?;HGNm@j%@FTG`Ft{)D2Tm6|TVU7e%fIfN9_=5!-da|TvJjlcFJb_6 z$&?3(;t?9|C?icFUw*^mLAxumg2UrYLt;Nva}P~S zDq^4)6fs~qi9tUHqO(8gcfRlE=^*7-hBgA(ld`S2o{eS9Syp&PCLaal2H%gAdlJ%m zU*;Xv9lBQ^#%$QcN$^Wv(h1!Lv5lOiCcX~qhPjx%Cp|~Gh}2iDF$&Y*+G-_b4k_IR znGU?f!Q0}QF6MB9dmNcZo~sr08^)Ur>46OyL2KO%Ed&Hoy zb%_G9u%z;*lTx&MWj=gnD)bTyQL1`IHfg6oT4<&v)N+;6oX70tiJ3=1I15CnjnS*} zZpGk6OevC`i!|hVt{Y;~UtkG`3Rov!oQM?WC{?5>m$YNiznM)^)$@j8nfZ{n9P*E| zwDOFXL%A-^qC}An0rJO*dg{0DVy(QI+{vbPt@G^-vGQ%Ik$=IFw6rU~y%DKt zlx#)mHh+5)NXOdnh4eYNMhOKo<%(YOJmGrTk0d2cG_^sA))95%B`WxRQ?+o+l-lMKErGWfTM#4r4`d zm*hSg4XB+<|I#H*h|=H6cnYy~b*m`K4(}DdQw)7e`8J`|H|1p`>Zz~qED-kaUltK`U+BH9oH%|ZS`0%+eH>GaqxQ4S&1%Ca)+MYYL-7h z!!lF0p*n2h@z7!?uJf-uthbb2Rha4rK1fW40Bq`AXjV{ z!y?yc2V=LWLoMAciq)3tL!pO@VyGpG_7>Tt@n#fEn+mHy|Ghw zp`R0_$Hy9c70s3kB%o9(=k%;Trz3_r%wN!ruWB6|KO^$Y%jyIl*@e2mh%7~n%w3(7 z?Ty{J_WhXrC}vH~-0o1AoJg;W$z%$2Q#>B_rA4{!iWXaq{D?vz@`DWEy~w~-MI@P7 zdsO-B;qY-0>Vsh9NCT)50PTNZE)rBvUE=|Yj>X8mne1ag2qnQh&Su~L&t>z_V^d&cRnDW(+dLzXC(PIHjV-KC_ z-jb6A3hzcHamekFcpvqS-vqZvuS-IE z6}xonBK3(M_)H)rEa;CD=O#q0&M`}Re}(CqV4M{hIMbo$OJe+o>?^|yNxV(w4EC}j zv4l#=EcPdbX52#*Nge&qkq}>a9P;mWX^M7~;+SM+Q?Ab+!!8Wn+d4T7b~qTN8lRr> zNpBfC*7f)?FxzCbR3>=+>1cuiZdPM82uH-4VmF9x{2!-Y`Vf`t`Wd$G4N)(Ixu&K{ zPlUW#c9xaXbzO#J-BeHhtgIV;J*N^=W?J>=b_q#DLXT3oQ={K*n0`OAM+u()x@jMK;7t?)N(DHgD?3vCvYbL=zpVUEw4wSxVK zpVW!3{xtS(O9n+X&^Gykepe5A_%hOCx0n)Ni*Yb_zFzhC!08CF1V!95=y=-O%b%k|iwB}ob^hFxViDcqB_k8H z`?<6Ju-QSomgyu;@S2vWAR$LCXuT6-&`UloPE{FG$`~GYA;3NcPR-AznUKS6Zm_r` zt#)?HMkua$fh9T@J@9g9M!1aPtDgm~!v+y8Ifg%o#t#AN%8`ClmYSjSv$KX;^JS$= zVZR3L#B}A|A&8Qw&#jcV42$H%N*JMs9qM9+eya{4mYifeD1&-NvWR1*q$QFk{k^5W z?2X~9wIJv~^cg~uXB1#aDFpS>44Y%d9kH&OKZNe!AE;^6#}D(8Q9(x%OTF}msfhd4 zw3Ha+q*tvz(?85r{-IGpSD%RmrkExGQ$Vc0Wur)%4gD@r%ru?@zZ;T!Hbv=koL6-4rlIwi#D>3iR9@wF$YA&P>2yiDfX|v( zo`l(vKPGA7#MLeX?DCogO;!@`J*jGqylNCRIj61F*_HCzYRRieP6uVEDM3o$5FOX| z)g0@m*{dXZ!<&hvT80!5ANzWBRPw50FgYo9SH7zUSAT);YU&KSfaxShXnOah@zH}C zzRW#nWLcv^=8vy@@!I(kNPj`vV!-6G0N0)R!~Tt-4kG75=}dh0`H)AyNNsOF@B86T zMd=ZWJ?-PxMjg1JFBEKrKkW}ug)5&mCg1=t)=aRJh70p~DOqlB&oo2?7xg$YlUb>D34^Ct49l z_@8kLzgk)=mhZ$QAj@((4H9q7mU(NUbGX>6l9W^uTrHJ*#%wSXu7i-&Lwom9Cl24x zS<~+%@S})@ay?~ksEiNzX-2(}K%uVN^@m6cB7Y#^yJH!Ko!H&CN#R$=Vt! zq}nN{kop`eRj1+br+RB+qp`kP5SR;XN6`=ZZ9rCRk@K9m{4K&KKdl(X)I|PBb(cEP zGb_^9EI|`40R(zNqh_M~l+2f2@)F4JPDP=JTU4~6@xoZNpTU8E zra+e};wAk3aGp-RC_&mUn2wrOm2#<0>E)lxKNn3xQ#~qel+k1k{)#vo24ya~xwIj# zH)E(;dbp`2AzT9{y=a&m6ixky$kyaE?eh7ADJ(|bFnqGCAZSID_|X?^02`(PVZm05 z&K?}$DC0IqRjl5KP665)pib79e0eJ%U0`rnyMbGwesO|+b>XW~Tt$&Q0mFL*WgsKD zWsRIJbBqrqouiWlNhFX60}wdFeCLsCSK@g_Prc?mm)TV&<)LQ2&|k1k=8gho$17Z- zNF^Hk{1ZyBmZS&6o&m&`qFc$)6}Vvfw4JC0^@9L@jHanxHGBgrQFe?!Xu5EcuLTN}B2wZYc866>lcDr4B93W=7-f!#z^1w;ZlHHvR~W61t5bTo z5>=Z}wOQuFOtzZXC$xSG#y!u!vHXuvM_lqaoYi;KvP2y?9wH4xdaq#GBh`)+#y20lMH@1N~30|_s0SrzI2mE28H^^vi!(|7Pcqmo!=65iop4m> zG5Fsq{BO--Y>2Pk#@OL4xuf`Z)GFKPK@!DyXAkd1kL-nJ7eRu~F_kN39fA!0sE{x5 zFnlkW1QfTg(Ws$3sl7W9!@!Qlkh+Pa)9A?19W4mT zvW`TTb>I@O8)wM;X|*Ru7gdt~tWyh(R?wGM&a({_MKoYeOUlKEzT=-@=fr3cKq{}9j)L|xMg|@g{OQ;V=5ol2LkgJYP zIdindH|F`u=!@Yo*HAm2{jQ!A^cMaVQdtgldbo3PviGucu>az8=X8Isy_JZFh;q*i zNxZGCjsH?lYz!aO-L%$z@@tVIw*C2mevg- z`GF`JA14Lc+P1~d3yYnzPoZ@pvxHEbB_0Ij;}<^(;X3<`%VVP^M%$ z3i>kt4UcL1j2FD-zvAsK>*+2>YgfMjWNYsV;r;>=w|@~M`PkXP2z;!8N7tA#bC*&+ zLw$bC;Z?J_PbAgp2q@N3wTMiZcp=WA-;K;Hy0mGBeI&R*OA%l)#7t7e##JJF)Ayl* zT(iWbg<=$7{G1!jI7sgS#>FqN{7n4LmUbNlc`@8?qlX1=ga4-yy2R54l12$~T-jO_ z)$2OVhu3kiVGUvYFbE4=Cs|j7z8~b%rV(TqZ<@IDq4Jp|H0r*YdmSw$qtFO_7YyWM zlk@G^Cv6kUKS&fP7LHxQp?+Ic*pmY07`g0=jsKxsH~GS%PlcmHEYH6A=*ENpt;7E| z{%vkPMm;gShKsK{O}dnEbQ#o)4L z^sC1EFQf;Qh4gl6e*S8j!89QGzR!22V%4I|hdLojK--jAz55fRTZvXk;UqC}@zUPqj7WZ4|UXNSb`62&_oNP=Ih{9LAcCK-C3 zS5lrlrL}Lf7&1BBXsNN5u%qT9zI>wM z55F!YfRf6Ay)~7_083>uOBmOLb^w_kzU;Y3`DVsO`9YKj&Az(Lr7K0~sWkM4o*Q}P zT18wYu!>G<6+eGYpw^5nu<+6+m)yS7#EkrnK=*%KvLx>ImHuzONxqdwCD1dL5a_Vd zxm*47;{5mRi%NN$DY)dcOT{zgdr?fhG4G^OoKaBEJJFCoN~)1lAYnc?iMrs58pjcOL(Z~n}aPNxox z?O7t9Mr~w}r&+h!zgl|r*m+G_FfO^?QM*s_-(&&jc6)t(Lg!CqamsJ(0HfGwG>l>m z{ZH0NMSPJaT_yH?h;ELgB?%iU)Pz#}-5$p9>?`tE8nY;JSENw9?o#8f{Id+&jiu26 zWz@lEYp*YD2MueNs%VVOV3gYtyV`U8Hj{V=_|*-`z|66XVCWGn7+O7%WFORH zfm|5OmhvsB23TwqojZFp7#mTPRe^@s1XKow$K$J_4eeZBVxt_jEArg1cO@Lcbvx{r zP&#Ci?;@-oHKZ+eUmzc-Xn#V7ci{RMVT`bmGC$@Bs@l^WRRjz(3i_l$RbwEDR#D}n zKDw`CNWzsmJ=V~7k>25!q%W6Z9qQCEqUGowzG_iWM9W(d`@tBHcHJnol_K8JXu=ng z6a(lye@Ix`TgE>Hx`cS1JCHrfqHS+5_6Lt}-NH~4k<@@>fI{BBXf3ANv zV~mT@3%9$JM7;wNSB{=;@4nZak`=DU+XhYA=}31yr9#AulG@*3J=9 zz-Z-O%G7zwCtPg~nMj3^eYS;u)d3tQtLY`71(ce`%yKCzEeBwYJ{JWmN}KDI^73f< z_ao<^(&d8?gOIMH!aQ`P2i-r=0-k1&DxN{THdkY>IIqHBbj+pHlmd*DOocQajriYW zQay5?8LQ9<1_Pi>`7Px&#!?e-nT%o+Kl9|UV;?J~6RCs*1@>_^L08$gsPsA_VZ9N1 z2GOJbB`$3NR~lfa1MDtAk2-Nc&aEb&ilTqTFDp*5dWGhK0D<;D`2{9-crZ zf%cmDBex zeJrBFbt&DK#}02A;XqDQ+eCF_*bUk<0Y{P$_Z<}zfdO?;2NZVPIl?Q0eqbmqJ-$O@ zTdA*kciz0901Pwkj{q9h``Lp<~e+?{v^? z(qF1NXL;l`R$@9DJ~Yuia@o+6PbQ@qo&6$jG<8%nXyS(<7(7i*3CbL|o%iK=h=xjk z1)51F*$beu1lpli{-=*BG*p$W{v1m^Rn^qp}Kgw%vcAt7ip4e@z_4Srn!yWpU zW2g0d%PFmuMFs{$Llh+yhS5}HLWr0)jBxYQxYL`{aQ*6uhor|MED*~UNmq+|K+_8q z44pb94mBs&d`&es2cx>=?hvV$iNp*~*N`oh^d2Rhi;VQn###|?oTMN{cM!|-t}SMlyEKa7CN3Ixl@>*3 zjP65MZ1K{0qx#gX4tA=qF6?oNTP=TvK6J%V$cG`X%UM9go+TiO>NXBHJU28q$aO>0 zP>f3GdPDE&Hn*=(E5(3rOk3{lj7Tr*Zl;!Otr}{|O3G~OfBGWNzlo`bj~(+$Dv2wp z%+ZU7_i`e&cQOiOaR`@1_1N*oH+~q5QMsUySend>PR)uO{~Qvb^h?L^38P?#<+5Bz zW31+Cl8lA#e;G^iHVCiD>y7fYNB%?oc*Pl@NJsh?k*tz}J(pkNUb0&_q&bCKf{EW3 zUI~!*VFytTvie9rDPCBwkL}$-G!ElVIn3>#8Qn8Wzn5+p1(TeTOF|^ep%y=k+X2hz z9o#0ulrO*Y3t0>$62)^s19XlRJ%KVgeTJ315=C~}AxX;Pf<<*WC3d66PE@dqc1ZN0 zcxcq?rQkn%)dUg zf82uq;aC4hdWMD{wNey(?hl8b`zWIN+?^t)QZAJ~)bOS9#}Bs^@%gE-viha`GYrQ! z)wf?wVPCQbTqF<@-Y$SQ+PkASmMv8S7&akuuC5`!1|LpXy_+~v6zw!tw;D|fZnNHa z(sYCzZP>52YOA$YP1$krd|8XBFtH=2&(79Qc}p2vVQE!oGsVBOW+$Kk>cfZ6_0N~m z=Eou~qxc>TCfxYSBrT29n0FDr;Re@ql;ot_-a{6g^R2&l zR?YL4oYYnGwuC(tdftonOW^iv4F|AwJava6UMMj+ys3KAkp5x{$uR}emjt#76@;Y* zQKui^2bL~H_fdTi)j31_L8M>n2{KrKu;%SC5EPq@#)jCF^xbsvyS3q9!T|q4YGZZX zZHQg$JXv>ES0S_6XskKA$G%J*RQ=H)h;cqTW|JW z7RC4#USi}yo$V(kqdUyniUv9K3mbQO)zFjXY=(rGotsX{^_*EmxJ{3-OLCRj3UsP< zmR)6tWAIBcM`T|HoQhxpfyALZj?kuGzcenU1}J78hclOfIlUXxMDVv(tH$SH#O2|vd|-7;uVrl*Ooj+5`LLB zBL#(Gpd1YJlb~&&_vct(MV!tRc@9@(uP;nrh&yPQ*k^kHK%@MIP9YEa>~Y&2D)~~0 z-7|}Qt6W;@k>DXgh5e#Q6wg^dt9BM?oLomm`H*6#UbSat0fl2bK$4oA`<6>avbO&XaV z`VtgRySOfFUGAfKA-k_gyw?I^xWoX4LJp1oT4R1n>;)tCG9e^3*gb+Ex+t8MOX?d6 zcgQ1Qk|vgf_#J19f<~O4$Mi5O+PL#au8tQ(JlObNyWg-L7kf_W+PjZRKcGj$=!-=~ z8+Xg3S9UIA*#$gNx;J*QXNs-LSahj!LET@GFjlNnHUl9WvnqyRu(2#X>hU$wZ)Bo4#f^`tuXyLeIcuBPMb zh%Zb322Ll+dU|dMXM`SJ;4IX%RZJ33v-X0axa^q^-ATh-wK`&1e$-1*v?vx@BwDs< zH;O|sJuB5G(&wgHRx7`uOAv%sqC_hJA)RkP5X-|W(kw$~`LkuT&~ub02R1XAl4&** zw}XKx=rnZ+sa$p2(W`5c^ip(UEbkM0d7W>aoxf~qa7l$P)xTzP@@EG1Cqg+rW(VnH zsXo%#GP;@Q8lg$E2hVaEZe=$7fHRUf9ZR90VayZ0Bvhmz)eK4SuSKW$DEn}9+V=;8 ze=$DACKCv?`fUgmA$}i>+6}@mxHV8J!^zq&>Ahhrgjj*9i8)hP69=8b4G%}1NieM> zn3+g|1cC_^0APP|2xVxQo^|%2`#K{lH@nML_ntkhSFP+`EqXBKAx(5G(+fFng8MH-nb2yEChxxs zty%uD`R_zvGMHc4xuu8_(Pt40PBT5NdlL-lN-=6gV_HAMRRT=&O`W(qcQ^_nd_cKD z#TNL_U`XDGq;oRr--_@<-&1lc3Zc%R$So0hTf4)lb;IF(?P2@W*cJ-UWZ-QRVg8qWTNI0 zmH(GaRrZV}XklaA-^)ZOR}sNT{$@o}R?;2mT?=uqs3wADn#yNU68}zm zLKneSNmIc$RR|z-SWki%C7Fo>q1!Cu?kG8~X_73>%WkRm^=Hypf}fS3>3 zuoY0|=dt;MQ{wjl7Vbr6v_N?q(##duH7)ne@ArqErMxJg86wXckqLzBhlUAvSxE7I z;m7e3EfNwmPH!|pBJ7J}NZL^9U8ZMw-aoL&jjBiIlRWjIsTtlGwys@Qf?euXe?pa1 z0`1cG7w4Bxv_Zd^B4S98s!8w(1!WL0OV-O-O(RQ5c|~5!$kVS&oQFED;?%y4Jbs~2 z!cmS2&3+dY1S-HD^&B8Diyd$B=kiUVG7A5<3Sn^ zDXf}Fhm&lf&h;b+kvEEx$!=Lmu2qWYx^Vj)p%+v>h*l}s==9_RJ&|+%9C6Bd65IuM ziS1sKKrn6_j_5}U>9ZwcjT6hiipG`(~#`ll+#t+3g#j(H=45bdl_{r7=ZRH{E)0+{RA=`WIm< zDMtQGzH+_Nistg}Y>}RcRDwp8mdG>s!j^ETkx-^nSw-cwB*Vl-l|t?EK@tDEOaJvK z>>19PGWBF!p4(1m((e}49kfu_xQZ0@&gosX@4*CI4!z=>zKWULA2`M&z(tGqJ%$8B zRMnTpwb4CLSR)4=)((d&Dq18#7>{H$^jnHWFdi^}2gkjnbX{=kN9?~A-wbKQ>>xcV zOP_DtNNL8{AZJoUO1S9!1Uu@cGe)hh)=z(L7Ry$c6WH5gXa5TH)zM*z2d+-qirl`5 zQznm?F+qBEoT^`>VF4-JpZ3rft8JgD@^T;;beLf7ihFV z1SUWy#J3U5DABwyd=52BA(*5ye-+n2TK{t7W}-WBzSRq-%;!@t9)bykP@5=q5UrIt zf&_*d7(6}ED@GxXI?bB3V%z{-Q4L=t z^LNbwGlg43iv^`5j)~zB8@OyVug_&@@ z3&Koj=jF@Oy>@%A4cO)IBaegA;BY$V+oTNI^f8{(e9zuS#I1dJ8T4Pn7ZWX0hNkBM65u}y#QWj z$-gKo?H?|z=1iA2DIBw^CUmmYZL&_)tX8ZFrXGwmKu~qkE20O$tD*yj%A?bBV^*C` zEV3vzvo4*Z=jh?YzNpi{@}beepiSBS?eN9%L0g*4K3@PV-H@M)(%W>tg)OsiQ{mdx zVGr?0=%T=NH71RNR|H36OFR-CGz$NnMaUU-b{+&mYE1 z%G;N%ujqz07fS!A*nun)D2UDp{KZg1xWdo z_>H#EC2Ky6N5=7p;;ietkwd*gV_z~3G?X?DGW!rn%=7ZkKQG%pjeweDk)tcg6?zVe zw6h?y>)3`NwGr9&2I3;x`dD3Qj~=He0w;DaBNju3z+Cv!m69JvT6`#}IRpSC476vm ztju9KS3uW{ro-4LH~u~LPScapJwL+Aq6wg!O8Z1l6Df2L4B#DKXXuq`)FlUKz$l@1 zt%G#Z2(ChIo~cLVlBaPnf{jVBT-Z1g?QD@j0wO8B!sS{S>+5?&-4j6E-uQh_MkVlw zpvMrPP*o9|^mu~+FsRiCBX_x@F7RX=+-w?(wNp{bP;|-ST72GtX0RaEFO30PNzeB} z@F}LHilih`m5Er@bjWUzpg;hCr(v;3KoFqktf5Nt4p+8kU_Y%OU{^IvLje%PyvP}^qIa*nb= zW2BH&gK^1zHvu~L6;q7M7ja2nqoBOAL2*{>DX*wnu0%}P@4r)iMr$-BIt~gl; zYaVIiZ2#n({}5eB5(hc~oNeHbOGO7mPjjPXMeiiWmr1SQ^*Q9X3N-o~477)1*2MMD zKaf8fd41o-$S|3=giXeXJ5QR$p$wT>ZZsL;HvpFX09oe|lIa8SYzlzj5D8B+aflkP z1Ek5(KNcd6E}QO?r$`Fs)}Myfp?7Y47h0@Ip^=R+|5Cvb3#DPS!tnq~_c@@JkIDva zWT0aUOovSawz2++9MlWESA=&n!T1$xnq2?e51*{sN=PH5ZtM-Vz^<%ON^3YVpr>@4 z!h7R%3oV1vCR$2bUOUud2#G~-$l^7`Y#)z?`-PAJKV&2`yCRdfr+huJhoE&I9&=mI z&2TnHYKv?snT{Tolr~%H(7!02#5R_YBuBAvhXksM^PW8Fc+St|X)FG$j^2fN@a-Ar zDG#g}kCcpwFgAsZcQ4V`k+EOo)->JuN)l7Ph|sJE0)j{!PS2r<)T2vsS~8xq(lcfG zA{Cx6_=qSJJ0hKNn}O176VH*L$4g>h&mFP0!Wj#KfTNh&F0%h$=q*c|{W2I$p=L~v za6YR_EI3J4gdOY_JsZsKmUU$$w+#!>dDqB#a^u*7itoH74Zl%MWeJ`dzV@G`uWW_m zq>C_}N2Xt;^WRJ5i^_*G#!Bo3uz$-x13!aT))(F~;pLRHsxT@=lndzj`R~tvx~Tj_ zbwy=|<0YePPm+(WfQEn)5Pq*!$k_Jw$E~e$eW_Wu3@8L3>1YExQ!ss_F(h$EorV60 zX|j8ATxgIVLYn|7#)ygAP@Yzt(_Ls_{8D9Vmt_5{x$K*h_(R?-Cg*U4=-ko|p?g9(@Cm-Z^sj~o)Y*-cTmYo6&J1g ztEGb`7E=;%6P4+*wEr4+>}E5WEWB@;?kpkK@KSl`M?G&ibjMyWjkJeK6;PV5W5H!8 z{R(m`C|qqh`0NsiS0k2W6MZDMjR@T;&5}5_pvS?#+~NJ7$aYYWw=2m?z;x~%yaE`j zE%xXY;&18S3(ae%!g^5$w>E>}a)3g7NuPjCT*_w#v?b?xC;w4tniqo8BlK?MB+Duv!*URqeHcHgqs%Cs4=T zKh6U4Wf3Q@V>=cs*!HU&4jC>?q+xLqj5=Mf=OS@JZvMLdxK62(X)UnP7!cLHnB>0F zpDaqfC_3Xb5;{zMB)!PjqXJ&GBxJfx#S&AeYAiD=Xem=i#2-@bNQ4*3qSeWwzWra( zIw3Ly9^pE2N-h$Ku5?aL(!&U!et?`Ta=2RcRDeucS?gVRB>SVl5^7d0p`ae!Gc9J4 zJ^pARXbgpWkNDMRt!kMX&oc-a2cAFsA_6eqJ|kn8-)@~)z) z?V0jg?J9TY^l1O+^_H{iB2TYQh|o73>NxUxE{ckvR30rmDi$oC&AMK$T~aqrx%dSn z0mQs`(Xmhh95{W*4Ncpg>S0erEG4NZg(EMa%Iwc&4j6^rJp(um+FiB)qN7W)yK=%` zJ@P-GX<~!_LxzN7Q=&KswI+ls@5j<#fk&arwaY;<5FuwvX-dP8XU&~nwz|39Ln*u@ zy)-FVa!r8`5bM$foOsZ{*ToPSS^YNzl9QmjL3gW2KQJT=ZlP zLwFj(Zt>^rL2$`|3n#YtOuPxI%uM*wS7^GP_8cVmzt;i1`{Hu^U_#Lfl(=rpb%<3YS-4gevG0)U(^LoU!-2k< z(fhe|6XQ`<6<7G}JTzKCK1g{GmSh6}5%c24^1aV-j&a#`l35U;1aJD@vO0r09qIC)0w;5tjhsk~!is$8#o49;p~W z{w(lao-apSQv%|nRtgH&I5@jo#sFesABb_2pQpJ)=Yo@WAbe^B6~BV_rieOCqyrWR_(;?P@;83_rK(DUBj(YsGY zb4lQYLOMH|;xJ`g9i`r-mCwJIKl~RZec0OqR&3?M@YS*;U2nIfrd(2<&jRu)3&>du zlIpM`)z6Oe#tr+d{IIN9n3_dfaDO-$`3xuF>=kQyWqC!+K*IYU4^EUu9J^I7KNHsg z>#G&9+Lm+$UZkzzn~qDPSimD<@XmBOj#Vwi+_YD6ETW&Xv}Zlj+qeu?Vf!f;=y>6MBeqZvO+m7 z?@kqf#Lemm=l-F$9H|H=^m~P%FChvN7gvKe;|h9QuPm)7pI5?gj3Q`reRmLh!q+?! z-VyxwN>SC&7b*Dq_$(L1lr%kAd#X2__9YvZ04O#l>{TE5--vW0@y8J1LDCRim%Fj@> zH*4WY@UQVu!^yCQe>?x(t^T=F{qIKgX}z}6*{)vvO+2M(lQ9)@oAjc_NDIcEarhFg zp&VhkzWm|C^0EqVlMo109#bR&y!~8KoEkYO;zha>-zea5o`RI#NQA)pCosj2rn3u7d8IEpM!npQKI`D_+AVK$74 zC{p1Y|A#urf~0cTF%DUKsYlJOdH@835kx5BV}@ZwN(9BF(7W{RCLAcnppZvw_j(1G zorQMzb|XC(6&@Qc(CtCaN#om~3Y;nn(CC80({Ub|FNZ6nlqh~&&?IHQSHAPzqmi0A ziBF8t%bDamD?+4Q6~zkD|0oVE#}H3Fm$IHLshu4iqQ+T9TM5_c=@Ll=GrSi&(wM5) zP>xQehK+P#D)r2c4*!Bk4@Q_{;e7ex$FftZ;s0uuZ$V}Hg%d=mGfoby(jW}qPc$_)*58?`t5fr23z3)ewMP*ow5QM%`) zp?{4|*}2n~L2wDk5KWDOURla9Bov`jMZj)7_ZoezyAjhfM7t?y7-mL7sp-*-*;)zj z4(2fw&SL@@&!5oS_3nD!B$lFG1wzQJq=t|X-#M>Nw~tN)-g#|#|p@;$4-lp$0U@XboZ4+A+a8j zHf6Ww6wprO{B!C};Vxm?uIjaHF5VTk z1-SKCMZ;y=3GAbuQ3u^RcohCjbeHB0hR!TJtBG+CF}7Ys=ZYb}TN3W*~y)g8I z&d6lw_BdXMg2F606ZkYKa*0U|oCZVw0nkv_LwL()%YwtMRuvTLF)pn!UfvW?oH>JA zPk0(gfLSq?7#&p7i~A*P@GF|XabM1};hBKj7lyC|RROsSaC^g^XOjaptg=vU<84nu zN<2SGQBej%7kC7fN)1GaN~cPa7n`n%4etyH6o(}ats7ekrYmaxfxWx&@#%{T8mf{8 zPBL+nW-+bstBRe@>vpo2(naUIjz$VSn2Ma4pr_DI>315+oxe&!46An9dpu9Y6Q48W!Ac89ij%QR~Tz!Y^RyVm-=7k!fJh9IYf)j z&?ObaH}f#4%@0Jv0cny7MP4N7wtz_Z(ROZ)x6&;>WZ?@2`onkW-tw+Wo7CRbG$D~8KqAiTBIb`i)3P05K#W`oejHG;4)MK($&8XtZ zSX%h@jzVFMg}b8SsC6$k9EIb7KlFA|~* zt8E$N)r*x*M4G#jZbU>U36B<8=d#GG7c*Y?#_$cSmu_5ea$n1Op&zPWeKK{V5u<#a z43qK6fK}Z6NXVL$Wo$?Zc!UI(Iq}F{Q9c_tTUj=PLnHHCQQGpw!B4W7brfTZ89I@ zLnbE5RYdDf)J+g(pcuauUxi*Dy@lFrv*-`-3GOksU6bQrvsE6z`HOBlI+RmSkDr@KHC4uPHNNU1VXbq7Qbo?HBE<^unQF6(p#etg;SW*u0=BmzrE=0Z;IYxd(|#g%F_H8agKsP8RjA=! zhP?z(ds3ENqMB~H1iPo>LB_~Zt(x(M2~+*nf&8ged+Iy#h(6t zK}-$T0+gQRD#f)W|%A{mgo`B?XOslq}Xc*-q18X%Gg9nXzA51W75bH z%AApv#4cNarqq*Ov{VSK)i34%%IMR};=e`D9y=RM1tK$tp8VA_z zei4EYbWkhr z2AmYAEO}m`8Yi0qR$Kci4z0IOLu0^MUS1JuO3rd;c?FBAC(pOaC$c@|TZxDqSndpq zDNQ{1(LpB#oy{cEsk53J$vk@D>uC27@-hYjHl2zdf$Y@Bxj`vp2e(i?qWL5GIWfA3 zCy$01(QC7~hmitwoq=}?n@a2`wH|=9j)GV)IIOUPjO0kJD-oL9qaj|(!7$~^*>(Oo z_0j0twAOa%y@YtA8!z++_qc&>`LW}%dR4+Wm(tg9Nj`}Age1{-bQzi8v_AD_QfCHz z)}wW$ITKdU#;=NXfB|NL+b@RXO*qAN%m_seilT1GnsmhxpUn%?jv8yg0TX$xi-dxO zjvWR6IX>NTeu&T?*AM7(5iPA9<+EivO5AfX*(Mms@<2~nOVT0;QuT8R9I>J75ntZ z#6~slf1}vUO9<}#twbD?fA34k=-~feLjMkz5C&X%;3c%W_$5^Rro+AZR~~L0`2Su) z-+e26HfWM6yPAY5@BBcy>Aqwga3P@{5#wzD1$Ige(p80_7GmC z`M?9c_E(-_9O3_79p8P=9RN?*+j0SY_bv2Uq!nz-|S-j-bP&slHq|H1t7|2gX}|DPj@bjuejs-7yY0H}JR z+J*$#H0y~lN&{U_{C(=Ii+_#`Yh>0FNlixqkfr2DOMT3$*N`7gQM}nq)KfmV67xn^ zI8PjlpA*+0TcAYjhBX}zO}OAgr6ZwKD?ph7Wv#qa%U>b2=^Kp>_k7TibFY%tJ970@ zlG|Bi9cMuw3_3h{Zbz=M(cmJhor2z!y?Sbb(mO$vl3BC&L>juPr~59GTDn1%G;`-< z&5YbjN$9kzvY;E_JFc}X*$eEZ7uZcNunoytVD6-`Kv_#I%NXrol4mb<5>nHiuEeCk z32k;Z?IG$GMV7Jr15cD*eEAa-Y%GKNn_e`%CDJQ2cT3D~QLkPqE4?Rjw+W%4jH@gC zdQnQ@_mvFPRnEEx(^9B!2WBuJ4**8wj0x~3ts7z6@N z^_=Xq+sb;7t@-Sh>6z<4{-^%^;jyVxwfUsC`Si)tCmX$uMz`T@cx!_-cXeaE_hh}< zXgz7It@Yi$`^2qxr~a^Cb*FdrMccqV_Te~4X|{7c`>)h3+3wI}NvO-OIG8td!- zvHn*MT>hJ)Mj$j920=Wl+`Rnc;QlZ9_buz+Bljn@%PE|~S#8W*|7L4_V{I+D{%dPZ z`1`Mzfd8Mb|Kqo`CEcWlfkwIc(`n z4ILh;0V=DzD-6lZc7ZW_B;pW4gXxg;%~6Mc|Lo22+cRh9=r`xrozv5uqqE@H&lv#wSRWB*KRwnj!&H(=Va&f zY=8Id!Op33^7i!PxV?w!sh%fimdqkD=a&5aqK-@7i-7}0nX9a+kce1ata8nuoB$0K(XmvD(Gjog2Dl5(~^8=cQh^3w3+Kg%opiQ7`l-vG8Cg zJbv#?;{Z#s?*t5?DwV|in76ywvcjv3=nD;kjsP`^b*ExQgqr~7=286lSawj_PVA|9 zU|l<@Qw_5)L2rW*ZGAp#(x2)R05f;xR*Z8~D!x8BxLGZhc_EGgm$25Ua~t}o?}(#6 zffil_@f84jMgb2+)J;~%Zy`|)rCvbcy&*~{5b>a&O%6*u7F42NG|+ZvE%Uw!HWJ_?f%i9R}SF$)TSOw#^%nx^Y52`(TdH z`W7vI*h|}UE?5;wzlb(qOBMYxm*f(#TuJMGmJhDKK}1eE{)x;^~CjH)xW{| za;c2=WD*CRGbvpkl7T)tN z5j$X6<8(7VN)fN{6LguZB2kj_Lw0RJM{nt|Q@xCxhV#tnW8^YcDQ`MHyC$$U%xCel zf?u(tluyOEQ0r0zV8l^9dnSO$Edup!QuRb;x)$wiZ1jp+yh_!i6IV=DbIJ}&)Y@?{ z5jVXwveymGYnh)y14WvBdzV_HOUA8kM)4x6Z~#O>0RRFBZe)gQA3l4yCUxkP~? zouB^RQl+uOO$o;UW5D+!w^!%~uv`W7?kX1RZxk6TX4er$qj3 zeSuHpl0a+haaLSka3cbr9m4ui%tasVbN)H?L$bm|dN-he)a)&aeOO?K4ZM9x*%<0b z`V&tj`QTU)cEQ*U^seCct~iR02REJw+lSXapd@y7?ZO@)^!nI&gSS-32JI`(M{_ck zc?P%>h_3t2$2Z6Av(E1Bat+PtqZr6P*4EWINQel_T48?}v+L{$5LP%2YW76-C5H2* z4F+kbG*jbf;)innJwGe8r?f9L59GzQ9dk5ctu1NqK&TtDo$&&$Xjm}b{_zQ z+w#-d;mOPW)9pG`_yS4JC!Cb+kEQ9zjjkQwRk1sxYup@8wGaEAE7W_J@9AH$Y_{_e zOVg>TqP1^~Xi25?Wy&+>3we|NWE2HD{pIl?mxd1LW|o$!B9lT}B>IJ;MWAI9s8zqZ zv^2XSPB$ItU)>O+_H2obKq1e8q9e0O&)g|=zKsSAjL@UVR^rs49kM!9dY_?#G47rO zBr%hNOVFQ=M)%v;w>o{|%8Rw$E-q10t>=i3Kz@YYP<+Dt7WhIj&(q41I3p{NOc%|{ zN?GGjBq{EmL4O?8!n497x@*Y+M$fr~|MfPXI=!b)`PzCUB!@kx3jZIZdekdBdSj5m zoFj1PBf`37?_glPqPiBaz`)19h>7v9#Km_-YfwdJ+qpBKS|G%oW{^|cR4tlk48egJw8LvM@s%ov;*sQkJ%6L(g z_#h3Odd-bv6k>JGRn6eK!5SWi$N=MwFo zDFz+7hbIvLn%`6G0yG}HaxriN6gb#_vAYeFdZF;hLw`I=$(>ht1b3VB{CRcn_?2;j z0kmX~pAP^Z;k$k9>L*NAw`8f)dAo}ZbUJ0x(zJ&igHqnmD%bbDm>hzsk}`$LMxN1A zNC3)wP&T1Y3~yDocvJ9Cr!?+H{jw7Sp@?2Hgvh7FuB_e3WC;E4Vi*ecg@I@<2(dn~ z7cv+KsT*uLm1?Cr_F{nO4F{DK5VJ4s?hiiJnkHtXyF zGpe!D#6AX}oez9Z)+0pcO9E_88T6L0KsmbWMW4$b5p85_f9e6%3{O03&J)`woIkO$4*;OTpW(=H@PnS-=M zO22y_N>~$JLtcC+*D6aZADSp*dh`g9nLAzt(wITjDj!N-_d}ULObCYT^U^2m)t9AD z4D^faJ`|AyFlI8dF>Azx4Ii;mj1XAC`DpVcq@7dClR}3?nu?zc@R|Wl7;Y~|_1`^pIO3H3U`un#tAEcUQ~!KAh#$`_u31`z)VT$nIYO^Cx)@y1;X{B z@UQREtvU6DWC6VsNEi1MdWKz#PVcHHb4@OSMMu#5^YCc}#H4I5W7Xy4(#u?X_9dla zVz~NwwR2Icz=EMyjwy2FHQ*VVE3M>ewCP1*_vCHo?6;G>>fz4O+gCfgXKzpUPOJae zJ3ZPvs2-o}o$j3NAH5bt(iUIl;yEV-9%bAoZE3jiY8~` zV$>*w^2zy6b^BgE0!p9CH)B|ht>C<&|jiQwh80Fzb;a~(eRWY27$K0JhxvwG) z2<5oI_)mmu3cz}FQQdPwNY1OtJ27w$%@q;cm>kckpm92!9n?m5laF=|PTuTvBnCC- zPmY>qkV#W<3;fRRv}w7W4<#T~%krTn2!ZOYe`wYxctxn*XimXyPv2I>y?1L)snu+3 ztd)y;U8OvT+mhE77Ot%?Ui@HpM*!cb<9|07YJwVQ&S=29S+~m9Hbjk`SyeW&>zwT7 zR+{wOWUVKQ)p}yqqLx}S$78j!!L77rk7vVfqqe#GLe$Fd8r0fM)Y>@^wO%}=*6w>z z>*WJ#J>3zN>hSC7%&|Q+Ytn42&YVAME}qn$iW;-pc=FU@!gidKac!(=$B1K*ksw9s zI#IcIsd9pRNftcIFF3$RV2-KGA?mzO)@8>7)&fZ@z0UC$ElkC3Sd-eLw{(t`C;PB}9!I-& zy`zYGz`Y54z}%B*NIp)-!MKWn6ZokGadC}rKYFfP0{)9%XqTOaxE0moo4g-)aK%5s zJu6&ob&k%YknatJB0z*qUWCcP=RR7N_;*&IMMUAG(B$`=ljHXOdtey4d~5tnbT^9y z(fJ$49kr&K)~e5r-8|`-5iSE>A>jp%q+&u4Ej@5i&q;@6__J6=+P+nwS$yuMO$rcZ zHpaQN-q>0#TUBw@Z~gH&UahU$m3Kz|rCZs0d$j+)!(xceyDcCYG+BaNz9vQPqnouP zDRaK#ULctBYeUM^zVSrg4>v%qA>R5J0ekXiI>uP=m+qtprdxdtBNE(>`te+-PpS z*{{BRUq3uK!McD3+NWrDVkb+GSCfg`tTn4wz*GaP2Y;t`qshdED<0Eu-tH;_=;3}l z{kwg5(s{XCmC1B)-D}VL<#y8dqsD6OcH$1YHFr=O`9r6)=Y~KbzN$Gp*It=8Td9i* z(1=<*+CSK<>9@(^$5NeN-uU&-er3X^N;i(x1Dse|?tE4L#lx^#kfNW=A|&KN36MDn z*cBSpRv#w*PVJl|_?s|s)Lx#pf8M|aKo@>=N;mX2Md4f3?+vTtyN#S`e9G~Bw<=pt zaIRIBANi3?m)wL7IxVH%9~BKr&6S;0(@(Uj(Ib9()MrCP@_X%~A~;-QywP;L_C~Y?oXb{Xj;`F{cWI(xHi37f+n)qOuae&~)cWR@ z{ml(vYj>t$5V~*QtHvSu&KoDvHQL+U!0W@M-02(?!Y$Zi6W|jg%MtOm-yX3{0^8MZ zt)Zvl#ELgoR7+ynHnChHCT-u0a?O(Ps3u>Dv_cj11vMR1%y*L5b~*e)(d>w{Wqy2JE<5z_^Qg4Ddwg=%-YypB$i-y^l8alA4Lt(x zmq21_UG6qFpg89tav}u9+fkPJg8Suro{|lT#VrAV&|3wR(J4KL!357Vkr&7|E z!~c_{%>;3yCIZbWqYv-S?cqZRg^=Ph%Ip9)0vjDXddwsrrVQ_e&Kf=3(q?J%n3)75 zAZxX}L%jMuCdLro6g9>rXzANpZ`6x)XN{^tW>WB&`f+HOWV-pOC-UPd z>spdpwtsXc9V^7~kd^g0&J4?{gruLI;sq4(26N+w?3maL&Ra6RMQAxf=%Q-Tjq#|( zMSQc9KfI!L%xxOvxLP(Qs{e!&kJiwlbA~Y(FnhqYMI4xE*SEitzNaOi{_lD(=%`Q_+YTE}k8?i%#EzQF>&>!+C1-(toot z4cHxk`vRCqxd6cKjZrbfF+A`?PXV@Ypx{f@6#)Zva|2Ra8nTaZ*e5)n=N>Rx2v?Bc zW-^W`Bl#@j5L12?E6ib1uv{hGV&yMstukxAR`Y zsF*-*+`ENgv>L6(*6soK0@kUuwyJ+ifW09`qzhTLbQ!vnt12YTfs8{T4)*Qg{z;p5 zxJbak2hj}fKt51DLdoLu=MJesDlJGwIQJ$)xTfHL1+5o}7AHaN>LiLQd`De0bg)*U;Md`0Zv|y^p;LkHn{)CXCX^s`Q7(oB%8sn|%&PW`sBu7W7;;0%y{x=O~W*?Un=6vh|w-%dzf@m(3OeBs-vRHXeu|6!5L4-rZj$k!Bg z0DWr;KMalXGp)9n4gTHf+FbBA;o4V%2l8uemBI>3ol_%cUb}E;7HC%dm98hM7)tul zmjWrNCRih>#;&zg|NeKZNxz^KWtf;E>(p}_Y{?H4X!xRn>SX+`_EErLt%eTP-Tr7b z1;Aw?M;jLy<%R7PdN4DvfH57j585H@>kzn&trNswhdb{(HlmeiZ6wN7DPiQ^<$yuj zd3EdLV6&P60XrRflmp-h%>z4cfRv_FZuADlg7;%U9ryG4xM%|erh~$&c~DRemOxCK zaFgu5ufKi2^~6NIn{?Gt&EDPg?ny>1m>NJ|YjzzquNjhWQ44sQ)72TZ61T(7JKsM( z+Uj3U$?%+<9`Ek8+sCKd8B#hSz*tHY1s-P9 z+uUq68qNit-24(vC{cK12{z8r{AlO&_1;2IC=qqDu!S3A3VU+%CTj2p+~ z4F`7&UZ}Cczj<}va}TQ7s*1h^Hk$8q&!0-(otr>oXZ=yP7yMj&9ccjPYchOz5FEy)l_)dYmP==OU5 zlEETS{Dy7a3)kUtBFAuf}0h@Zv3t)2b%KX08kTNkzFR;x@r5+pgTTC3JX zc9VZJ;Rmo%r)Tw(T}+Z95nvl9B(HX1xXfWmlN*XRN-w#a7)GT^FO#SS;_CrA_CG_s zxh1ZVWhvE(J}wsp$Yt!)GRtWzn;SR;`+*C#v*=Sos_Sm78Kvn&SMMB!Jc5WGmf`5C zN00p%hcKt_3=AU*eN>MgvPe=jvX*m}a%KK5Qk()>iu1s3x9Ck1x=r%VZV;`u*17kB zJgf`qFlj}l-*ECYk`+aoO_Dw`EOTH(*LhR-2A61_L%QPrxP%g$N1yONCRhvchTZeiG`eM+fXTE>aY1vB_tI78kU+4X|BO=O?Yk`1Asl)MW1w&Uz8P;b8`Ia z-sy7g1YjMKzw;QaLqzFfNAQSKCv+GMCzILdlZ#Xj{@KG7s)eyvqqv=)adA=b0)wo+ z+A;>Z3Ii3>5Y?IdU8*nc8!F;pCtm`lCoeMEfZ_L zmIa}bS>MLSLRFBlJ}R`6vU;k8lnM>j1rnRp%o%^XeKsx4L;cB%AS9O{TdEyu9;FOC zVK0u`Su@s!{Z@rlPt4iNllS%3&yB{`i`sFmT}w$&sWzj;GTrGVW?Zp zTC(!hBz^VWrJX=0ie~-+F2mv!Hfk;Egj9&&A@HjW+XsmU=T8C-y*Cw$iPg^~ zU}#-u6ER4|u%yQ6(79_x-qcBeT2I#RTL*hN_=!B*j|pV&=R*8DlKdW`eJ6e9JRK?136wu{@%ZkUU$wvxrhiDJ+T zcRGxd#S*2y=pQ5&CHd;#@?Sl{Y8kw?|v> zjf0T~=z*LRYzZ(D+LG;xqt|n4mcLb5Zu_UyJT#lN)@lx%A^`gF}@|LFn_0EtS@^k$N^{f65RniBA^4tF;;|3N;m$6hdvhWAq1Duk4AIAw2j_I&}TK)?`bN=VxaLUa)%s)t?# zC=Ob_Vq;aPhggzT!V(bZk}8WA11Td-TwtFFM=fxHS?ad8UdzI&SrAe;+5;i95`AdR z?L!v+pIJpnYv~VNN5&+or8H+z0A@BK1z3R*#pJVePDyCZn`whmAB>b`tft^mu|Fw-*N zj3!=^pTC;maHJ6sxg_6&O&+;t=(XJJHg;Fn z=Jy55=y9#Ce-Bro$mZr`K@|*vSqFK{b*9{h3-9U89(E- z*fQDrP@_A3sa|`JejZZL&v&Z-eZdwnGgRC(7=`$beqFdi$TiOunVJXQjS8`ZiNT z2@|eb1#;0XipvME`P;@(PjUl{j$Cae}y@VOj208vb7WSym9}7vp4E`D6p^3 zu~eG$5QqD7^wqJ`6{9Iz6I;0>nF}ZjMlMyc2sqSnPC68sqg28!CO##~e$l?i(AGt% zuCtMe>JzF*L%-&8snrs@z4@qw<`o4q-T1e{Ppi)RaHIeB`q%xVIg8)Bzr6KR)}x2} zN3Va~SesLKMffMuePKTio(Q#Q;(b?R4%e1 zGisC*(#f?i&+Hzwb24vSlZ-(Lt8BwUCRtPMJ=kq(myk}6wwBH?nouT+v~8zIq{e;^ z_AkEWdgG}-_OLQKxZFG3SUbzx)C;z-zhj&W61Y47fN4c}zPh%40jNtFdp}(#&N zq(Vhi*p=jMc_BK5UF>D{L;&k_JqupKt&1x3EiI8JEwd3eewTXT_1{qrP5wCQ-ao9)0GjF*(m3gk^X^Bc6YGU9A8CP>dD+vd+#=K zJ7tu4lGiEafCvh0Rx+DYK7g3A3c7#vrx=Z(Itl$7H}Zkw~_Pp+Q+Pvs=!-ACaTB2X2c1{mF zdpj@o7j%ecibW6c|WpDl;}x#SyI0$dO_ zuNMLlnwS(uj}^buiI}+k&`X1|g?cmV>LEawN!O* z$eu%zykdU&~WG z9MuVC6=yYVReVO-v}A8G>1EBYU|tuT;z=(T!h-cT)`l5-NA<#BqG!XI0c9~5xLIxz zhlJ9;*S&~NzbxqN1#CWSRIZ-|_X;Jf-=~XL-wV(r$xu+r_nOMsN6%v41=NkZ$P%EB zff+!A1nOI9u)EkBNA7nT7*T~Ue>>WLU!TNxTWb>G!4GGx)?im*(xgr*aA7P%65ObY zA`(F8k-sp{4TruCe$A1IMqY%3&2+*cTSZWsanSdKc#vF*QH+86h;2ZioXnTPb28S8 zynjx;ahzpeOyeqHW&<|s~{i=B^v&#aEezHI5KiD{)iX> zRu{gWB62C^U?6!?|D<_n)oL{Gea9VT`LTZ^T(7an3megLuK>gKa4aNzBUs_; z6g-RE^27nT;jU5W(Qw3$*eHZ34!YKw-a?z~v&M}%vBK~??0)WJ1G*@a5R&#u8B=-O zuD8@JxIaLl^DRYm5}tD?@*BUKE{gzQdx!67uI<))I-a%GRgQ!*Y zMk%2%^y~duyShqVMB8V_Cv#6g6bF+GC@3svr=N2naM27H(fyDNE|Y#q82~?|%ialv zJ^j3{dNBLlpWez3PnKAyEiERG0wA=8w#6jivg@S5q!9&lj=ThpBwKD7{ZMk8*(b)X zk(7*x$ynbaLnV!Heno`yq;|K0p^L_dmtubN`FI?74+fkqh`+)t#U$2 zzX0B;z|L<;RXG*=5jO(IKaT3XZVjhOExEDNw0n|T<(<$-tdrb13&O`)D-Wcd4iHgb zYhHVuO!D6N%<5Oks`;7L_mrKM8xIb($~N?N8N-)7%BtaSHgJ^_1{>*pkW+nGZm62v zAIlaj3LTXOxV|Vrr?S7l5X6lKbkR8pg9iZ3KghknUe5XFel^3I8W^=46f4)9Bk>U21C%t7weTYIpM38qs3g0K6^OQxljGi$Wy7x}x z4m=Fx>_*XaL_vLFt(5cEEpqLN`t{OTD8;ireA_-l=XT261&AK<0jj-=5*etwj3tY7 z4HO-8-n4?=9!Mee^~)F96D@lW1xncO>lh-^O{wej!XS#OB~MjJAi1ga%Ij}#H0YIiC^U#IoCBHiE<>LjR2T_1eHpc53|9*K+cx08 zkLpvm5A&Y3;x}}lbq^u|CKFFmba~OtJ3op;*X#*R_G4$K-xn06$`i32BMw;Zuo=@m zk=+y#9af(REwdxD@HLtSy%iz&j1DA^0X}F}wsz0<-{Zw!Sr-QaJv{n@5O!6upp2Cp z%?7LShL>{m31B^RXh$B5_?lRUD`34jvJ9SQLO6t0er{~NtRjr{+G@gBy6?N)YCrV^ zd$=K7%Dg*Yr@Nb~@s$%gxWMl=HVq-)GJ6{|-N`H~(VjB?Aw80xf2KXvfPpmNK3`>f z=bfy?l}c7>-A-L&zxhVO=DMBG{>RQ*BIRYfWhcCOok)1H-e?*P!RU|MbYU$o8xw)f zVKE4|-$&dF^C1e=jO+INcjuzw@PCD8FcoZ4KmBS*aK%{)b6K)i>#E~-JiQor&sE30 zQ1CRXOP{z`UkqXEtQOdc8=DdL%#5bi=QU+6zFzxE#vV3Dl-p`i`-dl| zJ8hL% z_Tl-9y^FRIP?=Lpj!yet=6!5KIX#5y3eJ@~juB9`l7bVBGZzRew)Y=YC0W6j>U@b! zpU?S7(p<-ZPWlhep})-@8FV@Y&sCXeqU2=n=xqP(A!FJKdz%AJ911%hcY)UmPu%#* zd2@K;051*~YGausb9G4Y20-Y>$XGlEq8KVXKlRa>X}@+@d*OyaI{wn@9lHLow(E_f zX;gcC{O%`{ik(tB+4Gn^pC~TMyd3!au*@i{8i$uf4bi zY{gGCsr< za%u=yA|5M}#*f@Sv9B+@73bt+mm)&e5;(T&!T%>0=!FM7;mkH5ADLAA(;Eik84d7? zX0-t~z^o>SIduT4b||0ZYjdE!asBJ51WP|22=@x>WWF$I(1lD5=Zu=|_tmwn_S=)a z)2hAUqq{ZUCew)*&fD<-WjzPV`h%csxrY2+q## zUJf9)=PeG2s2d<7uG~o!^zXA?=J`y7qQpGgs4Pfn6}$(5eCdX_zH~^RvbNIkwLX7# zlDT517iMXZ3U%2;RaBl8wJjU=WD7~Zg9J^w%(lj*9A*ofP+>_`9ePf);Ur=qJv4;U zqm1}T>}%yA0hF-u2M1BYN^=4!Nf(+})C{()ohP#^F_h(%Q&Y@Xiyj^xJreVsE~S(} zxF9xVcnQOw$_;1!iem??rFiV@P<&1A&g)Hyyz2E3F9Y@`=VVV;9C{{pqAR!^`4!j& z(hJ3=T%X=$$$bVsc{>y(0OK(^E-WL;d0|CmEKl(mDy#NU!H9x6#X&HPYL2rrzE3&^ z##>GH<_TPLvhvWy;PWLuJWiA2K2u^DMX;gw3hZ$%mZg*_O_`Z`iF$Nlm}s4#T) zBI0iDoF1xhE|A1cS>xWpgU*3gp%dtuksgd;V1AE+ie()s8DcA?qHdTjG6v~Wi}UaY zj$|AMl6I6uyMsSN}Pl zgUWea=!rck8i%TO(WAZN>d#w8$4B^RE*zAZ2z%@W)fo1RH>!@Klx=k) z^Q-uAGWhM@K1mGV@7wn_o9p@bzj*{HkbCrK_i8wsgp@t#Q|ZC*{s9JjDG|rJ!?s0U z&X-lqbAr1g6?1hE>6=mH7!h8v)V?f5hRr#J-pN8Td{Lr|Y2;kldcVi9KLG!{OWp%_ zUh370RhO>s=V(9S8#oqP#A!#ZvXl{e7265>^P9mk8C7>q=PM|=VX~^=gWnyp6UT)% zl$Z1Z%|^ugkv|N^ADrW{_sZ%?f~O9~-oWSuk)VYhkU~HGz2!WAj+O^YjV%(}KE-(Uwx|WI_$J1I6USi^{*RB6N*_Lhi@iTY3I}ExVK;!=S*1;PdT}Ta^ z$ntR?_6RSQa2;VUj@lA%-|GEQC@58vZAYQ7QQ3Z-`pig0?AwpBPH5H%7poJQb)v=U z+?aK4=F};qT&hs*RB8fC+X`Cvk7}8zi5dmeXd!D2`ybYmS=o(nD0I8Vj40`aWVBkd zS|Tf}LCQ*XQDx;dCL@((G$yjLYO>q4y5Xr#)l6k2C)C2SxJ{FWTGmx!6j0=WNg_K& z0YNkuEFD54UuhL0&Mgo^ZWan?!-L%SjT`!if}<@+Lg~J?h}EE6KE4gkSY zLgXyt>8djNg$nu?De}ZyPEj4d?d76ff)6zm`C;q6`a`2N!vFLWg#hh9397mWpN0SU z3;+G$ja)!(y=wDGZ}aJsr%yI|8;x$m+wj%~YwqgCdhf}4v(b9eT3hS8efNo5CuteJ zbksF2o!tiTNsT8@*6A<&v;Q_58&5V?|D(CuTwh;bd$PXKg!I;0V;#!>l>?XmrnqR( zWEceTta9`6lY{%eMONibHN{c#U2E#qup`D(LSZLBThVI)!q&l%Do=!XjU zRtEEou2+nH9D04!f%Ci5*h5hpT#I$!Q%6DHAHby<7f`pd@3CrP?1iI93aHSb$ZK-Z zJ#8X0*n$6dURj2 zZgCk&1|f+PHDMCvuz?7;4BZjMBUS_K3lkc$`d}d%3|WR^2abL)9da=C$Ih?&XK#+* zo;f>5zd66|oSyC+o&ENVNHlyG25iod{Ly6S17H|a=#Jxi=sHwB+&kTU12uMD>>uo( z{e}-@ulCQ5_S$Xd)$ysbvJYe9tSDSU<^^KpPu59r@>K`-9k-YPP2 z$RMKnPu!UuiXA=idOkGc-SxbQq`GGJVFg=)6$TK-ri|4t#%%@n0S?Ah3mglDT9^zp zf+{>#YO>;_F(odX$(|jNE(aR1CKRZFjL{No!a2g$6-KfEvZRQ?Vjg zI7FJACfdafBsxg(yYk9=JCoglP{vhwo=rQ8)w6MyfvDm@VlXxv1N0G$QQD5ok(-!yWVR5e zwWpKG@SZ0VcfOS&koElx^!IUNCkk2Qgr@QH)+$JFc!DPvP~C@6^@(?ha%b1yx7 zG>BrDxsWh*3NTNrZ0`+h71ojrBvA%SgAHRkQ-rWXiS@eLTn^7GK%M$@1-rz{qtZNNiTaS3 z(HN~3WQoxYkw;EolNwLI=UfJ`W8`AXrGN%T?nF&0Qo+GAVrn>+Ufx^^m^cPFBeuVg z=c31*fQ=a-`lKGD@&xxB;;usX3mueLRu6ZM-oDz|J$rk)cUnC@**o1i+dq0OhD&*w~HSNZ}DRbWeexWCuhKST;Pta9=^iSLO&=b=E43>yHMDj zg_PLHwQ$Tr97sU#S-_+?PW5WVsaCJN;Y9I#5=X$;FkKj4B5uL*F_zaXhAL0k5r#W- z0V@y{C5D`+e}nZU)-c+WNo)ygRw|s*c>(g|k`Lx07`<31$e>&>k9a~2c~i*{d%?_o z1??#xFB8ONBnX)Z_Et}t>`dev#lP4J8GG#ueKx z7S6(Z=flS(lt$3vE^G41kOyG}xGY_iG4z7bh#qs{WEKD~Axc36zx3FtUdH6|s~;5P zOM4S#(ztlmUid72;)_BxcFF~@qr`SslU_6=%?w1Y{%4foBJp*yt6tc$KP2 z2dS8>CSt;&IZQASM74aG#&A9YMs(#5Vi5&HRd&kKQabJfYQbmv@nj13wy0#6uB3W2 zg5xaib=mS#a|s7hR3%BMfG}A4K19(%MFD#3yz)a>n2C$Bx+@@3F0Ys)AQ6N_u5^#p ztk##BRuifL-ASYOBxCLeuo()xR_LlFnS^;S1uGa}7X?fh#w#L7+H^STZz>HVbM ze1`_*n~rW+jBFlQhQqI5Se|2_S+!7|3#weOgA%+zGdF0hQmt1d7_$3=V&uG0^hq4I z$7p_v{(tr1z*_>z2W~Gy{D-$z*CWl+u@5AFm!Rn^sI zqv6~J;WgZlzk1|8n5ygMiIM}9F&((1c^=N|fv{yIFvz{gClV0?DWl6|8-D}xWJ{)e*e___ zKw&0El;wB%@7TMG=_;jr)sMJUcY;HuuVKtb6zgkyNv_)q1&rH>y2D*x-H_dDiVqP~ z3BC!+d9$@jeG-vaWM3&I>=UeD1XMPU?GpbBYZtmiWyHSh&J_?+K9%kV!wKv*ZtL=j zUnk(lC2>p$wmYJpQt1&~q^P?Rq?ouedYrJuh1?47x-2Pmgv493J?u zeSDXia;>|=sfRKcS+v?4U(3A6r*8Dnik3!s?c<+YuSb9UaTkcj`AtCWXE7A`oAHw;CuaMBLygd#8@d{ zbE66@6KNrkYJh^aZa`bDwQknG5#Cw?-r6_9!zkI8Fx!L$1Xtk19xz@M9oA$2>-d6*Fb1wS> zpRL+A^#+KxyNvEYGI}Csm-nO>Y_$Gu-A5zDO7qF490MCALV{Ib-OgUOGVMH>HHouE zGP`Hu$~3wsB4@yDH1kxcV)$hNn3#xg66yYpP(VS5e;ku7az<~p^{N0cq9~e=4DRKn zM`BqaS7hmc7oz;$;#+J+g9#Cv%=5`-*r}o#fhqUQ*;chL8#>%IHz^$$A6t-u2A}haYoWPp`!n`+s*n8}8yo4r;79 zpJkvfz(SiB`sezF{@yr$qO;If`uWC_3j+v)=dT1HOwge4mj!HNEdg${y4t~D6#Og1 zTbG=U%Y+dnjLy5m3y(j&^I`XbKX1-kqG|kIg>NJ8$vOO2Hggj?pV_-LCv;u$*Mk)N z14H730c-{O>aE=&v$WY*ZxYi)XPUG6&BhI+^;h}N5DWIBNps`wjth9LYe@1YJ`5Qa zp3(+>A1RDMxZ0pUSalG~4qa^m<>=twb^beJu!DmYewJDgRMr{Rwr`|0gOxa}s)O=C2{o%uvnDa3uQ*AGl8Fms$KFY7L!$P~Y(! z;-Ae;u_s6F-9Is*i~muSoAraJ!x56*_}81ze`EJj06-fI{?qluhu4)KL-8^6bm``X z{-*6Jees&fK`-`5;${9!6bXH^1pcxMgzHav(E68MfVQ6eOlGY$jmrEpRb-o3T#*fHWCHYBG7j#(*%#d(gilG0%`D3x9QkF z(?!B3&1)x_U=}myDK$y)-Re_Ef2yo;v(e&O@U<$B(kQs|<%W#K-)I8M$KT_AL^Do* z>HN@_MAl=U?o9mCl7DH(^OtCP4Er1hL^kuBgc6@khdn!sKjS`q$m2_z-VXo8DVzL3 z{}CXCxs~%-d*3y`si(@LuBZ}D#m9ZZq^;cJa-pImY_G0%cz@ElFUp?@5U45*Jj}7s66IU<} zC8nQtQH~&2_y#T!hyQr;Z&QA>9!xOGitgaO5k>5o!01DiYi7?GlPzf?Kvtuig2f0GBp-oP3a=P z!;r>ax;B(bO!3{N8)p<;bgn8guiVw4k50k|oqns1T?vIn+vK3zKqbRwI zg(PuaYL-csx*Vwo5FqUFHE%d1hioej#kP9vlv-ut3T}h2@5lh_MF&nyKaxu12(jy3 z`fTxu4=H4JRpA3i1_)&|7MVG5Xmv<3V`%ia`-j(yk(H;1Mj8jqq{V_zkD!@K75=X* zoj~l5v(yqkd|YaNSl&`kTJYOy{KGPdewBE*?=qNha|r6iXGtWC9|nH%-Z zOih)d6DbBq<&sejO1w`^VW;r_zP>PDLq^iQf zsW_B6q9M`OP<>HJ=4~pf&P~XP$lA!nuB*O0N@WH|B?61XiEQ;KlC==Hx7wim^WGov z#Z&%Dp0^DQIa>ji+q_l;LypGF?iD8 z_K|zyk5KduI4J&mr*{f*QItx2luE{^2r}{mvs3CP23Ih?k)=|<^_&T|O8&$XN~Opt z4t+e8A4V-?usHFb=1e1#CBKJqg@(ooA*&_7k=ddG$Y-gKq1y7ybyX|!GH8g~K&Gs2 z=+B>9@>88BzDYQFbCY|E>>7U{amHWBi>W$_YvVkFiPJ?UQB{V;>3Zr@>|tix)Tc2T zV%^MN;y_CF&CX(u%_tyQxI{5z+ssnT#@U64+&2md^4~5bj+}j%Cs_Vp*sWO7aOBDzcMZ(NN8;L?2sY1r033e}4z;23JzI36D zM4_yvfYY-}5$l&OhP+;)AhUmFQF|^kCK5S6mCtOT${zU&vjGhyzliCRzlh1B`hxM( z-})Lu9*@(J+0$R*^avJEm4>~{=@Xs^;tIvrYD*n;;tbUf4E&>i_!_sdAu!e^29dx1 z5P$g0PWUhp8>p%YuQDowqQ_Xj#9(zi7Fji%s!!%1Ovsx06mIUVQ*{$Q_x z^r_Y%x}pTh3d$eIPMY6hW(9L8zlpmPKh%d_CS)@$=;|l3n&MMT0`r=x=0s8eg0(ci zyXKD@{R6m8{R8+;3m7t=R>IAuE`0OU{O+1R1pFIb=jcxD3F1c84`0#*#GJ~nb>nkg zeQu~PeKAw07WE_4yG=5#@;5T1RGm0e{ngZ;k^XEMpEs)thO3sJIa>LPcv|`6PF&%_ z&X+!s#pN$_BA|M&SYFQK_$)H>p0=N4fvLowIPmfVh~MNZfe62HISjfS^8*NC(d+}c z>tyRvF<27x*qWtlEfk`oSgZB(=B6k2|fsg6fznkC<# z5HcHA{vd;UL@vY~^+l(reP7n-y1i+XQGwDE>*4kTpv9R1Pj?p(FBQ-)rc>3gAvpvU z1aG=Jk-?&nsuGhBk^ZhWHa7E~%rIZp8No1f1}}JWd&$U(K3zctLpO_6Gv;3u6c2^= zBb(OeVl}S1eklZa?o|elk8C67$vGrnE~1WqNe^K7H>)qEOI@=L&mw;HgN%hvRKr$9 zH4_MKQwEbEnaThmTx>D`(*DU`K+4V-u&9=K5Ysh{MGdpYDcnuSA%=Q<(ddt?1AxiN z`JCpBo|JwhZ1mhi<-6|C9rrS%N5V}kKS7IX{d@9*!4-#Y?;1GWbWP%a?YSlm!$ifb z4MJ5PvQ^W-4OJu?wj`kZ?x>JdQ{tO8S4#TO!^5|At|axr;5PE0ZF~i=ry0l%5=r$- z;$GC6lbf&SUG_4tzb%HO>BNGv5~;aWdl}NtUQ;N+rA~#c)XJ#TnpbHxtJdoLY8ki{ z66aRSKoyXfTaDG~{>Ym9KW5E634Jn}Vqs9K+GwIG()$7a$=W)UZm*s<9H_dakT@OZ z;$keJ3RC!KYKy*zK=b0zy&FWCEpsW%DtY^5UJ)iNqH20!bb)IlP<5sz@#TG z8%9gQXe|n(rC|(XyuVk)i?|HYk^If&KCqW1M^sRHxYRlnPzgCREwjzWH?wZwO}ntX zgI8rm-1r%n>4tfk_KibDA#yS*teX|gU^-lDa@V3&#lKnGN3h3TcHBwlKZx#(+7(f| zLS=Gh#i(u$(pzsdZh#5SctA_|(dWubK~#*$51d^+tK$;D1Ml5r6WviT&ZliPVNV5R zC6r62Yp}SQ{+mLjtUHeQiO=5%{^U_FTvk?$nwzwbkK22!Kodls`r!1OMe#zv7e#@@ zC_yA)4(?h2Gh;8I*xF_mF42crUPs4rZ(fqHH{-5HmYy@yWawp(b@|!UB$g4yuT~rD z4bo#*8q0P~>DpZ=u@|y5n&~CrYLs#C_>CY@cNCfR?%>{9tl}Nta9ndex5f2YEt1ZB z4qoBRj01NyAqjHMhPaxT#q!C2DrfIss&O^RCDT=vtBIm&q0aSF9VP}>Zesk-Tfv6e z&!^X|B!uhLW=?&<+FR9E&s+HoCTHd8^+uw#jR*Jl2^4-}f-n{<)Prcr%U)qL!{ z+JAqzx8;N$y(`OyDyq8x*iC$CY!X!C6Ve+f?c{X*iRGVcUOW}$)hDDsUFF25tKX+v zG{?==jVEHvR?32NwwYMD%?GaB#3dUWsb!4cK$uKcjRHZGApo*E!+3U}>ttcYR1GMl z$vxE0lvY3ZCksV#30a}Cxp9+)N^{h-ZT)eDL0TrbwZ*l*v%#&I(7q9t0V)cH-!xFR zc~nD%y9Qz8+(P(6(pCH^D^+hIB02HTLb6OODC;xdfPdDq5HLAUy~GQ_vUDS=wFLXAb#Fc)M~(K@?^8W} zsIIQ>efQu2*|C8&gX~Qe82v##BwDf1g!80T%faIv8n0uc}3d`mMtyWLeg{0`vLO{-1q%P=|Is|RaTdJ z7K;t=8_9N+5idN1g4M`x2NN4!%9=(pJl*!Cfmf< zq6r7gXq`74t@P00tGi;Sg1Jp>2CPrr_{O?(u#_@k5ArDogZEd;-o3NgAl3scVQV{_ ztEXCSC0JZeNuz~0n@F-~Vju&?r#g9Dj2`XtOj+ThummXMT*8K)D*FGdaG4BsU3O|K zC!&w^{0_3SpIbKPN|mzxGtc4EF>AeyF3>Z+tuncnd1-tj&v3i+w*D^W-i?-HwH)&0 z`r}G!11pd=mwtp|s1(wbKKdzq_yt^3WtXi z8v4Ys8a)(OAO2|APY9^-cQ6&aS6UGDZG9|vD6@E?hE<0sycQUv8ku3gcs$YmG(soK zgH!kKKO4>ceyY#+&*ZSr&DluoQ0BGDsYYv?JQj zEqO0T5_7q+E7|Z=o5HkM;$Lng`T|~ysS^DQn5j6lniC&UhI)%Axp;W*KDU+G%CL0> z$|ydtTkqQ?Vx*9rnp`X`l5`|t8B;5%Xo89xESJScKoG?Y=3H}wsKIuCv&5Ntsa>R% z|0+dTE>)#BY_2UWyS6NKE!SBzw`8xVS0&HQYn#hV!X9$8q_3%((>2vzQf8{MeUySr zgxf#`IETF~D^Hzeg-Q+b1P~43L)a67mYPF%Vu_rtb6`Uu!xP-6t`~jGjm8YX!3Tm} zOI76#S8EA78F^%H=;oHXV$LXbkm^Lcrez&3j(1jsI<>yD;u5C#rNptWj{o}WdF~%+ zVNi0o!+6l# zK>hipy4RD$>e91j+c?@BVSo)HDal5b7#{Tz=aX^gPQ4F;BdLxtGtb;0vr}<#ZO&Zj z+tVKMGsB%2vLxQbo%y{tlC+8YFT9(vd$gsquThu%p6mTKk)mw>@!;9(a3YJIp{8mO~$l=sV8)D?a*E5%Vf9%ZfiDZ4>x)@Jtf-e zM7X;k9MlR~8Idlx#6W97g3f!048Jh20PJ9|8{8Ot083Fb$WrW8 zYS6hD=qnI1<=18~+qQ9xEh9NZ9_9N{Fm4=PREZvvS4unH1l0x*xSht>R8^?hc@g%k zW}NCk=h(c%;%KC&wCJa1B*{kT`Ri@qwP94kINz_Oz+`Hm+M3@$T;_^ zwH3LAQ*bSBvx1mV;8hZW$mR{=i2)?+1aqgK-Z;}+*y;Yf;Nu0!Llv9pWab=!=0p!D{o-f*!LUDuD2hOm3m1LMn=jR#GfkJoA@|cF&&& zfRcKZ-wz7s_Ff1BtWv#t<$%`Q3b^s@9Mx$tP>XTZQS+BO|;laks$JT?T9vZOOOY|o)776Vl>;MB&W>DZ2c*bhUfWN zlM?+69E&RQ+hc01VwkZlG>t*=Vl?s=9N&G#swtbRCm(!>V&-te7w!*66NORncFud< z)MLZD7RJY<@0P^?CBw<+9~3xUwhri(<{ABvCW-xBgpium_Lk*G^>iSJV)>fk%brT( zd~}1N56XEXc)QYUx6pByKMpr}QY3O0={!%k7KkMFx^CqeEyVaiknoEG3B9pg}nfIYD9}B27ftu;sy*94fYFfA2F@ zLa9;Y`_-f@^Z5~X67x&F%rqNRst4wl(>|8R26RiN$E_xG)A+v(ujnFa==tr2sWAZ; z-LrG>m|NVxf=CBMEHHIe-55lZIJ2n7ZdrIBdJ>H{_sbdfo1Vt!ubYLL*^2qS3?VAR z6frCxBW;Ozr@}&pfRSY(M~y+Y1a#`j4Wb2 z#7Cb^&P1ABV_gEq2#TpXwD#x9Mh96~1ek^+;xe`Hj8vK;~G=SH33-;#2)lq-On0ofQ3)nG_q% z{%fcPy=)|>ksn9uTboR`=n1sSXg8|pia)J;=wJjY7l-r$Wsu9ICtLq|qCjCgPueI% zgqnbSjTjQsCoZtzK7z!(pbIO3J%PXH!XbBKe@Sn%U~*(X7PW+*<0M`~<%fT2pb*&A z=uxyE*X7+7#E+NoZ@%p2}pD_1E%Sv3!YHWx~>ZU>@CpeO0bP zM1lAa@?>qtvO+qIG$X*cc)#sF7>qAQi)>NULFn(KJU6k;s6?`oHZl*|;)tB7so6aH zy*K^KcbO5E= zCLj8OUW8v%qiUYTH7+tRxw+m`KJbk4c zLQo}F6p+VcY(8>K^2wGgGw4_bsz9HqkoX&C%OZok-QZC>-_p+@z;9~S#%Bv31Dx*#5NSsc>JR7Ew zL(I$Bf2m#U(&dK<@9iVB^MAF#&42Q%i{s5fd0#;o3Cvw5I&L;`ro$ioD=z>jMX zYLtKEBW}P)&I+xzpik_0Nd}n})-c+W7;S>2f-YR49Et{N0CkQc3P~sds{$U4X92oL ztmD)c%q1u{*{EX598aKMQQBUcSA$YS9)YC7=tYG)s3E*37YA=3g_*{?cuXR&bPPLDLoQN`sf*`{O~$9k|m%{?b$ z)Eu^`sx3!VBUIEFDn=Jlhrls9qRVVpie}&QMU_hdG@*V!b}RBNE<`y8!d*mN(%2k7$8)= z9?S!BG3|ZneMkqM-@9g+%@9A$}B=qA~` zjhoo3Moob}_zwm@C{5Yl+-RfpETL^R#?0busEG&s%QKl41%|lksMkcTL-CmI2W$!R#PO%G}P`4V<=F+D zQP*wiqjLBMFpht+uph(A&1bH#ALNcZw9t$phRL!mcTW^WT`#uuqu_ig4kNxI8RS0G ztAJ?aF`>}O`iau0u)gqyf*dLRZ?dc4LS*tyd~wBKWJ&ENIJQ4IxDUGL*~xs#O6$lL zJ;{LP{bZoYR*`gT^}=@o)u4TWUS(&1)qnZdYm|2Aqrh?DtGzw)8y=+~!lupK_hW;n zUJwB47oF4!9}PoK*wumwh-*;3WLWa`X=;3}I_slvQDfB1ZCiwx{preb+3d2J(tM2io}IB{YvN zn9BdO%rpKe^@bG?!7T3Jw0j~C*JAKzMUo&crj|xdx)4rvM%v|)aECS{n!_OT>Ls(| zB5&QP*9Y%wY4XJ_B6#T~^iD|r$oAgsbKnV&0guLRMz!&3t;PWLS!pJgM|Nj6~Cv22mClm(l{l?Mz8kr0b zr)yVNUtiCswPPvlZByIO@JtxDX83pJ@2_d*|AAG%I>>il zBwWm??J?ORHI&G!`L&PQ8idm`#{YK(^$tdP-j6e`Edn-p09oM_OZv;N$*bc!Roq`U zfB5_NbPGwlj`f^cySUvIM9N1ZD@C?KK0Dx=Z!0}Q(5iPVTU;McVwzv|9~S5Kp|6)K z6@oX$wjdy7>eBS#yM2Y=T%NO#Lnx{CG$2%kI}J>OHB6`C0PW>p#~Kh#kEEoO1V*zdc7w1H?dbARJT^N*?k=Kx>C>P$_eLzG^P7BqF@87@ zZVT7b2#O6fwfYoZ!9gd{C#3b=JUI{As?iAj`Z8;6ED~@t!cW~z6}1-gECa5QXIol2wyu6O+&TCvd65S>#t8Dc$tJvvUa%6kJHOmL`Vg zW5&pb1!eh2z0>~bJb9%KT27O~xFK2^9-GadDyrUy;@|2zLxp?#5v0Jd*v3@xdyK&g zfngEfZ0CE6Fkih1tuZqBraDUuV%CJ@iJl-GN{phDpe!Ef&`L#teZjpn4x!-0TqVLY zjt_f#);@(v^_9tu<~9DR#4f_l^kQ+szZ(*vgC5aXg2 zgg@S7jl@}v3?lH)Qx_`0*eS%g86sw_Gx9H7ImyI4cw=1)BI8gJiQFC6AB8x*0{Yka z_gIYB<*-zBPNAw$v!VpX|CAR>l`hi{HP&eaX!*s-^JM zTQuBVSG>leKXY;P-a;}3eEP8DFBg5n3xTV@8c8>z2P!$rA1f|Rrw5g3w&dv(fr3W4 zbd+-rjRfhDBXcjwTha(1$f$X|)6y5ib2u#S+)FPjcK_xJ7msk5LfZG@D4 zm_cFR_;&H&0p+TGd z<3J&&5_JNdkoa_f{|t0f2f=@8`4t~0lAaZt7AMY0T@%K2Qq5j2leSTOG8J(Muk0&7 z2O1>ai1UK2w^5NLIS$1yM}!rs1Qbs7lRB*upc5#iS?_2C%j(@H@rvu55@nT1l_638 z;tf7N{P*otr=UTkzt5V+7^3G}7u`*utRJjMOAn2N@En8_E0IF7?M4bG`k+~H(6~>| z3epmSR7n>up;5bHfIMeoe?*fm@gO`c-D$rgROHz!mzwFHE#BC`Tnjxb+&iR}4!jn# z7jwT9XbvJ^v=W(RSVKh7r%xC$@eG>4CSjyhgauHV(BRudD}vWWA5GO!22!Z^Af%TI zm=vvT--RhVh4zRcH~9dq#`#4#@dnz6V>1sa_x zc!@rOE2l)9k$+`a0VZNk1_UpK5@mQ0j?+8-cJuVl$UQ#ZWLY{L;BTN^Sqo(UOY;~! z`I6m2U5=W8jjsjtaJQzqp5qHzRo9B<3kI$QuQF|bvAZTy?`8*}y z4ZJc?(86CRyF)JeUSn2@2qQ}OUmKBLwa`%J7WRNyNYBG|H{c9>)>B1V_J;a$62rCf zcQN2?FJr>&?%;)RmqWJ6l4o}qRx*;fy|cTkrvANjq=NQ{H5WF*{#~L9Z83GnAf2BL zxSv$K-S}_R+FV#PRPkyFQ00zru zA}e5AoqsE4lcmu!r_;{AT^Jca34yU4Gx0`42HGF?B%_IN<0th>1Z1g>o+4aW_A08u z88>PzDnUbI8E#GCe0_CauDiBVFG1B$$AO<+FRjA8@HUrY$Ji?|f@1MeS&02N*k@75 zteR5+H?*ZtSe#%~i-RGCEw4;^s{4+|DvHe`4p+28~kxznDBg3;6*qv1#!{Zv}(CbG|{0|e)#IZNW;Ls&rg;;g!XY{&Nj!O*|bduXN*WHXWCW^(kP{ShY)o|I>8WoWM z1rku@N|_AzE#G{)-!to0Y~DH*ZwKJ(f?lr>;}0xHKc9fapL+H@)*p@UABNYLmJ7s+ zvg2nA85a%hg!Ru@0<4@*zczikrOgFZ>o2S>5$(rV^&*Mc)P|APZ*Eo zuhRvoDtvv;i!eN${~%vqP4n$2LLUb){~%8u2?_dFf3GjQ{C2$B1GKp{93nCkB(Gmr?CIk~UzI-oUftRd1D0V%C1zBmCC4xGuVUUg|<2GHK*nWwjpj|3r9d?b~D7 zAVQJ$)fNiZ%N+J9Yh)gGR2VG}$Pj)F4}bCoD)03L_RhJ)SBdh&4fnO3$K5IcADRdm(u@Kp{_gbj=yAQXs(b>ss^8Z`CWj%hQ!9n`pK?mi-VjsL$1v(&8FyAC zP&7AVqv4^X^pRlTxmho*!FD>C8eNf%rf1W@`^=|T z8!o)N_VIx+Gr+&>vUk8-r}XLksDFOZA4f|~UBH}-*d%=HY|FI`!yFNmACNwPFAX2atL}L&GW<)t?(5y+p|feD zN=JYFwl+hXF^_(m`Fob;}L` z5JJ~WQOrF_;&CrFBx5}^l~zh_-aErkOn(OwfIfXk^>|jIC?v2ZM&>1iXN>>RAohd! zhU3XlpT5Uf1_19`HC#U;Kfa z{0rghjxE)MJ)yUu7Z+QOY(9A<_A-n2w3Eki@~z^+B_7c*iSvpIMhXDmCbrCsf?6yI z`1??*-m$zHa3WKUJ;GAFy3S7UHw%ct9%Zj_ki<^I) zuz0)?L%GuKDBIBNP-WC(i-=#^R23*oG1T=R8^fntLu8@#0^M?=$O5R7F zp-85f9j(?HYA*O{28{R)Zu(9}2328-S*v%iHoQJ|uu2+qSSC;~+NPz3UXr!N<-OzK z;(ILzg&x!hvWsfkV;#WZ14)U$7%8OOsx2m7Mm=sTAYprlT89ec>Y9N+Nv;~Kk>X-c z0s=OR41|Rm3u={%wFYn4Tb!kEklJH$@MK&16Rg$2{E;EWgis*U1{M1aak4A|NV=JKld z@Q^~%=?dxG%xvLR7Hc5;O^mQ{L(Z*LEy zOpKfn9bg(`a4%Q_*&`|2>dX+>+_x#ck2bUW)Ycs1m1c%~ru(54g)th^hv%w$dT%;_ zARmNs*Ooirr?WWaC4tkc_7k3Rh2T=yf@=%0JXligj<)(7*5_UnbOEf#ADUGw%WuXj}_ zE`r&@a)nw1zGj&cEz0kd%cc5We8oal2&a_|zj_YWH5!q$TvOGwvjF#&Qy6mmgY#i+ zHiUnKr(EUlqfFAJFn#l=D1ou*Hn5)6g0;m<;y*L%=$)N{^q1E~orOk@*k~+*->vy; z^LtwEI~pu01}$cd#8*6A$jK?^J^K(B!A}(?2|P8W`trZk-qB3qqv)AoZ-5$|>U5hc z0|&TGdB7JNCfCh;n;Un_Al;4>Hq%!{D!ZJ3bW)hDEV6WoljH&V^{9o`Vl$|iMPexT zoP#Pqu6RY&w9ZOW{6zL!ND@ga_oFc+FY_Drf7r!d;0z1P3568J^$t&Z_M=rqfBbG& zVtim*e|LO~BFyi9rR+DrrU1!FkJremd{1G^qUGh@nr(YE{NT9HeorGiKJ)1|*9EKz zHDAQH=16*#E8{}2gtcll2DU9^KgXpfh!jBLUxToplsMf|`khaHG@j#38F<<6!16fy zC9=#vT)X zOG+lcePM_hH!4$++y&4Ysdnjx1rw(Y%Wm_ej;!q1_SxCpugOu(sKzzB=n>S0BU27@ z!rVporgif6KmZAWYVEIwv;8rAa#IW0G>FasR|r%#lWf+&7F{4Z6CoFKNUO5RETLE5 zT=N;I=9Eli*|F>-BgURwHu6jX7kPyE1pS#jG*Hp!8l*_@GH_tC8CbT4(LqC%sI0=0 zu1Y72to$JgtG+C4+YSPDizdozqoxdwu(t=O9QVU*w_^Y=HEDfzT-Sk1+e44bpKkUO zr0n0$3Ctnc9(i(#?Gf2RbncDNhSzt;#~8c^i?;6tE{&SefZ~JVuPs2xbk4yuw!nCP z*V6aSoHZ7GRfT%sq5OyG&*=)l4+TWJazG7f2(WERZH9=Ic*GKcOPeCUYpiif>O{;) z`e}bsK(^Wf-P0qbl;YTJ^|vc+n(`9X7jI}Um6j~*a9qe9qZ^Z#Ti_;E9Ijf`zl^Yo zK{ZI#W)zQKno%V`xwKLIel)wvta#7NE@qIsn+5VK@QxyU-Kkiq{9qGI1d~)PEn2mZ z(_`0Vq-)gi0fz%3;fItfC6Wp_&fM;pgXbnNn1^#CptY4oN~}&}4-rMhY>Bl~6tY)3 zur}Zp<#fM0kINGS@;Wy(t2?dSUsPUBkp=keF$EW)#CfCm3Kg9ha7L}gFt z4?U*)jJv$SP2amJsKr@^S!SSUKepr3Z2t7#-9+~lNE33SPZ&T2xCv9f*zDEo6T@1$ zTTjqDVq;2}JtixW+dOGnePH5LaS8ve`v?z)S0i%x%dqfB>BRAb(C7E>(g*yaFEZTb z5LZG|8mkQ4vZkevo}|gJ*L`jtetxQ}+&WJCwtd?)czTyq$kvf53{Hmt9+p7K zW~H};EjL2y(WqyO0ei~^tu#}N_As8jqE@QS^jUJikiWIt%WQ`^M+t8jEa3|= zN5T@tss9b}>qFwITV-|dgF)Uy{^e!e@)4}9C_v%lM+qm1H@NSmOP)6t3Y~X+1bjcK zo0Jhm;5_{Tgo9UE5-1@+PB@`s&#gRRnn1CzP9X#{5*32#0B%&sdq8MTuot3y`@6Q= z$9>kzW0lLl)dPlOmbh3&c#t==@Z*F}F|zwYlyqIMV=I=L=zwB%Ey_!&RIE1)PcuEj zpDM(Mykq3mWGRpuqh8(fA5+BB z3N1&#kBYXm9)Q3vX_4MnZkuc2lgtz06ERZ$x3Yb@%Io>q)#2ZhM@AVVPs_P=ZIg?K ze{;v~!c9dbmzOJUad14u$<8+S2VhivxWSHOd9z~@cVM74mZd_|AI&JJ!{d4~)Kgf~ zmWgw05+FFE59kij0LT~WJM9isF2W{6&HGLw*pDSu_?l)tj62sI`v)0AwRsjok`C$0 zBvx0lWBqYO&F37aNOGZu!do}BqgQx|SgfFgUOTG27%9DXHfo8V z-bG7Abpxt2ej~P3`>}`TC+W4YjG6FRo*aUuypPB zHGjuK_(0Ln@{~7m&@`?Gp+R`OiN5>Ld7q4QGxRixah{E4e>bLsfl5!AfdROr`+V0a zt3pZNC#?w2zK(vyC+~r(QHp&OEs3ERxe)g_J{=~>!MePAo`d!HXwiL+v)|z^{BnV% z?l#kR4}RRgQjA{w98s~ zzUQ2)IND30QdSV)bP$`5#)(5yls{^K*+vLT)FVFgjyLR~k#*r8RThWE+=swb#{j@) zW6@Jpl1n<-_H)u*!NwL!-?w)d?S?|nx4V_HCA6hW&_(`Qp71A0@65C zi#Ox~(^v*cG90$1!6-SOE;P{(!xHHwR=(s9*Ap@w&it$hM$)QV3GZXzn>*WO$bfTU zB9fOIw>@nxqC0os)o_DPhB~eA7x3<%+Wyjc*Vt_JBj)@UA*9&}?YD&t)<9l93UCJE zQu)K<$`@0lI;wTWC!GG zY|c=uvE0Q4H}LpuP*%$Osoo-(?)}!ad!kI%F$i1RZe&eH2Bosvjp%&z^Vk}xbI`;1 zcdGQ`STuQaEX2o?I*+d~1CvEad{KI+xWYm7=x6@U5o)RwBdciE^}Xy~>0qE%j9+mU zUY(+SE#ftFKJz>hOPZc1x%kb{fAPq;oqA5OCd@A8Jk>$Z-L|(`d;#A{@ZFlI^*js~wyD)wR{UcuKCJk`VN_n_)&T zL!}dU%`DJ+Sa9mO+O_K68TIp<{=P%WOhe0{!MFjWzQC*vzSwpg8oA3gX@}JKdS-jz za!a4lZhmgQ-fb@8#tA74|0wPn59u#EMpi%N=}$p2?`{)s|JH-G#U~cS{I(5O3;hkC zM&=w)dM+1jJ7J`Atki_a|KvxER~TbH@BF!OJ2ww-`U9xtz z4(l%hN@MDF2S(|Ll#)MY9n&={O;j4#FR!t^n^!w=EvMuwf4n5z%bE^LYFqmSg!W;=Tj!RxTo; z(6{<`C(K$S2zs3YG9})huJ1bIeGC62aHv;}OS>R=b=0`@a4etqQ^O)YdxcXoqn(cm zznyGQr#p?KzR6i-yXR>TZlTaF;;OK;-t2v`1aEcd;iU|7e{iC!n@%TJ3hILK`eS>e zTD*xod9J-xZok-{AN+C*&KJ;098DiJH*#}77Rxr*PQcaP;Tf8h?++CAZ8KtUt^&0c zW58CI8BNHj*_pNl8p@6F!)zmZMZCNQtWeAW`kR+?C%S$2!z2|77`&WK*+KNP@wP2C zrG0lMP+01HwJbm0Mgg;cfXH8hKU)bMz#rsCwEo*liah@e2^r0@G|5<_)QP+?v{9%L zt7}|q3HOjh00Zdi%d7*%_{LDdPp+R*WPoGNB}5|N00M2h?=%cA?`~SceaAR7s_qsy z-OF>Ujv-Y2bCe;JmRpNA32Xf@Dtr-G{s@RIs+_|~5Orui+}Ha6T5x_(sOT7zqCVRG zX{JbJv+~>H?tmBIWAuUGq9gkx0<-sEb7QW)v-DIel2kPe7P(FFN`*JgiU}cIMJ>a1 z;wJ>@3`0QN-l2s{BDLgP+w3N|9JS*U;Oy(|-ry{6(ve6nIS7s0m;trg@C~o*ziPb? z7D=nTLNp`CG80Ipr&fx|1FrQo%8BrC<39b&E%C2UxJm(R@tJ+(o*!Ut)<0CMRQBgu zM5^+AV3Pp!=u6(4^m3K@T;-DT>~KAjCN9ndCyNwIg{UVb!bvdWqyV^e+%Dj1y(rWj zU%5Z?_bqAz^L-rk`bD_Tdtg=wB0gpXOr11ClSQHCHO`6~^d|D!#;)_opjK6t*iLXL zd~Pz8xm=j&g&wr6Uz-)8DWtjj{{jiO;4;XFiuZ}DiTzK2I~fCW7)fVL3$-G?$*UZ- zDY=X~_1x}@1#~QpvK*xq8=v*G!c705lqcBjjp0)CK_S)7sy7q3@WX`7S;B3aFOHZi zvk_o-xwh~9^|XWbIX`&1$)U6o4&cwh_(zD0$!7823x8t52l|Ye9nMnHZpGNNavm|L zcM!dwsy$h2w!{X)Xn^dPfwQT-70H*HI_HejO3*WFD)_l;prl~$=nP|@J~0?A&mkZm ztAUf0&>4Ghie$BrNc3ymJ{9P^1;N)4-wPo_46&l2A->P!G-Z!8>3R)p2*@{XU>oK5VzBtTPNt&VS68#CKJcdp4zFH|GJh`*M% zH?o5eU959-oNO?>bCq^BvvnRAkKs6v-M6wML|t|#clvb@N174sW$YiZ?T!i$AX`fl z24a`*8=i}<=zMLRfwD);Z7>k>)dH58*0u3w44y=KXznQ|^JvwA-pT#>ER%cC4Wp+L zS5K;mg(j48|v%$ z%eQkuP3*P6!|2<+M#?NPS8Atw)-w3leD^XSMY5fAd$3d6OM3`1y*(xKGwOpMzuq79 zhAjYD^;?h_pl>aVgMA<8@B4>-CSUb91DiUg>|I0gEly=;!vK#m3nWZzjhvprqnUba zi;k%Xz292hnzru8$fy|hWJ)kRAkLqRkKx!BXvGofQREo*Hhv}TK&PSDyK5>^3s7`9Zg+>h^ z#-}xblk&2C)%bn3^) zU6$9V)3wh%DlsTMfsHw*7&G-7Yn%MZ$91gMB)^R>-Xv>1oU3BVIc&`8JGSn-TGj(r z{?cev|@9a6TEdp0eZ4<#4&%Mp=jbXchwDA zDeCqinp%Xv2Y1iY$QL-p1np(6i7Q-MZ~Thbf(u1mvGr7*upYey^At$dg0%>e!}L3N zGCV)GOL<@(ar^JQ;H#7KrsGEI`#0#SnxYi@KV)-Fr26amNm6M?Kq1|!Z56Q5;Tq28 z$>We?b>@?seH+CFFA4fZQ`Js!$SBI9jvtD$rOH)OQcq7h-Rn?}!g&Nh%A4ALj> ztGyZ$@Q|p?vZLg$vIFm209J<<-L|l1fklNZLl$3W^Fk~Wp`q2LP0FtR_FV!?W+mMw zuC}wr44LJz3l=riO$?P`g~U(ALol^t6ACI;P&y@hdU=#~%$?t3YJ^5s{I?W)&RmP? z-VZaIZ^o|Ub=o(k+2$u*59w7N`jczpOT?-Ll5i~AnGNuTtNT7klxEiuq0o))JF(!l zhdVhj_LDVwQ93A@v9p8{l5>2f9@9J7EYC-=a&NUf_k|lwVtWZ8){OTD=R~d&jBb;m zb10=6>v?~9Yjw08Od1|2L}~9q2J(dbllq*aGzfmFhzX^~V7n5dcK!JPHT$09;0PIj z|1I2oghaj;5i?XNK~}lrc|BNAVx;4IE6V<){tVtCJS`Snw)FPEAJa(L*lEk8ZtGax zo%^)8y@dD9Kg{fMWZRf}TtXr0mE^~8bTmHGX8mn+@EKc7!J-T)XJMN1ZXWaSTF&D9 z#A%Z~G;8LX<+LL~s6T$VbM*u4V}ZxR;g6-TMU3MBw%5(#Oo^K8(vc2Gc0KA>VxE&j z#aond?cM|WyZFNBshV)2Gqc9|6DjNZ1=6+vW>D;PQDHg=>yv+!FE|TazXed=@w1JSDx4K&*s$vwZ_&ZlgdhAha;3dD-hIdY6Vbf<(D}esbIw0jsE$|J8wdZ3Wht~2d3h*FG2}G_Ar~NM8o^{nO~a=rUKz^K zMO#G{RANx|0VHlOud#Aqm5(M>9#9G>>-r&^ALaN0a3FHU-_ z#Le}5XUCPr{S@L#qN1HCU(1gINK8V;kQyFe7wYQ{VnhetNKwO&?t~;ihSsn5?;c zux;Ix{mn3(#~NXIiI=6DLHInZye8`EShpJvRni zV{rmrb0shyHiF=D_LY=FU z&vtQayI;Nf;rGzTz+sM|_3+Ux>T9rPoHbDMbR3~9opJrRk}t8d zagzTrwW{G(d=-yKbw7P(sdf@=(sI3I#;8)`==qYa9BeeIEs*^6PgK@G2>J?4}u8`HO* zNdCq*Su7VJT#->_m#~S)>46VyKkaV4}IFrr`Spa$?Q)194RTM5!O zm9!!q6d$oL(q*oQgx#rZsLe+teq%8ztXb?vPB<~%;2z&dj7U9y1vq%^7<(i>Hc$WH z3O2Z3DsB|#4AiZLLi-T$TNtj`LQ(`MFXT9mr%Wce4T>l;g7Wa({Tf-k1(lkSWrjX5 zKiH8*L%;$2jZF+8GFTd!*@1sF6uHSTnVn2jHuX?QFo9SR|$WFwll-n?U9_(^%`Q3=pwXq|*8WdlQ~vQKr|XYpFdJaO1IqiLJ;I8WQgS)~y}2g4v{FWai;5gUp> zgr`xiZ&+QEl6BWOs_;E9fsop96Z@%~A@{g0&{bp1_2}3AwfRcyn+W*Zi-bg%nog=6 z3>UzP&!rii3(Q@@FINR5>nF)_7_tIr`YPUG^sMzo<>*kIq-xflyj!(NO>W3qyeTOM zvpmRG@xW7%co3(Ho+}u7xnw#?R^(0m@$PhS{bLfV21HjTUBhhy-JshjJw7Kk-HbHY z8o^2fjkWJDCqHc(?m@`Kq8VNWE-?DKnTPZob8J{Qu#LBbq~NZ9gRK%MCP`M=$fzyD zAP;A7c;A3d?rg#r({H7$Spe&jxGVtV`u=RGE*aQm6A23b*%8pZ&K$j4?dUim#%UTj zo*f?qWJr;TJr5@veOEHQ2FkuC&H&;cgkRC~&sXX~9byqCx{}u?>@}KJ{-CU=dI=Y3VKo+H~k{(v`%lIzu%)>xx8CuAx73yD=ec- z9w9fGw&zMUM{$-SSqB|Ty5+#1%H*euD?MDjLvE9Ai$5* z>q-k%1-WeVhkjfN{a8UgDK_%u-I5}Pf;cJ}CH%b9EA7~@7 zIR$xCO=D_6|B#bD!-?HVGfkRWc35D6lYhM5ic16MIgWSy52|LokJ&!AnyhhaO{cUD zksJMhVfKe5WklFfwQ7&_k(J}v;H8c1&N3sXFW$8YC(VFUWaRYR8sDl_D^I;b!cHDm zi0pGSXR)%(+vV$Z4Zs`_iV@TYLtdr<1#TfjoHaep9z(0VMP>eDBR(B2Jb zM`i+3{-XRZ2cXQIj53KBf3-9pK#i9#PEFA5w2A@=@ol5h zJDDHRLB`iMR-eN0yt^(|eEd`_SzTN_k0^S0`ODMoPjZ(9vFhQ@IVX z=f)-;kV(fkLYRdTHBhP1T%NwU*v^FG#Fa5nAj7meJi#<#h9?^l&tXm)Eja1G5n0bT zzHu~LUDVtMBnmlO{p@#{mnmVvi|@AQ5*EBPbjgVRbqH)-WG**&C`$>!DVZUd$6q`5 zLxTWPJlx(*5bKFo3A`fw%xN{x^g!fv9D*_%}I~p z4Z0`s`<2d;phHI>7kwsziJ6DNq=PQeT zH7i%3OGVLCiufM@fRwuFP1ntOZgEPrC>zUNC9%`eMYQ1M>*2B@t}sC-L( z1K8C}uz2Tv1JJchpm2eH1I%@VY5F0v0a(#*vX^vD5nP|yMR>Ajm%*8vT>_@zFCzpa5UW<}pgIw_r{FD^hxg>jplQkh7rj2MYruo#Y`?~Od=IDX{bJ04u;ebp-^ z*LN}03|_WFzuN=kqFyX8loJ;g5C?u3#T>;p4)3Wv;HE&vgT==SGFVKiuL~$heu>Wx zp|E`Ybx|yX6FyJILrOt}|C`OlCdpDgPHQ50CX>;-MQiksa$QM)EyVh6_J z_28p0ANyF7odXJ#fPIvKsy{kze*qn ztl=|`z(HjPixS~MD0MoHfSJjMm8iGt)UdJ~}=- zKEn8H$2+I(J*RX)F@L4%c{%lRXK$ZuC>&}ear~ru89(*_S3KZ}_agvr5x~R|2a-KU4t%H?QMxgMzKPI0BrYctE3N5mV-?V(jx@m;~bY0$YyHPO2 zFgY}3>64IED6v{tq)z4RHOi`L?&x%W<~*c}J!~otcL1sEyl5XEygl3Nob4Z)yBO`6 zjZoAjXx_a6G6LD2nT7?T(p)otSxl=Yp$a%lSw*Y4cKlpoh9y52$g<^^IcaNtos(nF zZ?>8=AmYvAo4v=ImBX0S9On#aAni5>E;l+SEW#mu4+<@L+!pZOJIzL|*;>}h@F=no z_0e=Fz9(#w?8_&8si^2wDZ6kV|G~=(*$RBw?=;1uJ+~2_A}>4W9>!>pO!<6}8QPMp}?7Gy79~QV^PA#2i6W z!2JOM9E5#@8vw3Gwnf1f9v7Stw+Kmp4Itehh=FWiQ@7OWWsfgsnX(JdSl7G?3oLSW z)kQ7q*lv(ntlfjXoujuW25QjZ+xebYXd_5|Zv@xzT?fO1b~xZxr`NUj1A)UmK}8)A zN!0(-6+ykp{#2`yu*mV&bH*NrB1WHE9%&KBS3c_uVoPWQt52i39mItp*Z;<}R!gt= ztbydM>ur)b#`Bvj@ATWGIak09utVpIR-{pu7l;~ zXw74}sxBuCjjUEm7*CLXG2=Xf_WUZ3)a&O`Mydz-w2>;3K^+-+Df-CBOA<(*l109x zY{bpde7cf_XPOSaf+{#IYUVVK6B{`Q>pVLweHR!FZ>gz@D4#h>LyVp|Ly}NfV7$$Y zn9^CI#H^8x17mQ~AX1uU)2jj;Y2bPAofFF=aXD6aKv+s<4j|KPDzQ;=Y&>U3M6+a~ z{o$ZD&X$OZth7ATfWz3QKaB22R!Hh|TV~lOra=|Ulv4JGd8#_+DG6hLnAIys5sBG2 z<6!5%f9oJGIm=u?BZP9wZwA~hbM5u9h~|%kUjW?haC^W$Ks}U_3ebJ+-QNPjPX!^B z7VyIvMjCY0rET~11*oE9Qd%+l{6pR%Ctnv^=**uTw7=!F%RwZ(U z8d+#ZcQhB_WX)X4bp}c>ppzo%Br$q>GZh-K4`!Hm=MHtMt9W=PsT1Xo%F>CiY z_&H_9fjV&VpFLAaS0!=+Grc?vql-B^GY8WxwjF-mIXxMH3vvp8LH3}`w85p*zcFz73Z$({D=qJIK-t8Rb}mm ztzNS!XFgN(svg$AQyMi-{CKoFc%cov$NVN zVTl!|$(Aq-dK^(4XNQ!7?&!a%7OXNAD#Q)pS5{M)UgC19IlW!D^wZ3}BJvjAD7;Ia z{!4es>a1Pz(hYC@ae9+*x^vuSHmN1y?JBX z#?d(3f2B`>xNSl@)IsX-A=|Ai%Z{GPl72eUv}qo`AOT824?776Um?#|B6&d$!^0Y$W1Sb1zvgkn0I>t!&RgTPCMxw)Q$xw%d-;eCGEqlEES zI`8sj?TLpbY3lQew|kuak?UOs3EqDfr&qr#VFII(8=4n6T-1zwkq>?$%2eFo(7Rc$ zZQgvpccQ{rSQilH*p`=|XG zcKlA^fts|m`ZX}}=J4S3V0ZUTJ*=qGcfpJTQoxQEg(+(GCSgdHxiyw#^Rr^=$AiP4 zzB_xji{1(Ku=OKW(RJ>EGX-R8VzGlmqo77^dNHm*DZH8097p02TPnRnlEM38H%!K8 z@SQpr8642rTSTbvCx!64m>P{u7Afd7l!lnWd zq(}Iv_2XjEsZ!m1M0`Qw{HeXv3(|=Lsr104Dio$C_NhX0<&L|5>Xh3R?(ZG^tx|{a z<20kf;-~CHIV+55E_o%NTlI7hl6U8{Y5!AacHsY>y^6zU%|EqKuKZJ*<;vSY5mbX` z3{5Xz1IU)M-f7+mCJ$7tekge z`Am0(ox0Lr-uUIQie>_W`a_n2iyKWK;%qd>|amz!{C+&BP4NdXoPY z`HxX*HU-1$v%US(y@T$~{y!g3B@-pxEF8E&Oc$oNSIgOOx<&0=l&s(zTPe5AvUY$T zH}6HxYls|KHi(@$MVvh0|VbcUD)StO!Aw8%>ayp%{Jd z5tc+jVIfzL{(0#@Q)dTnxA%_eikVDPw(@1^pT^htZ?pI^3+>khOKZD)g(lA|GJ`(P z?EC|g5FCq&fYnd;t?ALv&+LgaCk)3|F;Vzxh6y@3;xI|~ zIx&pUrPv>0DDQkQhP?g*qgF4e`}Aq5kdo2`W+}BISXSTE6Qyw1sx%^4p{puXTTR&I zN>m~F0_9o}Z7hWMNztQe%?K!q&7$_gE}GvLN()W77w67>Rnoh>vz{_%rWKRCAAW@_`ePLoTA|LsDQ7g(Ai4X*ilu;9~!pA|;Trvhf&1 z5!7*au})KHFvn>O>)R~%mPj$m? z(&5+E!v>$8xv{k_-9 z2Kc+fLu-#JpK^381Oau1Nf(nTD%#VjLc`5EPZl&iQ|9AFd$Z*{DZ{5(Y7$1xa2%8$!P{@Pk52bMB0e~(sO|v*Kz$41i zF$U6vJ|;p@ad_}{?;TR^`|X1_CygJgv$v1q;1YmzLl+${DLsOq*wF_sh@GXe8DOIl z4x0aQs1m63`^qfsB^QrotD4-5deJahPOsBkjFFj!`A9BJZuzC|0}e)jd}%;`v?y&8 zXU1+8J~V#K79SMD)=(b=$CS?7?6YWd@UmXV3-uq}W_J2xM};QBN&^}qa+Yv}J}ojx z3asFB3~JHEtWVBwuoZsO`o@9({eO-Mu@tJ1D&a_!c}``V-^>6ifBKQb(qNTQ?J6Bv zMaFXre}4|$0*?LamF_s-upj%Q2sSyKQBDq}k(DS9&;b6QV4SenmqEAj

    uRE~CcF z|Bp_)H-wKr{o8o+I}TlS2%cc~H0ZVzrzh2T|TVe^CK zB86b8T;F_0OiR9o_$Z<3Y2!cpk2 zP%iDRoPRTA`bv9zkBdaL7yGy>XU0w~%m~=`$YTn_cnFlyzI;z86R_`uG9bIUXp=*E zIJDhrEj}RX(iw60U@P@rke0_*-SzgTe+OZb!q;w+4FWMWwxB;azwqLu8`zN{<>M$zEp#gf-#^I`X?A~BlJ9JNjID~ntOuea z;x+a~+$}8wE=FUg7laD)OZ>)q2D2!Bi{#rD1|--bZsZ$$2+3t&)P$cUg^N#^0<0@O ztj>>1AnfcEt^}$4PFYz3f>Cfd{*VP*KQ5_xa0KKQ1Yc8*WK?eq0M}B0zN>f{a^Ttl;*NZmtqm zFdmmw68mMo6}gR;x00(QTdA&8e3H{0Nicbp(!`2;-*2Dpb`Q4S@17iCrs6_br})V= zk^@oFo-Hyc7&%FzSWobG+TUCJ1nW! z2XU0&mQ?lpUUptYm8eQ@%NmS4A7lpD#L#o5&>+0+%C{x8F|2J;*(SX$sfAKV$)t)} z__m_b66~m1h$*JZDs|P{3DwX0{faib{I;sDQL&_MdBv2{r&bM=-)_jxN}T<>s**@mUaThbE|*tRnWKvpWvb-zisEhAO3cEMJFe;{zb%u$WHg%6 zYC4!v$qWp0RBg||u#Bqn$%XAYxkS1p_2)0h{^85#bN#5}T>DA?>9Z%#o^19vS9`1e zroTQ|ch@#I`cF36tDPsE^>xqn+$ZjeW#%lWy?X~vC z#>V=Sjme_O)86L?_nCbD_|*!Y*prLJMGUCEQXkzdWru_h90%_J_s*G+{DDWitx6!Ka6@kKbA*yfJSC8u+b%W-OK5# z5{hx}c)?)clY1^3!*E!)!O%TVmKU+wfUdX)xSK3HOC$H%^T$xH?JS{P4ZL2WJLGR3 zFJo`?T{mau8St&k;P+(^tEEPW?pHtd1m5v8`}+5y{>p4^Ok4l>J-7Z>pRBF^VdH*= zY=8dC*Z=M2{~{d?m#>rIauC*N1M#uBxn4Q{YflRNe{FTG^M(IEz~=?b;`1cE8T!eE z4;#SsXc#8X;o-|i7il_vzOr(4b+vr8wj9OhEA3~`o~`_FvcGbQ;SeyU*(0f}eEF!o zy!r@zy$ND4qNofPu|IhE=&wQr^jD%AB%odteL(tov{F&S&_w(d2~PJgS524}j{Ni@ z@?JiI?mu$i+!|rPz?YA*bg=XkOTSpL$NB1W=iBr8+;06Vg+JhXIuDq*{?}L6pR5?PQ>Yg4H?WAm(d*}%v(vXrPrvyKtRF14Wbg?TnqEFi{p%E0md;4X z>sPQ2xzbru1AYOghR{z}VjlzT`B-zMpT#J)t&H3dW2z(;d%%{zdULpQ`j4Ys5CaEL zFurk)&R*~D?KqE?R#pzTPxek$R^FVxao$Az*XRfF!~1>b`}O7Zm6cs1Q=-YuA~y+= zCCqqDTun5%0wY|(dgAx>vX^>~oF5l|dbPOtLN-o}NB#)HLB!!LUZg=f^k1D}BBl3k zn7P9jEBuIGkK@S8`YDQ-ujJx+v0^^uaso35*omVdpjc*jef8J;MbKYgtQf@w05=0h z7;_?GF?jJz$KiNENzJI$|8bKje6gY*=t4;r)05px;Po$7LYEiE~_ZvVm&k5$-tG_2&nL3nSBp(T1ghoJVo zWyd*0UoOfA5nzU{*oZknXCBUx&26G1rixK5QkX>9(8H1m!^bc+j*9_`#&G~mEZe;= z@4NifpZFHD0$(e4ma0x$oHu3vu+reBV;>!Dx#eVnqULS8C|0BnG-PnC~ifd{_ zj9F#-O>{c*B5#HD_#P8HB2$V-)Kvo4hb0W_Y7k|i=eTS_@FK%BdGtDqwcxCTB;4{I zDRu!%yO#thV2`T3S_ZC(U6eznF{%OIgwYswE6u5RXea+TI6Q)_&mJn`Qr1xH*L+AU zRyl!vJxsS=Agf?5g{GR&dCY?l<5S&b!}bUF3a;?#>vO2SQUEX86+`J5JBH|c>V}>h zd(QUJp4ETVU9vqy3B!yXMQ=QR@QQu-UP!`GoJ>97uUOQqjz}KiL*W?65B$`RUokQ3 z-{QLkfq|Cv>=r%l-?_(}r+iwUKklf}%0`blzh~4PJLB%L(eH73$AK#+(N7zHd8*0n zr?)4b&vg7sRG5u)=1DmZGGsksmVWMYxD}mM8&PLEJ3oj*%sP7yEdBbvdAbRKxRowW zHwO<&-Lnq)<52g%Ezm0^Mco1S;gFwFhzg5T@GLpYs0*|f4ZZO#h%E;K4z)8uDRd+BuWr)~8v^v= z!DrCWBR%j`mdpfn*%Y_s98y#IHjBSyfNHT5uSDz|x-DO6|LMvcxl z9#hiqZok(wM_|;IJ;Zm3z+cO;mgkC5z-)Eax#;(zvHMFl(*m#H6VW(**v+|2cRkxW zo#Wy9HdNJkES)|5ByrFE<{f0SIZosqW)aK2dO;6ua-~NvF5TF?bwKZj7c27NmR6Z_ zBertFR!VEdx~)sM?-S%LA6>PYn)WxlCp*V`N2hy-2ez`kpdp!{-Ci^YeTr8GInr#H z6RI;!i+d73F#(hdi%#zb%Ap%$7OJ$@z9(*6mK_J1mGC?PkAh%e9LXdLQdk=|j!-f> z4R-)gopF>TxSPNi@>d4d;>UgHT9c#MaIRDV4cd0Y8!`>R?@{k(z>m7#n1F-+EI8mk zW9t+3;iCDZ<@j#@qBj15(1HEpJiQPYW7|_iQQ-L@Cw5FD0S^0(nLsAbCfTRDFp&5> zqtxQjpxB*~0X&}6-4=sQFC_B)>%LFq%ht82IotBu4MyQ#kB31YsPe`!sU(e@U;S8) zi%?N#;%udU`2>pUY8vYivFLG<^)J+bY)An5o5bn42@KTpVsyz4=#m{1dXF(jXbGB2 zRo@&$k8?LBCgw!4cYJc`4l~&SUJHb@-i=eeCnypEW}E0mH?2D`0$_)Dc@HgJ!I}ef zbnvc`vQuKv4N9e_D2%n=rURcCsw8;8$z zcIP<@0PD4X;a&z5eOEwId4SM(W76b z>?w{934M%CScJ3iMzqi~Y@!Kdz6d)WEYN`Dp}r3E3GR`y#CcV$83} z-S(VpjDYd=5U7GD27iI1PwN=Y9mcS*fjwP-MjD6}6u0PkzEwOj+!VG-v`$6rm32+b zKQwhVh9W@L7$Kr%7SyOyJa!W5po@6<qYlhZYs zNJj3BLjm>7F{%|QgMss=h&vx}N7wMZ5czM3Sz@c0szh%&Un;T7oAJBEZxYu%+kvmi-^`R}I`fZAi zUdoPX&QdqVINfx>U_r?O$MH{dl2e()*l3eI82)GXr*c?QkVFC@>$-j%IbP(mlmmM} z-~k^i_NEdd_7hJvTI3Eqt9pzQfMhT)RurZzwU#-t^=5pQ=`PImCqLB{U2S2y%-FA* z>%znsVsLsDl~1|_OcFq0(kFrLEaxyd=c)eZ?(t#GnZ}~C&fuOpQs<>eIi)a<$RJ&y zapS-o?$ukVyC7@L)Egq8hVO%$tZ1l%Zl| z3fZ8z6%O;;#4SkN=p)EnwV>t4MoMN$#VF|7w3nq38rk!CY~+S6p7cdjn=6bsBgIra zY-2X{5MRa z(91gL8I-)Xa$Tvi1URA4vh=*-G^fQETkC4+TWj3wSQ@0&lJz%HlR6PU&|Oq9mTIpZ zn>fmqHBzZx_j;-VnI;-#M{Hzjjh&&*lQMyOLnmqYgY-?(0qOufN~j^=1*jj5gYzH+ zsq+(D0HZxhAUVE^UXoHgfe!E7+DI8VVr8o8nu4d zV;*?^4$B@2pZ?`~wsZiKhWBj|1%-}Eoq)sz93!&x3&5#J0E<15Ohs}4=qkaXI*p{s z`#tU1H%NB=aG-8sLucRyLsVm`vC56ViV&xeU_rsTYH2qS-r%+fn}{j_4xQ+X)+yD% zY4`p14@Ng;xw0!$Qkv^R(WO$&+0ihNzZ(G$Zz?CSx>mYD)rvqa@ z$E1W|A})odjGB!AYH$(tRj!pQzNCV${A}zdi4Yu$yp3kk#Xgc^b`<<7SqzSH1jp)8 zHWKni5MtyMc_XI5DK@XJOe+YaB~%4C2iJMP=G7vDcluEbYjzxk7+gYJ63MG7<3e$= z=37ngWSXxt)e|-AhVT>!HE%)3`dIr$TH!S@0y6MDRz}_AthI} zCWl8^lp_ELUP;NYh`GYC9&im3<80j^l4}|GE@hD%u)yazPsj?zw`oFWr^UX@n_hG_ z#&NCyhgJ97A5oqV>WB(dh%DEMTEWt(YRH8}GQ8|;gpvNp>ytoEdEh|gakU*r)VfDztViY;8~81$_#a; zviafFDB&3bH%i28Dn`f4$(GMseCniYFYN5JdsS+*@-3j?1bF)p&Vs;=l;W3+g)5OF z&IrtQOEHwUw&tPR<1DR}v#`z&wt66P1G5Hu%cZN_7$zDj>6PqUxh#*8c~tC=DGm!~ z5*$X+hishCqISnOE<|Xr1oq02Iz^IN!Zk;UZ4s{od=Qb&rdcsf}!QbF2cXgOGlW9 zQq|_xq7;B6I_deNmh-iM{WZZr{@rp`q5L9)13>GlvStS6q^$C!PzL}G*8>PhFO2{b zv2Qtn^MVHfU%&mf>HK?fL3AA+lEVhHxPYVT`~)>%pbOS0WGMiA)CiSgFUw|myDKYg-k#juP$Ha8RB?u>WZz!u#}tJFey$7 z;}*Uv)|lmL8B86^BuhX`9yaiV9AykE+9!K$;A$7y$PJfJc$CV55gapcOORXd|ELW% z&>jc!0s~LhI?1|etTMs73DdO$fwVLmV`ht2`7AU{=&%u6%8o9Dfee36A`W5r=+*z6 z93D62@f=u(c%GOS%>_e$LffGQ81jx5B zMcNbS00i^t7f4*$+?7$0>%RZH$++e4T{=#>i0k#!q3Pg(3wVh%f!& zjkGKiP4#3`sz*cp#OTsRWtX(*OkKhrunLCM3&v#NUc>ts`%Wn5AZ+F$(=JT%x!G;H zN00q}bRLq`i=tXyuaqf=9BR%(5CA->kygYhrkif(>7);`E}GfXF+3Vs41lEUo{z<( z@TANr+43Fl9)bJr>HpLyM%FV{GJx3Jxd|~FVYP52Zdt4jLpQp)6YBi>Aq@>st=Y$c5sN;>5tGSP#f&=%W!8GSf7QhIusbeT_7+h%C0KC;8)xVXM-inXG%V~A(!R)Thy9^QEWooFYRB-} zA9e@=XIWwwn71o4WB5xboW)tV1P7Cx`+{EOyu}*zNynu&Al#^zj6|Sn6Vg5z0)p)< zM|O?Fs0TuZv{-N^`}%5tVmIJ}xY!V+;4BvE*+HFl#e$cH`{e#62|O0mDs({t0XU%{kqG$_MY`KJDrPH;z6LoGKoTV; z_mLeJczNKm(7akInS{RnR-~i+I;v?xEm(JeJPvd5kS{g%McVf|;>1aybh@0fCA=Lf zb|BUv*;&I7N3Nj;!m}&6YE#+gjPJ1v?2MTyX*IAa^0E*9O-?}qT|(z-LoK6}G5~Se z?ZGBtC}Z&1@$~bGy8S$jGrtwG z8sm8z`^s-|YBZhF$fu8*o)t$o;l!bqeT`xcwe0Eb?t>|MW z)#fkyILk8Be=hWV(u+S#3p?L2!_L20vA~f`(4*TWd6CSD0HJ6|vL4;pyDY^ec;=>C zc#P4H+C2)}OdbI!HuiB5(iGDxEG9-Vh~0CAHF19xqwRAa)w#m}j!LfdpTE}e9=v44) z-H(zqS=O5z%L;s%0s_djPb!TiVKfYj5ksSk<>XoHgvZQg>tY3ok;)T1+53uf$`Eh5 zc;logRd$%bpp#q&<-s?Sz}w*v%L5aJ zG%lU0Pzr@2Ugm4!k@^XvAav=w=n8pc3lGKQxbA)sM-M+tjN0V`3pO5N{EFsIcfFS?O~!?FR#!`rOmotmV&P) zr9L7Eg7z39HKgdAT1=cp)EXv?poY!3sx+vZ@6eN*uzvyPAYpVyEsP>zJ8Pg$C>cmv z$w(2-JtAwj1d|Z~WmBk=d>9tFsz%BaPMBiJb&|-Hk)hg`MwZ(RS-Ze08FGcg>oY|g z0s$nA$3Bvx;Y9#HXw{l3v$ENLuN>K6nlLI4`9eFrI0E?&C9DXIvfSZPBGidl_(2{s z_!<(QFeMA~IotbEUHqPtqm`lHqUiMlevy0K^DY2Xtlfp!$a9Z+7=cE6aKx6_^CIlo^U^0|f>aKjwa>lvz$A3z18= zR3BVAK*FqYLuy&07F-m;+@*BtFj{X4RVu8o9vr}>5ett(lLF$DpAor%sF5&v@YZ5; z2@iF&qaY;asp*7wYucl8^JLiT%3V+1^Ep)-s}pmWwChP9GN?_drVo(4TX*F7%`S`JQkv~l2XY_!s=IF&mWy0 zzu9~HHs{Ass;JVdoiD#($>HvVx=s=l&wW!y7FsgYA)*AU3`hCAppFuz%^u&K-b5KJ z7~c0_5LqmQS%cfacvKJr%<&A9O}L?beux|_xp4U=pB5y-jLYO2jXr`U*Qptmflc}q zTp}Hm$ApelztUV6ZyPL0Dm<@J-cPo?_Y$%ZJDp;LLZN;t_bx2FY?$f3r___G1`MeHgB%;Yzcj#?4A&q_NMf# zLSngyQTgb?(~|PeGZl^D9e}0~l%A*~Lw2TdaLG6!b5Gw(wEG)|8H`Yr5G*eCm()EV z?C&Z6LKO@nG`Be!?huG9IzWoO1m^}eq8M0$FSii9GE*Xf*!IQsDA5$HI?267%FuiQ zM0bc2LNcTFgd_f?ZUKl1noC{|w^ehzN@B%v3(+a&UnGK3@Qx%C);)>Fw-mn>q6bp) z({jFG%sfzhU*y@EPwSH&#dHwIJq8BMk9`VgTPfL6=p4EnEEA~iMHGEV@IAm#{0)AQOfB?0YHdlG4e5*kZLTkYq-f|Tb6u2&MRAhhP=PK|M07>R*^*lh zGI zFCRU9#7COyJ`5-H@5iSoJ+*_KyNq8<+!S^#rX3q%o-L;TE-llvO=R5fnQ^~o#{Hfd_g&_C zN{hg_-wVe5UNCO{yLp#Z*Qd1F5v_JaYn`V8&3BsDr&C&87p<;~);0v5H}5p3%gf8v zZI->PPoziFiS&qxbXO7S5fka|U5Ipd3X$$Ik?t~)PHzfieN(!JpLA{#<`ye-F2f&_ z44sPRZK67#$k1MJ=XBpW*nYoza)uG5DKB~jEwhczPP3l?qm$D zt~boG<9yHdnKxr!-{tA6YBo~_)S*!V8#xEDBYm${(8-H+Z>gJRQc>=Qc=_PT@CeE% zFTT$#QLpOiGytjhlySm3_bgH-J#fk0=F$0ioy-?HglUDX?<2h z8J*XSDK&~=LYVImmiakXCJ4Pi#9j-PRi5%jHExRuls4NV{TJ!2SLwnOEnTwlSeo>) z)?4PbP?vL_^6j^r2Xru6a#q7IH&XvYzDLW#%~R`&4ExSjagcdL&^0*ur&s8xk`hBm z`$h5U!*Fwxk6Ly&aBTHAO!94Az^T8vJ*^dX2EN#6V(I@HnR~aKQA0M$2A!5F2 zpd&YvMPy{KVm8;i=|n_a-2xGN0~yPMTz=RCHwasjQrNzVj4qINiJTO$v@AqMrDG%e z^9bYImI6ZIKhUy9qznK~huDC%VN8-b^H^SzF7(HTO1dK&?!=V9yd(mBh8_H;$ z!lZ`73dkYOVWydvFVlMJM!jJ`*==e~eRQOlF=b)DTeNBdEZD! znv9ifZSzB~=l8PnS&&KjfKK6$0uQy#Z zFrEd4s?x{DBX(tpyYlhZvf&{$3py>ws*fZ)?2dZAhZ49%4SRDfX{wZdkCtN)eq@ja zjH`O1VnR&!F#DJjm4zdBJkuOejgL?7vL7FxbByC?CMw7BA0L0*i%$c4-C@u>@Aqd# zqNw%}aVhQXu{TSyHzrQaj5Y-e{0G5n7G4FRcN^4l%};|;myG^=e6C4~i&;=AqO5#u zOy+{Ue7r@^{wcFyPY=RF6*nRo#d{}kq6m`s?12W18sI!VmL47kR}=clqCAD^LKy2JB$ z#(iGa`S^I$J^A>Uyyg?yEQl0UK0Y2*M?NANjrbKcFjx}V;(?I2jjb+4;nI~GW7sB# zapY%1g_wEs4(%RUlYtZ+CyQK!15nSMe|#Ur7!W{u z!!|0q-;dPq4fw@K~075Xisbwj7ud_9SnOuca2Qay_cSY8keahxFcB$EvJ9a<_i27FP*{jN0;tA_ zBIJ@R)1E3Q(ns`Kn1RSZ!5qU$8m8DB!gfYhpmf7KM`!FYh*1mJDJvFHB}Cu@C^0AL zah%2H^^Sw8^hevtkFa&jyFzKYf4T0vSnf6u`^!&biBMPGHxS!9M zoOj^%VBz0%afXXt-1=R0)?}m@&rjW8sM9)PXK4h}dqDl|BAw)#*Ds0&OF8Qevu&!z zZO@H~n<%Q!$;*&e!p`rBpC73hQ8K+6&x*;-Y57w$X$pyf{k4=^GH@TiayXoa2<{=m zc2aO0WzCk2P!on>5h#R-0!A+T=u=GrK*UBZ`Xd+sv+FBh>lfH9ZjpT+^(hI(e~S68 zgUmErB5J4z4=+<(eemPZA3AUl%0xJ8A`mlwJP(8-$W9bgUGFl$3^#WaR*z-aq z81BRIt8E<}g~Q8Yh!r|rBt|R{T=T+6mVyZ>aPUB|Cu0r3qcp%EK1Q}d8G@*!w7CW| z&LY5}kYZo21yPm*?g$%1atW7J%(zgl0tn*VC<;7SgS5dxcw44hS&u@tR@OGd?Uibd~&(a84#*NEPa(Gf6gnZPVrkJ}Bv z+7-(+F&3#Q?NNEn>}mEAr@bmCu-2L`6$A?J?zEb#n=qDgown+%<2fHPjO;wMSF};* z!F8d#q?wbNPt&BG`a|KsFhoy~>Zq>x5gNT#iK*~}97TF&j3fslmkcx`Vt7(>wBfs2 zHkqtPO|iJBMXN^ zxJppu8rpH?)Y%W+VmQkV2gIt*C}2(uGK05SjE6l0*D~gqk!>4;j||2lpsH46Vhkfz zrV8OMVz01d6RPoRsYvC_;d}|orXL4_HsnY=8nqKUDT2_Os&UImCkq4g6mU6gDJ_SF zNI_1~JB|Kx6u@&_{V^xEzY6Ph!pPEv`$r+YuATuG{XC0) z?&m4GxjX3v9OuW$mdroZX1@P}XDO`9d-dLcq#o4Bi)>1Kr6={r6cGmF$=KqQ&WGR` z;YP*c)43vaA(e^A80zY^yP*eL)|o-<`-U!ih$50|POG3MHETAv;g&a-*O22aoNkt4 zCFwyT<1l)WYu(@?suvtf+;>nwDO6eokvB^teC4 zXbgvVExH;jFZ9n{l`JZ^q{=#IEH8t5)x5~7z7?)pmdjoT0K;;++dZ3QMiPs z7G@`g>WAv8aHg;~yxn_yC_{G|FvVdA_M?2LLi}jr(i&uH9lxmb0H+-1Q-Ji0*k0y_ zj2y=kheIYmSaOu8fRCCQ0`82oQFF1f;}b4vLM|uMv!i9=$5MCxT#T0a^Mpv5=9yGT z#wvdz9N03E24%omar+snmQdcK=(IXoE^XhpUnORkOLmq6p&EmKrMh?8oz=v@h<#h1 zn`zccTx}9J)923uVJ2`Yp3E}j5cAw97aYF5sQ|nWUeAeJ#J=}yGy|&;<>!8cnhV8Cf7ogRmv#bgeHVRW@Z0aNjtc_E|R^m8)DJ&7{L>7byh7UE0A z@}JX+#(l5$a>l}+@>>2Usvf0`I%Pf2@8saDmL$$~cs=A!^>i}5fHR~|xq|w>4$zHV z!Q*(znYlE=tz|hgG4wDi1Vk`qR?mv@Sp7xoNKB#{P=Z`N_@T{(l7n=bw!D>UcCR)2w>R%C)|FvMorQvJKkI)ERYz9XN>zGY=b;#*RO!Ezc- zk(!0n4f~|kk>)}s91vSN@A@~gd$~MSEyopc zyZa`-VwMQ8>|d`ZWdg zyC^G4^ov^c69IP^WOx;$$Z1Y1v!>gmDElyL((sC$VL|2`9@##par)g+lOwvt_RUSj z`qZZcs44FYafb*}1N(ekgmbRp(P6XHNJAPR@Zp;?T#?=o^G8|vtMm%Ur*A@oG2To$ zBZ7c(GZqOmaN~*5OrA~Sv@BruhSmL9m_`{zMC(%?TWK^`oBHQxqi8Pdnp9Y^2DT|9U3DX0H&XK9r!3E)ul5!+cS#*(BgxlKNqM>@{E+^KV&I8P3{+Wk z4Qo{-TH-oY)^wsiPzr}-MpL@>Msr@a&vj#o$iff8nygqMGmrsqd00EBqoCk|8s`NA z1zzPrdQ)!;r9c+U7;+n^@ut@H2`3V0YiWR50<1;7S@C@RqxPmwO(&Evytu0yFKhJ+ z%J(QaEuB-SqA)x`^QM`w)XzrcNP-u&`l4*A29B^TMinFvMnnhHCZjd+M3x^QLsVc& zzOL0j(+oG_c&J!+DUE~kb3bO&8l*qR%iuJFft>1K?i@>(VUAwhre_zy`Nh%|+DlLB zn=qG|q*14C41%zLKmDRs|EOfX2QZOG;t7b!<1FQ5rK2&71FdFIem13kJUWj+`u1M^ z>@z=+za8;K=Q3An9eaeb!Z=r7CM*m|kiq41rG5}l|ubiXA zp91SlKw#BB^hqA17Z+Vl#I3YpFR9Ug;k)Cmz*v`O_tC5GcDIk5qvJ!=87OdX50Bq( zpBl~*4%SuJ=DLm-!J6`Amgf2GHC@aQ9@G~@j5-f$TXV}D=q);_VR&WkvgDbHj>UDj z$kv4=Ob<6U4xyN(6wyt_cMBu!cEw;=PaXlUjJJdu-1HHUW>-3%sSv59W<-PIwMHa> zke!Adla&&bDGOhaV?SutN|$V6Gi~N>U5lf8F%Y;>>rp+JCwKehQmI z$+l3L1{5>O8EOr2-eQZ*G&)BYK;01(wL@W*F78^jSudHP8b?yP6rIpp1iM@i3p~aWg!U>!k zL;niT|KwQaNgbl#dO=B-O3Ujd>5Al!J+v7>7ieYwst8u*IG9MMWpi)S?4;Svk5kg| z!x3=)gi>EUQF?Z{SN+5-33mGuK%bt7SDrADADK0cQFI7rwr@J z&8hUHy2m>LrRQi3fk*=sx#!-ayALTI(2?}Oz&6HU%_%lm zW~>M}RJzbx4w@vPTW|17rU9WQmF~1vIu5eQWOD^FkCPclpqq-3K|^nhZ1L79&5$yL zrvb9YU_>@gu27i?#f&7g;$hUlh`<7;vrlCXTj&mvp`T1}Akf3SFD|F`3p4V$qUNE5 z(|sA()&Z_0uDmGu*mAjuS1}5g}o)UW;d5D`Ls43b{8i-+SY|gZtsP55Z@ZmF;xj zTV=qSUud`Ry)}GpO;lC+-cI+qHH6H(@9m5}w+d3;_jVSaTg}&Wu!$0%+mFG4P^mi` z1Hu}gGXL4P)YQ{%fwPaxtwbI(v2+l%rgbEGOYr(P#9w)m*LRz?Bz7mos}(;hcT?yB zt={~KIJNGK_&)BT&s9QyMj{*DYo##1-FCfG~r0y(Wz&rg%rML-n;N-^4s@p7sfvDw1g1?keE?wpA4 zT3#U1%1c{!8H&=OxU~}A5%Iw7&m(-Bqz37PF%t>NN_S4OSeFyp(Ld_5!gqq!STN3e z$5Oa7?Sqp$)oIknQnyD^37v+^1BzV5z!uyXig;T&<4WbVYwfQni{bV9IA+gTo??(z zq5(r}J2uFgsTR?Oeb&n>Bos2)7%d&KEv7 z#^Mh~$~9mx%#w?8nUO=6nsT!`L6ZbLnE*u>2!4?Q&9?;-oU1Civ^z6XMXsYon!xoi zi6)lr-Dv+UcCmyMCfnpIaHrORKn7{yNS7iLz4YMXoZzfq;_Tx|8fSeBqhMI?SD(X5 z==2a3hGb;d${kG_KrCN7oHHzZ5Yy_YXEt@T3mvaif?V^UH= z4U8}pV*UbJ6$qYphWT-Ey4^rc?<`AIRhqYDs~P#-T+(&f{4LdPARnwq-G=8U{WuuY zjXW8|)>m`j%$8jBAfaxDdL8$p@r|^%rWV***4#FP@s$X{hy;XYTj#)oy6YYZdoE zUzudf$o7Uv8wxLQS6)iv^_J+66?i5WDaVS)6@-^k%UB!)wn6bVXquqoS20Zg7RD`y zUb>+VEgIT#*IS_iQthrYb9kAsHKooCgMm*scoYSm{qS<$Q04?n%abq|E+Ii;tQ!%P zPkQHx0F`Je0}BWE6J8+ZLe2xFqO97F&O=l}nL5geD&QbFhf~VMopr!gbdkvo_J(d) zD<96J2ItJ|X0WrKWN3Mh!s;!#*p4ukhAoq#J7}}H1Os!&oP{RE12%%(@;ynKkYu zQ_frk-5u&DMVc2b&QpY3i#PCb8f(L0CrnZ8AG0Kd>=>iBqbI3Khg3m9@3;5&4|jg( zp6+b#e76fkq~eFzZyMC)5$tg>pqVF^e>IJmBgLb4KaQ{mh7Y$0NMhVo@F2~gDdy6# zMOR>B?m6*stymz_E8_~uadt6)gF-67w>Cr>9Sth+h7c-|nVfTR*V!s4m1u(jq-L=x zS5m+QzLkfK2n7b`A{u&gjpaW3p|9p?ug*fKEgs(S#Ag%5R2S@1+l1Mt34o;#i+CD1 zb2m)=maxKUtM2X0uPr*ztZ1A{3Q_6{y7mf5Nm{u4QU|kpARdssVgI_Bg?vPn|3m_^ zSSdO@@j*ayQuh+XXjdLp$sSZ#uQ-E@0xgR9R7o)AIAt>TMEwk8`Ggcxvyt6O&r>f8 znOmMygpT}lEv>pqncPgf0<0=>%LM~n4y|KcwCHQFy(z&c@$JY(v56KOK-YnOz)PSw{MRT1+${MkwaqHU0*N=00*7$-()L$q8_ zxWg0(?HLCk9#lrr)><5lJz+#aw^x@LC*KT!&&w#E4pv5#pW|Apo6Wnx(C3ud`!*(J z(l*~nw~YxBlf-hDNzBCCm|MG`e;1KAZpY`%-p!TYlYOHNJ;#xgB+rVIA{-im(mv|b zgh>89M-rMnA85^Ffkm>a%8_cUI@c~7o zG-ODaBj3|$Hz;zcNKWu;8mcgLh2 z0eZqab-tdDC|-dO+4TzKXqow|-hxn(B^jx~dN-uJrVIJ-LujV6lYuu`sU7>{A*?9R zN))(n3J(a`Y=xbf{W(n`)AT*(~UMyg5;6SmK9#d3JFD1Bk>~8gr^tc zMeYy}A^N=-&o~Z@?v)s7VkmElXbePzVptXiDQj&}Cx-@k45~yhm1}-cuY!6JyK${~ z{y-S}IR9Q;;KfLwyXnHhWBRrwpL_a+XTQP7Z$V0L_^Yntt9Os(clDAs45~%n)bj-J z!qDiOdOq@HdHS}s_`74VkI7-iWx#CU%axo*8$PMK@#K8Lc2r#xihw#4=vJ#xwS zg3G`YE7%`KSFje9mtH0Ug%~C|njG=)LA*&&tGxJ^dznaf91RA7ap)y$Y|bN0?m>6` zfNPyPq@{wqlcfY6xll;u81VAq$e=0(&rH7~LWP@YHPEgAxKD}?(5N91BN66FVgU@S zpu)J|NIn46>|pTUY^;)EVYm9OCsw}MC60m5<1#sz*nTCJ&OzKx%Gtd}wO(ZbX~Gn+ z*r(KLSbc=C!(pfZ04w)JM+KKhC@{W(dl`|@HwEHWr8O%;{6xTFLN0*;$%i*h1B%#L zxU)p?sZ42tWNsP&8BLo3l0a_2tH2CpG>%LoLf(<#j+tK67^pb!sktN7A7fDXhf6CX zBq%%wqDB!YyJ)$r_r2r>M2V5}!H+|KDD%DKIA;VRWg-ZNhTC%d<#SDMe}@$mT_q)K z4W|zdGtoE+l2`N#dXz@Aa!ru|-vDEKB^exrqIj$gmm3zJGp(2TE7mpP(IqcF6oB-l zF99-dfyMVq2z)ef3=sI_ZV=pIa2_Ik&jJR`NUWi7W~Dm-;DtF02zhB{4F)&VE`_9@ z42Y*n69E5K%O2IUAYB%eX%wB?B1_=ul(HuhTHgVcgZ-EDhotj`xa3 zj2OF%<~e$Ik@j{N-(m_4QmmRQA9Z7il*h_?_Bc;_h+*7ENi{1CjWkDx4q**GYnB2m z(&AK*hCnPfPGHPuIBk9$3wt0%3UUUunaIJ}{=Q)WqUAK0eAqZ}OPV<18Ag(YqsXJn z-Q4y6B+^k|X*xORlWWbULWbTVWCjm^>Qq<2AA{_mH-2p7Dnio|v_(f75}9DY+^H2bWi zp_avEl-rQx8S(%+u?>Uk7!p`zla3^JjU~-yM1Ux@K8$-O$L%H2Fhezzj~vn^X~LEA z0PzmM91SxCYb zhn^m`sVqNcm3x9%`*Q4$C>uCSf%;(2)_-(7rWj-*&5C8gC5*PW3CTS>7%oDHNr zpi<51GZrPh;>#XfVkp1V<7%};ZFkC?^D)jekWEGGR|?fVX{zsX)GOz8ipME9Q{}#T z@}6dVm2zJVc~74mrpkS_{H9j2)P6HW)yUO$}d>&P3;K#NEs<8AjE;ck#^s z!Y2Py*`(5X&nDLSm8?>}N+>Nvj)Lu5Im-Ui|3E1tzHm%~V`>*`tvYgr>5Rk_i&2_$ z%8Ui&yC0hq8p13#tn4V}A%cnvIKJLsb|r;L-N}hUqj_Baq=iOtx%d&7;TK+bpS+Of zwUQB93{a4O;C`B|NiN9#HKxarAx&7!u6SItu3O8FZU2SbF&GrEW?j+}1GLV(F=lpA zoC5{-r9nUuec1r%Ws(p3E3*@&=3`0QmWdg9IjM(>OAO*12+j(W^U5KANFHi~Vt2=YTxp1MG zU{v&DCQkN+8&2njL?QLKp&+I@+>nYU1=dx09g$;KRTmO=G3&m5GNnvqy}9DVd@-^D zE$8zL!wwPH|5hfLf%6>qp`848z)2Jt1% zTSnQthGD7eF+zc(aNE!^YDMo}%*S6K7Zkldc1pvg7cnG^&E}1tZP-ZT*u9yMtmn4A zd3psx*L|1FJjoMA{5DpHWsMZUj3uz-=={cO*7ERG=$hZu-O?#9`E-2oR95ABK6%C| zlEOXNK6x#b!%Sm)nk7e-|65J(#XQZ(5F3_H!heVtUUDlZyyO${y(?1M1Rp$&RpEvK3PNe*a`^P2djUT2p)2P~2?jafPH0u3-+%lV1&J6`f zbhC^f%YTHyAEBAwd`He9kGdoGdY19)S`U!1)5{h^!8QEUu* zK{%T@jTJtMFnyr&J0Oc!$e2r}1Ub5knelOPZwQ$yNuB0$WLS~8B|=s0xg!=@fL#sS zm<735OK*3qR(){(B^M)6oQ@HUF0S{mAQ@3%?4Qp>hBl_C`m~H4Yn^*QmR_WojSwB> zFwQwyddJxuw|_1@zILceF zB&zla^CZ&zX}sASeV!X7s}g71D!I-lL0T8ty-^kq^A@xvDt_wJrBHWY#3e&n)W=*t z$i)K^)-E+I+>V4UQ$(e}bR^@kumy1`o)f*%cPFT4*9w0b^hu~4L(jjS15zmrJBxc= zR=9Oh4^a!f6?H#{`1n>vc;T%@_BazGE|*8LpOhgWuW$iT8s^XlQNooIoeXzsn(VU4 zDrPyJG%e-nv+q~CIBVL4|G<0;RSAzL%yog*a&PW1rp-Tvn-gJS zik>lotrlzlb^4tfl0W|QbrEy^}YWzjq@ian92 ziVdPu=!tR3jm0o)%`22P#<4#Lu1VyepMj^oC;l(=eS|+o@)xGpezBrp=nd_PV$>-7 z*}#nqP@Jd$7uK+LCPuX~7-HF);WyLP%)Dswd^VyypGK7*o7MgO!Vh)cd68DFNS-(h z%C5>0`4l*viZy3tiWvydCrAW>y$sL>*a%;m@H?b4?;hwn!ynnb z()DLU&-H_K>*9Aip4>NoJ@zSF>u{K-Vf4?%qJ3dWsqLK-%NpK6#fUB|M|}m ziF}vtu>KUu8V{EJ`<8RG4I2<@OY>>JWyAzbW|&_2zG*NKklf`oYV4NI zauJmu<6K7Rw3qucoXo5m(^u+8PWyeN6MnlYqdF%bizsVmBI1`SGZ8AJ=U`y1^*lWV zp)^w>BM%Xe$%ifwEOyRTz%GLG3l0Iusg;J&mCluknJk5JhjPd#m^B_#-%_y4j04J~ zW+W__#GseS;EP7ecq?Ge1R290M}0GO6#Xt*ZiwHM^*%)FPB*sV9)>cvm~7n2&?%8l zv7wRl`ZqH8p>RID479bpn?|FcZ^TkG^75J?(J9#x&Rnj%=>ZACAkX|cCuD#n?CNf$ zoCuUK>`|skz`YDCB;z>EljZWlQ5*b*44O`JsrSY`KmXI8d9JU9HBRxr`&rzVF0}lRFr2ISu<|-6re)~2^_!yVdN0B*0!^ue-qBT%s zW0Xb|Vt|v^UjCY-T^Lf@lTY2beP++xwg;$ zcbw}f!cGucqiUg69gkX^0ZN}>#oE-$anoCdqri;N$4MOrW1pkj;?5WJQ$8o*;uks* zI?mP6Hs*QuRL(qaDwpwVX4)>e%I8YyRpI}wcx;qeQ3dxgnXwF8B*GO7y+I*IfO#ax z`2Ws}K8_X+$uuu7Mt1*VA(&kY^E5fmmLCT@~URholIXRt}kynP~P zYSS2-0;tA@S75l}J$6O1!^fL<5j+!<)?$U+Ss`8CgLXqonsw7rRtTN*%1U_~n=Q;g z*_cT0NiY6zS88vXQL2pun6Bx7tHc5!6IaTl!S_dLvpdLE(A-=d zW^ZL#StC_t&K*|c^PNw?I zQl9Yf9$Y&ghC)JNDtQ7i<;<#X(5=1xH0Boo#c$U<8eH@ioBpVy*H(KeH;pU zS4Tf|OSuPR6;#SI<0_|P{%EKYVgCfvKy-=3j+inoLKFm+g$wnY&%n5@`5>v|PK{je zGDv(QWXWqlivz@_zfx_uVd4+SyKjsfCw7b^)tY(zkOZRuj^s+CL)siX z63iv+MqXn&s~)AOKm8mOE1Fo8?j)Uy*Gmqaw>YpD5c-*%;7w;oajIpcPTYg@dX1^t zi?Z+v_qGQY?lLnn?Kb~|CTVN#bkHOSpV55tv6Q<#e}pE8hxtf7IZ&UO$YUs(O#46@ zloRzZq56HQ=NWn7h#J59EC@iKJelugqipDsX37Zbs&9Nm;wj9$9ADCmXdXl}gMp*& zYgL5ayk^7ujAz4I0oIi`;r`y2gd~AEOoIgVIZs5%;<(RSL$v9c&-Rc5^`EglaFC$; z-yRrwbJ!j%K%eJ25b4${HwmrZ3u2fSw3WQk8Lj8PSqPNgeY3Ctea>@ZWpVyAW@jF| zNDkEJJVOt=i(s&;`D`LKr2mXPghGMc{}w{1o5K!b!udRx03rO-;ZTgwyKf&RoX>b- zKBdirQ96&k!-n)3PtU{d9CXC|c^+(wAmPuSrQ-MVT_@(_N7$A5 z&XIul`R(HO;vzr^G3IlWV^IHbXF09xm>@sN^WH;g&*iMm;dUcGYoGJEuUttb=S{)V zxiLQ0yv^m_BR_NhDQCo5EAIX#XAWCVeri7FJ;$B|f&BYZqKHH~^VyOlurHg^Lulpo z%+2A}q&Rb5Hl_!eySZGe6ld?t<}{}Xg1uyx1ZA~nNW=Z?wyE3=*u1tZ^(?^t;tPC6 zu}TtHlx_MYf6%0%pHo-(H+4DCwV0$A9N6T$-R&di==kvM-u|xh_VD=q_9+Zk$2<|C z7>4OZl$~GDibt5jXNvMX3B!LEaD3xjxrrmZw}uqn({{w~`4nCcumS^Mmm(Qc>;=Xg zazpRcn1htdpN{y#?pzqz%fIx85qb?mThP5!Ism@Nlv7d`4`J|WI)1(afINTsJc`bT z{xZN?8Lyn9+esQl!(`>rEB%9Nj@|wT%H5oa;HAXmM7(gav!~ck6zq5sCI}cVP2iW% zQ{n1{WMb4J&o!765$IZ>;Ah{)0RfYEAQZ&GrSxU1LR?F@Y=swY*!M@!bRwrgFtqDy6VbCd`(jT}ggiUS9sma3LXp zLC8*oiGw5^h!E|Z3gQC=k|UDor=g#qkKPLad>*?aS}o!Es~2DWxbtTFbo<9eqV~(q z%0_3s^YkeIPai*bI#1iHo6nt%&CRtZ8|{wslP6Es z*4ImlJbMPOpFG)E2YAoc*4vxwtNBuYbylx822VFV_bGkbaNAy=-@)(BGk@J%_u-q{ zSzTRgZ{oLJuk-BbQ{4*mwhf)x+{A|xI(DxOz1v)Wwn% z{k_-6+sAvmC(rrCPd}l*j*U2?IX#0g@%;;-aC3XnrJtKd475Iql5{bRKKLPJkU{Ur zq1y*KAyU&5ja;0Gh?prSZc%U9Ta;&g(I}=|d397uT33NT^d`^zixNiHz4T)wr&lX4 zFkqj^1ECTH5VLg8&tv}_y7B5c^gfHD-p`5nO14<9V4LV?BmH>9k4@ruA(F$bb*DnB z5A4~qyF>dcK#RuR0Xk{Da$bnPOpz~gPp!Au4~$@np4R^KeoVlT0Y+kJE>NC5n;A^y z`k-~uUwJVOyjLJ-jG=JJVv#)1-fs&VuIKl%bJ+dPyL_%gnuM)xl-S$uo88xE@80bl zymQ`f@9!V(;1>LPfA{@K$>#fa4yiBTwnX%P*MTh!HUkjkaXG=-8U*KA?635)VCXqN z``kV)OaewbWM1SUu=?7Hf>Y+p;N_nxLZJA&2xS{%u)r~;Qlpepgf->|Nfx9CLRO&i z$<3$-o2`nlz$3$Cf4nKdI;RmQ#-O~G9#9YhcYuPlW4O?e=Q}78`C`vyEx{5+jKXBI zF*nU2K9R+lp5~4p$59MoS%%Ni^*5s-$G*>@qOW2Q$d=pz@C<(H_-VgvLp@&Bim^PY zqqxVOQwAp~;J@v$6Y)L9vm9d8cg#pb4dcp>shXT+&ZB_=H#%fxjQk^ z#@=Ak@&%fW^N|zjS=9oj{F^?kDMd+Q_$-O0A_FimapvjeE6XR%1|=L68??X|ovZ1& zBM;a#F;|JXuZ?`h8Xyw`W5l=;i3y6S5^=r!QIK-I)W1$CFf{fPmz2zi!<{5x$QsmG zfVgQfMkAE2-6Uc>PoE~5U9qFe8Ro#$sosdC9)gt*W{3b27mcRr&!566EYT}YJN@^2 z(b)YZ^UXj2GSd&t$$5tCOCH;E#DJ}Gdz zMDW%STH=($_OGs(55c?xk!9&<=wo!yP-S}ZB8uQ2-e91&b1}0}(@!je0~vQX6uSzb zTu}o?I}bi!$QY=h7Y+;~c9RQ-;ybz?0Qyiyt=p7Jk(mw0m7@%m+Pzsw6E8TU6>NT4 z3V-SFCHXO)yg&|K$rX_@M@{a>CaYawu34V%9q*o=9UnN~Z||SwIi3K~QR*yyc>*); zxv}SLAMIHctW}zuTbm?xtQcLcUs6tUP#ytbeylT?>1g)a&qmHB$pgsQ2H9|nW5rxy z)&;kkh6OCyKwwz;B+gBAXM_x+0%{lun|a*TmM5?sSt z2FwNLVL3)bb?zavDw-`Ua^^}>l{tGicuN_85F!rCJOfsfHqAON_DPXFQ?O`p=h>7Ll{x;eUsP8uV$cF{2L8Ue=%M3owuUsmC4CS(@q zu@*Al=j9wGp6W}9Nj)wqz>UFAS{5{izSo_=CnSAKwvgi%v+xPok;)yff=?LtFB7;0 zD~t;~u+9+Giu(^;YhM<{EQ=!cYb*yv&4OlI^301jgBaoljAQM5!% z^a+PS%CxEW;i%uwVs)ojW8zVnHD%UAi6GwOwWw$ai=D}`-a!=Nwd(Bh-t}~WFD@%M zQQ{LIX5{3zlK6}p4)~J5%mw`bk3*PcC^*leERii=DN(?^1j2EJDhcf^g1&F|*0}9o zt~JTmlTc&|e@Yw#GNUVMp`lFO5#?5|gH)by?SOA}(NO#S$CLQ^VJWxhu zY|P^SVi_9E_emwiA*&tN!!(*0om0R%kE5%Q(!t>cRXubicSTnfawec)h0CU^J6r=p2 z`WbHx44{vrp$AZ{Zd&lTi4w3|tjxrm+ms145z2K83yUTuLcebT)V3IK9(^x+9yv8c{o`s(uA4!77=I{eCb&HJFR}4?cvzXjg&|XglyM7?#*@ zujn}1J)T_v-%jECIAzrw%p{6;c-!aDS(`df{LNM6EO1;FYc540NWu>{+UZdiq%XCmfr*(mkH|p!D(M%3J^CSAR|5S(M>45vjppQaUZR%>f<`KM3<>@ov~qa z$IYX_B&D(+UIuYQCw*I`ynFEd-tpnVdtz~gD>U0^$pB*9+XzA+o_y{1GikdgMYpm` zHo50k?6{R=Er}W!VEl@$Il4QC2XBF@=}`$q=jmD8l$DLetX0{Bx*Fq*s{pc}Vyf*x zB{elwCh}sCme3f;f^W^l4fiA>E+^qK07BJGSUqr z9f0xj8IO2;?JZ8hY9^CD9CE`~!_ZkuoNwrM@p&3&{x{BwpY~UH0WISN@rxBvHrIQj zVczy`3k_+)q2+1@`nEJ5+))QB<)dP#P~nwb$hZx&QVp)m#-qC6(hmC#Ty7Z?=!$l(L>_my%+P zlkOw)MauMn`dC^=SC^1^3!SASXYb^2Wp8(fDnEPr?Ai17vuCSLL);R>TW9Cls=WDa zR@z6*c7Z3_7dBPUiCy3=C#mPOS6A1T+e@9*)wb%x(c#J756kUhAMEMU1n_F{50gLg zo_neNr2q8UlV?vh`(LNIIe4f{%C4@imao>xpbXZ-vy~rC_E%10H%yRh zA34HE{Np#SC;z$K`nRWIS{u{W|K?iB`rml+W&JYtW?%n7*dJz| zzf!*qTn|s4Y*em){GMC??akHAKWyCJu=VG^eEr{k{$!{$>6j;4_E$ zmx&`Q3E>v=ztL&0ewqIV_{?+uIh4gkp^b_1UwiGzllDe_{x{b*H@?V!5Aa!eJlD_4 z;va*7=MS9!*%f;B@4Bb|INCi~T>K+E21#gPfBX2|ZukA(!Os5n$;rY(b7OaIp&|3q7hvV`Hnm$ z(;U7S4+c)?<29EX-_S+qV{}HKkT7C2jxyU^UNi@Dy!+;#hX;g{lf8fLE-bV+WoJ;^ zG?E>{+ZeK^qjav%jqH#Vae?zf;}tzSGT@cPa{z*H9H$X-HRshUXL#PF%H2^mOq

    (7tyR6$<7BNasIuyKxrJiqsa5SjG33tYRh`; z`F)tHmrlF2Xgx<4!xWx(7Jqk)$LW^p#L*ejsBFB@6q4*4f8Zi6xDj%t#oJUL9nBJp zR^ROIZ2tpj^1X)GUPTZ(sfm9{Za_i*LjuSYj+F?qf8oQk5^x8ndpmzS=^pJKch8RA zY@hA|%+*zmr6i3rbcaKl=we7PF@NYT)=es&0|rX#lVuVWNmO0)SyerNZ?;ugj&5 zDO%;|QuGVTF8 z1vv>Ugjht>JAz7zj|nfXfHc?hVr1A2 z^rVC>u1OYXcukFVw}YW8oJP^*nG7AyePsB;G7mr!q@jC*e$~S$`oMmCP6MVQBNxuK z&Jq-JlA9#;N6qJ0p6WQxU&P1y65_{!|GaE8bXDoUIM1E4P+|GG)p_*Q90YNat_;z0 zh_;iK1JmcHR>FJHZtf#?ANlSCg%D`$gEw8WM%;Tth1eR;jEaSjJb*;3`!Dn}gP&Q5cBa0^ zPexb(15dzam&%7jcu4U0rwcz7YX72e{#MzJAO#2$1e|jiN{SdwRQw?fHUj?wA4*if zV+Cgn;A0z|^gI#I$qu&+Eo!V=c3MbwO5lhCQ4UuG=o$wdh2LNti^oYIpI?QvEK{XI8kPJTEd;c`QEG7tINlmX@|_mR*{mtb!}b z^UAJ0qNV-qTVZNRe5@};xMruE28U$wTD&M;ymWHsrzzc5V}9UnlS{qT^_R1D{oyc< zvI50OTv$M1Ax-p2VCTEDgTHlg?lr}4BaJ#u35)Sg^ElvO=>z1}qHj?8rVA- z6Jv8B_UNmWXYs^OB5nC8thPYga(>P*h{U=2;reiu(Ro1i?aJd?Fee zy$Hgki~@FqOM`5v#9##9LVePO)Leb`O=>2-IUg3TN)%otz_7M~0-)S|-)A{(K z3UkJMQglXCK^0Hwx#KZ5JLL%d8V{D+qz^PHzUq)@67hDyp`XBz7BIRJ#_Y6y`gYyPc9 zws6(Gxe477dh6rNp*t@v8ojq2D{m$v{##!EMEi}Y)X}URGaK|w-a z84$_K`YM1@Mp4Te)H-gQ{b+n6`8Yx&R!%#rybfrJn6z#XGC-F+w+8Mo5qlk)t~kks z8`ENP^!`mdnDFJ-&Lphx(bo#*XJx76^RHsL-(Y~=U-A!2%9X(@E2r*iitvDMuXb7W zK;?*wuTE)q0=(({b4o{C2C=Gb>SNn_=%5$#xeCl%yM>jzV0C$U*~veAfO2z~^?W(P zXo!hGm)euc-G0`~;Hm3AP46Y;{#6EVMJzCL(rGoSC`lard!(-ib!%=y^^kjQiCh)> zE>Z-qR)xt0Id%gXhAweC8SJ+lj}rSQ(fGXcu$ph?jy}}4<^N+3D-CRjyG>hWi zK}>?pBvWLy%a5ku-(Fr0a-|W+7sSmGj2cJe*iC z{%mN3k+5r6rE%rpZ8O6W0fA#5%SqfNYW}f*TZAs^>56mm{_ybhyOY!1qwWFzd&1a& z6%OyQiY7iK8dDJHRK{*JGD-5yw&Fg+=v<0Lsf*$)MxpjWsAF*sLrlqt%&vDyk#lje z!@|FE{BZ=p858Jm%#o)D6cHw6c!ESk3|QFNV{-4#O``7x&|(GovdGhi+c zz)qxM_hOuHs7FVGK57;`1}Y~`PXj&XnR|oLEl~yoA>WHen83j~_d_@cfLy4F8B^>m zEc}cDc1%G$eLaN=1WChU_PAwre&PKhqcGHJN)dk`DiRPLJ~v>7p@DNW9ap$}+JP9B z#gD7Y%Muog+0HJZSVm})bO7fo*be#_^8@Wi0(59KJHkS$KsPtUzM?({|HQFrVHh7% z$k5;d!@EoAnimevWDxRQSJeNIr2e>wehE~gP+tn0i_yinIRKI(2xF3s9B$tjl*8-Q zq8>*^lLmSP-YNGE>h01iT3*X+ctZ|t1c2X(r!Zy=Ym5 z6W3jWb_{kQxdj6?R=&KSgq2Mhd1M16Yze_@&|=9j-QZb4Tw9M(Fe&gL!AN-ZVTdDy zMUx1{8HPA02{JX-l0x;4mF|LxRTvqGfm4_Qv@ya!?B=pl2*A@mjP=Jt4}U7z8Q2=b z*cV{^M`J%0*@{>f1O(9%+BAyWkHK|iMjK%`1w;rluLzRZCS74yM6~80wh;!i7McmA zgTeYOg2d}nSfaUokwFKZ4v|iVgd>!0fFF#|4;;-UXgTmolV@;rR3`5ZSK>+rInGj_ zbYfV^kOOv%B*PqbUkU^=6qlLNiKxQbq6vt;#8#(rp*?%dC^Z`V&S$-a)L;q?A9FRz zy(0BV6zdJ`5GWY{#w1}b;9m=$a_ln|;zl?;57Y?JjE|!|@Q&68w!#Zn>}Q=IG;Ee_1CL6CH=izOmslsFU_^E8n%j}APhS&32w ziwTl-NW{n$ft@c&RVX%fP0!KB2z|UbM2vPArh(aIQg3>W@Jq6D6^QhL!K-CFa`DO* z1u4KDMmp!53Z%VE`sme7@=P)PnhXr$8{3h#=Wv|W=O`en)F8%?MS=G7lA-5{S>zE4 zI{2~klc@lCK!(4t)mJ8DU_v~Dg$Cw@1}hc;+mKbsO$lCNEDAD{mA5OWOc$CAoW>~f zI7*Z3M3WS9IdT)ErY~V&NiE5sI7UAToLg~SuDRB9c5vA)aZZ@59uQgx#E+-(Xt7|$ zf|e^7vH%%gjXv-5ebsyd!oegkieP zV|FFwN_B7(L8kTAi&qOwK70d`6eI{fM@sp7aluP)G=86iO7UIr-Cf45t1n>$^Od8t zZY-J$E|<}@`J4#SwmQxk0Q#leA11S+W+14_Dfh7(q1e$P8+%v*;5K!(RrHf&Xge*ad&J?Amfhy_u z+i$+_zBxQNJ=ooS(R(4*M3s%AnH} z#Ym)Ml%X8b5kE&DdWdUmUzto)p4${s&dl?ffTnphp5esZRP% zrQQqRNgND?+J_=6d(QsxnSF833K?A4JR)XYe74}zIqWsk4gvcm6+?+X3YNT#1qiXyO0^1fy7cNdabZrF|Lw(u!MyrS+{XRO!E#XhmOX*Dzu^w`+oVxW zg}{HOK!8Ex2@ATM`%xRhZ&g3;4?+Qxzg7MCP5jglcuuw5i$ik-PdtX5{~>?!_o;I_ zLr*>mKIz_TWf}x|JGPSlp^M&uGVce1b)&a35rQtdl0VDmtfBOit%9|S$rr>%{04Py z!dOP?MGdLZ@<&32D15O6*L;5NiNY%uW_9+7K0NUmGY4lS8^e*%w`9%b(qj!)?2gbF z%g|W*dk!53Z0JU^Uec!X|nJs&NEt>Bs+-d+QbGI zut?0&Zsj#Y?30n5^hu!&3O8T6L1j&xiUrm(l3Zi&w2Y~;wDCo$5eok5;zsydN)tjJ zm&zN6*TB3FMBV?D8WC`ZzkNn9q!4o*z%2wsLSCBDcx>`BsF zB0};x>AF_G%JY+sFHah4K0`?}O1foT>iO``AYcWrrO!IfgmMDTEFfL~iWC|w3rjx{ z$ekh#i$fdvW08WE={RTh>l!z<yhMQUSc6}`}iad~8=I#%tyT=D7sQ881?z2URBaR=WB)<1P8r^u?x^^gzAleqrq1g~vuw zjKP%r_>(H8PQC~scla$UB4-Y-(q)0XckhDzK(9y%P{5OBkz`4z&%%Zh$h2VO1z!S! zjylq2A@`npGIFoc(%9kH32Ke-#Rsd5OJMV4LS5loCu$cQ42`OWqIM{hu(`qIYx`C8 zFUsqafKaGjAh!S6J=N~cG#;mcOhED@VD-|EMB&VS?>x4C4`6-A^@{fWU?^AgB3YB0 zhf}eAxHml?r~|WU3v+J(BQ#CcmgGFfzZ~ck7XqK8q1O{3$8d_j^lu5-mKV7$|Kc4K zxFLEJJjTB*$8N*!7*+D7*DCFq1ohY)B=5*b1ycZuSm=>$=p8M%Esx~@r*G;C>6;vR4W(QNr_e$?BGj_?S zIJG~jFS?RV8MIwafkCK+>9cDx^O;`Xu0rAmflOGH<&g3{w<5ocqa30I=29p)kIBCNokkW-tSc#%U zoKZpn+`70?N$xxw>KClxvtJ^dh#4Ta~9RP;j1nugk_!HCDcLPo}nw z0?FhZeUNa{#2=2TIx+=`rOg`KmpNV(?rGX?)0G+=@HU;9fkT-Kl8+1p1~rKVh$>|) zfL={u0lfaImuzv-d&a_@H#wC~Q5_ahV~S?kG^S5U)zp+!Rr^EP)DU(JK|~Y^qb?ml z>`@n|NeXTzpRpu4y4|bhXtCmG6|%#v8m-Og42upi9l7eO{Rl;b`O@h8K1tROA8mm;Q5#Sstj4WD)KFE3e*u&sS zvgOCt0Bv#ovF?^WE(*f~;{iv{vHD%R$7Q{xKGg1UX>U#9uIjPui?9NN9?>_hTQvKG zorvE%rtqWR4vV$|X@K$s2XEDtMvLTtASWeD7d??_ih17lzim-GlAd`@5pRV!6;K2}p>3xG?Y< z3h!gA=n!d{s!7Xm%tGonLFF3;_bHcm3hm@uVCx%otbt#1A4GK%w;hDF&?Ue$>WUJL zN&RphiwgM@ody$=62CT-$>WVZ1Sv{aMcZ*6$I~Z=oGD5`by=;g@uJSCW2Bhz^x3SM z$qDdy>L$!uIfd9WCjlahZ%(RN?{|RFfj!DLva6PYZZm6EmV*MoRO{s6MgCd0D zNy$EK>r0GoFd7dFt*P4%>!PC^fY6TJ0mQTua495vix{V|nD)XZQPl7l{KNvnw*j7i zT^Njh&zA=IDL{z`4X;A4ARV4I=j9vj%AM@8LH5kOwYP8EyVqbdX$+I^!d zsM$)dtDviEy4m2HMWUhCrD9g2__WxtwvkHeDo{cyT?MAranrhqnc)&PWsl!ig%>fP-{{e1uL+y6KbFDkEP7H2%7 z=Ov}Zf1?{_LMA}s875beR$(yu{M#3771rgPsi^2!i2XE+L#OdIy;*vt0@R5*OA_2q zQ2h7BsxeMd6qJs1>PfA1w6eF=n9HNCDWFL(6Gu^Xs$QH?4{@(ERT142hT3T0Mu;0- z%ENZPU0lFrVLZaXDwnshr*Xls26e8elZ23t_Rb!s{ zc@gt7X}9hR@s;$TD;evMhf=oc9F46PD%kd&SWkU@T^=_j_{*b77h!_3#}s=&d`w!E z6xDKKT5*b6red;!%BI(wK!19cZja|F+RXW57yS-vpcmTyEbuE;G;)RCTH9-F17BBf z_&K(}^hzAurfp%9Iy#pEWNn)|=t${NM*Ww*!l9tIRV^vWOXSciiD#O-- zt=pa*eImJ91-CL1$mF=Enczmxf`InIAwM|O(n$C(b4l4u6f5=rjHGv~Tfh6Exzb!r z%)}52x@2MHQi>&mL}hlS5*Aq699F;0)^NK^&i^p%sp7NKUj1v`|8N|d&E0Yb{s;)1 zG$PtHW*^I~@6qpjJm$VOS>2OAvDCcFr&EM_t>xs~9SKTq|4T76`ADGAN8-4Z4Q?jU z-v*?F{2YAP9H$BM#WnkXrRRc*MK|N*-Of0o#M{WcEH~!CobCz7=mMo`JP_W#roaHX zk;@XCZ20Ta%ym>+Ug1xuQO*rTx!dr<(j}so#$dd7c{7nt&_OpFA3za!c za&EB}Ym;ul&~nT|6}LW0&S&PPt>0N^Zg$(Bdt8&n^`lmRz1}9H@>V1~qpi{xR`rRn zDl3O5t;1r?+lnA_v@ZLoY)u}F%Z|aezP~~>Wp|~RGg;m+s<1pY=D(biS!aI9^JM=k zbL!4IuQQtUKBRD3tBUqIy=quiA1ko!D*eqw$M3f;cnW!mi#%ih)edttZ!we0D^0+d_2s2vI~2fG7yrqaD$NAM!e}DR$w6{e6OU%2FYSfX z6b;ilQLXEEl~&=OG=9r0?1bH)VU?%k+9FY!c2f-TTY-J!jqC=XPbcGq^Xs1->y6HC z%n&3U{k~XwhxkXwsS2?w?Y}vYV zQH(Pc2CHrvX&RG@V1Tx7)Qmd7OftMPx}uHv*b8~V{8u@_thiHbt( z>U^5?0&|NER36i1$2m)6N-JK%x;_O2@6&K8VVvNrl*KEKvh#~^mSX-W%#`7DR{#8! z7%EOFktNM=bK*uLhP=c>o-|J9*iZT{3cJ8fshdDE16=t7ZwiO~Q>;5|@Gz(L6g0)ug!AF!hFq=&cJm_5lT z)A#}YNJIddFOwhN04CRdA_0T+-5?yX$Z1skI}=}o*=}%RIZHDqYZK`_6P>2+3d7o8 zVy0Qn$RqKx%6WO(^lt|_032svU1;h7yvR`aZY=Nvi`*Mx*z+C`H;;H_@sI2%v|#sm zft9+^K+-#^NY;>4Ky*4jI{>NrhvVIMC(T0rM$&}Jjenjs+EmGgix&M*0`2zOc={;` zot;DKPe3jI7CW12W3AcBKU#0L^p6em$ENw?NwMZr^ZhgPM|;)!)h^a;cdQa?)-S6| z?TwP!R@d53tY22Q+Ruu$J67*HZR?lSuTBZRI#%yG7REYOzd9v2>sY<(SXf)L`n5*$ z{5zc@jMK1UAumv97qn*0yy5&IXs{`I@vYNoHks{cj?y6hm0yyo8M!jCjyV!#q9u5d z1(QIoIGTt4F{1!BvU2{nT}SvjR|R1BisyTxhf%!)-{eb*DF?nO z1|<3%W(;G35sg*@3EfS@nun$_mi8HfRoi8y2ub3jr1IjSt@`ptsgDdT>$veX9p7}X z#w>y&iK_*}^(equk?dLyPwZk&fiziHJ$Fy;Do3l&o}llfFA#*9g`$HQ8|yeHI5D`# z^W_=dv+7PDKAl&^<7)d%GUW-hNo~%m8PUC6s8U8uqH96XbWK!F(&&XvUiS;8NXjxP zC0M~BQzBw_c;((;3QOSR0cJ}KeifNuIG+RM*%r07X_PFvQof-qh7$(oHtJ(Wb`(8f zZyHADAOi)7sD^3KVTYhpM{p9m@Hutj)rC)KH<2h}e}HlxC4`}NMOA2BG`?~RG%E#^ zxwv!7Ix&~ELR;O%eE)$H>fF!d7IfVefb{@Hq65y?Ul&favKLT4w=Sv`%(JLPC$o|g zt@QMjI4k3kJBzWQ-j4rBK&#Da7$1`($o@^*k(;*Mm|H5E*ySEL@zTUtFt-o*f!mx8 z0lcL%E$N7;oM|aX_S8z4i2&M7<7R4$00M@&BMFC`#KD*VQCK#9%H6*u)pi2&R-1Nx zu^_F6?JkOu5LV2(Ejf6qELe_8ydb*Ja>{NvYs|ppSSfr#UOH*2;r7%<4d2bX1Y0oz zt(dtUs-u>e(Hy?YkpQwhuBPTjvruA^Y7H7}Nz{<+l^K)qo{f!P=Gdz~%!&JH?sze- zyQDseTe(Q(_!uXI3ywFV$0|3rvfiXJVGt*Hn%EsPhWR|Yv+|gAim+$oyW#XPS2fV> zFpHvF?DSDXOC25+$&5vm>0MI;{Ex99BJ-! z8+(VVPjz)<*EWp~ilTtE#Z-CYk@#l_lOX#!U-9gSV-P%;Npz7cMx8MB>GWWwj@)Ak zL`SM9WM0q^Bthz5QPWL0U1dPI$O9Nl=o@)cM-hfj$8_v5ram1GDWR)8YQSi-F=cF| zYaw<226XNDsUMFJ7#(16{;&yAlen_k-{P@DAdHz))EfqrX_j!Aql>R4<1Dctwyj!f z+J7!dfXd#}6fzN@qME#o;^ zfGC~1d{Z+_qHg8RlW5FcUN7hsd1x)3c@GgXCmlIv;g$2|iiWAF6dfr=GwLME)OH_T z6Q>N&>&mi%{8-dLvEjY@ft1}oN)%|upd<{QPc-3({a-RaNo5s$u`GV9Fx#AoDOg;= zTrwu@s$+8}6E0jZ?f9c`G85WuRv9&$`OE9cT8#Pb7T8vY4e5ku#l7za(+N_6o3^ii zw5{bM>p?4O@~)|oA}pY)$Td-Ckb zW`A?Fx9V^D>w|T7ZF8glWTU;>dD2;5_gv3?;;sZ?f0%jximbUJotIab9G5ThZLGp) zb90^k!YBW?v%0$RhxS@~V`F3e$;M_Io_97kH=g`q^%moq`I9B78$*+E6s42OO?#go z+-LHE4GngtH)P%h2jf1VBN_>N*hUi$n@$vaXgnK9TT)WjT)1K1CmTG^cRh5Ual9-> zb5v5+2y0qxN$YUZEHWhG*UF|&v;`ZL7dii4;LpEu_ader3n``=KB<>PS?S_4hAyOo zhJry0{m~0as;{0qe@>qN*?aE1g^r<-pzP^$=U zt#0`CN7$SsaJLsBlD1THzb~ln|BiNvX!5$+G^3gm0zfoC%M;*)p2C9htYKo}U!{IL zdVsSI{4Ata$f_9KydBZ#l!4b9^g^bgt0Fa!fxcU06`Dhc@%#=*DMP~nNM{E0xr~0& zD4Ec@Rf8>mZLNuVD?;|OfW55=oxIJy@#XWM@tN!XPX-6G?*D6>?M}P6|F5rp+5aEl z^8l$aPyca5i80HhVS4vKDLPM1kI!~at9l~dK2Tpoa?(Y;0~iLW?10@EH_ZqM@?pza zZC2J=>y*^Ohn2O)xe8+}OGm!a;=ksK|2nBd`M!vk4V7Sr-=N_m zle{Y{At|m) zD`N##@^B?p)C178@box#C%u&sV_!Z*$V~kvrsGs?O z3iMcXhcwKV@n7k12;wylVr6RHVi^>oge9qX5eQ!WK#SRYB}DDiN}N$nV{>AKzVTgW z0yD>6tCMS;;>dt)%2%oq{V1y>M>eri*~ls@^o{S;Bb(Ts$Z%#dcfNhVlKibu3LkP{t;@=ww!AK`it}D)t2+;_SPeQ_1t;%ZTtwf zf@aHk^m6owHawJ@4zm>Yc*ce-X}sPG?YEZIN9ug*RJb$b$6144+vCJ`t{veSkB^#- zrH1@Ut|tN&pE@QuZqltbA&HTv_$%x8{Un((_5IQKx_@yFV_t_bugWn$N_-IGEF8#@ zBNANrD&SAU1Qf^8ef)x0OiFQs<^53<-#o&%JHx2gLy@c;cV9ISMNQd(3EnX?Y{tS= zy97}JKW)Yj_HQ3`&S()h8V>`OC8>}#9s^%QKG(OQnHqTEtD#{j8{KKQnN6O|kFzNJ zh-&z{JM`T|)WEPpjcJ9NQz|07-DZ)^aTId;(0bv)+H!hXiqR&}HrT<46J)o6cEud_MVx=(y|aI`x$Y!4N$QUhI{D_o z4{>UmY-{XAxll2}qhD=;WP%}{N%;!_G%c1|+5|@WwCl`&RX$9=K@t%Czqi$S$APGo zI2yy&TQ!b;;mJIg<}HU{F2io@4^`)D?cbFU6-~*G>@a)Z4!x5rcMPDAr`PJrtvI~;8F36Iidz)*GrkXW zWUn3r58FQ4BSuLkYyR~(!hl$z)LqK$h~*u;j$B+=h;ktml*XFE9f=#pBKDx05t3v% zO?Ka(>~z21J-!E0+`}s>4(7Tm#>0d$z7HvE_-09!9kGfCkEdMVu znbj<=`}Xj7r@G4g$*kK=ZbGi+h92*TJgoa6FVP-WtJU7zn3`x9;VHQldyoKY!w$?@ zUR_)e4I<5SQ$I<$HFPGhh$ivzFMNyRD0Uvb4-yQKr!77B5(WcV2+dOdf+l&SjQo^# z>^>A*Yn@8PmL~dO?dfJgO?4~f6{>{uL_I-uEcpigtgE~legeuXbDT9)^7c3_ z$}9Rhi}LDYW1?)C*Su%Aweo8C$tbVps;rt!JQI{v!#c{U;peBUx&d^91` zP*Z}EN)9zu3$iY?AT?DW>-Ruew^4!Ig{J1A0=X3zo4X2R+K6kYK%@d>y81)tAIj7p z_o($a68RW%YL7DfVDPQUhHHqF3QM4-o+nYUS=8tm3a6%&g`_rCri3oS?o5Y`Tf3gg zrX^G<;hxJ!e!HjS)U<#y=4^`IS4-{;(d3imK2`qUf7H zlDcRqGMY-t=$k*1>11LvkZRk?$(Sfp#7{*0?+85zsqzpx=S_B7WXjn4JgR2`zKvLiIcW>fvj?yr=|2 z55|jbOF4JvN5-f>C4Tfd#i04}`FNjs`v3Rb#4mdPPmTZ6-fTZ9`2Vl1e)0c*h|d@Q z|Bvhc&(4>)a=K;FB8?D{(z}n!VoJfJBOZJ?U{s zu;E)u>IAFBaqJJG*v=GT7l@J+OLS2c8m2W0_Ai2=moN88MF;rL-#zEb|AQ3BY1a5J z?I)e~T1otu_80#D0H25B|0@_X^*R`3BZmnE|2X|93n}Go=YK(_e!X{!(W&2W{{Y`k zmd(_yg|`cKl2%nPS873LX)SzoDoP@%o0N1!lVDd3#b#w)TJEw>tf<6LF zK?gepr3j`p-A)?&nECoG=7@HBah6i_5Gd349Sl?A#!dqN^eBTT3lp1=NHJZ){K~q-n&i}4CX)yAW7J}1FIYF4DzU!#~hD93UY{A9hIRKZ2RBm@ygBwrsM+q*eFUrDWA^oCf`D2s1#-g{~PZy1!V`f9`sBf7R^0=zir8 ztAMoY0jY#r3Al4>z$K4lW7W;A&9ga*3^=hHYOc4FBpcz?u}I4%-S2nzi=oMLSi4D!Xxa?WCKN4vLBv4T4C+c#_D>Ng1k@66C_ zXMWAD&Cu-H{F+^#q1pB7W-YUlvry0l8nW|Fji*V|bfkp5n0GiLA&+0Sg+Ytlg%gwC zHHrgXaq==vQ&PN@lMh(hr%jdiU4Q=@6{5eJ~P@tKNMK{)b7WIGVQ=;b$wu0kbIE$Bp} zTyjf?7dXV20ThETh?CT5h!Wp|nC!E06^G5kxOSAdOg4rODTKD~5g;)J;I3%|gY%cq zms#dD#)guw7&YL50sXl8Q&WZ+u#Y9OKw%Yq+x}^*oWvK(q|ZvUNE~yytqKbNy-IiU zF*R!@#bGY%hIL4D9{2Ez(vgKC6_H(4*P;seO&ZyhLDMnH#9K<$HL?7srSqB`2$1!J zJdOconVAtcW_{FO&Qe`X6uit$Q*t*@xfvT|vkuGb3$T&9_2$lT*#s-tDf0)V3*}o+ z;NriFJXYrIJO`HVP|CVNwDd}FCGkt}%b%9b%>mBN?JdR55CT=lUq2Ew;9tn9PA#1P zb{bCiBDiQ2i46x|tun(>3GU8ib`zFiWKRQR?8xLA0W`R}Zz>|p#OAq?6$({tdWbUPnwqe*^7xOX^rXl+ z(T?Zt9E0hZj=5lg2KHP#hB&9^wLJ zjUBE^Tt{L-(z;U8R)A@E)N*Xw%TY*TdO6}8;sAJl7iwaE!N$MMqkH}{WVq4-}$Qsw`Pr8?l+!&h1B7$X+>vr%H??s|p!Q@V1 zDejJ-qM|@$u>*to^Uo>l{Eo8FObmVD>;Q=84aegiO9oY!(LWHMSN+1tQSyoG#H_qU z)xYyDFnyn!<2Im>@VV{;6|=Z-+j*3E`81%rZx~-`ELUzCg+u(}7t6u_s?R*_KPbUs zi2>jg`@hZhlcN2{#uxj)2l+gt{RiGVonR7CRFAqtVEL#wMGYi}tl$QI7+)+SiM~{yCNLo87nDXZxoMoi@qG$2$26Co*9-`XG*RH%8eo4aP%XB#cR%S1+BUq$4}Kd$1Y2Yb0hT8g%=ZYq10c zi#cH59UlG-lW6aq?rrby{d4zNrM<fc)%bf&!rdZeHH{wdwxd`e1oOuP$JB-E!1 znDl#i-W55?#Y-0T`xkD!@E8kt6jMyRj8QPes!ShU`n^>Etf*(J@(t0|7NVu0p3%{U zNVlw3H^l79MrUMC@!drfL3h#x^A}*jg~b6Cs>>8@aycO}8s*{_kkzUyCsQi3?QTgo zMILzmiHB)Yv-7jqKM#OE#tqXkSR^^;M`rqD0EK^z^z)C;y=*j=I&OxE_gS}qDuIaJ za+*#w3YM;frlGS{h!RKF!A&Jrq;q)aU!kK`Y#CRpA8K2&&7>1(j*;=JA4l?gKN^jK zRQwX@sVj$h6{D$j{ArFMH@ox3xuA=(&O)y=sc~mg-z)9UJTX|eM~EuC(Jge(SYYEU zKKCE0cPu1WGduLdbIO};^vN{h|D3KDEv6wC&=7EwjmtO?AWKKa<^JgPYN;kLef@u)_@ z5jhDw`f}d?6h3o5|1%v}ZA>};cRI!M|C6=#%`fNw2lzbP`Cs}&_+d6e^)1i|I+gXi zUXWnIc+bIGke2ac90wybN5U6v^F`tVxApMVOQ+LXG~NsY*q|NKq&jPhzdP!M;c)Th z*W2{h+ex;n}E ze(e0z)E!}UFI4&9-(Ym-VlDGmb4xhAK_B2h4r<`X33&rykKtH@0ptVEckz06?8k#B z9=TzkZK9yAx()r?cio-Cvx8Id{3r&P5}>MhIexaBkD^zZVxYr7S!Ka@w+=cy8)iPIA03l$(W`QL?w&nUS(o`o9JA zYq(0b%qPhO5Z8yYX9G7F+RuX!?b_^Tu@CG&(K>hR;8`NPz{iQ7dC^krhF+xnK+pY9 zgvQMEZXAc_6x7IPpF8I?N-d|r05kEL?q~}Ov?vT0z4t*#KBs71($8WHLSz6a&~Adn zE6JY#Oj-g+&4T8I6P`3$To|5rM?olnZrLxOAL<2-iztSANHODs1Ou`QCML$GIn}~r z{JX`E6el7o!mlPs9vM1l!Z56c^>77nRJ$xZh6-DsoX;K$sxVlPd{KELbT=*DCqP#H zaG@mC5qjoPdS-*Q6T-wTo5j5)&Xht$^Uv~jv{M*9BU)=%Z&hCdx&<$G5K$b2p48}I z57c*D^%e|TUX?>OHyZ+DKFGt0{ZVvTH7vU-$4W6fhTjQ&6&>XojoAR^fT;-pjt6@+ z&0V7~MCnyVuX%UYz2J&sPVKtw9J)GYi$YAeMU%2-?KZz0p}u@R{m)#_f9dcx=f6!j z^%c*58&AHR{~qA;@y>tgkS!?^{k7yIh_Bp{?^$K_gYp`%>-yJr9qUnfExsP&$eKyv zNqHT!v?})rol%O&h64%hX$uR4M*HZI$2NLjCn1_2-Uh$_%qo=9PF!CGO6IsGB}Ucqqy?MXchDG1Ux7q!og3(BpaOV!OY?Lx?#TG<<~@(rqA`~+u)-z_c( zD9{5nm>zxk@!<8|!3jE)i6TQcKKE6j$3q@5jyD{NA*l#6JPuLuA~Wk<81*hn>*WZ_ zAf+N=%;_yo_{dLPR$J(2GP}O>jLlt-UFMEEg>jOKhVqB zJQSN#k)MO!I2Q{V19OFVL^cK>K<4^yR51?ZSlt?r1zwJ6;&}MHg)Qf!P~*c%L8$3{ zjTqEcxrz|fLPvRxOAX357;`*DY8X5K(Q^m$fRYak8lL`)-UxN8QY}Q(SM1fTyiOM6 zQXiudc+k+#Db4r`DS^Lw?!0w_VHRV&Qb%?AUyuHr{OeK6Szcal{wsXcQZ=zl%`FYh z?L~3gFrNHwc)rNorK`;%UNq71D2b#TNk#%CMVw5jrqEEtK-HKZnRqGoE{_R?fjfGm;AT0Z{{Jm<=cY%{q?%1SQ8j&;M*6KXd%| zjkPD6h4cTD)%7pu{|ESdwDZ5|Op(~G6`#}z8qYKs2|OBzPU5}M8Ab^2W4J^(gQ)8p zI+6Hc_pvhw{Gpea*MatMIrQZ=inGojjz-8*`xksn=8{gIwno5#8{V+oFU(hg2|tEL z=$AMd$-KH4R1(f#0s^`35=aVVj!olf3ncY1bQbp->@dS7$w&b9yx6*gLr=emT_AgIjx_{EZW{asn6#9dxRK zQ)HBdbfOb!i`m~B&W-~DqNeTr*#XA4hd~Or9yknUsRKs_ioFki%WmDA0rZb!bGrQx zj^X=lmd2_5C4=tSy=>tLvq!OnxHG{B?_m9Y+T^gGlz=&LM#470_w?XT?D3A=8%j9e z^Bu;M<6{_Y8F00i_+g*u5RtOPNNEuu3L<)S0fm$CFi2Bu6Obq9fV}M!JZ;H{-U-l_ zJq{op8J+Y3DFj`>5s3G*+ea9Z^u^t$MFF8P|9JdompNxJbZO*tQ{5BJ{Q$dv(;p)2 zKGGvnY>Zfk-b2q~0wTQnp)P`28U(|kK--GadqBh7j@ft&ID-{4zPL#!rmO~}JooH1 z(z4q1uK|1gAO&KD*~YBd;+IPq3S2>Vys)B6KW3;2VweCVZW<)8_LD*vLf`iYhzGkS z%*SN_(4s-=pa-%;Ud2gqgsRs$47-d=3&Sw3OLrJ}6n_Uhi24CDpaJ`-B?QEa!f#SX zc%u;~5ia}1KDfe}TZa1?pzJ3NTw2^VE)Hrg>irCqFZ2{+DBMl^={4Oq11R-CK-&+( z4>bHJ=1zPA+$jqa)=2R!IeW*k5ecrP#N>s#a2H7G1GE#PYT=i-VeKc)g?^t3V7sui z02vrQDAxm87(mx6Nw^{yz>)~uYeX+vhf*c9ebtp^4J^$KSZaW=^+EGdO+k;tJ%}Yk{3R@O}%hxE#jEwE8d=Fucw&)>{Xe& zOs!`!W(%$QMXf0*&LsM0l?9V1#=s~7z$vY)&niPKQ&yK|tg>!NXo=k1POTc2RTL9k zS=opmP*z>f^JEBxm}Q*6K_srwZEMTR8bjx zq0j*>1i)Tc*`4Z|!v3l<5K4u-;XKxlTKT0SWEXl_-LkOOD(~%`qZ!~{aBx~4&OlAg zVsRl)UjJ4s1AMadN}fooBZB7Jna;v1nqekT(u#;^J0rs{Z$)s6nypOR9xB=h%Uht} z;4^O}UG}TMsoQhzjRoYRI8y{TXwFI5Ss|= zH89K8ZZ{*8A1@uLov72??4p3(sMc=0bPSzAE!vntfXI;tDEp-_*@SG>_EvdYqRD$4LiS1##lVlNa4%>rTmwL#ikMC}k0 zq*E?wa`7rDluN3d7$~<0b03^H3`AT_4qO)_FV^IvHKjh9jbCDND)^e|D26$US@F@_ zQZL0Y`|^4HKHR(fD914JhhD?1R5>6jX~JkZw;$gGDOL7N6shcwcwf?oj~CVIW8c1g z{Leh^f6;5A%m94y{cmS=t=%cy|E{fezTE#lz(=`(I)ez^YVl@9uNKA&9dX1_`Y+IrsL%K z%F20=USz#x*j83tmrJZ9BW&G|QJ+nq&|+{XQpbJ?R`HQ_?4OUi8Le~II^A@E90RYP zZg$r?MTal+ttOKzS=>o{tzc4B+nuMyy8NIV0&n`#LfDDBQ6?PB4Z!fK4q*N{EBH$1 zw#hI{JmBuEgZgg!bYDv*!P>t#FWkrhFVY0WQ94AbP-wU&f_RGK zw-5S_Ys|qB!|)vV`s$U_SnD`T@Sq}}nVcD;oz3-zny(q!*<7F2jvmjspLTn&A%D=M zu7>t_%)}mWgXA?FdPB4{pfEDP$I)FtXK2a7Ver8p-k8zk$C%RuDI^*wq&1h?O=XOT zsa#}>u}{9)3nPEjAKx^-hTkH*DGvN`voUtQh3+t2ngA)8i3*hJr<^HCBMCDc)j_*P z2jE_#gJ$4f(1DQ4(7|i;7U&>=cSDD!j=?Cnwy6Dr$9{o%W-gG}+M8SW^D%V(`ss$> z>${u1EvvwqF3|7vIjb6^ z$~|s6W^f1u7ZUj@iy99#R45F(C$5b zvbOPTP1q9jBJYMs9!WRPK~!f7yrl{I0*>ih+2hCSM76ZSi0V~JrzKHh3tkwBoD{4N z*zqDa<4SPU``{@Cq}=9W>k!xaH(IQLxfqEPVyE3P$-x9!Ip zFs{{We{*Afv$wgf@+T3(P&O?=*i-iYhj(SuT!yU0JPuarS#lMm{R^jo^Z&K8#@OnE zH14cD$8sq3vPPX-FH+`#WvkS9*eQFsR`zgx>LTx~ zJ?lN&Z1*Y`xeLenXHT9!D=Dx>1=gPS*4ABrqolw(LumU@X8vrVy?I}J;_*7JTBccD zsZ*m;J6~z7Mx{=^(t3?bYk6iiVeL6(3vU7sd)B0a-@?022{+r=Y{SlpG(#QeNZ1-f zsskN>w%MdtSZHkvsTVsSIclNG1y*^|q=MLiwJlw)BRa4O(uMnE{fX$nnuM*ftvawK zpov23TewoN18brKg(|3=Af&-7?7;e#F1IE+@N5&t-$(g{IzPv>SO;iNoAgS| z6Rb+?z`E!_p~}TNZzEU`J0P|jDz`2=u-0+=YrXc9d)QFKvMOvS`K6G1v{rnyUa=Da zcJ)@&-gu5B)n>Dv-;CPpc7e70cC=={=iO{|>#j4yz;P>F)4+Kz?5yk7+jfD~Libm# z0?xBs_nl|<`=`456p8#U)Xfb%eU1~R8hDZ)g7d^Ku$do%vuPLD$PZz|e!p(cz#X^) z17Ekq4QH*;eOs7u3erqYoN@Bze+$Zp%H)i6u{7Q{NB^ zf={*;!)lwHqf%@#Sg(3c8N&}@no+6eEPLLue=JA=(t$ZO^vM+fQtr~V4k|Q= z33ADHvE;g0a;>HeIDZA&JWp#{5TrKpf*?QCf}q(jy(96C53x_5^w*yBp0(Q;DtpZ% zSv+65qe`#$2OF-pzSeoN4oABugXUzhjTItgVjI5^`%9Iiw#rF*wY+8tYkZ_H5!T?y zuce`mEv#{+5@D@SsXa}Ef*QEU*V^?(D5!!WROiVQ5vt?${U@DgQ$#5205Hk5HO(a3 zQ$#52Ku7dhExn=$wf1!V*@oNOxT65^xH&<5(9^eSKkC#L3}~drT5m`pp-D@A-4FxT ztBbG6;;XjQP*J=iim!Ea@lJK|wkV#L3Mz`PisE@0p}aWA3tT)WF;tX(DoRh;d~48^ zmL1yVO+67!Z5TW4lj`D|#VI4XqN4alamq-{s3^W(oH8vq6pODFr%a0x#p0dflxd-& zSiD`FvUYXpReQ>A%Zp|gILxiv*rtmj1M##Hk%3rEa{?S=YtL`sSoZhDd%4ruKH4k2 zzfpVt?6+IRn=R9b6`9wIm(Je7Y4`p14^7GOQt5#pn5% ztNl-Q?$2)jgUMcs@gLSYYhUbt9^muEq~=q||3b0ccsWhUSthozj2)(w4YKl*BuL|# zA(_S>`E+LjK6x7w_*iB-?#e*ZMl(hRRGG1M=oQ+?IAeDZwsKG0>&j8ZYE)erjI7*t zyH?|s+1)sUPD&oxY=Vy8aL+CAFt$=mG^m>*OyJLG5e*_@A;zRg*HzDf>SF-&oD(e_d;@Z?wPc{}1pfm{GzR2$sV3e)l^S@UHA# zNuFml?OjaNGc6Z%S*`w%suh;Z+3E52&aMXcBDqOcK49P&kT+lHBS*f!2Z!A^@crQ3 z{%$Gv{-YzT74_qTM%Z+I_S5bk{}|%?2_s;8XHKy?rmj3Hg?QLK-aXiUziWY8URD_R zYXD3NzIBsqFbJ;yD&S=Hycl5U>{r$IzmDVJ63(o;Uw_RHWm?;e&PkJ3XoIE=gSG~J z!E}X36uJI*elI(BUKZxy&F<^7ceFEO2w*WV{7L>pvLpCls|5Ikd+B3J)H1*)C$|PG z`ixPBN+7@8+uy${=%M@TO&4{oCGhvR|M`zDeEHjL8sRXn@&`92RvG?+!@Ev%e|YeA z?_JmUHKD_}$(Ho>==kvMU2#GC_b}+4_xlwsbocjOzuVcFS3_BN6@=b|w$2XzzIX8E z*3FIFaRS(OhknxcOPYDVeRKl&-rnCm0hV^hL5;Ey+AC{?BB9+CJjdBCLO(g(KE3k- zW4XEn`YELVyEFKUGQ6CAcW0!e!!ned?%x?98Et4Ku*LRwNAQ>K@H{Rj{qMK;-yPqD z=tUFP%xp?p_<{f4wSnL=scK>G`;)me!RNjRw)V>)%F3pi$g_L+dVlZxy~DG*(_RGA zZf~C+zTew9F$J5{b1`Od-;!%`HM!a|quK~M4WhtGY26a9jl&a$QSPvDcye;IhuUe~ z05HxEu=TG%#i`x<=i$L_u5T;&PuGv*D5jbvFVTCq90Q*ZkwZxFbB6hKpuPLl0&|2Rgrx^vt?D4F{hmLDq4{ASV-QjlKxj?G zloV}Xinu7FWQ|FNISD8}r)PYqpKFVPf)oF_=4_y!H^SIVx9?#_Bj^m8YpKS=8Xgy11cam?TXkRkDW_5 z4lu-~it6cj8HLrPoRkRWIaI=WPEob!IM1D<*oWPpqqQzc7_{=;kfS#`<0SLFXek=I z7DNM6zq!YX+r^);=ORIQiI`6^yj-x~@5$6bl7*ZquFHx%nyYY@;Cw zyxAx~G$^5o4o^>;O=rpJRFPDT7W8+434q#JC9s&D7gr=MbYCP15la*HjwBn8qd2wZ z`gHH$AKmwR2MY_0_8Cqi%$j`Gl*?4*b=9$KHC+V>r)~XzXWIv-XYZZfO$r-D171h8 zN2Eb`Lmfd#A}>x;WFi(~g04+k5<_0hWMN?) z-v|){0kSL-xSQ+U?u?b(;rPOxp_bYAWbDR$Y}4*OKfiod_OyHXtlOPoWM*^j$SY|K zO%RHY-*rolWbN7N%#}{Qn;q*ZNQpC78v1LU87h^md3cXU6X(%c_#uq0Ldq@7EJyMg z4&n@_tkfr=ZcA)qHx$bI(c*&hXlV&W*`?w65=9HnMvsh^b1G`pJJ?Lc&7kIh;-#^t zh$p=LE;J;wQ!MwG{pTozwdB5{6I$S=^8oP6S1%pc;=&U8&h8$FlJ6R>+N_{=@XPtu zF*|^p-~U+;^tkp958jmzO4vqW0b^kJQ;rJ1PaYCQ@t-~(0-7I;IU3a}4MqRbJh*XIe%c z7IHF4CS=xUoKa<<8BD~k-uL{z3tfbzPFF;*?i#c(8z6N+T?9r+1~1r$KX$M5>2BGQ zwN0=K3!Cb>`Eb4XR8Fw&(!}{y!!Mm+TiVra$Z1Zj)tBHb+Z~~~&ONe^P%yPH(xdHn zyPOMylWuYv2J9)Tx-bb;(vO3&kja>zVl9HH@1TrFf}2U|gcqfY9bXu$Tmdl3QtuFr z1w2wp1f&F&heO7;r#DD+ZV+;M3VxqkGuYbpn>WXu*V`w%Ha8P1&kW;$3kesG>|?P* zYGSc{BIW7xO5pNSYR;){&u%^6c<954G!bShFokN&DJH4t`NQZcH-0*`yvMM=1`=9& zFKbT6Mx*h64PCxTo)$&CM1w)%`{;pw410eLj5q_5_Y1<-L{^RP43hu9r$X ztR~Vpmrk8Y-8D6x`EHr>n(ssvHmPrKNaF*9u9RF$Lq9xEFF@9a-5a?sQJOk^v(x>4 z_xNP*@SxjuUO4S%&z`Ne_YFbl{Rx#|fyU6H;<1SosD^;VXDO%4e@RXem&y{H8NR1W z-?q^tfHhzWI@edwH4vjrl6u3Sr^`rDOnni3`hNR(kAewpA0Kc3qk-xh&eYf9d?QT< z(2P$xlO;wTrPgH(7JSu&y`aIRn}v(==?xHceDb3J@>4CfGwe0N$e*STmKQM7%4Hx0 zPb(u8{gX>2;~<25hX9Gt`mV@Q2X5Ym??L z-02H)6FG;nr~&iJg9hZgq7 zS&}c!&~?T@H-l}0tZ?KZ>{{rL(hD>n;5%@xRo)7}>mnd^sk_as&~XlrR#LZ>`0-`X z=fsZ`FEsXl$$ZMU=Nv?MBpkx=g`7MS!xTYunvP=xFZ9un08VSe8-l&1i}+3-`i*HA zhVHKc=Z(a~kC?pD=)dyNc&zZvU5EY9>mN1${zBDTHuO+s zhUOw!O1d2rhIrg)T1^X=7oy_@qq!dsj!=1d^3#^+3SFglxcIGl({RbeBnvT3?wC`g z_9F_F;Uv+>XIqi63wvl2)lx=-HM(wh@5ckX^Z2|&MX2*|sCPM)Yd;lPFdgm?5eel{ zO(Z!`WgyZqW<(XRl;*P37>3oQ0}ux&Nqviu#PB)s=oob$6m#S;Q}l2sk_Y=tHIxig z^*J|g8NGoAZV&B87_xTwaaYj`2X#8iZ^~U2PGu!I(A-twOH}aDc?|15O@ObAyVcc= zLVc@Hn2kzvuIci~|EyLED>}u*!(C%IdXV&mQa#wNKFZ1iu%j!WU5&2BKQW#NuyD`v zBrC5jEA}s}4`D{hz|F#i+E8O*`aANp>X zjSXPAqMl!;a3DrgS)kVYMBuTySXi^1RjhY|rd$(BD+?l6P{4gYSfs9B0+YeF-7!K> zrxfa61Vc}j`ErW-a*DdQQ`7^gdA=M^W<8$V?sOtg3=eTSF>)S!;ZOhJ{HccC_+eRC zJ?8WY^PR
  • sX}M4A&Xz!wy)r5p=CwV!6)#--O)&^WrR|*ZPQ^i|@{Gd)F{sZVO~VQGq8kk0(2}6JPq8b-_zo5@Irt#8 zp)k`%FrMFcZ@S|5q>jxa<4kF~W4YQEOV&BX0<-dbz*Fay^W7PlW&E(SzkPCo;ks0$ zdZ94E@L3pyOD$#0N;y354uCx4210r9qK!#Gw#-7wMKDM)l-9MQ?8>^-M)%$E;n@+I zs0kw)=P%Bx^Blk~iEzS=$>ptf&f_u07p2x-l(s~H0#*4Tx@r$O-s#k?kr{*Y7tt;G zkh)r0amnF$n*#0B5FF$${W!t6FA1g+{pVg|iS7AiSFG`jB99}-B4V-N*Ck-cB3)KT z`wNLU?JQ8zJ+@!B(Lv6Vgv4*LX)_TE6Y9e66B0yE-ZF^d{;ko$Q-!av`=v9b$}$?L z?AP^&r5oysP&1~#lKZG%_*jhx7G}o?)Df;SjHUm zQzLGksdmNh7(Y*hA*6yY3p!kaXXe%BG(b~3Rz`B7B_f^|j={xLKB3&8MB5V@%Zn?s zGt*VcMg^2fOteT8_r$7fPKC@#t^$)cS=>x4DXw4x$s}PSF1|I&llhs3b3)Sq@w^6aOVpG zmIVB71-f*VzE9zb6(NkHs}4Qv{>t=b=f;vXUa01H>y*!zZ@>aA$%X%+i#GPPHVr!keh5L9Az z))IcInHQ$-qc-#b#@y2Qve4^FYUIBXfGv2zYrA+&iO{Z8Lo^giYR&?p;w>LvMigDS zt(Yh`?J1IC#Xg-|Sp^a;Rg+YTjkzmt`dJhmOICoQ$_e7*l&+SnBH6B5Hw8hmI8xP< zsZCdDfNmkotmfpHSGN#j6phuEVpNSe2r&&)@FW372^7{Vtk^6OZpvipylgY0P+clM z%pgD9lWZ;4=v{>CN^&gU=b_jRC>1>B&@V+>20PBiWV)N|5sa*bwGALN#0q zH`5i&2GwM|lPK(I*C9M@VfuhZ&YcM8SwrIeP-OQxjc?ehj=h`=!zG=?SWd$U4ncFT zuMMEE>RIoItOM{1i~F&2j3!bHkqo0OCokne6%9;WVMVS@UZ!3Y0y3_A3V)QkAJDn) z(hXDe=woX;XMpxuG;uf%4DQ4A^W$kbJ{%KniezRHg*>Nw9f5o)_lab@xAT1UNXudlGRRMkGIM-_3SuyDD%L~_ z6(vkUrE5TD2Nt>hov7>^Saki10B8jRU6Jdk5$ZdM*zgYtIyXInzGVym8Ac1B>404# zmu+^PxEexdg;7J`cKgRRxl?rQ7p&Kg1x&T6SYx~2Y2fUY`&63%NBWH zJMIv}gr&YBXGz)-ET|=V5NDyXoef5@JBAL?mvcwo3x#KYf^+BwenGTvJX$B#IsX>wo z4U!;>M96!z4Hji0RrNv=AD^yVVhHG5j&(Jc)7oFTN8vv;0 zYghnO-u@dTBB9ckt-@o|fEQ*^&3t0F-`pxNUaO)c1-Z(sw-;`L?6&7qfXo1+Rl=(B zoEtyQEx#%w@0KGk(0C4A%1A+X zFqK(AYooiyD!G+iNf`^%>L}dK3V{+0{>6APF;dQ#XyQc`vUKc{rTtii&G_1Yq;|nv zUr}$tOE6kkHc#T<^ zKpcTr#8j~tzf34p*@I$RiC0Pm#k_1hM+LZ(#*s|!{9!Xe5gfx%HswR10B63( zBaUL>c?=f^E{aaPuI2$P-bK2Y#HwbIE>NiF?${?H(rkUj7e(^fR|a52i%i9%@m^7o zl|-VPda(40tx3&z4EY0>ncY-rH5f635EBo<`^q}MiMB;i z!=u{uyoOwtc)DtdqqMyO;ElZkrk=|t(8tUYU&$03<`pK*b+Nmou&6|;KE=@i4#tba zC5={#FlJ)C)}TI~%vnY2SsAelh4o8^?q;*Ho2`$yEXF0vBHu@QB8`5OMfxO?weN_9 z-QUBkqmmUQmew5B(ZifE>~&_7`Lx+-$tA_0W*F5ik1`WFYjTB(UNaXc=MW}ysAaWv zo5K_?Wp**`hLK0Jf`7$rmWbWDid#KO26S`J78?QHo5*XS5ab_z_Od)&(ty1y~0I zYQ(c3|336XyfF9_S0oV=(2RY(i@xchC(J}_VZv^U0~Ag^1>IP+{Et2Wwye~T#y7m5 zENWX7X_e`GD?4Bs&e&I3>b$UutdW0B^p+F672P&W0kwwove!R1+?%S;D;u_Y>x&AtyBGQ-#dSRzjVk4~+K_C?Si7HP?`iQOwo@LjudmSWX z6)tRwt^9I9=z2b|-_+mIHjklyMHz2|X$KmL`KgD7{7VmTcVQ+F%?>s&M@~=zz zIZG?aY^ul*j;`HNbm^OdSZ?yfK1_(HZT~SL=)Me+HvfQZ)Gr5cWgI;*_}lh=_d7(( zA3o<#F)c6U*jm1rV;kswu(`SZ2lhi)&Hr6n-+1ze_F8*mV`Kfv#%3FyuXWn%PyVnv z$39ekGC0k~&}1A%>7;Vg-scA=5xdlW(trBw$+IV${ms?hs=w*457yna&5iz(jrMBi zNoRfCgVXX8cSX?sN`dU>J7D`$FwRl5{bqM({{&P1>*cikop8y zgFQGFk7Y_Ek%vf?cXXLtZX{~JgddIlFb7_xHf=64c;r#rp%>$3#c9_Q%DZedi(t^~ zjT)vO{v|$v^TPRs1OxJnelHU@`?yh{wfgopH48FrHpXZT9&27z4L5Ca-YSO0j9Ru` z9HxonEQkbxb-qsde$4jqYFaI#Jv`~29lYJ%JMO}-zvUoD$qKBd4+Y3Jhw4a;L^0~IY$8*)rcirvRCx`oIr@P(Lz4yC?lwSRd z75t}bK(g|T{?YCOC60$;#`9q?3erR@Rh}Z1&e}+sr5{pCOnzHvBlX<)oM9QyP|ztn z6f#u`ADSpf)iEGkDEGW+}DjpgA<4~Nzht)C5kl5 z28H>f^PRwFiT$uo3Fi%-XXIg1g?QL9BfXT)oP;~JWsKoy`}8|RvjdyMx;Yc4Z+4H5 zyKndQcMlGoj`6139b{p@+l4Q0&klA@DMMT-ah-IJcTY}__jXRLWN_%l83!Qzb-VJb zrYN+)2-vgs%8fDYV&b?xR4;hwDoN^H2dFIdM-KLb>~JLm7{~7U$dwfv9s_#rtUr6U z*(9?Yi7m`%_jGl25p~N!A2GBTB)y>>*GIK)+_sz$$tZ&@Z}(!TN{(*zfOCzQ(O5%5vqE3Ekx=|$i;+QNze62U1ib)mtDKsHJOeCmdNlqJI( zVdGui{1DJ5D{FW zQV9o3sg}{lJh^}&c%n?;Cvb4S8TFzepb|zK41@FrcDVk93yf`)Am~eSGQIEtiCi>6 zSQPl6=i?dQzh=MEmeb4F%U-Zsqmj@ZVbOjx96~$jQ{hsAQdD2jyb!Xf`6`U%P#}@? z-Hmp0gn?o5AeJuCY18G<3od&5z!sFzK7!u^4^vAAAzrnQQ6;fx`gYy~{*axKK}NlR zXO}^YjK^>=o*eBR{C#`xRHR;I!UT4|({2kFA2HZVHyGkZwOlqnD7}4p_OQ9naX4|xT#Z!C9mt%)WTh(+|8FP~*-pZZqPwQe`L0eQsr zye_oV9r$Vgf;3WV9n`3`9QnN|dm?HiS#Np`dxV}3w2}G@5xRM&TN8Hm-32{$L5!cHC;O`+E;eTxqcI;=v zMh*exL`^t&eYUrMx_8js+5cxrQ&G~*!hsvaT|85EXJ39*%gH*xmaX>_V1jd9?Z{Pe z5H;63a?d=+6Izi*r^)+VB_lgK!TCJdfnOzqOZ5O{z65TNz;Q4;4?NMqIxzQ6PIgbv-tE1~k991wGEwHBLN6-;9-V%7yu1AdcsY=D z+0^!ll3?(7ns4kN0d(IV9vq$?9_--)*gihlZ8^)!%eBW)4rA7)@6={VW2CWgdr36R zQlC^8joQ#lbRs2eKpbVnM9EB2P1apuRFLh|frp%hj_-XF)Ebb;#anet&GQHUrpx*K zka>uEKM;_Z@j-JW)_EDK1RQP|;ixoc6sf|HYhxtxgryo0VrDE89ST8^BwL@dTcHUz z^=rzY@Gjj)A=pi1AR|C2igqHNgs30{CkZU^%8@Pp2$J2|-`zeqJ1Rl7lH!z4J71MU z+aJQ{Dik7n;tqU=ZqVVyct}fUL=x`yD8fr8lG)Ih^E{z*zzB|GB8WX-}7SR1GM+9SF_YItOSYRw5z?eP^*Fc#g88+{H zBz$)zHBrCG$y=n$>-KvMG=mETK@llZ7%aix)yYAQWszeOxpWyRsOi!-2PKJ-aAeO+ z#dO1J`hkYn-FeOFI-_vOAC2K`AqJTalkO<;{2@q(fa@McT7#bH7&WEX+dlzT@Mf2o zy&*d9bWc!!1;-*k#1)@#8b~s*NCJ?W`~dC&@MwNp33EM1vJ+fv=9G$ zzk7^Tm49!{TJW)IL@J>#*Pft|SbdF0>Ji+uu_#B+{GFVj6hIMp& z_*RcX(-U5|!bvYYF9n3P2+kQ$E^x3c#ETnRaa<9(QU3$xrzK%eU1(H7>+bKpK0EmP z-oYCaE{cwK1PTJKg2<96YH|}q*IghBuog+2?!Z>1>CF|~XGugB>{HATY+&f#2>!$Y z?)Un5+>J4>HEz6VI=?o~Bj{eExiZQ^`oelXK=t4G zMjau2&*RzVxM?fmCrZCaiDTFjc?VOH7)TmH;*;xbiUb5Q%QCSGjSDYAubuR<8tIB0 zUzL=3bb^PH{oRwDQtTsBk%J&5E`D_|Z?Gr0O3Q(cvhe7J#8fyi9Nt1jBr0J9xftXm zfF(7?sE_8omrlsgoQomy073gfm|eqgSyTtX1H@BUL%(DslKlVdz3E~bH?lDL{Ux8G zBWGgLp(O5_BK`RhitQn(XtPY!t2$m6Id{2X3xw`EuQ<|pU@Yz zf~se8MY*{I#T1jQdC6ZrDh|f1|FP85unl z_}J#5cQqOKT|dGHa8ZQ&0^%kFKOOnFnS&gdQgUO!E=*$?&j$vCek|^Rsk^~ABu!K5 zB%_|+&6+2qgS3xc9_)X=e{`k@%Jwa%aCGAQSj52%9bEN^OD@O2?=HLGd;=}49?8QK zRQdxtQ@DV&4A@*Uce>qw9UUrGeB*&?)ng&s^PoHEz!UfP7oLzpo&$&=bQB)G(BpJ< zaR&`x?eE{TaWk=V^yUq0y58))IXeF5Qx-pu!@ThLyT}LcLU!?5()#=1THJ0s){s0C z1vI~%9+(I`)lMi3Fl5<_26n>OAlVe;>>ej=Yq03Oheal3rki08+WmgtYSIGrZ4`3s zZ+3f15k+ZfReM%Cjl>~CFzufXkAA@i4WtU-;vJk%nNLY_O3Y?jrwE%iLa}koWEz~a zc19AOG3xn!n4U0Oqf2ycD3)F*#g{3FftVpA4r^;`B}Vzx@!lS~+7$bQ0A{`DW++VS z+*Aqr+`Ui_7_R3rn0(X#Q@ZO0XfiQY*-MbP=(2Q5>$(-Q-J`?P!@a#-g`GbfdOaW3 zVwMyLRv@b(w+Q3ojeG8-!V<*as7HRhl~HC9Nci{`wdBA%7j_ob6{7K!E_vCHHP6V$ zvKVO`fH_4ZB~qt^HU`y|YhD*J3@hKg)!@;AM1pVV5erM;wF_|#Fk>E_wCUS6Ug1Rl zw40mpRiJm?H3-q(96OJ3r3hi*!8NY4pk1M-IMOq$HDB((ZtorL?r-y%0xI|8K3?^q z$Izs+b(Anb*z3yNP{w%Ie=2opd&-~s%`Dfo9!Rl z*Mlf2ND=3x9?q6=H*Z`6_Lhkc)yzU8R3gxUnG(bU$g+W2$- z^DqbEb6Y2cp5jI6`?{2?6-G}7Ew>Eu5Dm?Oj1BrFMohl@Ip*Qx( zUo5ErxVuWM!}4MZ))H_tMEl;4Z;y_7DEOV`nxH-Y61#Y`yaid-f}NwoSHR}luh}=u z#C@83y=Rub|1~7->^2_&%)9?zEjLp4ziPGe!~L&&xc`;doFrVBf%FYVA{}*62fb;H}a`^~q|Kyfc*vO78Tf9%b)i zd&kyGObvLWNylp=XRXU!EQy1(#Gy^k19<{ONgU1(F5PQkobRG9oO95A$Uq01ZLxJw zhX5u8gh(+YYPpk3cpt&m##4w3Fm&rva5SB8Gab3(3d?|8qW<;9Bik*6DG9HSA&{m502| zHHr(g1O1%WQW~Ztz`Q;fP!~fcRYg5^u#Z!^!kqEcN~SN&Cx+T-r|^hfHfEgy#weT} zo}KLN+Q%)$MO-*;fRl6oIxGC9Jt6q580YMZoOIUzoE3h>EOq_rS1oU!^Rz6L_YZCj z|9$j`p1Q|5)EfT#=#i1ZZQ;MkD%=wO%T?6Vi8emnIyyW#jk}*yY2xw3dzW5L;E$sA zp44z+Su9|wqCO}y_6Ltr4-RC-yIkNNJ~KbJsHUtBJ9e#cj{euEm6P}XtBn?xzYhZ3 zmHzh!`P}|<>ExQ*KKK0yT} zx)BDtIvMvP3ih?@z4r!a4|^OsZ|Fj6$>~l5w6xJdL@@4x?u{=$yeqFu7n$O>2BCIm zOkP;P_pRs#22C3nNlfbN8YFT<$d5~8qAqj28zRJzQO2E`$-OHMx$7)oD^Lx6Fq!QeT6`gTwIhL zlAQ_rc!GZ?-I2YB(WOwMlP0ZRo`t#idJx|ennH$!K3%TE z!zsFui8n{kE>3q4=*0z_UuYVlnR*#{mlNh%zE2!E!{`cK0~x6lO6nWPDFGQfD<$WL z{Zm-rVJo(M_>cC{oBdPt-D6atcOa)HyN7Ld1&*~-g&28DG40v4&<1Q;jnFghM63DS zYV;rWbF2G5?DX*}%z?%{KP`#x#t%VCC@hGrPn2VAiK%u|(A>71B~RrROA`(=CLI^P&9d z?HLa}mZiZ5^Ndz2BqK*Gfu3;7VMB8Ec`)iJYE&Gx?yKi@vSbQLtZQ20jXooLI@zbt zne{UGMI+ZtluAaO#VKep{?F^q?%u2IvxC!5>566Km%(_xa`0IUL3UB}!Q#D4{=k0^ zPcdd}`KsJ3!?yF}_y?2=>T*#IbtG*kw{r~S+1Wn8$ExU>N6VLtgt!VaB{dSgeNUAIT>&?6_ZYZp#@#GN>N z)#{^FLro)!mzLK16Z?ZF^hE|k=w86i>Kj@9pYTtHKYKr~bO{?>B$2Q(Q4?Y#h zXuKStPd7W0sN2c5EZvC8WLJ=r%Gx#DGh@-!7owQ-`W{d7;*u*+ZoB*I)DJvdi2%{P zE7+gm%iw}+3DMV$i@81AvF!vtJy1IF$U0~%z2pLwSfQ9~Nw9O5D{Z|xumhkzm`jVj zDPCnh#zUe8(l!j1;4*1eSYpl?%%Ih~3l=!`(9XUD3mjlz?7L?H(d|!Rg0n0T7BXil3gVw_2kqy67bz1N5dnbPE5su~B!fJvRu&D+h6h z?eGLW1L01aJQm~CxXMPo-fY$DB9!Axu?RBBPMhmJo3dO}k6O5%K)L*pdXhvDR0~pvPg@3YMIGzSw ztObY1Oa(8}5Pab)HQi@5N}`1C z17<928~_<3mjWycDX5$vwOBGxB-@iH>X}?EL0o+|{3V$(9u&zx!sj;i zKYE?J^!abOT5cxae^s0HX6-@$yNA!i{ogM@uX;i=r;T1^_+bUUP)~#wqgCe#duNb~ zE>~lBG#M3TQPD&c_(NM!wj~%r%m(;wHyOUuiVH1K44>0xAfuJ-dCQbW9}3=LMe$Yr zVkJl8Ib}66f4PSS_r20SEVX~0&#kWi7dLtTQ*G3$srA3nYCYJ0?&Bk#SvYT9pFF62 z=ed)FKU?{Ldgb)OGs-=MdWYU6I{l#c1t;F1zq;M~&y*udVv!ZBco9XD@ag*c<>lpC zH(cuur)yqsy8eGdSYo1a0D2=nw17VZdi4V8g=>>uU$pxJdfIcRSN_0vgKHX~ z#R%^Kyx+3s(6tY1&VbJ9Ro+KJ>v)X8m^08?4y!iK%mny-A(F_&8wE;XrxuCs3*0DH zECVb#Zo7{$MeqxhD2nC^^lSBn$X$yo7i|E;On~@DR7AIlA4F#|fIH7s@h#Cm^gMv~ z?91A-VL1Rd>A@l@p>7}mc`!+{_PGYqIt#44o;V{Ia_i%%X3U%h7|`3w&qu6RJ7QVsLY+r zWv$L!?pTm3N=`rwub6|PKJ%DAPxXz;@+1Qg@mQl>(NQ%kX3bQ^vbCa>O(=A|~khEB{EBe)f7ZbeML?3F>AdfJp?BPljadN0p0uj7~~{=)#A1-w(nF zbut(xj}~ z>XQ$kYtZ*b?;r~Smz?Pc7Is1af8+50&}xLQ8qrm$a2q7*3&hqO=Lhmb6h!R28H>@_ zkt#_Kw?<<}I!#6QxB`HWt{`D3N=`U-EQBZKbja^1eAhJc zu4}{=k}MY9lI?MTw2?*s8H)_AkYy4sRr@L_3Fga3tXGn&$zl>!6<6i6<+{E6O!$>0 z;akdQ$Ye&wkjG+{U_~)o#k`CKl>l1iM5LD{VubWCR&6n`lKu-%RjQD|COnH5WDijq zWujcbBm^6`Pzyp#%p4tANB}liN929X8Q9L_Bn{J8p^0nH;j1~d=WyAlv^}@>d=AtA zn7p_c5+t+Cfi`r!BpWN3Nj|0wV_5fb+?7jP7LOyU(z2%}Emx$hR9yPw3j ziWgPtTwcs>%FgzhYO2&%O=q5~vAWYMyRvHTOOxv4+$n`1R$mB%46jvUW?A)_+2Yp# zmQp@uZxXZX%@p@Z9bvA5udDZ{+;cnHoSG?q^ocY_DM@rnYggOYn8Z)>%50{ftuE-j zH{8YymfG*f%pvP#ZfwoHkFgb8beINedv8!I5SYv(m2?CFy8Z<)&^Q_wifm7G&b<*j z-$uu$GH4qWKm>&E)9yHcWpqNZc*f~KsS*OkBXLJs_+Z$_Ox(a7Vz2@KH;nFp;SvA_ zU;(uX(rIx#kc^w##!xiELPz1Xk>rXd(y;n-o(s~w^PDXF=+Vf#ET{@N@1-~r#p&=g zE>5B1W>%1f+PDsF87+0PP=bA(v}>sfvMVH+c*xKbofS}05jN1+;vh|C?p!RbyW(t@2F^mCfBbBElRfhgk>-BzqWixXrS zH31WbgP}J>*@|Q;m!v23*qmRKobfTIw->XO1qmB}%|j#bqG>R~g+~cQLubiAL5{2q z4IA&wLxb18GzzMy#4};0oxM)StMpc8Z)0GVZe|Vl{8_fp;D0mG?C2&Rj7&zDZFD-t z$gAut4T(pQknj+YO)DbMCF6_}hpb5h_cwoVEdUS=Wx)(#P+W5;Vu3PUp~%j~Rx5^& z#;ORzba7pd9+Hw1MHzP|He4xm(XBI@h=hwc1{g>9qgK`#j_f~Ocy&k7D&K!4lN%7h zbI$<|_Kl*?kT*DTCd5EQ<)gx2eC~{jYdT5OhNc<3X=FOe!kDoH&(Z!%R>e_MT~yJS zg<~yP=#Vp&skk1fnhJ1A<(y->5QT&yzF>nTqf)GSB{!<`bkCITJeRJfM#zdpFK|!dPiqkX60fQ@t*2-UXuJk#h1N)Oh{o6=W)oOYx8QFv zrpyw8c)s2-b2ugg7b6Wm;g|L?JH};`qV@+2EQe8x2Oj#F!Cz_=nHPyqwYUvNw!uOAJ^vq$`Np zy#2~PN=FhD_0700UuW&w7(P`c24o7#M&mtNqMH~H2txw9aR^(%pcMNOxk0Sl{^%X; zZ3v|7+CoF5ZA}?83qp*S>Y=C7t7x4rB2b`Rbv#2yNyy=A%!q=J0)*9GL3H9SXISxP zhggjSPytG(5@_7K9(m>RdS*5a zDTQvgr0YX1(k(?PX;vEs7_aMIw3RX9b=?Ypuw<4n~Z$Wyr_7_G*k_Iz64D!p?@NEaoYJK~{DeBBy7Sr{}%t zN|cO!>LsZ|Io`;YgEVai(+mo6b{dcAC?@!vTsy9*A~q--6g^#gk`lN^hADAX7P{C= zrcqo9m3ReoP(M&o(tB;0;zgi4M6SkaZm*Y}sbQVY45uBL>7U~fyZRM&Z3~K?p6H&) ziNt@koTeMz%^O#T`^hpjL^bi4Mi|c$-XQ zV}?v+C%t9!XT|(kHGkIB&vC!)cDvJIDnVaHwNyjMQ#$-J#2~nOTGWlMwBE*P7!E6) zs-;DNm_aEKq;uAsjJZD9|3IP`)GWSZCe4fZr`ywEeH z*xxE6#u_F8{63F42@_JFLLITtV4AcV6|wr!_feQ=y^CmC#8gqgHK{N7HWZniYkaL> zDiAVHB}U^RsBzlI)04dbs7Ubu;Cir=bYG!RtBFV{oABfg={kiOydfsgi1kkWNjOHFaov zPO=W`)?)b58F;n{EW|T3oh?R9&q$RtS8jVw5-X*vrV>*nN9l5-R4Gk=wIpk$sSQvA zKnZ{^UC$rbbNVpN0-1$#21XC&1MuhgIwKi~)tv^Rm&~S1iOEE|IFn2?M@gmPe6yrw zwMTjMa50%PaMNw#Bu*t=rHhG4n@Gm9m{gj*sL4uk>U%Fh5v@D|3>)*V1R4Euf9>JO&;m;Z==L6xJwpuXYX-==xgmO+CVjq|i#;Xm#E z(7$R!(f9|Hryp3lLtgD=cAO&11Y=l@WH$YfIakekyHec{KQ@}?53GWwK4EaB@4eJ_ z+P-kYP|KX?-woNFNaC$ZB2gs;8koekh|Q3k@4ND^_UE@~2;U6*q%ZInf3d)*{oWm% z2e>^V%W#!6miQ|<%$-I?lF*K{6xhP&ebHy}H^}_bwbIen&wxo&u?4 zLKOC>6Xj$=MNbf17^}-#o?uKZWtd6<&X*qqV9kL)#5bVOjt1(R@*6F`3h&eL z&&hS(@U@&boi^u_{-0F>GKlWXJD+q`EE7gy{LNFT`d8tdI&XM1&W|Ti+wF`&u;aJb zTF*aWhTvZke)5ahZL6QOQ^j8}g-C;!KVoCKOraS@`x%f%MxvN@A}NvwKUb6@#)Q6O zY4t_e&9t8wFh2g>_eY`~`gz`)!bV8_gb{xi|Kdx)@oy2`ghBE^P#&4U@LOyEqo2Y` zMc>(oKs-vnxj+2%f-#ulU~tmaFs3MK+4)H|_$?p1zRc?mKJ%sr&wT86Gp}5k=5-gF zdCiGuK6XW#*F2c!Cm`di;cxP5$tf{dtVsAgID(&UUVqze(KXW*<4FUEtOk*T!0q(W!va{S9!PV}UBEMp6@ z_LD@6_8T4js1!pO#T8RBYGz(&&r2qw!6pTV5~*bPTNTwv{0lRP>|{eU5U`y`lZx62 zfjATewy>wRF&Mj^CFw^tHs%CA3QH!i&0SB8f@`g$u)FaO>=b{)eVh1(xC7_iTB<>rc68UyzBS}{nc92lXT2vx``XK+D`mrJq6D$Ov8$i!WwG`522g9F4NP308CGrK z;=P;v28(|DRaH~;)3$cFoNGID%Pm;wGKh0WN>Ex8m%m_0wQn2({JR>X2%+eqZnDgL2 z`ImV{#%m;Trp8rI3@wS1WKl1)q=d;7Rc#c{no)a%2h1E`60n=%-cQ=?>x43 z#_9O*9E9nxEdqI3az7AiF{LSCm1F z@BiM#{qO0>d;v1|{YSZ(y8qq4^7nB8`Y(R}d)fCN9MfLC^musa@dy9h?))E}+aiF> ze*aOav}&zX{I6R5;r+*beC{v)7rV+jA&*!10^f53^m20^Okptt)!Lba-n2Jfg$2Gh z9?GzB3`IV9%3;^k6bl63`%7m!q2rCh zDLPTWD6`n47x?2|4BACU5l1~0bhZ$90~gH%=#32nhM<#z>i>?4?6#obu2zcVxLQWU z^w9fteT|`Y7lkIu53zjpe`{5N&xz+b{}@f9i^rjJx_z|s)~N+OXRo(ba=srA)|_g| znVhdVl@>WS=y~XvG8CS}LXUeLxG+6K83*MQy$*pO_q&AX(7A9!qE*ii-QIg3k$dj3 z<2pkcWIq8$xuV=10O; zS{xJQ`0#amyIrg1&h`&aYt{Bnp;9T=8kI)1UMZTzcFbbcTD?_FL2A~iNV5(_nzd%V zQfpSLwT*hSwOK2d8!3=_`AWTBZEiNo_43BX#wJR;))y6?_)7|r_eI586XAAL-rnK% z%Y!{pAkR03+fmMP98{2*4yEf6a4iQ#g*ZO4D9v{tQe}Ue7G%geQ80mpe(DwD@PtaE zBmK}nAE7f8QH;a=QG`Ff2KeRybyVW_3m; z*m{!H4e`_3vWnD;UKqsww_sH>yQA_pQb|RT%!Tg7O z?69gpX&^h5pe(}Ls>C56|1m$SQKXaY$TP{wGaGHzwMOB`tVWs6mJ?NTLSTDH?t|Cu zll_0~9ldJ5Jw2wCcA36E@&+~-wPm`((VidCX5Tp*#Ubv+^=j&oWRcm#jz@Ue>1{A% zUWA?Jsp3T?rdb81n)G}kD>^SQKu|jQ84v*pBm&hiTnuNQGnuW7MRMpj7+rsG5{56~ zhZJNq;|c{S$7C^bplxAhhCv0D!`{@(l7gI+R9N<}h_$4*m;9V|p2P_8S12dRl=~?C z&Iu4NUO0tX)mhcWU&1Mzh*8$8ucA7{7(3$$s=4G7xa*EaW6+g6C*}}P>R_`4Lp1PE zOLHcmNP^Ngdd~V*LCJ0s}HJfz$L&5w}^NOWXzI#m!eCB#V^r zHzmry&I%Yy=^3yJhzLWd6s=Q97ibGiq~8{rL)i7%Q|Px(oITLO(vzn!lQ=gWdsAlK zP;p2~OXxw0Su{qFTu>oQVaD(9SvT!e@H`#2rm)8uoO974&E=e8l;@7Pwp=i-If=qZ zii!aIS~#1Aos$fvG;)3_udQ)Wa#$-PFgZ~Gn{94NFp+SanOaQ*N#c2+)R>rCu8Mn*sX%L{H#30-b@pVzSyc%nYz(4Z;53(8`HzRdSLC*EPU#gXM5I*_G$Sf`vi;`m!CqXlvY9Hfe|Wlg{QdTUDzsXW z_W1AI03G2DVJ8Beg1OLr=cp^f4xNmzITYXm-(X?KK~|d=i3>%t7$Y=4jKf#7+nuRD zhQY?eXK}hIYNV$VpRGV3kAxJdILa&?eF0Q zIlzaV(mOyD#eH}^>Pnx}fK2E`E4@@T&`KC+9yzyEb88H99%n{Xq}rwGxsy0 zeY(4Sx-Ai}HkuOcSfoh95A68|SP^UqK1a>{dJ0u6Meg;Fb6X0Iw%JNc=F`E*VXjt98>`HaVu|KVA==!FJJHMpn%v#W@1e6coVN|GWQJp&Uv@% zw9iJDc=_t3CbQAm2a4A?8Wo|ESPY>zYJJW<>FC;iZs=YCcHH$THy{^Esqm&3awgAP zHOEXkD9e0z8jL&t3AfCo&UNG&Y4n65S&$M%Q6!%1k_StR>JTdoWUEkjI{}|kI$K<* zwB(>Hx7WcJ33zIKjV!tg#V~3MsG1Jn40XiqV&TVM5>W$y%7#V1Vd|aooF);%5{!P+ zxh=GWG6IJm<43Wiqf=R=uRa}t-{>1}pD2*S&UnJ7y_mlG0^qhkO#NP|=ZD?EAAT&U zr5n~buj|seE1@xTF@=Btsga|U0P9QIs{WE5fD; zWbx9Q)=AP8b2<1E3P9WP>V3~SAB;P&)??Uc?<&%2u}TP(!7mzN&h~eiynq`n<5L(! zSZ~y+e?+UJ!*|FTD+~vXDhm%ziNIj!jPBUGVj!rnU^_r)_%~!dE8zGU7*YGuy%tgA z!fE$Hgax9?6rUP{c8r15gQt|W>THv8;VFK=0z5`8AxH=H7=i&~WdoKXgty{V0^{&f zvnF8TLp4bi}>i& zL7rKIhCiTSDUl%g==6f|1PNH?04S(l3P|d;79kDR4neZW3nU_b#uR@5B9QdlqO(EFl-wmB*?2y_;sartR5RUxwy4+;T` zwanLZg(6N#dho#=;#}u_CFUEZ5DU+D) zy69wsv_-7pz&hc`oxpIRZyniJkWjD_AtM(jCYej zb$n1|P+i}j(o3v_lt0&{{9DE$bVjg0>x@edEuhok5Yvbue=2x*Rqml^A&p_J+4FTlB?Py$t-M7`m(0JLhPjOI`oHFa$V)MseD7v_q_>MSg2^d+&T?eMHp`|0>0moALOf=OK^ zTLBIjn*dlcD7BZ2IYeTNVWk9}2v{POVpKF}8jI0H4@SI>jL}Rn&R){#uD;OM6-WWB_jG9i)8QSXt|m}OQq#PM@&vE7B=5@qfK)hqCK(o@$&d6dN6Xi!Wv%DH#4cBNON*gOZw zgo^1k1A2Nr(u~wIRwgO>jAIJ9HhoEVc+p>5Fyb)J(z(r+HMK2=#+sS-C!q?8tC?wk zK3kv<;=1eM?hL0nOoiyu0|~?VJWC}M3mEBEWDCIz&I?9jQLk+x%ahsR4^;imKfhSL zq$wfJ2?XzFgCO5Q8i-f;+gMmo_OARW8w45CGz^YC+f-};o6!?7Q)cW9|k zOA$tIGLHDQ60|!U_fS9Y=@t`Ot`Hu1<_Qf9gk(twr+l8ozGiNb{+bH(M%;M>{Ss8`(5z*tQ*{I;8*jY|ABh(ZuMPbN{IZu?AW!$fl_)uPdc#l4e(uK_O` zmT)DmE4Q7fBoM=e&}cMXF^%$2Mr$DVghd~DkxA1~tCSQj|Gjq_U+_0QCKV6~{r>+r z_KYR-n5MCA;5<>^VL)UlxsLl7?UxED7h1587jRJJ@Sq$8)n2V|$qskH{b0mxVs4rR6)7rnPAY;Y%NjQhC_qH&b* zMEgYy6~muZl)YP7CYn63q%Q_8G_h^04)dDlgHhaY#ARZYTx%!2z)y6INxIWAQ2-{)rAj6Hrw@--&cH6^8Z4`;DW_jXQ)XYXYDpJk*~GHD9m^&h z6)_+vUv$}xD7SPo2Sk%z=!u_U4gzmm6NnuV zE)Q8Rf*eD|Y|X{QwfIh)p?EMI$w{f+7s-swjqUN&u0v?YUvrw!m_vI&$>>>hv)dj6 z;b_t#wwD8H>5ZXgmaHR2ump6P_ZFoO)2&M6;yB;V^GUqHH{r zd+%|p)pz7>m-bV^6mR`VE2KCSj)aj3?>ypbd%RU94;RHXMV;=-;FdH;vo(pfc(}&l zb|WV*m5Hk2T)*$4X6Hs1B{tg0s|BO6bIyLdJP+T~pSytqb1}Yzwy>c{Y}DL$Y!0J? zeR*~`1|ShS5#X~y&<#fCv0+c%e%ElC`5Gand6wkVNj`@f`u-fZgEnG_W3xE~nN17< zV|juo)MnDelvDWymx}xc^NNoX-|*>*_ZTq6`aW+#>8S!A2R(Pv?&1opb5p zS7My>H!7BBeFbg=0nfx2+c&)rFSW2)7PTAn!xBR<#=j(+5^K-GeOX?i$zB@m3*jwv zHAd3xtWu@4Rd*5bdjs8yOU==(K0z#{MN)|tSSgs))YonnbKPgp9WBjv#Tc}JSU4b& z<{n74%H{o-{)=&)HI>v9mGdTt6XzSIpX3kEyHtU6;knl-))(tP4^ojwI3b z;(Woe)NH=#0+C6<R4%+~NFZSMg%{Kg{+o^K+Z? z|JQ-d`_DZ8KjnHOwfwqqL72G!7Ei2~u^Rs;T z^dYBGa$LlD^6TJ&L#A)ZtdIF5JT0D50I)F$_QkkAHp_a0p=$cqKz^l5xrDaqyFjZr z@-7uzbnvCHl&PyP6rR%hE3uVsy~@QTE&?okD>-hP-ux7uUjs_!^kST0RHlX$Mg5UuRHE^(EiMR-26qDEx zB8bk->DwcFa~x7)y|w3Unz`E?*rF}Z-544^`jX4QhI z1Ffgf*y4WR&NJ1M=hPFfT4WAJq|@aj*ey}yV>~Idc-ieo6nakd&EBV+?2l1;R(EXk zJV8Di)iOyCo9J??`$_VqdD}u|lPdFQID>E)W8sg}smvL-HM5g}HypqB=Jjt5x^nayv&sY6(Pks&9^Y@lD2Q@K$ zPaxaQP~rojZT}ir)*hkC{~CY_Ei%_+>%5vn*4B)Ih)lAeT~Gy5A2tvvWVLQq6P_6Hcb1{=* z!Uyg5LVZr>X45N4>sje=Jqk<+^%;m1XClry*mjKNCH?-euhe@Jmri(IPzvg44)zfoyIi;&JKgcb_d*$K4V`|PzB*M)Swk7?ON5^!PDxS1 zswD-nYPgi3wGU)YZTBUET4&I*6Js9OG*{fmvLMw&p;RVi5H^7}^#T?snm|>5IPWXn z0%d05T?G1O>CjgaW>K@UWEFlYZzY;zZ)&_5T}kzXn~k@!v85%!Mo&e@mhP_6x!mmj z(V(gReQuTN5B0fK{a;0cH}?T^^#5A9m9qc08nw!U{(ldj&*uL>W){3O(+-mfX}|nh z0i(3=%fHJG`E|d7aUr^Lgl|P;9D2>Sx3TL`cz?fr@cMZFaJT(GXWPfyho}38v5);Y zMp3o?MTPCmqU{IaTnMjUZOPy*IdNC~~J#vS_xm`#eI>_G!d%tJ@$zPbeBOw6JsSn}Ztrf4n7_jc;_2pXejGSKI} zQ~01|{W(6jss9C?8`yss;D?e@+o zs)%Ai!DFq9X$ak+cjxGPHc!s5*^x3nq8JY+E@)8r6e0wOVd$g(Vcaz*yC%qU7DCmh z;)mlr67$M=7Dhe5XYJyJB?Dj4#5p2h)Zy;l%d^+c`npr#;8f_6E%XpCTBK^lwPF$$ zXQe=Cy*I&m!;WW=8)$5!VY(%h3Zj*@HPQK7=ePA(*?=y5*H$Vh|B^~d+&$be9CBbC z;RxB-21p0P^UpreC?StbgU+apoV}HqOC)Qf7vuOGM8yBPkCx4BdXhzX3Lk+GD;0P? z15z)WMC6EcSwx}x090>}ZgT zMGMd{8z}yzYU*|m!OsMbVR++VNE!}pEO?BgGOORrFpGDbsf=e#1@&4b17pRDMG3%h zzQH)=yhSUnR`Eru1)Dh(qe6?2Od-IwQ4RI@rHz88Q2aenU|#3APu!&n^k^$>)^~PD zuFV<-ueb0Z;IvBiV#yJRu{F^k`8${XUhmVWA9GHO)?!rFI474gMoeZgda)5)ABhyj zXLM9M`1~{mTcV4<8$*-6LS^7jEB+n!;5No+%W}lUtD<&2H#r%o4RJJ|>R8Z;lKuNx zdj5S0yVrM~N1oGB3yb62%K(O0O==R@tXe!+kI{!s_7(b6zN*l+xvU?C&XMo3x_Fbj zrPrDvu$-9eLOM37f!|;U_it*P*+S91X4IL zmX5llHMn*rE=Cy#sY<7eUGjN3Ml7IKLBtc#g*zkXw%sVAOD`PjLWl{~cWEE_InN~> z5$HwCEm`f+BVH?HRWg>q>Z0BOh}t5AO{%sxoJ7|P)R)@`8w|NpdHM933UA_~jlN&?L;!$ zlkMPqiq*pS{6uuy!N;7hLqE;1){&+bkcvnx!X|LZU`A(3SRtV`Wf16p` zyK2W((JGFFd+MCxpY-{UV}@T;Afgxs0icd#!5~9P1|ZsZY0zlpC1{5j&=JqREWqSZ zmV|=8V;J4x#9@qR+ee38fgb{e)iR7eKT2`%$o_aR7+(s1%)CB7WoYJy!k};iPhzf% zx8(Bo7;jGl(^){+KmSXg1eD5ANbADCd!={Bg@{O&*|`6=YLFq_Su#VGjKhm13ky zvH|)6Q>Z9{@Lti9C5PmdLXTya4*UsN@gv#QXBX z;{c_!9>I_!h-DF92yIdS*o&U35SK=)B@tqirtzg8da2H;p_l29z$$yX&oio!hQ0um zUVGZKTgO_wSm6B%sCVa7dS!37go2B_fhH-q5W#435%<2N7h-L5E(qFCEC9kvQ7rH{ zbi%1HfUg6zp2y$slB5o`{M6ab8bqS`EbOcUz`(+UFvdY!C%#!|OvE-tCn;}Fw8FGQ z$)M{A)fZtTdm_yH(2XemAym9uy5bV5a4=gVnjUKOPN9|{CTA_i$V*af7DM0p0Du}2 zcMm6Qe>@%aN*M~8P^ieE?d5n13T($iFj<@l3Wpb1vTeBky@k>3kYj5}5>BqR6J* z9{{5Cm<;h+r}A9OHZG}f8clUO4jLkMl0=bvPCq1q34iAJs#TavC`zxTF@+tB?b)d3 z;R+TELF9J*pgSD`nxih$IXw~n4qAMl_-Yoy4|Y#BHU3&KT7UsJbJ{ zPT;xkwoF>+jxdR{vS~g0HrK69hQ%&cEHxRj$HH=BPELloVj&IpuQ(O%gy|QUUw?(E zu+&AH_)=f?gy7NMN}-y?ak7YDAuK5fVL))uPeH-tBeFhX?`e9KV=uZkMv^2~sLWy| zNm8q~-2{{wT%v=yhp}AlF4h8rxsb>(hFF&i=D?afu1)W>e4ugF#KU4knwZL%aRb4T zEi|k^7Zee*n8<>6QgqzQn)nxGo;%@@#W?bh#XO&Svr<#1W<=r8`lF4n$ypy^A0J)( zgf9Ze{erMIpAQ^tKWD>Wc+fn%+YY zE*KOL{%+B~6H!^de&ED!zg3E1%+aDa{~+!bnv(`cu!wZ)14X}5FAC?fKEe5^^2^#n zQ$SU7k_bpfv+}96k!;SN-^y%u)@l6QF}X<$?Ahqj9NOf7t*&nAQ4@I(Up!By$pKvz z?@%NQcSYkGDuyJ|#`RfRbVw-TVZK2=U0I&bqRrO>bT+rfXB0#M=j;vQSHUeS*U-?(gbmG)k`DBa7QRW-V&X!R%nDf$5 zHM*6kGyCyN*WAy@nvx9?hTuYq&v2I#U&RQ-}Qxj&u0CYKKC{PVQFqtUw~Mwx{+tPQsU`Hm+q0R zGl)y{+uGx2ESJHU?jTS^8gV~FgA@Cu;g1CI-S`skvEdyYfaHav$8@VR!laHz*F^}F zJUOQ(JDv_=5{xHafX1)$b`V(GBy zaC7)(`8H+<(d=`y#dbA`rCs0DB*z6fhXxeyv*=f#VM0#vG9qQTvvYM`*w z>599(<7reqO4rC}B? zDUcR^HuffSmIRI6;w4?-FBfy`-fkf=xETCw-GCZO?tL^nb{1I$Q>hzk;}OVvy!`2+ ze_8nHIog&wTfpdniZhT&OuU@=d%pES*ANYK++kR9e*eAXtgo}B z?Ws$?%XO>VAb&m05$c|v-h@ug}6cewJ?=ToK9LX-5*B$bw2ML*%R zJ088|j=R6;HB!)XQMqUv3n?r47*9F`x^~=dG^JQ>9WQ>VmZ-a=OpWVi?ZGiWr=2Sr zhrc$PhhhW?IV}!n$z#e_Ly{?vwSrD+yBiM*6*3So;b*hUQVp2HuZw4DNf*&~uSfmm zB-WC8)I5omc?lqW7IX))!!&cmILAIEM%%!%p9F2BG$d}wXmci zLGSURst%JrY0FGOT1L%Lm6EEC32lzLBY+t+NU1zug7}E1@6QX!4^w;upCUdMkycrC z7woCE?n2_qT6e*1Zo01I$LYaNVU^HKr8X5#qJNcmTJUQea5>yPOF#=C}S<SX5CPT z-e&G8W=z(3Hl8KV%%u9Cc*v8rqq&K7CkwhZfQwpsmOmy+syJWF-Fop^ROY7l1dCgf zQ+1b+x!IC$zBu}Fl!+TyZDt*9%o38z++V-pJBZI@qglY-l3H^Xw_RhUKc#G=L0!%? z@*ox#s(WLF=yqqZT~b&kUh)N1nk%e}p6z(!OI*stXD;hz9`!A-i^!_G2=RtNXPmRPR;(5tnz&!V3<6cC?D+|KUFjTVlrTaW+- zT8l2>rV1#iwG@jngrtbjq`{KQ`HPz`=N1YiSE2_#vo7w><}L0^k@n{Jd`V0EA7Gj< zJ;6U=)wrGIehEIxT9>t^7g+UYjpC9jhBi)1X##Vi5-+X*=rJ>lVV$9?ExzV&ncYiy z0LjFA=GI#`y0Zxq@H7ez5t6H!3+@9NV*Zom5`T%_74!>8G-(_i=`{#NUaX8HeAYL!N#QExSx6-ci(8x{C_XT8=wQzy zYCn(x?p3%Z5hy8z&*9PTUVHy=|8#%*VETqlMd)`;>|r6>nXIhP=|6+y<5 zA)X$Gsle)^>9E5?N#Wnlr{gI_&q_6?B58*dbc$Q?N-A<2WK0KQvJJ< zW@pXl5Bc15gaZa&azc@KGMSFH@QPxhPq`=L;|H_6s|W^C#At&W@7)0;f85H$|MON^ z@By8tfGYij0xyerbMio({9y6m8i(W4JTuuJXRc2 zINI_W28x?_oH!OCbs;AtNEhyV5BS9!ahNs^78ZCCxKO-Dz`-3q3fusF)(kswle$~n^z!^shey3h*{qdgx~Hdu8Cl5#or-dE%gf2#gc5)Rlzw@ByXC8~Ud zi@Q_su22cWc#;T*Q}I)j?btzAe$nSH5~wFcI>>E=L=DaaQT`<+C*IqEeq*BZ)cNVz zbcCb$!ug#ss>lf<6RKLm+S=MLX+RY#pt1y}CsboVjs;ScAi?w~#MEI5{qCevE5#&6 zG|1?uYZAC!j)06xiHK$zA}3!p>ngIQQYI+ECOKDIMTC9vuE@h-~v-j<804hlxYu=%V1@fQHP0V-b*Eiz=8yiD!{D0%SW==7<6; zUhu-op;|U@W>a^1eu(0K>W3FHOs5|*E^DIQ&_4(LDwo6ZTS^w~OoyPpMgDtzHJSmI z`@@2T03@UhcAA@29hMDu0Y0^z5|Jbv(+g1U34Pfz@_H(>Cv)Q_SjmWc_ZB8ONS7(f z(a7x#JSE;>!F`VrsySM-u@=L4Q(wwR2LeO1fJ)3{QXU9d69=$OcbRf{1^6kc?fjzp zCG^8wPD~j~$&XoUsaW6cX*J#Z@pSL_5LQ$2I4VZPyA+E*toZ?pUP%EV&(lNe@X(fc zE1Vu(M~p7$5HwG?NYBHtt%=s4NP5l)z7kU_fDvt9xIr(8C^fH;CGyVzM~nX)U}-f7 z{*=;f*hf>)_nt;`|L|b{5O+P0H8S${4!2((?1=(-#m|DOF;k}pUf%h}xdYu)zjJCt zAY*nl8?`7Nr$$>?IsBeQ=j0=lIU^Va*Eool>Nr+3jKtqP(wAU8S%@c6otgpO>|iO~ zoT*-lwxH8Xx0HgcO(%qR8ThFgH4`9*E;8_QQ2SkA8k%&sNDHRMas!~x@tZ-m6B|vU zbJ#QmN#oLRe)SoYWZ^&)(<$)ov{$gt&LZB>N)_QHWT#y$=2hDE!4KR2JQ39tY4XcX z>imazg>$rOKu9`?nvMqkJ8y7}fuq<#c~QOg5_`wh7ranAt4;+6Kwl^EraPC>7tg_Y zyFVC@g963Mdhvpfurl@txUz~HhC1M9j?s8=D?Z`Uib3+nd?wK(CGv{-mKSG0z_?|i z(u)`HIs3LztP%mAubaoPNGdF?t^d#b1e2WF7CN;)70V}u3+lDOg@XN1|I>JG8T~I?V}uEw*PZ6B43DWXYC4X@u@% zcoLMhvqMQjzxe`@EM0Sa-A%KPZKG+N#%^rewvEQNZCedbY}>ZeIB9Gr_w;@4zw?427KsIJXA>yuMnVafAX&b{GI z!-a~hAoJXju5E6}58pR3b&2&Mfv}%HJq6HHiBMWYx4!_!VErjfTZ3b3((ihI6jj}n zHWhGQbBD229mnV3>}iz30ct9v$%i-zTbdY7?_f+m7tiKL^6Q#IgR0u`Xbm9yO?W*$ z()awKrU*&pPCh{%{7m=y(KiK|IEHYkVj5XNH}~)Tj{Hm ztz61hw;oWd{8_X78?Zdwsg+U>SU=jdQoAghG>wbQ;Xh)Q7#b%|X`@xDyOl7dPg1GU zQOcO0NHP0N*moCXM9OYZ3_sokwYnNBM3Yg{&>GbHoJ+2{qjH|dg(1lcV#RjFkay?^ zoCk5<&!2!Wc7)oNa+QX)e1a265#1;PvLDIP|zE zWG1Cx%N6g(^vLA_>!s7qlZ>z3-6w-C{u$D*(Sat$Zs=%s9xZ^_@`rxsYgX%*HjVn( zVc1*G)3wL_lII8^APoCOyw*P_ zh?9;Zc;4x$yN~D};0*gf4+@49+Y)3#8FGOP!76>6HGp)cQxZO&>8U=xOS&;_@}~hU z`@ZS{%>b*uVWa6C+ruEp<2fxqy+YU-H~rV+AUQ`rn+{~SZ?th<9~(~JZx)R&aFh)_ zQncd0K`}VYTrz$cKaNM`JJc~)28|fhHAe3Q6t#M^l&Cm4z4Yvoyhss=R`(uHS@)$% zOsa7#CeEL!cATg?gFs=a9j|R7$2>pXAbrxCu4zt%-+en4{yla_clSn+im8~WPH}$n zd}yD&k=tyZ@s%+B$7tsqWb_|=UCL0f%o#N%&$}f-VjxJ2S�!^Gf}~ z=b41GCEPfP*01d?39v=?6`59DTu_Go$ZUeyr+>$jGN~ftvOOp|wxU%L4kU#GsIX($cCsTxfjguKQ<}7Zqcea zcn3s$*_7EgDSRp;MorJ~fAM%CXe5X{m&GLU6*KLCC=*ZkeSnS@rJ;@z6~<-jcDwD$ z{Mb)$e}7J9R!dmP0h?O~y@@Hq7DWP?9F=U`#o7ElUYLGIVXFboLv`nnqv>Gx`tov& zsqE^h{Mc;b4Kfidx^93ZL zmMrqSjYb3`EWm@%Q*!zBI5+h?qC@uf(C0z_a3h{tDxGRXsb|$|<&3ZI1v?Yf57H!4 z!0(kPGuLEh_xLWV&$-j4?YgIazhPZI^h^A@F(Bn_!0ZbL7a6$emsMs@UFYZnfs+j( z!x5CI+nkNF0k$S>K*@}Y!?CxE>OKom+W(Gzn{YQCMu432D3-rEe~E`F-Hy2xvy3CT zELer35yoi-y7%1+g4krzIDqB; ze0Q|CgHJsLuISP^pJV~k+sVZ|hP+Ul7?^3;Nm&?uCs!kIv^0awQKOM1yYCUCE^qCT1|>=1~kYQR+3) zpMJChcBDmZk3rehFaZ zf*G@vCVJ}3G(>S3DA8bY#8WOgFQ>4Jo=N};)nuT;09gv>e0v%L#}|R;?mi%6r>3DC z&kyyA{J|{<{4wjbrZ|=?X$qEBNj5cxI6azZJ|Q!@yBys!^OZt1N&du^NouYujl*NOXYE|uU7o;3m>w~<6-W{iPZjBP+78EP`ib@ z%8%uLIO+|zrJD2!33Ix9{(j5OS{CmtU;ht>H5z+GH(bB%9OvrUCbmB({e@%`XE&Oo~d?z+@wLe(adB}HiR|4 z`=Uu;c7=eR1ZHZj-~FFpdEVemTZ1m7^ z+$KTumWZA+U6+}jqvP9@75!+YkJ(s@*4)^6{n*Qtj*W%nn#+`C>=e05`DDe>`Ecg% z_MhG9cZ}FZkygRh3!mFW9Wsu@FJ%kdqf9Ca&nvj$O(G7wcD1_Tm*5DBxirHm#diPz+ zj|$Dleh<^}?*yxsB-D`$K=lfcz-+4HB#6`W{ynFi;bVT>@Qn3ZsKer|m2i(<(3m!6$%TU^Zh9m{v-URB2g-MPFs`4;}c@UV9O z-rMZYgOq~$o%Rz`hi78C;dmPJQ#oJ>{{dPIP5~SE+jmy`sPI$ld0_PA>G(t^G2m=b z$dA#iG)u=I9BHo@pUL}WEEa_2>GVPfqpAB*G$gR-;|({7xe}r4#ZDoY!DX( zJrrN|G%n#^xxhb}ZQB_1yQU5+T%@tKGy6t0ubcceQnRC!T+IoNBb{oZ zoM{47^eW)}fDg~xd|fwc+Fz0BRHRI#c-9>xuZ$l;ef1Svb!}FyXvH`(ftGQB$8Q)2 zq|}2ow#`wphn%sYE70L9G__NKP(@oUO-+fJ7AeUZtIR+xma*b0s2VO>{iMlKr4fs< z5y$#WHEXIjEKp=YS138RWz5eUEL*n_Tw$xiX{rZptHx={9E1@=z!KB| zr(<0og;23=(n2VsXR_)CQTgjaBbJrnZw0VZ1xdu|TeV-*2FSt_5=WVURJidlSs?}} zmdxiTv>2+49oZNd{6|oPq^Dl7!s4qiae5RhnKu`ZjhHio-;&Yh>`0tivs*%EwStST zc!&|}QUv^@^9Bw*zciD-R+KhSp#jHV`l~iwZQ|tGqlv9+4I6Dx%Vq_LnW_(oqt3|I zEw{5#!}-@Vs(q=9ZQ>uJqic!gBZROWOf+6o%&S;KyJeDiip+Gi%_DmV&pHOqPwNM% zrY6h7b14dQ_0IO}k1VhvHCv}5J@5h}jehZe#K+E95Hb_j5+y+L8NO?V?$p7#R@Y`Zj6gIhO8e$&mD&WGhYFCG zdB*1POe18b{tlQttOx=K-G4wDfIH|k@0J)WJoe7H0D5o_(uc3Pcq}z zmbhr&57oMWd`&>UM}KhKcp%ZNL8#MWpe%K44WQOV(N1CF>e8TO`O_fF{C|AgWomeV zalw<-py@GcTl5Xq8QEQP)<1{JU^8?p!MXg^mnh<_fV8$X477R@6IVb;d?wK0i36LK z-3N#m;6QMDP<1vfpuZCb=W+nrW1uCor17>cKp#vUU9kmV`OugjDJYA4RMD9oQcbqhpPf+y`3X=30e<1=NyFR^fLoGlGXX=5_uFE&|a z=RqWot$pZqHB*5wi3#LsVJ46M<7(jOP`dOm{jI1vdTwTHY+R&PH*xNXWga94Zj}PE zh|(^COBA3`KUVaQrg2X_DGt2Iu030+RcV^G_CPxgC8=Kn9}S!h-eNc!SC<8fxWAg& zx|XcgLJ?(f8C{CtTR8W|H0CQnJ@cFAp1PpnOkh$g{-tdAK#ynk&ZR0>up`xg%BQrN zKnthn8_qPd5xvsn+US91W-&hE+!h-qiO0}RSGP*QK@BqF-y4Cr1GSvqcWSAE zSTW|uAZolCIslCn14iO6Vx54RrkQ#?sexqu#5s`Ul_REqjXH5I1zF4h15!sp4r=P) z+N5*At{GNUKNUR|H(9+%&G=QRx+$Vzkvs$hHQ;Os2W8?M zu9^rCQ=x0dUeD2?b(PYAhGMKzW9$ES#fA+C|Ae8n+S#YSfpHpUL2w|y@P2ko+L<`G zVy2fYqx<8Krq`;*C&b(B7MWq0tFv=|$mTZwOkGPclZ_t3=j7gb#mdgfKRZKz79&;gQx{?JS(!Pw0yb^g zG@=(Z%mxDCJDRKp=o_+tQ3AKJe}p6&wrGGHq#9s0N)n#}1``y$#l3n7$O$~D0&bqR z2BZxuL)Vp;*G-B+1;!`>G#keL8Mi35NFT|iL{^g>*U6~pMNL*o8X!=_yz%^M~| z$MLi1(iwpAOdZ(5U)0oR!%f+0?IzkjGjwg64cOth`Dx5s zfKdlFP#6N!G|oUQ?9d9Odo&xM_@7v40s@E6Q>djsqcFY zr2zb^M(aQ@strMYWP$?0__03>l{pN^?#dWi1HN1mzBU6)*IDZDX0rw$N`P40>W8Jq zNi=mOs90fRX9xNO)T52RmAm@Sq!q*1fK|KQ=^ddW+G#r%t0qKHASg8R&a$#xGyaJL zY#;sR;I06Angr<41lVgGpk`D6{2_=@gn~?6i)_3T{p&frz#mbB7TGk+*k{G059&rq zHQMVzmO8euY8nw!*DR20NVV9eLlNz<^0Cgj$cTnvnt!zBY}sfWpxU_AgY%Vrn{KF;gV(CUHEhPsAm)W1WFVTTnjGgS*`^N^=Sa{h}njUN`RU;hq8+@1Z^Ha zm*R|SV4onU*+2niyT%iZMMWi0tO5hP0$^_`gBYK-0bhV*b!K&js33$^AV#h9$G=z@ zgRh7w?9*U*sEqLhZQ49&?ZLG7F?R$|)R~3yb|iX0wVf^>oO%YCvgHEojApd{pYhy| z6~9y0zIHN{2|TCGl0T@*Bv;VRvrAT;k!<4JtZ9>|1YnvVo-)ZIfqf(tCu$1}5jur9 zfxI^$JI4d+bXiNbfDq`^03RaO8luLV?JeaU!o02XYwigia6AFUm)U|ZlqS$7XZCT# z|FMFn7%I2JzTsu@j034#p333tP!l#dHBtm@+-e!R|5!Wln_|;HXdANVbcpbYc5MD9 z^npvaF2tgK66%BO4WSfvdC)+EcG$fNaB= zYGCAD@wY-bxNPbt?V1E;DwZqvu%yrwu*}`GbJXGv6@eaDgR2EiMSlF}e~78bx9_Z) zW-V;BPK@H2Y#zzLpj_BAvO$Q^HWP%?p`7&a=3klZ2i%I$X9 z-cj8eaHjxtNzZ2hlf5vLwt;{@>f6i9YXR3Wt3tYK!~yA7)L1>BFB1BY@gb+;BR8D$_&;L;M+>esCO zr+WjS3IQy3_)}H@MF0@=G{DI7*3hTSVNKYqjxV!O6IomT=lR)_z=yE_(DNTkkX&gO z|J<3cT-bonb^C|A7aLT z90t4&hyFJ?r?za?m`s+MW<{P5Mab)U{|k@jrJ1^os5WhbAbZ8iP1)xE(3$_@!u4;l zA86MPZ31jkZ~UkHLVb{K85p=J>T}{O)wlmebN(*h4Xg+;V2&al7Av9tKbap^B{R$& zL(EbytMg{VAfpR5NN*%%@PBT9 zs9|cz19fM@n4{mL{+DGfp1B-cn`Q9P`P#q*+z~i4CVs|63z)1xRJ8hQfEU~Rr}|^c zs%J*o1X#?@BZs6+27ooEzGv+s{LCC*$ZSAq5oD>*up{qZDLL;zl-CK{LSgP$S`VIT z|Nq2s>jG}>>|inw;kZwVYS)^_ihzjrEj^8-8w`zK>)p-D?^^$d3zMGxF_uw8;s$mr zMr-Yu%nM&vAg0lptr0naUk;&7e&8a2|G;aQMs7n2RBqomn+;0V@}IyOE!3>cf1?fD z>Lb$bt8jpO?wX}jDd6V`?I+zTuiF1d@SyZ2PBS zij)4!EnW>5jj&A&;6yLm*!(fopb^W+zOa1^L93%OVfzLA@2DyDKLhKp9<+pn?BFsG zt^D9!ZCo>em3r+ejXk8W3CKCR&Q+z4x$*H2&?Y_l){!Z$K+Ur21V8_mGW9$5_k(M5 zzvI`4;tfg_?AI>M)q&OG?E}k#4MAemCZ3e&?c%?(=Ihr$dTC(`+@H4vuv-gC|9>2N zl(D4-&4i7}E^}8Ifi?d8sRMscY!N7tL)^c#qWNv@UrqGwX{jT|Q5v-|u#Mk%WEuUR zHCI1@s?C&*g4rttc>gC(0%Q@pMT1=%jWluPKS#n26{kN_2VovZV#L;5`TwnLu>J?K zsZ)p%a}*~*|A`_4`5Qrc@;_4M```amN`OGLt0|6Zz*?ZkXsJyD(?mpI&)LUG*${$8 z(s%(VD$@l@6ZKr%k@|UC=6#MI=A`d?JqFfO zioaq5FVdSW@U(WPBF1y>^(CI%FlnTyZ^*@`3h;dXL+ydeQcF`)Q{n)>$U1{{Y=ijs zG>~u2M1pd{v2jQ$z2Se_vqE~a1gsTihvcepQ4$gm%?GvJ(sz`|>y zm8?m@c{#0mFI(mjr0l#I( zH!~}Of+&(5WT7(=uRsWM95>C4PzAnr5Rt_d-sjd&Q>yC%r7k!1qJMmoG;)pYt1=X4 zpA&J{v2f<7QofXX_k5hk**tcI#KolfkUg?n^!4F!q3Cfj_pSPCZz>f`Xx+Mm5Rb z+U7;FsdiGaXw~K*AmEFjA&tp-1C&KaUc_Dux%Vc*BXJ_(su- zj}1}2P%eE$^R1G6w?U;c`ON?PVej(Aes4UNzkPFw$8P)%!R@`FRA`N0mcMvG9Qe1pE*zNQ1{${l1BKU{g0mTb`<~r|TD+GrvWb z4s5-SpeWi~MB1iuvRQbvru4&Tt@7eYt#9)rit_T4Rg!{qXqZin&b+aqMAj!g^QN3; zfwYxhuOts76TN*#&7Wj>PPrDKiQy8Rw+53etTDAdlzam>Cm73$Qz(8$J1;Q2#yeX&o5EK_V6F#POR* zpt=~m(T}dr6Nk-h^95OjPO(bFf1~TN5SNaG5o4X?HVnrmXkVu&Ct69c0&05}Ulc=8 z-KFWk`F8@#_X5yg2dF4;IDw|E#~8jAi$zzKi4GZbyyZU!J}Mb3!?Gu#9CFCzqeg^_<#WDtxjB0-nj1!`s&cs8*>CW|MqJqNhS8R|hS zi`_Ge;ff0_-eX6-ah9b_=Ew>~|U4x`f)bwLS zWaykt2(Dc^CIJ&lya8Vw;LzowI-eu1hPYdWu}mYQS)7-!g2Z!Qh^N}u6%E)RPlZCl z9l!L{p@-w);={TdTlRF+0A~>EWy^(0mCJRBJRucoTBz^WfxZ;eiXZih$L1ZXk8t>` zCm(X|PS1xWKLnSKf;16<632TT(q%rV2e%`Z3lRENjU}l#r%6~f?WdgUe z*VPBg@XQc7ysroOCD@cHZBzlWZ~)Kvj)T8=7USJEsK`Or$$Cbtg({BcfP;TeKEzDI z4JF)g5Gx^CD_s%`EGlozhK@z{5Rm{PvhQvrJH+_?=IBu4SUOm)U>?jQC3_PLJy*W&&m82+K$Vn1@FVWhT*bzh;2e z<(Pl>RA4B;6m+4*zU>D#*m9cLGGq=M$Mk9RNYJJ`8f@@sr<3$yq~UHSeTBhQ!l;d8 z3=IZj9~Dnu`l()mA1{?dqrj?bd%zup3F;3w?)?b(0sX=^Sx?kn5?V9G(+O6Njgtkw zr*xw61Dk{PuH0})`Nbw|xY8Vv0(o}XJiENyRojJ4CB7nT6X3!6IN~07p;-U+9dO0l z7{wBeY{S}INpLH$m0qGk=i?A%CFM|s3rXN-3}75tRW+X*gCLUs$!``|0u&$ zvUfT|CMTD? zk{e4@*m?i*GfP(}X_6Z?umBgv@!VDGc}O+}5FcR4AhBu3dAhrSU_2zaU# zyu-Oojc)Gq4WBKeUDKf-k$UltWCz&^YVJQm!&qtz7|c`H8u|dpT(uU+8xB*^UiUhQ zYgoaxKeEnNgkdj~v3GxqVC>kNUpbwAoObhStUYgXdy@YJyFPUY2PuiG@c>^|DqEOO zfJr0jHMQUMF_3eyFZ|q{rP-FL_;Rs!b9S*%wC`uuqCHXeYW6mw1I5UxwUP6Sr}xhp zGow${^c_cN5K|~osuuAG76p7mh(g#8Xz^Nzs9)=EVQo51*>;i;P5O=GF9Lz$)C@6@ z;|7sMXTALr0cAIirO|()pB`(*qJ`dBMYj98&Xf%(dZg{iZ-=48@Hi8E0#7NMA5I^! ze{`oy(d~Zlyhwnc2zFx-VH2Sup$v;o;%_{X@g_!gjjXp4Fb|QY1-rqeu<*Q#Jzx(6 zu)OucU?kW%l0;54%4HK&zI&Ru+KEHaj-YIBm_wJgNeSz&9HP|-V!s*MU(q!S9wm*D z;SA*O^R(cagj>w6#A5didj`k9D&xGba2u5(8h0^V&OjNjqBM{r(@iYI?{xb2GsU#* z&Hg~O@d7?t@<;C;>~p>W1#S<~-v`eunX?fU8%nTIP$Nv%l`3t*R)wuH0u-H@VPeq4 zRo)(vu`c$|dhw5Itke7MNPdV=V&~bnqRtcVtemmP%sYL#H4mwTKRx-vOl=kJj5Zb> zXz#~!3z>QyS4alV1f5fgLe5XmW3nrVOi=G!8ScNFMzj}1Of|}C=~b$el`>|_xCun! zHK+WX6Mrr#xgy+}rJVP@{)J#M-h3FVb~qKsrUe{4YBGU`*vAQY7sSx~0(SKACg!=P zpx^E!)JgMC@Ua1N$8!ALn=^kMR8x|U%aZ1E5vXc6$flCrt=(G5pOya&|5=zYW<7S4 z!pIUzM3^poM1ULBm87m`CO~595Ndzok(@8HP;A@(moFKhg{KXQgRjPI&?9^hsTetT zzR~N`tKu2Xmje>f{CEgnoJK`v5u3dHl9ZgxYY#>s{YUZ~Quu2u1|k}3*H^3Vw{I29 z{+6My!aGsz*v`)W>IRw-jn9yw1U41U_0XvZ$~=@lUF0*9V06m z9tVyMzWSO1&SRL|_x68*Y3z3J5)u_-bM1{ATxVvACIoiT?3l~tc{+d%Ag#|;@JleLfXr#P{7oGC;xhXPC0vq$ z5wI_I)$BMJpyc5+Gh=9bn1<**H`Hj*e>-d6u|*G~(nxGqaPdv$rzh9k$diZ7QEU_} zx<37F#}$u4faCENn_p?}In<{Y*ZeFbfbr?$D9+O&4Btat@huyvr+k|Dox%ltSPN@; z>EN5}Q66>AFZdNwH%go3R>mh8sAx=sqe4dVpBk*!LHNz3MII~sd9kIF>|{Uv3tU5T z$E6YU0C!sfU-(QqRAy95-|s=l(FpY|l_J)Qj7dQQR(yReNRWO9^rCCcy*^K(+}w-b zWRMyUp5G+|7dwF%BPvBUVem)GTvFv{UJ8dntrS~h5L{U?Uw3qK&2H?Ns!3eXNYoZI zhA=qd6mFt_HkZtgeuk66N)q;X!xxL_uYV75pexB4IvQK-mYWwKFJGcAiTx9P7gX#Y zTN^AS5cxnoV?(X%ND*WS0|$opYUm+kQ0(|O#GM#TV@T-*>@FR98>UN-4l49?m9>R} zRdy+iH;G?Jhd}(YjqzrI-aq9F+55YGVbYrtGK4;3_osw-;Fd)`oA*Mz{%Pbmj}Skc zY_o#%EBKR6=bB^b9FF>Pb+;FW1<`}WK4|1^?|>=(O!KF(GO)-9K$A_5Er2}zsAtoX zB;naRi%MeAcoznfP=;%qX9~AH24fayBxR@;PNr2tK8XXioIlG7^vCIi2tAy;l3lV) z6rjhG_Xs$if{px`P~cm6jDXB%T8?QF4cPwJ?H33;6yOL>$u9Nv1_dkWcgMS-al^pb z_h-{<4I}JN6dM=Q3SoNYjDEox**1xc|3xEW7+3wir*COt{>EkFSt6@hVOM6*!mybi zwj=^boFDu)ms&+$ib7~M>#dN|#O5&z03&~Tn7m)>b1#Ow=QZPh$JADp<1o}ulRA8IF6@oyAkFPi1kVj-dE#!kN=pHLAG^GW9TNk1r2^kVB8SO2Hh?Dzn>CD*1(C4B* zmb&*hvR%c13-9^L09q}+H+NyoXzt|*AB-Ch`6EI4w!W<4chR%DI${*8nv4S1m?!nm ztbC}MDgD9ISIEbS#`lIbJygMYj>}F)@4mc#b_A=35avM4D7+u(S)>5-hBp+JLD^JC z-=YYGd~nNi?&oMF4feK5qcTe2jPyBl2}p5D(C_o7zYJ)mOc-_M;nK>KujizpZ2q1W zBLmyTy{!r&yA3EHj4nI5)EHgmN8gz|&eOv{zAOzDlw zC$N>3<#S;H;gJ;ZJWgn&GYHvxP({nBY)lRUX@@8H7OA1bNdQYt7vM@VT5^m=rbLkR z`Im&YedBE^iXEAo8`!%fsB3^gCGt@oevc2UMkX2=B{dFas1^h^rP08PE$vM-rrLCR zf%c1{q;euPaq~9Yd_diZt@z+k9ae(p?jg3QYTh%iXbxo4-Mt-q-{Ayv50n(wj+K~N z0)YtlSrq8O4^T>WXJ!Msig0KA!QTQuLf+Rs~DyYE(2u6!tMmE$e$L zesobpg|vv4J^C3K!pM*2i3QnWkDKxmmPMuxEG5!95w zU7s)!kLlG_*T0fDpq+Tri0La&l*pBwS>)9tY#R7|d87t3+8pnvG3d7zoha2y^Zo%I zu6AgW%w}nGukbHLAukbnagPBL(LP{;xU$-d5&GiYqC5bA^%aD9?2qnpS*y5z#HJyI za>_U;S}n!P2qx(_msv+ILlR+uPSAe6=aM#w`2SwJn=K+uGSviSz~aeD&m%o1Cv_+{ zkAyy)%Zq2g%2sG*IypwtwHm+=alV^Gv3)-x{0&^Kz34;Ki4$};UoOwD2`?_Tw)X!{ zTqN;HL8$?Quq}BMX%OI_5 zD;*}uf()x~pk|MQY zj9B#h&_>M`a#GemJ_(XrrE^b>3XU}78_ERVYM{KMw(K{4uOCP%plJ3}N0%k59&PA> zjfp0^rb&kliv^Gw+TE%`mI&uwx5y7NmJW}82}hqLM{=GX;I^?3O~)dU)PN*N@?$#} zvl|$yzkcNl5=`@mzvC=3KlVx$$GiECF=(-_-5(w!*l~sj&^}G@;VHp_tndahF2!O8 zMS5uBXH{H2G!R1OZ~2Ge3!oln-kR+bW&qt^qyr!IS6|~zR=gKqO`3k;Uyg)%Gf$s! z@_s@uhZFZM6bbJF4A54poqN!h+BY+(pPyj7(C@%^+L_A7t}0AmHFQq!xweU|xMgfz zUV}R19 zN<{#q9`xh&z{nXB&H}Ma3^v$GLP+V3hmpauDyxBhQ}}t49s$4kXba-_xjVG0$IpJe z`Mfu$%2)l>hPJP~qOYG74_aDZm8^vC{F@`kd_~0N$3NR!nAV^7V;U^OtFcZDGg_ZZ zE-TZoq7Gl(>NmDIS2y^0yu5^JW-IGrCDk(flW^KQrI%wPZ^4*qI`vDIRbf9n_PcDC zz;vgi(s+3}3cmSnRo!mzpA#-u)YAWO(G!43+joVY^$csT$uPK4B8E2rq3Qzrh&B4z zuM{V(Z5Yd;ETR{G)hzFdywR#D_5*Pb%CkqZtWOqx zOprC{&uf9gR?zP5(_Lw^6jXjADCQb)*UHNm3z_MwP&;;<8plCe!W|Yw(_}GMp>XOx zD~_9XUqY#*Qns7$XnopUr-G;NL8$aJqRITuxd$LM!HNlguHkhM7;*SY%8VXD#gKy_vYui)torQwY32e}=R++s;cKq0ZO;=0R+421fq8ea&B z{FFwxfM)T>a3_S^kgl)tKlE==0*thtGwa3gNeYa3)GmB^0F0b5_0D4ZBg2pS`63t8rK4o&K zBx9*cfyFC&=sJKttrG$N&HFc;3_f(dvJ!Hp7Po#<5rfSPmx*Vw(B9-rZFBRZgBa6C zMYEnpic{{1>hvL^arTZFBy;SU=*@|A(Cs$C;Nhuk>8*cMnzMzcrvo9uJ+ok%cc*~_ zy$C#g9qQN84b$-Ul-(h)a5hzxxJrLyJEz(u_Yv-9))}|4!Ff&Q3UfI5Fe{H_!(j zRYaUFb?$>Du6ztB{gWFu^961{C}+U>)C^SMW5RPMv55 zL_6n`l=b8eS5Bnu-FRJ>F3IA4nu@)H#h|W6Eh^zDk{8CbepaPk84UJGJPz?+ZO0>k zvb+*9&JIe63THJ$$#PUGB0H~xJs%t~8oF<17byyoOoVzz%NqKUl zi$0xl?^d<_g@nbyRtoDC`u4pm2Df?Ysa4NY%vwwqH5(IerWqBLBMrT5otAVu{U}Yy zOmz5L5=lvy^5nDQ(4tdL2hSc|wVx4D5K8uO5&P-93=7NjSo@0j@qNi0EfekO7_jWR z*o7LKcRW@uMrrXLCkx@-j8iqViorYhzY$y=nUNaj98evYEH4*s)kXnCzNzlKB@5m^v>Ph-qV)^jb?PFb^kkZM@R>|D)>|OpEw8&7I z$gCl>V)eF95WF~!QK&2o#t?GR@ecH`uG96eu$K(e3OcZGO=1vO@it77|FEV9y(N@b z%coD?8=^HQLOmmNJk(8Pbe(fzVWwrWfQKce0CRm^0mWX#qmL(MpiLz+ENX{;R|FI3pB>j&FQjv7NeWQRRf!Pv}vEBjgP-EB@p^w)b4MNdpp{iT7M!7Wdp zcjd*HRc)W_SY{e{fG)3zn06KtHbBiTfw-PVWOSs8+ikbZ-Y&neB`w*OEe{pUC%V@;O@ zL9;IKbiuDAc4PvF7AGh{jN!-AZrUJ}|{ zAqH#mHfiF>LzgZwo$wEeg01AnOHdVZ?Fmr=_`C548NYY{a{Yr~^QWOnr79~(_bp0i zJO*OjZ*qCSya@HMK}mMWAH|Rq^V`RnsM2XYl;|K#MFi05Kg8fUqPq!DT&EfygZb*d z-vJmcB8r0;gu$&j4n$B{W8$!oB^T6+CQ>DfU`*D~AEtDO@b=6W@K^2<27eIJ)uHWh zP`lt=u_B+MKxQ3Jz|qn;FkUq;il zJ`%(RQejz?#Zh(|XgW|IVBa@?qsms$M+DB?u<5^@oHzR1S9JNcc0BEGN0^{SId?|D zITKP)_?UD{Dk(zWLfYn;!Q+KY8Yydy=N^_ir>5r115AJ41FB$71`D_q6RQtQkgTXK z-!^lnP4JDIx3`9Wa>5_}R>ho3cd=ZaXHGL%beo9h4 z5b3;fJhAVrrfu<3n;rd0eQw#lKm>7PWhbyGnxfP>S77$=(M=%{=|G*@bVYt;T`3=4 zt%y-^EZN;G*$3_A->(qEOMB+)o-l;L#NzEr+Fw|j&Pjf1X1nT;a}+cM$^H{r`#Fs6 zeE{pz5I)cD(GdQtHsq_n<(ehC&KoOC^XZQa@kB%*Rxf9P9kvt3C}&=hmKenSevLM> zr!)N%#FDJzpOjta-Jtfrww3`5-uC4e2fy$<)E808Ut!c@)W!os=pC|)z;Ja?L_H5D z6T`|K$avh8svSo(SmAzQACieRNkWx2rwq8QR1`N6z2L{jbAV8PCRnYSWAAdFPO@b5 zNVt~ap@rP1nv5wHmN|1}z*cXy4gc6A_hi$YEE%IJQbq*C`zphbc|0SMZLG-at zSYEC3IyVY_3PyJ{_{0{PEQg|IW;i&aG%X$)f$$H*v zPKrUf^L{bL^#M5K-v~(;)L2_RaT{a?jEkiW_$4!)7h^tq4hg303=z)Pj*?*zj_QL^ z^6X%iVV#}vkvsM;3#r(PSP@1CZ?%sg4Q`$K<~n37-*#T-9Yc|)(QA!f#q`+-SV)@7 zSn(^~^(E4(SZrO-Z3z$u+)Wr>c^xGkiqDX^adk!tbTo&MNW*Vto_IClLKJbYFnoKh zrx2LHi;}X>L^^@H;=8{`aBxkKzD&cyzB!6=Iq&C&4!8aps~sCxpnHI8A54kTPlXpn zDz@AqfGwS!M>kwg9c_kh5|qGcy&#dF_uw9WE6Cez=P!XhX~h;aHs6Yq{_F?92Uq zD5Pz(}M4dU6)V6zA9sP6PC6 zu4t}EJSjNIvLq+zeV18NbTQtsoN*Vx<;Yl7FB@&$g_rS>!i(3WD1wzdx&Z!jq!Eu; zHqjxQf9vo^JCCqmUh31Tm5eg9r)$l2B)pVl!W8Vdwc>B}UlbO6Np6$*#Uw^%oyaM~ z?1%I$#@Ki_CB(SK6u3rWuX15Z%=z+MD#okYK7!WVsS()TeOX%fbby1qV@<2naGt_6 zS@12XZ$o1jJCIe=;LlUHJij=|}L`jxxZrHCv@eIHYT;8mV9 zbs^`&oJb94d;vTQ{Lk8_AI(%d`KK^o+x_~>3pwxY z?u&-mT|eC~IQQe=%8bB|TWHrsuZz=mdBZ;LN$=w8qzVCwW^A*ibY#q?d%;zi2q6$+ zinL3nM$JXdlg4Zh1HCk2kVd9citI7-0pdHAMmZW)Z#i1ITK}ZzWXUEIsvuj* z`N)VXo*-Cxv#h2lY)i*3-tDWK=S+;Cdtf)2$c383QwjFq0dGSk_+7Gv8jcU@47p@M zf$J3^WQdgUHZ)H4Wb_VR&H|KVA0&tF3Y%j&^YyDBJ_enT!YSMJr{|EkUGb~KwaTvj z{i1y~@3BJ=WkdRGaBR18xnr$dTp!9VhOR5$YzRiE5214ljN)|0M+a6Vd)C}s{aHaS zPhC|gaS&`N!c-;yhH?f64p0mZ9-olWT;Q8a4C)N_Y!L`wF`(2`)hmNB7ROXwJ?W_t z93VwvdeU~W`zm(5(@oxoY>|lvRQ@%w{|C1~NWb^?GU2IA16;!`Y9lCZvySU@Clk%P zDw<|N35r~Q6k^EHA$rN2b}uv^v!lmA-eCOB8#%Ctg9gZf<`6xuqT^EuD>O+NclY?U z^qjo5wpMi7?eDiS@uc03*Ys5uoyK0!ES z7k*FfX*FPd*B_8UIPkjT^AWO~@=7bcOp^tes!CW!lcH-0=^h7g%|FT|MyQU39S|F*N102Kr~fZ?C`IOAllpQ&eTUAxNVHE+XXfAE6BhA7IArH6Syj7 zmM<5RDC(_HAv-{krCE8>)>4>Nd&h^{2ko=N8!7Z(E5C**Dnja0$H}llm{B1NOYm^xPDZszlN0jdj1LW`JhMFBao%-Dn)O zsf$2pZNG~L8^G3ua@MiJ=Q$|rog5(Rd0;VjIbI1N36-cVQ!rVI0Yp&lzzJP=P``Vyo!KKXFjOx5jjsVNsp1 zi8AlQkz5C*-*57#KJ5sX>MFlh6b{I-iO}f}#&qwDe1W64Y=9VDh!ZdBaW2(_Kq46v z)X`Cd)fND8f8V=w^6R)0z}p0R6POOJ8w5CB^Nx<+Y@dpSx=<@Q^#l#ycSSpNo>6#0 zG;Mb~MRCAkHdK^xPrIG*#Qk+D;+@)XG5SW8M2oftKe;tpF&FQ(r0tTBDkZ1-2RrVO z*B^5Ie%KxirUey*b1)vA^FJLQ<|_`9SvXnNUQ8=xFYwz`o`q%rXx0nzYDrd+T9t}W zNUgYtsFa(GFRSgY=MPL+SE3BXans7^z^>wn7ng`JzS1!noVP(JfDqI&iK#%6v_UA2 z#!@_vrb7-PmpfNk@+|dJk9W>a_Ye4dhc+B8C||Iy<9Vsnv1ch1uU?(t4O{>`r%+KG z)y^$BQ!j!152b_B!NJER)shVBp&X0+d~P<(lNp34h_W*+mS&q7lVX9^mTxRQwdOao zd@HH>iH#r|9lUPiw(jUv8+W_UpKA%wqINf!P>tJVk2^DJ9~>RNHmeVou5Sa9go4^R zbLQbV83714y`tMtx1cogMTf>Bw9<4#91-=}61`QOvsGNSht2vbl)qUIEiyOVLtq5N zvpSO-=3!Z@rm@U~XE~6Q1Ch^M2x#71t*&mtzs~Q^9S*T={Hhwi>c+1PvtHBqS)-a1 zU|ZsRm5nFGq|985?f_34Vz6-Rz^;E9aafoTwIa~DLC?M9h-+0DxW;(_z-VDfc7`WU z-o9!|Mk%Cbajh6*V-D9<%q_MOKqyf{a+RS9`pwAbtRTTk%QN6~8Ltl%Z2t>4bsvH&7P;R$k!;Bu|}{=fO$=sui8}b?0ar z0rA5%l?i!dH^9vOk(Q_3UVzHZGv^Fu90sy0YNM)_i6SHcAR=ipyU6L(=k8sb))KZU}E^X!?qQWl+6r-J{) zhO6%2H#NEDz{hpxMhy(~>3hS$kxmCbnTjj}3oHbSTgLBd;&(mqdn56?(v%kjMk3Z zQtrQaA0uQk0dSs+>bfB2mr%|CHnnn|uO@hi5^L0zzK%`wO35X~>yS!oZXqld>E5kq@Baz?T#ZFbG8e%uofNodo0e zeh*J(p&^8(1-*n8Vl7~de^C8e%Bm-%0q-9bYpW(EJ-mpCyb977{($a)g;({4AWYs35(5it)@u5KyZ&f!E*>g6?f8o2WYcNj3Pb zpqeel9$~0qy9b|zxa)XNJQO|*=w5fc3{aR%K0XRE zYZ173mkBSf3KWaEB4SF02i=%phOw`8vGW2S9NqgpDlGXH$KuHLjYlOlHldEZzjG*J&(s8J)%Amm#OKX zSjuqcGgB}nI-gUHgak1d@7Qw*VTW|U*{te0&!kBHAiI+-Z)FJ@TY_hsEA|z ztzJkqQxxL}JIy#vI9bzNr&(MY&7rBmhL0Y_RM5x2V3Ie{&3@o{$O`;EPnInux}lLX zZrZSOi-PaEbfuwEfOUK)D;r4ibB%K2LfXp!p=Bs4h^ss-uoZXubj zH7w^^?7=1Wnir%0!YS)?pR=CJQmVcN_hU?QgPY^d$+qS;B}>d^Wvx(*PbqFs`3cl8 z)7d$e4NEU@c%zWXE;vtKNLo^9N^rLh(V}V^o;i=t4iEPaU$>7A6c%>gb|i7SyMfQez6VfVQ(1Ke6N?XL3Pa)D~R=wK@XQvS`|w(d3*j&LPYfbk8l* z^gwq!W^4>uGEKA_T2uYi|Eyix(p8tOn2;>l)n@X>=epW--&Wi|xf%DypWC$m3KuM? zCt$Pfzg7C4wEvbL-hbW8=l<-!cu0sj!l#UMf)uK6(UvNs&ZcEv;Evwu1*AVNRk!Gv z4wvGhcrGUM@IeGfMsYh-RgQ0`YDw{Z;{%Yd4q||I;UA=*A$mqb$<7b^ry#dZ+Q-|6 z|7ah***_%{5;ivQB2W??=*&1EyC_*^FFe^0nyxND><8DTB`F0lmn)yeSNe_YH~WXV zTqVvmE8|lN%C_e2oF3?zCg9VZ?Va!TPI9?wM(yq6y~AzDuVpp<_W0;kd-v?k+gz@m z0fUOXJgvUoIsGx$m{sU>cl#7tYtAi`%e7{ep_XzRXL5MY@QIZhI_^k&6f{<>z-Q2~ zJWryFb~GkdEELfch=&0I7KJt0@3|3~@2Q;QU7?Uf|K{jasSAVeke@hangLzLksNj6 zZ$sgXR!h@Yti|zqB*l! zDG>{o*o~a?f&bvYA3uIc+oja1_<>^F9#RMndIX9`cl5Zyr|ZzLB(eOV=YF9(aE4IP zJ@@!QW-fjJk;^?n$C7w^1_WLXpFxVnhbhBHnPn6uD40TaG$(IRVe*B4egWzF>PK+o zS&ZBLb?OIv7Q;x5aot6K-XUU(?cl99#LHaKb2hcyczHGbj#I0ydd9Sn##bVaY5j{%EmfbD>29bfY8yY4$NWWtMStkVehr6052 zrF>4mcoIvTYXwbh5?{^dJ{(EL9D3$HvWFzpGom5i%`#B<@4Zu{)$tP}rSqX@H#)=v zswK2L)(5jxJa(Vu*Z`@%XsPm^l0kwvJEQ5?FR1p)DNVC@ufBNCk%@w=u3Nik)QJ12 zC0jeG<)>Tugbv{2&E-3hsug!6k!9^nqRg#!h&8l%%xLhvHX+(dn=8e;AOe{tg3(e; z04>?tc`bi2eq9z#X>ih8^+M%d(QpN5V1^EH7brFfJS;h@#5jxy*aY1HM~jq%ZP=h^ z7W}d-X0smyl6yj9t{OhEW-z!x4zbHn0F?ixr9r4Ewj7w*`yCV1Y`!GcP4Q-C8wuFf zipf5r;2)r!IN|6qwK`-Vo=wE*Xwhn){Oa({+3DVo{Or)`4(|p`h0hBzWy)5Zhsmfb zSseplMy!0XcQ*Aqb|tq(dJ{}xbO2DeEX4|2$wBogcJ6qI7aMg}-Uk;(~mbJ4Kop^YF+ZP<6+ZDl1a`@u>FJUb6IZQ2YEjmCz zc{E1jezK4p9?2QXGcAo5l;$nFH+N{cx?rhi3F?X>xema1M^0To;y*olV}jp5a`WUE zmua8RS(QMo?WAreSHERgW%f$yXlp|zgOySi?xj(&ugzJli&o1Ez+Y@({&PtwyReRz zg~bKhLi6RhK3~uZHg-6WDaTsD+YcDo)E}L*;ly;5EL-O7glwAv8Of=nge51ahF(Dp zfLbCOQ{WBUE8tr!fmW>0jr_3hd%aYn67Gu(u!rLajX(^VN}8rS;Q-oVPtZuDi|%yO z&l6q3?qVjIQY1Z5{gX*wJr{9nWhF|D3urQ_NY}b#XYj`46X%T|qGb;cgP|}G1~2r} z*T4_Q@!ji^Lr=zJI#*T(Iv-S5a`kN`3Kd?CE_rE0YmdB3St#Z%$fLVs{)p7gG!RvE zo)B16KVL=+d`a_OSYvn4`3M?_Y4*{~cRINF_F$NX7QzJKw9_aR!XM#lXQ z$ZOV;_dgnqYU|q7vp?yJn|9) zlO(v3BUUkj@&hXx7OG-p=8M=sj=X_dpFnA&Iw#)LUKQ z9C{qA;?{)Zj(Xzg_xViR4Lp9X>xFoSXv_~5!|O2ehT`r{0JJ+wl$yA_b`PoW$Zb>y97%d22E^r`F1pbl}J zH*Q2en1Lh{6tlDiqkcT5SJTlL#k;*vXD4L#CJ!AuM~5e;66gKCG=PKhIq|xF|C%0% zUxHdH-$qZSoq^xgt`W$#LV7?dyebR-p{Rs9JKugwu9yMU(cs$o56Be1k2@Rb1}hR? znnCj&s%~j;Q3*z9=L5gf6&QwDU4t(D_#@tyIr<@XuOZw3&Gf5pI3>Y+r)G7XpDtI` z{Nu+yN<{2z@qCB;`~0s_SDyw(#=~vwFXeNa>%Y{`k{ZCA^}pFjum7!DwfeCB-@_+% zz7a+}kUkXCK~KXoz*HhEfM(jjN9ms|7oY=s^5Kzw+q8FlvVU~wto&WEkBEEv_m#}j z?Kj*1eRS-kssQKxXB_-}M!o$*dp&yHFWcp7^XY}g{Z4&UcevUR*UF7Y=iq{twUgQ2{2$Q#YGLR?Gj(_?LsbWi+Wynfcj#=9r)*?LRieG zA~!0TFlB*iA*_H3S`||@akb~iw@1gPJlj~C+dVkY6Vs46SI4wB79Vv31cmgk2 z+aKz+LhThb^;I)#JhGvDc*IO$fFTaCi>qYC znSD9g+c^b!oEm!$&XYC#$9pHI$NM{{@a6RE_)xDZ;~q%KEO`>0uvRw9Wh1n5yS=yj zy4^0?@uz>cPhmYvqc#rabY4}~75afTVbi-+J0tVFNg50?=Y^U|CIp&I&}vxJhJLmM zAiG;~;-%V<)(6?lLi2J46fUp`rFB`buryFyIZG^L%wQAg_kQSiV5=Hk7gnHxI~_zT z#bg@`TSKhvT4sF$35$(}iV6D74 zjsfSK|5a;^RQ%6Yx!!!(|KGz$JO5+5XFI$ecE*FS9bM5;ApfL0_x{!MudDhIl-P4z zJ7RHh+{t7h?Vh;N;1^<`hC-^3Z=L`OKbd7-={XGE!J|HWZSX|xRrt-83S5X0@LYx$u~Z_SYg_xK|8_TlS;J>y|7UM=wAO%lRG zOc{GLNE51L()HN(z5RCIVOpu|3uu;{Z@RJjPT_@gbrlg`Rp4MxxjqhLvYML=&k2_=2A`Xw*yL)lAO%NIdSYWAR+tCmy^lxN>uRo+&=HtKR!RqMwzwz7> zN{xmeo!_ItdHnRTga7;8y?p09{y@vI^L6Fp<9~}YPG=>Solxx~{9~}+M@(EvpM)AQ z9?=Y+toKB^h{W3^9giPWVwlH>JdwHs@?a6+8*1W2c@{uNDSRBDI1fkIx6>0jRwy%C zXh!Jg&<*?nz5O40?uagf5j$(=-@x#pH|V2ZS7Lr0)6R8zX7THA={?pD#{+TPfJf2D zi6*1+VTX^QBYii?cL-@F>nR?~W~KXM5I#>>iZxQl-VYz~xy|)oUcgw^{$H!s)As*H z?P2}Dhfgds|NbAZ{dci;$#diW7zXwd<%f`hV!1}UgUjm>i8ykd5&3<$OUIbwK8sOw z@T?h4*=8!Dg@J6t6XS?qji%iW}OJ48!yFY?q9Sqvxw2yZBwTmK) zFmWX*xoy@S%ReDH$bg2h;%;SCq6b^Kf>A-d!Iwb~v!ff*o$2h|yRz|Q?lKS{HIm%G z7(I~jG{_a089y?%tC|};u2{70-Y|;AT_W8zkRDBkMr6!S??6??c}#W!7n&4Kgc4|= z{uMfCW6Lj(>ysfm4iKSD1%{N*Y_8)xAK%eGE1#XF|1u(q`gBFAX!ommidL27ec zG=8y>`Zjwcoc3Fs-z~^5vr)A-O)aDMy|~$|mhNSaU+I&aBkJ}C?s=FTmd`!81A(o5 zu$cW~!3)f061`Pwy2aCAvg_7}*FPh|I;8Din0+Hhi(veC)X0 z8Q6&*<9xye~1mVf7x*82?T%SLab1F-cx_wE&YYVkHu4<~~~ z2v2`p^JmALs5fPO!eEiEEH)E{UScegXuER1$hn<6=@emM>(u<|o$z%3A)hlOS}ym5 zzHQ0$UYyZ0b0mYaUs?DX*HO$5mApiz;4P>{-&DF7C`hMoDqZvy(&^h){-YR*i}Cm! zIp+V|%hbc?jy^Zn|AyiFZkxTotj}&^-u|zgj{i}wR4f0daUTTui|c>4GyXf9n=^Re zCgWdkHXg?R9zM4>{_3q@s*O4OzjCXbivQlMl^^Us_wkW-5ixr03l-ZVPSM_UUebGY z5tiL~bMpO;(;Isf9bHswzeFW!9I!IwkDML8(uT#@WB<#eqXYJ3gpScXejG2MOx!S} z3zvBH;&I5XpgJC3#4I4pqElXLpie_|+bZDf4BXNAYIlIwWYFBhTxF zj>fQz(uErebU7}$3#tAfTE#1)=vqLx8@1bAY@^*)+8uR+s%z$RK?LeM<*6jXwxWKL ze_5v-S82D2+E}bjnQhQgWl%V%sQGkJBZCNj;`ra{{HJHZ={9E1|4OUeOv(Rd<-z`Q zFQ4^wXaD5L+1b2uIAE2J_fzFql6m)yk5^~M{mA*=4>A75Ni^;G<2+P->EiX&@d#n8 z@9*t6n;V;(PfvHM&W_l2c}_2Ip=#lcd+jt!PPJTa6rCSlSL*ABl}felAQ=1)YD}&J z|NH{Oa}}{%a~1yCaJC2Dl{>-<**g~=>`(kqRyp>1e#o!S@ZK0+E)$n(!|^oedX&=f zM{aOUzFXK)+XV_I;X4kY497j?tfNHXMkZb$Zl;OrwLQMbZQh8(7^G}rrbB-73=TddCDCgj0@B;&Xeb(D&@9L* zA4s(pBUDWz1PyD%kqAMTds7jQDuI*2Mc{TLoWzBE1*F)Em&z+dUP$RHwb)PyK>Yod9(;JN7lk%Y@z6c)t#i2d`e6U{-r>$3=R>b?|#3J9T4SV+)wrej}ZNKz7yn^EPkt>IN*hPDh;jt1X>XK ze|1*QLZLXap2v=tNIImJtDSGZ6+W9lAN*=XDN^e(=wA}eot*BXhl=moqEV@%8F{96 z0q2ENfptIsXcD;RLl+S0dfK&$w4)TH+AHXav8XytQkel`a&d}+v`@lobe$+De1ou0 zDukunQc;cV<2TNwCofxH2JQrucf3XeO*;}BqSs^Prt#ZvSDX(YA0sf~wHHa|oa3_? z!hmRx68{|&|0#<~9+I7Et5$pJgdQ#g&JMXk#pqWchS8_lVC?oZk7dr|%b_>yUIdSE zHK8Y4mtJpCFOvAINo`3)uVy+#L`21Vul5hkx_k~q57B~m4twDOK%;Yhw&nSOFrQOr zWnB8Tm`#3Q=UTu4+PnRs-y4vh3l)i@{o_WnZ3NzEziID7Vv))jegJC7K92Szi%@O1 zFS#krfrFNx_RNKSZLL){)+)~aDDuwnDI4}r%YdxCE4Lf1Uiv*?0Y6nN$&|>5d&7>` z!!gw6Rb6h}uoB+JMJIo#ER`H@?RK zQr3_14c2gtW1A>aJtIod4JT!3XS+VN#vcTZhU;`8m=Zyx3RRl+Qg|RAgQ`^%rPNV^ zYI166QI%3{PHj!9Vye-pYLg#hr}1O*&FLx?sx;?xl}gpBbGxdVoY_^?=IpMjM)j@| z|JtyFX`sfUS+_ZZ_jy6%MDn8Gr6F&*?H{QaAa7 zuA3zxwZ32&ib!$V`u4(ou#loOJjYOLMa|^eT-|`821-g4fP(snmg35=8p8M`2+oG z{y4vyKg_S@kMgVegZyfNF@80Fh+i!*!mkz>;8%0Ux2}z^e+x@@d~p|=LkG$!YNZM! zX{8FB=_NGuE$ex9vvR02o0W9SZdR6P8it5=MjT$OD3*nGS!(ZFPlOaf+VU=lD*s+(k&R5;mx-QGSudb7X7 zP|2~XMnpXyp$f;VvAv5PK4F&Zzm+Jl%|VsdRzzW(NOe1-R?Kf{eu`O4RbeLkVduMT zW@^;1#4pH2YrH!M$I2goKQe7X)XTVbVwmF$r>s*tPX8qdM5C1nM08JzsuVEpiyWCB z#86SgMw$snpLJ=a6d@{(uP?yy3#-2L7;gJ;p7vXh72IZ7Lj@GvH*91h1EzJsq{n^k zC<^oXq~B=o6yCdoqT|57j~@9O3B^?8a%QEO&^L|TS~a0Y8oAAS-jF7VhNoJ~XX&P= zdfR8|#;1DgXX)mr)P78_s!p#A`56-|AtZN{h}=mD$Q>mfcPz}57-r(9goQk|hX3+# z5%f>?xt;qT%GJBIjXC$fssKI}|Eq=N?}Gq;@%ta~{Vy6pqBLK|RZIXZeB+m(LL3y) zDfrH}4!X!t2_jhRtd`NN!R>01#Sx*aB3D-W77(HRqm0)wue;5<=8SfB*9V1^VK9RZ zk4!{tRB6^1r3)Wh;0Y&4myX+gN8?_ru6F#0ZrSzl+!9o`W}TA6X(pcqzIXirZoII* z8-~*%?z{$m6v18$`|OSoeJqY~8&mxo47T)ShkH2qsrm~AW~4});o4x}Ja?{U4L_9m zsr-w*uOj|xX1z+fUagiU5)B}-LBi!OkK7gd*QP6p{-`aJ5~b3GQe{#_s6;B@q?+%y zaI5g)^GEsI&ia4pk9yg``^`aGxh$r(ri4e|M&2b>p!mVuormqcKa0f z;=4yboV53TJU!lSf2YFL%qpW@aq)s+H0G#3O1R9Z`XFvT=V$r(ufqLKy6I4#*3Z28 z-%P*%sWw{={=e_zBm8>*o&Wwl?_GIa>FoRO+K<2I2Y#pL2hab`fB&8r^PSS-@0hN; z-h7^P#hK5e1Cgi7V`qNX3%h|o!AoIJoo(lwg6a;O$uyXZ$)5+k+OHKIe>fR<@l|2E zlI;yA7cO2Y3Y|VG&GAKIe5UJ;#3k^s>yk!^29)kJ2)t4ET1M+!%by(|JasOjX!3MD z=I)Uft>?dIwe)E^lGoq!JLJdgsk6HO^@q6sAM*!(2U8Ee{=k&_=&TN8Z7iu%QMlf8(u1S2JlMK6B)MrCdqbe`@7g z`9c2Q!zcgU_!6(d@}68dqsQI)53JPZpU?Z`^OZGcp9PzU=WA=ucCROJ z`?-TpP{xE5=%e2`-srs_jLD0Q^WF`7vOVPMYtGTc8#ycApAgWijiz(RF#=A<=;sQb zIN{>*Hh@h~w1U`S@E%y(py>KlQ{9clhb$4srHJ^-I-MRD^ZCY_BSXkbV_DxH2)cw9 z&T7|L+1YvC9XM<2f-Gy{#Y#TkTyqX_*f@wVok)vt0b>zq0v|r%h96JE3<42soWeiH zD_R%(w-ydY`Om%_J$&x@b3^%WVV>+Z=E(n6yC6%sQ2No)h@DEc_fY*2-(udaGJqE9b>U?2M97VyzB3 z;O1H-|CZ%jVdX=$(kSoMYPHpkSGzl__3CzQb+g{8uU58en~mmfy}7%$@o}XH09F*@ z`;j*cS4y^&53Ta{<|e)}g^HW2fY?ibyir@N@9gb0cbb*v?yK6zm3&TMa3UhCNk{|; zTZ8?PD3Z_RV!a4*;vf0k-l%5+%Eu{MCpTX1Z0_v6+*sXsS>Il*RCdd&8@rXtYPq~! z-r1}`ZoMixDW@QIMyvwi&paKKY*$~_-gmFC5cPaxxB2pAb+fvaf_Q%2jxGYALTE!$&o_P;$d)(D)s3B3yQ}rRm#x+MW@UGE zqq(`WT5gp$c3*C`_IB%a6WP~;amO9xbNn|sZaaDqrMCk&qL;y9AY~yK$z88AJ^vk7 z@#l{E9;bscpwc&*GJTcLV)SZSQmRx8L`xzrLbb^gV6d14%v2J{j&+_my2f!SzU@Bux!R=o+R zBt|y@qz!8tHFL~w3O0e}EdaHVQD%M;5z<7xAy{qB8K|0b@dm(R^gaz#f`l^trrk6M z_i4bgIMOPIZVozy?yUf}ll9Y>$j+Tvpch*27Lkw30?Lq&OM#g!AD0D{DIb>rw2XYb zA=ugS@g_hQl8-k8JWoC@26jRDxIC~-`M4ygdGc|2a5LoNQoxpzk2eNAPd?rZ@PhL3 z#=z&v$9d|g)HFdYop?dydlt?9JHMxX9-^0p=gzC`gOfcAbd~U@HH_WR{v-Et3mpXA z;QR+2z(1V-+{5Qn$DjA|v)SMV<6mo)AIASaKDRXfDj0;VjT?+Ve5*c;|2=$ed;F6I zM{HxZ{U3%pb^otcZdED|_WyhM{D1$?|NHE}u7(471N-?(Wv#s8c%$wZAG|(anMVE9 zjg|j;k$)ykedtFCBlDcP!MPWO&sRuiTj4{|?-i6ff&suIC?W3Fgxw3^j&N-#&l}dd zG zlHTr!dQj7-@o#1LM-z1QTR{@w+%}2}knrAVXWKJm zVQ7~9+^EhN=p_-o-Jv$Nx^po6JX2-v9QjNXn?Itri`cVuB`i(RDJt#^1d1`LYimEN z|BD-p@yQ`MI2gHupD9?%+Qo{q{>hNS5ccG^0}7+9c5ek#2ZrKy0Qu3-o!km2ioSAt zklMX4yhUikTkGb#i`#({4Q>Svx|$r_4uf8};qCP8VtRf%?5VEo$dtQ*ijj#1x5I5V z*!{Z#%^pE-1^BRYJ7A`fcXca}Mq%U+Z-vRBV-oo9-Nh+i*wcw*+Y1za2mv=k%7l$v|(LdcWR|%=$IB9WYZja7U3px}DAiw<9%zDDZ9v zAqsHU9iY~BJ2O5Uj>pkO7y%&q!Z&kKv zs>4wJ;ay*jgC3juG5guNOyUB^umw1jo5B}z8b4b%)48m8fXWAr(he?87+-9xRi3TS zERh%H^0@1~#0~M)zawyezVdb9`yFwX`PPjtia)F8P-{Dr$(lIrT!Wc22_=0^g6_!3 zXU|t2e_b#v^Tpr4E=atJkDot(teu@bUXep#RFMN>B%XqH^J4cJv?afL;CBKyxK11& zPzkPi7JAeS7xcaG1xA&8ww_KGLoX1d6I?HP@xD?o#|IBGRbX`i(x3}^{0Stz0mAq% zuOmFl(9<(0|A|yHj`fx%-U(Iy3)0Rw!n;SFqadcYL%1I<#shq0cOJOIqY138{%?Hl z#bcA1{RNQyGcoO7ocQWw_(8_rOvc`XbbomXd%yhl7oXzk3p5YXbt%#>sRtYSFV{7; z6MJYZK)oB9jL=@SB6cEi(X6T`fUVbCyu9qo$%O~VuH0Pz7F0}A0!B8#9m%dABa2^*G6CaGb?qGW|*>xjV7LE6JGpKZ1igkj_e@^~=+Zcp`nV5_Bd*gq$YZbvzNV5b`>seeu?-In_Jr)4Rt9T;2Z$TPQW-4t{?nqtRZ ze68N*VmGyVKPQ3y{A^v+5riKJ05hwIQe1`gShOc&H6AhiRqQ=m*YfVgPWdI0VKs&# zwj-Ei0=rvE*_EbyC(?b`{2mx!G4)Ba9xWULDogd(v)ED$`?b*7KC`%gF zxLrR)r@{e-yR*GZvFS( zo9%;xqn-Bd-pjMsTjI~(3K_tP)Gcc>5{>O1lh~EF$6)YbkjG@YwA>%tnscWNkeLM% zFq+w-L`x9wgrl~11+~K$_3~zUt6r%$x9ZhqYpY(X)t{|r6^=F2qb)T$}Vd#HH4H%w6jV_Ih^UkKU;^Po|xd>kydwg}9IKY&}krg`}rv zV&45xpPZ@2hThCfr{rqY>xn0_i6ReE^UIr>hJx^ioRJU9-NT&vvgVXc_x?jpt%t?#3tQ}x zjLpvTUmOj3C(-o)W?9R8ZZVKGy}UsOgsXdYjn=J81;iU%QL3X$a=Q$(D#$a4w-#r8TvJ-~i( zbN|mtf8A?+e-L>=czYhE=lTDxHd?KE(*Jk0R<8fC{eR!GWA)qZbo{x2|8LXL_wD-m z7BlV@F=6cR;AA`vx*o+LmG^O>)Y-`Yb?WW+Jdl-rAC|!n^;bKsTI*G1b*Hu6SgqGu z&DCvSz18Y=t-9N4RJW_IUVVJFo~o!Mikx-fi|%By)!pjC|F5pLdR~8P+#789@ZZJc zKU*DG2HbFn|6PH!iE9ag!T4u%hS%)*o!P8yZZ>Kg<<*VNm%FR=PPx9iv9b4ZwOMVv zdb!uA?!A2Va>34AY+b;qtV=K_4;mYwbJbN zSL>U#>T0K6-B@jUWv|=xHv5|!US>a*K}K7oXW>)sc6!yW+gz$sCE~^>9*ZK zYkt;Z&5>Kv`C|tLB$gIyl99x1fLlRj$!b5k+8)-_`)z!u@k;Lyyog6p98Ih6!g1C zjN(0fW)wuMh|i0KiZk(f(U37JJ~I}dU+&ssV|*DHnUNtrH%=6j<8vcH@jN~=66shV zUj#y561V*N`Sk{hU-Fqz&|{x`UQ~k5zxv9UE1wsOM7);IjEo-ZDL9)IGER#mWft#$L)v!H+S8NjEQr{?Z_}X_tA`qymK#2 z4ey6~6yN8r8=;Uucif1IB6P>?h`>X4-3rGly5m-~@J4ss5=BJ1>o!CXr8{m$#a+7N zc4VlgyKaV~JKcFJ6cFl;TQOot-Emto?9?4MBO+DZbt@vc)g3pZ;$hu&J2HgUUAIF~ zx9+$R5vc2qTZvbtjYA_y5byN~4mz|6ge!zuX4_{^H}mzV1jCaCW}^ z)``RuBnJmE54qdZ=wci^b&ioxLub3wn+^u9_a5`!((#P5J080JD4%~Mj0es$-Z8(> zQu^Xem^(02qiNvjDWE3w#zQ@YG161s(_<<<1+5d~wF^B-_Da2=!0h}q^hf6o6o2}Z zp={|%+?0_t9r^h12=5%Udp??}I^>YkC^Ymz_w2P1_pN*$iaPofC=~M_KnnlIYm55} z-SMdJJa<+`WWNZX)>gJ4AFGKJ$Sh+LS@a_({0$QD`?y~aC5l@yMA*d@R>qTPP2{bV zoWlFD-*cW6zY$3#=bP}il5===a8Po}RN3!45}5PVbEj5xKG51R2>_9PVP!7}aM-6K z?`i^+0*)9>haE3)#(hWN?fk9x^9WkgadMo%gE1XB75w!PJ3;Rv9q4ozQ75q{;ZMIn zpFTLPl2d7xoLaTyG|MGtvjV@%@TF44f9iG2{{Pu~_P4fmWPc`qg-TCL0tt{1UJbCD z^5}98(0--dJ$u`J{+w7LE{UCEI~3aGf4?&$$(A2^Km#o_*|Rj(V>FtVMk7bF34b=> zPi1wr$f{VowvKI=sRKe-u0YKSe2Q$fyoo(m;D69~v%CqAH}QioSKvQ_Q(1!!73fed zm)T7%3rm>oDIyAJ=Ha=Jmkc263v7w$&mWgR7ijuIsFcjIudWCFjz4Gs@q0fULUCe& z0oqVgM?vwA%`2$G$w3jU;a(XYSC_-b3O&|h!bKjVL0rrWen)=`nov<|6Zl$=Nm@A) zwWvi(+Jswbf<|9*UEHEBByr`q$SKkKx zF7j@#V>Y?bFq;?okAZD%+n@8Q5o3eQ#ugOp5P|ogG%r{wE`S?A40Zu3lb(W@ z-Ny?g3(E8YyNC~PGZ9wDktzxOZ?Gzb5YdDR?I>s&AWlzOY;vzvgL8Erd$}zj_CW1)+$*1;@VC= zED^0liyLKH1qEKR0A?x;s3M`>|K`UQvdXPjsY=!=CBb&Mc>zw(wgw;UQhe2n7PLLr z;fw5u`T#e-Nrfb43P=PS&+}NzgncB0JLtwq7)QjxP1U$l)dnu5i)r>`ajt0LH+P(e{U9e zGVAx7c!5Hz-Z;$%@KYXf{ z)czlw^XdNo0Y1-ro_XFind>xp*4DK)Roo~2iRr4m17y-;4A=kaYPFKS|D(F{$m4(a zU|1(Raq+ps{U3Y0W%fIPyqIQt_BCGSF`Kv+q&(SsLA2C-h(qi~t}sJiR%LXslFn}t zov6gR?C32MX`#a&>I$$Of)r(URtGRyxZEDitF`JxkOVlBMsW-hJ~-{Nu7MqdVZ80n zMs*0IZ;9w+SslA~djy6*M^#4Z$WNl!XpT-6vE=KNnYljSblis7NgB*IecRgkzP!0y z+1NeUD^>Sjua~Nu<-O9z+U9O)d3||f@Ac;T{$90u6OK=P1e8DdHEilF!CtK1T2e)# z0p>P6Q9t0>(O0ysrFflaFRzRf3yhk-Tv>KD7Ry^pLtEq|nSLzay;5ByZMAKt1?34c4n-_qxe`MSd{C59pK6^zVV ziL<+&0=C>5zrC8vX`9nU$v>JOI86!QtcSY~Os$-tg9dN056u)|G$i(o*11b=k89$>eim?)1=X4ywa&S z-N8S7npB#U^Y~;%Dl_hjqaV{{c@vIgO1i!6`u1r5VN%55 z*|oYNDJxIhL*$MdJM$WdgYRMU1o}*C~Cv8GzEG?eQLWEbcV1TD@Us zEq3fi?Sgv_$X$gV(2bT~&n;Vi`)WIW4=>ch2jGX*IhQr2b!qlgGr!F{3J#Dj`eyb6 z1-@^KL0zlSR1M&3vnL;PHtRUYW*nQ0WbOT@s~qCLK1yql${QILoPMZ(INbex4FwyX zWKzBym6d67YGM*5rW6e!k9G>BTzSlyG`IDFSSwWl#t_uHDwspXrxi;u{v zjTe%e8n>ylICU1Epvn9_L!0B;*AY3%l9%1f7pzA&Ht^DPt;eW*X!51>P?%yVBf^}1 z;W8-i@|xA`m`MBZeZyL7G%A~w(k3rAOV#p_c(|HIS0|1Eye+hp%~lZHbivx%&Rp6VvDXlIW7cb^}ERS)cz4X)((>cQ^% z%KAaMw7XtkEmc?6*GhE|y;7yVQrTNyt<)_odbVN6GZN`2Q<#4x(PtSjLj^BqyV1%p5kE&6Ul~)s>Cq(#GcNy;8NY zTrF*E?7uFpRaOsP@2^(&Umv_4b29B(8~%eCwP7;tZq0UJE>5#{Q7pJGO0P?F-G10yHZZ(e_E-mmdjK7{{Z$sOqi3D z>Y-VBva|U#+n(#VVU`_7UF#;2zaXQU%eu8 zPHV-h+zSBE;4RnVh7c&MEitnnxJ}n-(Zk|WQ-+>6`{VoW+v9hKNBi%MgQNPJ(^m`Z zlk~fGceiK%*xhA+#9Pd=o3IH~og5s#IeWh^>Ii(-yx^?#WhM4N>kQzgBSm$vr)gf~ zV1~KebM~Rl*_r3a*7qXoa-Bs_wYPNOrFC+IS(v!eJIK) zhR522xPpAlX_Z<(s3e|ioedNZ)do{#eVjvinEQ-Pq!6Xskl@e7MP{z3m7vgwf|$_6 zi-gF`6>~5cNP#{=o<{f>P5U#@?_+9GSeiW(>c7AS%pq3*H&T2zbipwK*C1(vIeyL| zA8kyUbL9cics`3|3&7e}G}ush3)?XP&Fj-GGP{88D%mc>bHDmz>}dE5poq}0u>>|Q z0*OS4%Ip@?zk`1?7(LGi6N<@SMp=joSyChmib!57#|zB}{yYwYk}2s6uuJEfJKxJ!O9_ zMoeVe(h86fPN-cP7hk8e$gm`kf<@WD^nox40Q`Q#;)~q&?Dv~lmRQ}WaBfkYZ{^c0 zEV5ZPEAno|c~*$k#PBN=1U@k(0G*k`!V*mQ6?@@-a^~0z_&&HLO(<#_%}!BH*tS88 zL|RJP3bPxLZNPA*1q+$~3Wm$g3}^z4V5J}4f-%_0UVv9K3X~uR82C~b&pbPJ1KZ~) zpB77cOfOtneP7{;Pc$VV{bkazPwU@zkgCUc8w^mHyftXCWqTGJ7G?3LzQ5_Ft;>6(a#b9 zWO(PKHzq3wEO2!-|Ktl+~~SsV*~0; z1BL#+?bHlQ|;M(X4r~l}Uv3L0X!+(rp7%Uo8BCoo}dMTGJ0htCiC@AV0S7UmFMOxGn&hU5RM*5*)v03=#my_&{P=#@jfzD_s-QmLmf7bH zw`QCU6>`!3B+O2p?8TG)+S&b3VA9?0L#lIZo~akj(2ES*;JU|sqK@RBim>})?@OCm z_3DL%={a3Y$BBTf@VymqKM`RG0eAS48_jm#xiBn|dN}Yc#|YRw{#z8xz(@L0BzH>h z1U2-fP|JWT6eYl*e%B7hfGl&XIzr5;S0-SPm$A@gQFADRf5b49W|hV$ito)1H-Yu? z$--9`H!0dxGYGce(}@p7S6tY^qM2&}smhvN;~}%w!R;mYb2{(gVNSc7S`(y8o|k2qU-u@9mV#Ow590&Fp#_W0CrB%6Cqdl1kx)(ZCABW?AfAIJ z1ipbp-*@Qk(h)d=bPsRPpy{Zz&j1zl8UEs2&|Z5%-o$*}GC{N|YiMOq2pwR~I++1G zBZeCH5*? zJrx(sn%=09-KcRZ@@uqE&~_rok}9fZORE`2$R8HMrGrT`#A$ zAL*(H!bzUi;Y58Q_nd1F zing}dzs_Lea9sau_o#k)s@01IFWI{~)t~-Zz(Cb;K}&k8P3)Vd^mTv)X%3U5o%5={|0?p9{N&zzY72eD3PJ#?Z11o?^wKyBM+e%tke zQq%UDeLG-Z-G0Xs$4{0^2QQv|-VOu=k#nAW_fCfh>0^YBT4CIss=l`EW*f`ERRysS zJ#kVcdI`jexz-@m{eaoQ9N=MhdmW6x2QK>pyK&Yw9jn9D$asbX-ArGe0trN$W*-W; z=Ru|0_ko!{Y;F8S>b0b)77-HYbEC8iiB$2w31XGRu}?_Nh%49_c&)l8weLQpJWIx7 zWy0f2Lf9UuDK(`kW24>Vb|)KatQj&OR*$2Z*lws|-j^Z@AsApKbQ`@T@QP{D>4n{6 zCX+oyNScJQYhHucUUED(Ff9%Z0^YfnJ5ln|segkDxg}jJA$$vKed^!ftZQFEiR~yA z#nV%O+v{=17ZAR-xg*Xd#IX&{)JFOLyAL{qTO50lFG^jd{jTUpTN=Vx91BAqQZ`gB zJUZo<_*NF{bwE{p90s%)OguL|(cgz}px#3JdM^4n6lP&9)FZ#&YS~R23ZX=f%gjsD z?%<(^fY9Y#*SluE+vO5GZgOH8tE;E9cUYP7RXvKh?U2Z1Yb*f4zH^`|)QQs0px$}MdN8HdDncErd>Zwyp!9k7DDwH4Na4if;S+Onm-0u<1W#2dvO zAxZw1R9xCYZ5539h}}OI2T`Ki4^Z?|v>?}Q5U)K@kxM@1^-topB>#Q$nu1#t0J{g> z9uW{o#Xi#bi&;jYjr(V39GW0a76=$wgeH#b??gQP)<71-BuZt3CJ;)4%+37m84mxN ze<`U1vH%OKXVIny$AtPW?^tnBkpahnon%`P{KP|Jfy+oKxd*E-m68cbi82`jd9Do` zvVKRgJ}nicEUAb^S!FDrFc(epnUO<^%mJ?>-zXNkQVgJGK=zR0G_WRzVmx{e4Cr>< z;DfAI#`r4wj-3TA=v|9Pa;)_}>P*1F*DUB9FE68uS?weeVw4OuM7I(?sz#W7F%5#4 zQpSPQRY=^7B+LLs9ZVP+CnAh^96%TeJS52=$r?V7jJcP_oqIa?MuCNS&0u-_+z`h$ z!wx^hp(7ns(04LP0+tBMgd4$g3d)Q!SGBOWRGnxS-*;f1EsrO+kk3H>5mKw)>4;XM z0AxvFrHTZKa`SjDyr{f7Of1MYeycwg7(7=<*jS>Gt4u9Ip@4%lqv@jxQIiPDBTj+_ zg}fG=l~5L90oxEUn9j01zhw);%xUfNv4-NL{}Q%{^O7$lOA&!frK})jw#+0X#u_9t zVn%?@z;ldojxQud<0~U})FummG4>o4h(k}!Fh&q=cQB;5tu$LK5&{Q06IM&}>;Nh;BxpL~ zC2Ty6?z!|8ti2<{a{?X)Nm}4U_dQMl>ClN!l3z-oegH8*_6I6XAw!Nc|s_SEb}Q$jN*+PLf>^3 ziY#L-kCkupP|4g$C=ed}z%<4_CxfmQM@h;G~(i*h4LH3p%=DTx`x=M{OE<4pim z&$d9WZ#xNP{%qiaIPOx7=JB;L6?MJ^Pslb7rmn<9p#Z~T^fhn8V9PaTgAqkAY>Cit zTM~f+R4ptNB2TXO(f+J#P3Df!!JoQZa>KN&l#i(n#VyOw1fRTZdXL+nA9o1fQnqbI z;F#JnspF^6trTrg1!m;i)DnIy_EJi8PxRi}lHH>{#*Eqtgea&p(HX+j(5m!i@+c>X z-gbo92;%wu#vtNB)vbvKeNx0r&0ZsvB*buPmo!xL?n$!Y-~-5p)6~9alMOuoaG)Vk z2*1~{gF8hblu_2B5cJCn87hetMK5JVMx1j^o)zr_H!wS*7+009k@WhbAxNLtfB%~r zB03P#prt&zVId2p?#YnF*s%r)U_nDb4>&}V&?BlLK`@r7^%UEHZa5EpRwa#&kts_@ zoQo`uSur^P%AQ7BC3?f+RU=^zFl|Mh_lPujJ%4iE!=F$zFX+dHo+*BNgGXWL{Loy{ z^5Rb`JO+ir*%1eNiZnawbX=1?B$zg@w6xUX?LjVt=?CWev;~n+=;-|)TCC{ofq4ixiT8lXi(zh=96fERZFi8=O1Fe@ zW;nn7)Vfuzs;+LeY#_|u;hBkB-F02HYSpTBy~i7W2j;TTf@O95aDWh@hMJC|4g;Rc zzok^z3y(R1f(q8j4X$3Fs1hD@9lM9yA8h!+4q7#X7#br=)G%2~xt1*X! zp^Uj(5AniDyOV`Wr9aXZ-)!5}=f+{WtE4={ztn$URI z>>XaVPryqDEN6pG(=#U=gcSf+iM%qW+M-yF&Y&?c*Fdu>h?bDhCZMBjqw{@DDAYPP zQK{o`9sVs>pxsPT{aFd$SD+yoH8}+{l;dz>!QDSx1Rub9(aBf`3{F(PX#bD>g87Mb zOhz}qF{K%qhYxX(& zRBq)1vIR8%?5gk$2L1nKB;jQ?ylIPuPX^$|M2%}`*BYzej9VJoHSGeM zMs$9MjOHeqdSweVKAfbWTAN^Q?&KsdtHl7lx5*xn5Dd+FtArQrk5Dc3^tTIRG z;tU;ou$c;CRAA*QHOR<~1AZZPHFKEyPGgAiHAO-G6ly6Q=LtM#>=5Z)TS?bztv=a* z{QTLogO@LD!#+k6FVf1y8a^=D8F>6Q%ex{Izfrr8)ppFLhgrHGb8ggG`F`gy%R%(s zMru#wzBCjYoTyOn!16>v&D`IbY@g!t90%w)z0jGZjn%|S38dfz9&o!GWMhwQw#@?c zyjPsWmBx%<_67~JXdN=Oi^`kJ@Y}vZ=(l052XJ&LJpihv%%1<|;1I~|!J(nN2Ty+I zoUh6`N9igM+%-65kHD!DE5;}*!*@huUvj-RciVxvtqZ%O>`&|>7ZJBbsC$HQ@N1D4 zHFD+{b|@e4HuG3@>Bk9F6_ezSL+`K+MlLK_Cr-*A{<{yZkZR;_n$MPCb9*K(NTRE` zc@>bmR_{XJJ_W0Leo{hLbOC1$Mdhh@*qU8=CwfwwS6tQmq29JbU7Ce9{%UEVO*pIv z;aQEMTl}S(tdDJthHF%^d4LTSg6_qa0?~ zvaB7qp-`UXiL+{gEVi?wAOq(mxc8JjHKRe|jatqD??8;Mosf0Q1Q7L9{xM9QyMf-1 z@G?A0%kdQ#fHtoZFW+1(U$)Bw6o;eEOqYx7t)Z6Y`87jM3_al$TggDEwIQNgua6E= zW;?-6n?y|K`C^t;<{&_NY0m!n;tXwd(wVgR*Mpab`$tE^7mq+$`~1cJfLXgno>&8J z<;oJCz#4hTq#kd$wY9Qrhn8}jqAWll=`bwu?K4WcNm(;v%D?SvSFGJ^X@ca zMX(mN{nc5%ngsYHMg6j-7vDdB{^uieZ9RB-U@orz*nbfcjt@i*P~7T4nClnkGm9bt zq#gOtAVfQIAE$13v@@g=o8dW7BNpy_e*Yxn#wSKIydAm{mQF4mIZFj-?kX+u9=h*G z8dUewD!O|&I`3VSpKC}z(2$NTZPVE<02B8K4{YH{Bzi4vB?T~uthQY1pvd+V2*80dyMtwQQ_FuLkOV;S42?9MuPEa1+p!60}6O`cSx8nS0kWb0OQ52oWUgFo?n888g@xc=S1Q2)L+o z4Z5|JIx-mHW>hHyoS)bjToWStr(`F&7g4~(<&eCNVI`$ssp%;iOl@RS3_MAI7T(!m z$F(8u0$pM#fmwkfJXnp~Un7Aa!W&mCgS4p`XsajY@U-h;@2ut)E_z)CBr zw(0XzG*5s@)D>9(a%;D5Esogaw(eWd)QUdRl+Z=&H#@hZM?)HOD=u|kNN(!)Wvzn) zEU+Y7p+G*U%Qd^HQ$rg8G7t^*isc~uq{@9j%a%;ZKbm9YgcnZ9p}|(rF149)>GM4!%ay%CXlbO zr@5sdj~@?@4xa8GKHkU6mQ^IwjPWjaXOn8?*3lwEMCddH;|mI|;#(BM60k6IdGQP; zIw=q9`UgDn0&6dcDD;FIp;RX<(m>`$9i)b_(cDA!CWDj))Bx47-_g&j0-+{k2ws#~ zxJ(mLDF&%$mk(btRENK z$6tSq8vwa*7pw2vt)iT$2!~G)Uy$>z1ry-W;=#2T=jyN7<6`zED;cSiW3Weeyi)S= z`yWUqkLG3+ymvC8n$_~Sfb6aRx(`{BNKS&wIoPN)xj`;nJVv%uGyl!rCKL?w z4*uv-A#g2|ApGVT(*__H$Ee~Fb!&?x^m@jFf&qdfxc}Ad42;Wv>j4ccEOv%}e#BAz zBu&;^^NWp^N7DI96H7tDH!t~PuS`->!@zhvPcxLnng=Q z#fRlN$k^g7nk0jH+XG}V(#lDtTtSch0EU8UXm#9b&1Ah_E08))D?RIh#I^8d*k^|t zD?|$zCp}!RqBr`>j8lf`Qd8(K5ExLc%ax@uoZk8?reuv>WsDj;wK|u8ioYqNMjW^$ z-h+{s-l#4x3B-g3YYY44W~Gz$uu@;5Ygm%6qrxc}qjmR`;AeUwt4-?ImO=?7^!Th9 zMW7_bp7buln+b}7iuod+<`o8OL@hW`lZwrgI_vS{l90>Ltc^wiiX2QLTN26 zF=3U9XVD-n>wHEx8e|A^SVdJNGDie8V%xB*MN`LJ=nQkZV_|gH>Le@8^sm*@iD_GO zxFZ2^;}(dB&${dYLSZq4YT6Ff4DtQyIF5XCJF<5rYb!Cfg$-@T*sT_fZ6{{y3bjQ* zRksuCjE=G8FBR@pX5UZJat$=l3*W}@)9uG!MPZ3uykviUmBf#Wf=Ei1;|h!KFMZ>c zX5eOIaRAzq!Q7V_hKHDVAPv>k%Bvn}o{sQ> z=_|7#Gcjsd9mAkuXNF!RE6LYP=oidGjxkNe-mQntjecGVsk689bT)n;SBlut7sqKo^yQJehcX65L^Qqrc9)NK0gl_B zVp+ZVG9sID#buBc&Mt$vzosAJ9xb|7p?8o9?{WK|D?+$2*CJndifz|2^!ISV|?2X=1 z4x)ubI_Q9DBud*KpC)x=6jsor2vMRu=T8a>o3wSDng5*R3rYr`O|lVXqr(VE^qV3s z73Yu*5=xCnZK?o*jkdmi+*0CJMMTLIBnG9h>=CPgvMQDRaFTLbj?C{Z+hs$O~Bp%P2 zEMsm`n;I|o6Im&XWsTTvX?;q`c!l#3`}#=UtNbU+8Auz~B8Igp{ms%k9tKvoq3LlQ zxP+sv;Vcz^S4A<}@VWtP(zA~(Ow=P5;?!U5DAvN;~k zTEOWhDP03bzFpQ{9duzj&igKMOJIgX0;9%cwzna$8YYso<~pBOjLK+XstADWdpZZh z1#Ci==n6!ZROz#52G$YgurRb7GF>74*fw`Cu%b1iHDu(R?KaU0i4)!yV#R3#!ileo zFfuh+W61b4xJc5RInPH+!Z~ZBg!^yr{o-l>ChfvipI0C4kAWaVdf(KLH<)ah=hf7381)FWXUx{_fa$9!r7Wz?gE zt%Xo)Y*`9V)P^Kr74J&#n;&-3`viIDwsyg4HcO zFfKR|@L@Pbiy>%6Q`PUXSynnSDjm>`6~lMuhVoC%J+m~|QJ}85p#T`%%T#!~pa@^{ z!ffmGWRcMsFC!=l-eaR^Cq`94g43<0Bs>!m z?tO%|T=EQVdy)JMCj2ETB={YL9i#VyWM0lrA%=Jpkw8XN2+y zBTWH(&?dzx`z#D6IXcWmra_dE+M(|tTK(#-DR5@iA)8k}{$#f8zcxP`XD4P58%Hnq zf9S!7L2@=o-VTzBe)3Q7n}qc`diMPJ%kQDc=4Z>9sgdCG#ybOY+itZ~fok;tWz_sL9$QG+B+|B;yC27UOKPNUg`M%WmsW zbjJB||Fo$cJ|(MdtgMb%9)ha|v-@F|j^BV~KhQ&WO6#}4J=I3T zcDs~SFg5}7hwVjX(NAuQ7}sODln@%4mKBhJ{HicRkB$-;sZ^LStG=%Gz@H;lFavi! zSGAGfh^~lh^Cuf;qBcJ;`M2fzz>3!<=>zyzgGRd&pozd4?>^F%=r#4o5bW*+!aFvI zciQE0#olP&Qh34-rN;)zLep`z5k*ADE-Vs8_0DwI_X}(|Q+ZT43d9JHvB56+s~bwS zJcj703;Vr^(b0S7fy|j`azkCs>PQ4{ZwGRqM{~7*G#-1P;v+#?sDQ@fP`2v~+seL$ zXOi>DRta`cGrxmfn=Y`P6Gtop##B2&HuA(${S|2;;pAcztyhncyV|fjBeA;+wBPBx zQvy#RE01X5gxus5TWil0-I}u|BSc>m+=Qnw^J;|W$;^vWFa30f#fsZr3me*rM7Ukn zWi{N*tY!F-OO?BoQjuVk-lgJ!D#JGq_o8;?fX?nT+zke*B?{H*stGx0s8~O5SwiI9 zYGONYM{Q5u3=MRu@$Q2O<>e#|BOmQ`*|yzKa3%XtKk_0+S<6)lf#8db!u1c9_kLmy zOeJ`*0cC=64^W`=?|&;m=~f;jX1oW__J>FNkB845{VgKPh{_8hcPX61S`sFWzTG7z zdU8o^S6ogMqJSMrmDJ?Ai^lK4{hBBOXHblS>g1qIdBs7x2I^+;k3 z(pbuNK22vAlwyo-R&pUb&XRmO&BnReYvvA!3Lxw z0J2+Cd792`T)V=Qm7x?fd$~JL>?Q(f@j^w?`*~?7!%0ptUg)A}E_c<^j2o{1lk9D7 zMoZCp6h@5BKx85|4vWlOuE#VJY$J!{=@>_5kYqWErjN75d6toY>0g{<>EX^hiXCI_ zsixNy1WQ}cK1-V^y!{3DmW-Dr1ox3ZHn=Vc*&Z!RoPbGMNbOI9YDp*RzQrHW6XX`ZHHQxu`}C>%0%yzkuuFk2{8^9C{L8gP4g0> zJYnpnG8-++3i35W&+ul1NQ)|o^W5P5Bm-r{qM$4W;VzIxBRvFdNPXi2rABoIsI4yC zoFV$*!>HKX-l|+(KaL)1Uv$Iz;VbJ=_JWj}Obk$;q*X+$;G33xK(k+1Q|+!kyb0 zCl&j3KbFB$vuA#)n>IiZ$7MlVW1kdX7BEagGGGV>4f5B9PWP^Hp=n_zSj=+J%H6n; zv`SBjDI3|&Ny1yX0q^eml=b3rXdX((*E@Ud-9f^JG$-qiz^Fr=-<|S-@;oEFtJxR_4VRgQ{Wg&YFWY22B#ShqDV%lbR-TmS(N$dX@->F z5S?ji7EeIH_kjwJHLC4Z&0r;HrjXDCMwQK~l(7#s_DPn~K?+AAwQ!u#VGV$Qq$YTZ zLW)jsQit;n3E$SZH1ucY_H?r%L~g)>LtWTqCHFF$=`UvbCl_wDTm+%hLZb){M8pNF zu}~0@CHP+OW7?@!Z3Gebj^X%bdC5RvFA`)?4e`RvC}J^@3B`xdmDubev3R^NC~{aV zSTUteg=B>wZT2%)-*@MQ&Jx;K$hA{Yv zRU23kLRgeT(1rFq!&6RUSdJ;&foC5cuSVY_TEb{#<0L>u>g(Erq~6J15)K4^yS_a( zNq84ND7oqbHlMD@`c-l{&m?7_7ojxQ5#FGN_9q0&wGo@9edo>13*<;k;u}SEN!cj! zY@Cep(`hpymO6|EV=uX*Ylor6$s9Z8d2EQd9F5(_)g|yU3~8sO4KnPD*Ke&@i;QP0 zUb7YLa{EUn0>to>@-+2&n!Mx$RPQf&6T)1E+D>ZuZ) zhE#wF7JR7i29r&wnSd-%Io`Wc(l>gH5|{!1#m&)5ZS2lPP+$n(9TAYj8#IW!&31a} zXjGJAJOvAHszld6O}ga(5hYdT;IRdosa!{r)&yr+hYim&Y9pE%V?)AAD{nLG{?Y`q z?K~yXkxRBJnqk>{J&PE(7-W4&UmB)Hv$hZm)lOrG!Op4s-bw52r7P^Fr2@trHc_>aGWN!-cf=lf3TPpJ4aHDt6%$ViV+uaCY&Y50Y*hO@`6P+aFVxAiQ!s2Smy? z`P)(>Mj-*7)U)1UB8`9XUXarYmvIZ$nP@aFItWvh}DTKqqE68P&p_zVs1^xl?hjRHuG z8~QZ(s&yGE$0P;kJ;_a)`J$C;0?5}5F!IEA`3^{@?%ko_dIJkJ~1zFPds|Kq#cruw6~X8>8dlqMN&!nU{Ro$iG6%no6x|MUm974UZ^*wJPjE3sESA zTh&WA3BgHvv75JkpAf5^s0W8Q$>NrE!78)`Rm%0M$5be!srue8L^>d0;xf0@c zUgit4&BkP+VaUOX=`-=VinMp6Oq_sQON-j^1DbheHurcbxY8NN}p|0!HiLg{`tVg8k6 z`B5%f2)HNQCMiCMT$7bjSWiRHo$f)uysx388Dz)-P=j}Z1xbN zLu8BUTz(OhKsh2w4=Jvj3?@k3O-^Mt)Y{=U_k7=&ydoCLT#8`k5YjDU*2BBITT$FS zXeLRvYVx>yVsU8hP+<2aVo|F;tvsV?G}Vz1Yw>^MtjbF>C^S&M*Ice{$>~e|>H|he zyBe@Ei1jqSYRGW!ES8JrQ5ddeLa9_K5&E07%+uMT`?w$0$9+3;I!DYaSH{Zm+x^#- zLU4LA#)0B6nvUbx2^Jp5(L+(CUlR3z3!%woa*xqQ5EKA4W2HXYripE?=R^@<+8baL?b{nxY&{x-^xK;W)`dIIrUuo$$R1|% z0^iDVwS6VWDl|L3Tbd;u-xVu$woq@fo4mApO7xzUvUH*>PAIJ()aA*qpPSu^RGdyC zNNNWiDP%(0L9ap|6C{Nsp%5_whe|bU74a+}P&k43S6+r{kSkbUs44{;^u$a|s3cyb z9D62Vy!M=2%Xb5@@9m{K3{uz40MJF{k5p~8`%~5UCNp1@E5c68^sG!lrIlvAs*aOw zS~CZf@OMK%9(d@RQ7lR&7W7sSUnAOp2D*pn0>yPS^(KqMsI=bp(_n1q3pYVG{W_l~ z7(h&GzGY?{3oq=NUbl6-r$jfVv^gYB3P5&wrkj8YMQcm_NR9s#%w0Ys(;YR)co*Di zWOB;pLr?L8*m15QvVRv)%~f=R0y?PRIZasxnE*UB^om^kNaA?1oQ*_;xik=7EOX!^ zQ0hq@8ir7uF4NMmx>;7S?A0_hE6`_B;#JMFl1T(GWn|^A`w;?fog-8B8K@SZvjJuc zDx;4=RZJmTA`i53o(%V(2G-h2j-XIMP82eUO4OwbGu#R%0`4jm__LUzR-dC*YkXA>)(3 zf|nn+SS`vNV_jGCEPYLv7gD~Z$@SR!nO??3u+AkeQo=bUxE})ta+%G|w80>82HbKn zGwe4;KRjNV*@~a2&fSfasdk#+Iirt~Fy=jeXXP5wN8HE1OoryLfjcHARIH`rmTZYy zm=QgZwT=cha#W0k$sgT?o9AU(Kv_NpS80cfS z8Ss0X{;5gN>TgdhzK$&U4ZV5Tsbtp$hJ6c0BjZ~CZbRW9v^%Ijf*e$6%)$X@Z(jO zygx!+GL1&zY6F$79fRu->RM%c<_0O8ZNXZCVo&C8IEoQxRMDtpkKNU$F2mpmYMyKgnDHLcjFig>>1Tf{i;&OIvu5g{DgNY-e{4@U?8*mt2DWBXfUg-XD& z7Cmf}NkDI-BAsN_s2-s(CW3G|b?0oz0juTSL4zok@L#dH2ERnTJ0ZEQPOp_hLy0^1 z$mL$0T$=l%PyECGDmOSi5~lMn%M0|IVE~Bxy~8MJ{g!IUmsyxISxH#i+!Cj(=ZMOM0q0#S#EOU&w)n+gPfe4wVICI0Qp z-Mw*#0WGaJX*AdBBhWSSB?`bTCxO;DM{sPQR=Mx3C&xc<>xsci=oF`=7BeO+mQhS! zViV9agag~CfG=uiD9N7EvuX|fs)5UGaOUGKnhGJF-cqX!UmP~e7iT-Z0>i4>E_F#f z)~|a@t))%hWf|)z9h3+cbwc>i3URGJMk~$R)Ck77v{SFzMC+)ArbC^XHr9?&!a?$!fUAx7=yWj4-!J9;N`WM43pa;{IBW7QaU;LdOsBWZ+1~!*&X08Xbq1Isjc=M)uzlxAO`EEO42L@ zHu3PGXuyQxGL~a9R_FUU2FpY1T`!ytjm3BI~h6Y&1#pDcU7r7APnJ z#`PE$g$>(NGCnf(%vo`c7W-gqFLr3h8R##_Mqg>+K_ce*q0n4KMot6Jr|pt4&@^sC za;|$AU+xmhUOgBHRt91CvuoPGvNONOS)Z=4A&!_&aZXuvWNDl-LR1(Dz0E z&Qhn(qHH)o?wsNtI)2`{HVISD+HV)>v2l}kX5nKnT3NLQtB*@)E`3F~yktyzRyPR_ z`qFTd@Q)Bn8hjzI7d@s^LlL(XK2I>T^tr_i$QwUfRyxe9uyz6g%y~(B+CPCG_~*0! z>;lv1R`j`A7xg!}Hw4UmxKM#9K{pE5F2H{@6+z2hIe!rd=X|&-&)g(E zHa8~;o}6>!Rj}J&c-hD!U?P#`3o&otIF^`#&02#~H=Yx_H%vQ1ilaS|V6I>d0pX5G zdq9E5PnHw3whuk_3^sz<3tNNAIY>I}G z?2^_2`M@%tf@WfshgkF)<@<(NW}44M&SUTF*Rylv-b3bjRR;4qt9mIH5v~Mt9a;mh z9uE@2j_~k&)2^CsqjnE#jI9&a_V|`gG^?Q{d*n^i3hx1}oCiziID1x;Pq8y7Wq>@i z08tFH${1rgC5Qxz+pu;C>zkcvwiE=Qwa9OHN%Tem5@T4fDl>bC!u31%`^35mYrj+2 zFJKCU%|LZ@PaOaiKmq{U$+SGlws7CTq1(zdw?{kxpfM=iDY4R@S|Jz5=GRL@E6|Iq zVaQ2Gn3_$`_ulKM-;XVO=TQ%2qFcP;?{U;F+<-I`WtfzSgYviFAcBng3pk~ zc9`84dUK-*p0`v-TGFwWf~Q-%zMS;kWzcLOkf`yd1}EM(sx={+=Biiux$&P~jV4}W zZf>W)^1K2_ezfyS3x__bvn#p<{|MsON{Ka@@DGh8-mCNOZ&YMxw?)GyBY3%9(4p zaJoS^jV^;mv%JXoafqbJ9U^58%ywa`+j?xZxasnx;qSe;N2W60Ei-5j{V{D-F9yNd zpjegiPJTzBEpefwtXzb2QDh~KOREC7?Lq1T^JKkBO20WWO~>XK9Ez_SLok-WrB zK&g32yGxit?`If#S1WyIV8%|i@{1{wfEwMCx~VrT=B0B!rn?n z0khgD9MX#>jZoZ zm6HWXV6z!2SrxDhhH;qa{&ow6_hao3Z!MF~QN*xq1y zbYr|spbJ*7Woz4{zG8Is$j3)Y3x{TH&{%+QrMXRUz!sMV5WH~^`OM$uq4%9x7A>s7 z8O6>v40Vr?%_El%T|CSZ7+VLO8VP8y582ca5u3KK-KU=SGQ{NNiuuBonVX5TX`nS$ zGsB~x>u?>-IP4ynJCtOQK*|zBXds>;CO>^mrz85!&e_NJxL8UE#muIHK<&(blH8v-2hqUv$-y4-G9AlR6%;&lW=L1-F`6 z$Z&W2@{LARcXYHnviHMq9}9Gt8Up22EJZV^NDhE=4>+z;OjSP29J6VxYFQae-*clT%ulasH;yzQ+iy(9h9h>YcByP&Pt8 zP2T&O?5pkckTZyqqi+nY5aSz;Dcl9JZFCc{(>OeraHLqtqrsNaK7amfTi}8m4##M{ zFytP90#Vb7bJYLli;6^TT$ROf0}}iIPe8E0S!V`w-gs{AJ)mZ@3nFw}^%z*8H>T~M ziTEMy)q@cGFcuOTdK<=`&fOup0eRiga#%O91>3-Xfy?ZuQtq<5bI21WQpkI9CI$SZ z_5`AxSzeo^6gpl;YFA6}fK+g4Pz06KnAxOSER3=)3)typ(5b}$)aWM>HgUBxyk_R^Rme+1y1dji`{|qA!Q~crKf15*p;FsK6-S#bS zkf)dEL0Qth$9edp!h7|bbV6)_S;5cplW}(9s{Z%>%gxW0GkC{@mwN)&>Lqe?o1ak_ z=8dKTuC>FSVM!ZG^*29*h>|BU1zoMM#wH!xL85?xL5^p8U!0s62oeYI2jB?IS?dJ~ zo0sOxN$>jopEG*%@Luwm`V;6T|MlwhE=^3dW8=Ru6w>Wm%Li;M}IoDx{7hZI2tFJ*U=NW<5G( z%jbA=3u2Z(@S4$q%LRLH0wMtSW?w{TLhkM+RNin`5t1NGfPjO|r=5RMee>CEXO#ge zO!`em(ha#65!B)5hKxHh49hya+pGt-c@OZoMeBF#7M2MWt%XzdY9c=+tm-yT1Gbad4BM5IvW4oOFCc%e>$(T=Fd zR~27ZYQ@FVP^Z!o$~h2pcwDq6tE@{jg5X$Dp|O+tklz(XHd-l`u%WL2f_LCY&uV_w z(SxilDrlCru96h6pg=d!K~BK(G-B|aoS^GvtERnux7q!AN%y|k-Lm$D?~u{dBQ3*= zh_tTd70KGQzgClki?&W&uEoA!?#dR(3RGa$6(WNsnSp_=JBFKN2RgeZ-@9sE;u!lP zFoBZ;ng(H-i=qZcwI!i{dGz$V7YBz=3~~AM{_xrUv*D}5C;Q(X9PU4%sxD-iyXw6+ zSqb*AmaOCw18;-Tap;6je4#X-kQ;g1ii)aBu6z1VULxF zmtGWu$HjEML^FfsEH~Wv6r50U2nc$-1l0*&XF1=VOX>tRN1L2+FI`ZmP>3F%L#V<+ zykN~ZAb39W%GvhmcJkv+vb~*r^+n%Q{pv0?0k?p=$)EFY#I@2KJzw43LT=eMq9P@f z7&c>eI-Hyi&3c|@!(W#+C>D(vNvY|R?c6YZGVBBM`|+d4-|s&)IIf9-hZbd$8t+aZLIOGcKpH#M8xGm?c7C5kd130F}q_%+r;;)!U1V6q5e=3Wigsa;}jV-Vi6lO;CdjFq35#+m{82d6LV(O<5uIr2(%zC#iGd+ zCZ|kuO+1}}RGoVz4qG6 z>H{$#A1M4xkdvX2KLHR5hMVG#o*sO6_-z01rClM-`mw)&TQuM#(a$3+q>9A!dr|DTMiq9Xi56gz$}Hqp6@$*Q1?rOES0?0KAr_Ea(%536O6F;Mp%^W~N$gl*k(FlNUU z+rgW1|K_HnJipy-?{tK(5@Rpq3*NHaU%VqZo3tLQl-Lcc)~In?;fgr##iX7Ru`!;9 z&4PG^yoxG>m~C*=8ew{FGzmtZ=M?T}tV6L3?aUh8N_}USFizmrD4murwiBdoccYq^ z`hVj!sq>O-#$;^49j5iRvc_d))m^1oe0IfK%DhXwy0th&Yh?+|;Zw*nGB*^z69AAM zlat)HVFT4m=Gr*j+zn~#260Ny9v%IeT?qTuPvi)eeNXX267Wno>}sBk@)R>sCG$x- z$}D^BOZj7rz5Lt;>``Fwx7GNWdq8Jh8QF=DX&vwkdz-%9lW{Ts25*i(&o%k%DFDEv$hothyMUcWlEViuhemtadvKr>8a-OW=_fpm4&W zja(-*inuGPnpoo%mAkWzyA)=s;=H+{8>bqeoLzOYBSZ%TMCL!!Vo^&CZx*RqBz1ML z)dn|Q8o3zrBztwO@v2OQpvtdb6T>?N zaZ4@4*Valc>gdiFBObhq0Z}(;2$K>9Uf(gzxAom`(gFKy8|@M*74?!=D&P+WW*&3Y z33;4XRWX9BZPo)LhI>uHd_x5~iSHoVnr+uP&6w|ASafy`v1;tihV(slzxvo7@!IlI zpWrv(m6=>pZr81cm2d*#finpl=qMWwVD}4zW@o*Ql9MW#f`>a+LA~na$6N*%tU{kvp!q zCArYqfg=v=@!g0gRhP{2aqse4Di%p|BMa;Km%E(n5yY&!1-+j@G;x%5vHAcg8Y^2; z#(Y4J;trQGGOG(^QSZq;cP-c47TFSOUould-vM-G=d)<5_Q}?Ieh^h~_rjF!)ZEDfnD+LSK+z zmj%T~Hk4F$fj8japXyQFeG#VX+P|w7f`{>x5)hrzkA&zBSR-tivu-U$q z``W8CIXcf5qcfiEwnt53Zf}kX90w6Dz!fdnTOrA32dQZ0JB3y*WP?*OxAa*CF&yG_ zhV+HQd^*2;;H#QJr=zII5FpkrDg$V?%z*o}pRdXC1|$L*YIDGdX2G*wYd*}P%;FW8 zBR4oOG~jl&g4i!)XIU!Sd}T*|?6^Wg9=kDeYpVOcWIL|q^8cS{tp*t$2bwYuBW=Gj{*zh2i(1;q+2z7eF1 zSmC~ag;u$#ko2y_aUrTeXgMKS$d7GBd`PU+e@$;XFp9ME zqWe!JYo6jm1qrsYts{h?(?Ej#KSq?Y(T~|Lz=oCskAcEUrIgjBh?UltMRqXnkx;sx z-3`6j^y#W^z~5}xVOosPNdb|gl&xfSl_f}>sIH=^Jb)&<3>kGmSKEs!%KA}kRsN$) zRUDlGW+p~wPRru_15z10Ma{;ag>TZgp_|p!=@7S2k5R2QRzn&U)J4(K)R{dm1bGMCICGTy^xs(Um=9-DX{|B>aljUva}J3Z|lQN-E6( z9_-fOj8bB4Q2Vz;zwXSu?HK?|B>`Qy4mR%{BG?RF-3aBWn#5cc!3!2c`^uRzX4aBjB$u3#sP?E> zPR5CC0fHp~0&$8FN=7#aFq@;V*bKtV+Ds*u0@EZ6z?Nvh1~_Zlz?&FV?isQ;I)mAq z13jnmKvvOG%#d$0(u?FAM^$DxhiSSn+eyXXy-YS5$fMCEqohx^nC?c?&gfBKhMDH% zGMzJt)b10TI%$1dDvIdrS~H9EyASMEOHP)nC1r^)G?0*`A5!(0NU8g6X}#;x)rC;% zWL@XRwAPhr<(;X+rK$6+sq?jIjeAqqixbAExP;?{@}hCv!o78*LDG_wh^>ylw?a@~ z`yP#s*QmN^ls>g;FboO}ePjR2=`^~;q!hQdhOk+<-|cwBudUCj?GFpyM&LGlaC(q> zv(1qWTl64?T}bSG(j2q^fwbl-j48Q_atv|p3v+D(IFE|?1*#8;7XSuw6K^=kL!Mv& z;*$h~0%&n%>L34{jTQr&8s580A*0BN<#Pz|HfWWzkHKPM#*rJz^MZ3?oB7EN)Ktg~DR3?!A<F3vcxg3DD;x>yrVjWVp>TXT9;4xPE_a5vtmeb#@8eu_(RhUw5eziK*q3~C3ug`Fe zD>&j43>`SrJ4@?%VdtUV%K(C)8J3R3e9m0^P2Nm#+N*B3k8rijDoRRmcV0=BbwzC< zaizI8PO=jv&a2XCV56@F9k{=+p}sbgvBrxLUjL_UiOFjgOQZ|lhQx4!4xSG~FU5cV zTdIY!HzLW`qIZlw+9mT9h}T;tWe~>o=k#0@swO~z4ef&FyeX+_4LI=@v|z37juzd* z`OLbbg$F~^FaamlJr&-B%F%9xr!9v7(9W_unu^FJdkxphP3M9*@Lek{)d7l=8j=(G1q#`*RL;g-*|8Yk zO7z4ZPk!g*jVS4%;(my3l%$D|I6jva0K6LY#kjzwHqUD@_G@&PaiBcfO5SDcYZSPv6r*t?m&+rEQ9ykxfVQ3;Mt;wECJ? zzt}w-;!%UI{l7Ln$L3Pi=M{*eqvwqsQbHH2!}oU{SC!%Qv(a!NCQssMV zgV`spxk~{;-Bv21j0}wl3$}-#MKc`BNeX{qfq_EOY^9MNSM8^ZVw#VJu=$3+_IgZe zZ}x9QTkF@BAl@mg2ltS8B^=D*ZaZIs^awoImq_-3nJR&DM{XLLiu;{npz@yIO{@W) z>Uo(jvJP_{BwV8JH{E3^gk79wS6+vDQ}soNbf;IqeO?WIEV}~ysyl+c>R>&<*9Y#P zt{QCsdEHAjtxP{Cj31)0g%?{?Q3IPc@8*m`f>*>jQNV}MSz0DHZ<<|dm@65qhXm@q z8>Yv6-JFqM>A=09MDMr)wp-9s(BS0F&?e_rMxZbjMK^{sa~-XI3h}JnBe6ar~09?jGV}%jn6V;tMzb*Uyn7$`dPw*#SC-vIMR@nk)?4bNF zoisBNiG_--!{dA;WB|b)w7&Ilnqu;4xA{m5Nw4=@IYTqkd=@DwA5_^H9R^`ZdacV% z)A5_Uij?FJsw~#6_fD@>xv3e>I9;H&JvuS}d#t8a|2l$N|K><*U$qg@9JgtfeONeD zi~+=b)wRa0wZ;v#d=+@Tp8q{MKiCj0(}T~E*+pyVkCC~>>x^}s&04~&*#s~BS*5aN ziDvyLotQGJd$%R_{A%Ns)yA=EI@e`WMVz!j?1BIa84n9y14{z9LJzoPfimQ9Y>3f zV?{O6#FE?i#Z`9Hy2pH=+~n&$$=Fqom_;@04qxttB`3@3EUTBL*J7%oC&NhWO@z=^ z6{83rK{o|9$o7@a3y#F4VU@KV6RN7_Uqb>oaA)>YKkmCZ4vZAQ19VD=V>R-Ndx1OUnSrKX;%#;)9IL=mZV zB_z@Bm2Ar$%4|9fawFo@CU6!`z`t`{Y5a|d-W|`IAaJH2aMlh1e;dS4WjJYnnIJh6 zRA4RJFB7qxi6q3~qAn8-IdLEsdljNdu&i+dn1zVX26zySk_z-nup`GwL5Z#mjV`ka z9Ykmcmf3hY$||RILPcDDRuqsYD}h~7EwVY9oNzWR3d&1X$W(8+$S3))WJi%s3o!H< zXJFi)PJqIW*pkgm5P~YAcgS+VA=Nh7fWKOxKLLbQE3*Qa-?)8+2_pbF?L@EafBG=?U{> zP&Fp~hoggk>@%Cbsol~rl%$?GMtDQ#Il}@HWXgI0mIsyQhMmul#|ugk_=c=U8;zt~ zD9U~i7iDXyH68|Gcz52)gq4vBOvb5kiO6 zuNXNn0vG;5L9=0ax>~$SWAT-}i0r6yKe|FY+_;X09KC9R9h&XLtu%tQLsK2AC2b#x zE9;HMy>1!N;qxc^lwPkPn~s|7&=B0$l_ezjOR$jb{|?N!@^k`fY`F50NyGcNH*Yk- zz}~j(yv>{I!l6cHY&!49wEhiof};e(2A&PiSQgtl4(o$g4+#uN=HDsxwniH#Gu$gr>HCY|-> z@HBP7E&}Vu;Ub<4eV&dv-?_Y5loz(8K{C(E zlWeq5Z^#AEzSyIzr(fLRst5;HkAHHd8HrEq46;;Pcgzrf}ZBmSc)#z zNtQwsSMHOX7|+tli4A{S8}J^Q2Sa$jvD3 zfo`kxZuKi})aQnK^li7W6}A%o$E)5hrXlSatc7%^9x_wndKMaNIQu~{0e-Rn4L(gw9@OaOo-GuaPA{y#W$AK1 zxlO1O$2N^LFBlQfN?y{suvw#}iEw6uBRBYKCn=mgrxT%hP!U@?$-Pv}Gjshl*k_0+ zd9ag>BoeKexqPB8wV_ysn&zbE11m@gz>P7K7T{clGs7D_8c>ePEEV5yhvw$m3b)Wn z2G03rHcaGkcxG5un8!=&!_;L{^VG)eq>$nS1q+~1@bb&nzHZ3=#G|eYOfa5!)!*B@){86T9{VjAC98|&oqF%#lVD5;`}Y zTW&mFoNNpGnq8M=E%9n1_H(WA)2&ynxt0tM4!?bl$?LK3MOINJUAi^&!A<0!t-;30 zrF$VW{O^@ap z2Ta%yodWic#63E?P|Y7_g?u}D9O4V+m|UFA5vVzw8Yp}r%mMObkTE#{r2^VE9WPIQ zgkYG__K3RDHw=N^s+<9nLF6yZ}Is6Ai177rjD${rYg{_Oj= z`LzZRGW6(>Dsl=PPtg=M%K<4Z%&mHo&4Ly|fY=v?dNZilrE~PIQv&`*NZINDP&^Jk zGmxOYH11jVfmcWS!=t|+J$?k4gw4q>17sR5shFZY zdO7^|$>YC18C-3MfLGWXG~T;PQy7E*F%sH)kZf^6Bo$CAE6buJo@ea^gI#SPgS#7} zC$T0#0|>qi4DkSigyo49JOR?W*-8uWr`KGTVk)dX%+aK%K;NW?h3{ZoZV>pY$8V@B z1RV|Te$ocJEADS^dt+p_ML8Qy=RE{NZA&|IZyZ}+@z)#m4GLGfM*2oFO&0c9Lb0jl zcW*P^CaXjO)8(R23`ZkjG!4Qc0k>}+dD;z!koBT((c6H86B?`e95gB`q=siNU%TZ4 zyAS!K%^j;Q-ONP-Bo1nYWH_@zC)k_j7@Av=75#*&$9YN-wvqw_17=mYN}*p9(pnOBvbNEbuveMboq(kd}r=Ta_{ z`D`>Cb`C z1uL_ya-LI?`xzL)wU>tQMcWYW^t~qON_UK56kSv_830D%GgZheCWz)YD#{X^4{hXO z?u+D6A!YbNX&jh6(kt$(4<>3 z1vUn76YA-q-qP86yg^}2XzaM9iO(}h!>Hg$C8{dDB( zMw>eOVzpPPf2OvmG(+Ri__$cHxh;009jb_;zCMYfT0r?Bai{Cf#qT=OR&)q!fuOa~ zc2Kldt*!1ll|39>r(%46NR`o9K55S^(RmGAQ}1sHVr_O?fmqG13lOW@1^%&m7X1I` zf5pF_GXl-l?$@I~{PF8Qe*M+xtDWPW?5phV$=&peuf808{pIe?ov-iQy*p0F>DTG) zsvO-WbLZ_*w?1~vzpuW!i+|0(@bAvo33BHSh1-`Qodu{?E>b zojUznnk!Rjx-88dr>)$o@8RVBL;i_e)P-+*`TW_zxsmfN;kqEnZ84a?EpLYM zX8Y{=|B@Z^KkMJ+_CMUWy6*l}`+s-mtKFU2{@=Ot`~LqJ|7!am0)Aa;_iwJqB);yy z+or+fNxGm&exlj%4)VnY`dXMXeowyB-Ft&R_=d%{j5gKeCeSXI%{s$!$b~F6Zel_B zQN??+-lOr)5GNK4%Hm(eo%j7iF*=*%=BAy_5Ue|#zl!jc$VUd1;BSqgq$}>QIG=GC zH4yajx>&cmcSUBvTh3n1d*9aP&RO`XiRZdfsDrYl#6Ur@sm?F z->z%w3(#edxW3GPZL&TW@uN2}z)_ z`PoH?jgtcZ?d$A9TL~6RNRx%uc5{Ho zEK5sp@lyGHP&I*aj!Dk8)u@H-fZe4IkXDR1^L0uiYqEW@Ow4`8%xTWn@jPBrm z7|_zrT^M^kYRGvmiPmT)bW0bIonhQJfkx{88lmFnlQuY>pgl+?@>bp zUsW!37*RludKu}c%rNR@tREp8%2rv0*c3uL#kk(86HTbY^F{5jcTGfw;sc6fh*C&f zH7+}FgpujZ4n{XE88bDGG%Yh@d zHs-K^lW_IgY_25zaoSoZtwR5uFX?!U%10oq=y~8#T#$`|`OyWbM~`8Xqq-KgtL#fU zDhtfFbbLW(z1ev*m0de6Zvei{`P@>Nv8{!p^h12|l4zuBPqU)8bS*ZeV<5=~sUBUE zN$gF#5bl+L!w;WD$MuTvWD%AsSnm4|{M8Ig+O?2^+%Is}u9XLg-z}D8}k}o zlOerq1h2$6#8;jK9E#vEo!S+t8tyo%GT@pn6}0}g@f1jZzcb^r2G%fa^I8u|I52~zO2dr z?!xyEJ9YZ^yZkSZ{~f%9#CM;D1W*!5fgRw>7yJ9N1w)H*{eleO!CqXd~#BZ88rKHlBa zKu*A8PvL0@b^8@tho?c(-nDF2%$QTN&FAg*4NMrFxFu<7J~J#j@M~)4d$p|b6;({AL*%6Oqv_7on#mo+v@?=knQxtYUVK%A6CUC@*(Fl6n`4oV*&y(x5RMtqY3^N(_Ic@pE_x8wqUi zJY2ky)C-ZBVW0L!V*z)kHJwS8bQ;i)J}w4(4&=x`B@Z7yu$ZH%5)d69dFw`;KAmXa zrg9})K?wNL+Xgq?t7L?21YRrLB7loSF2A)Bb{K6C&FT{nw(yyZ+PSM7%kSyXm6hc4 zwf%7Vctnz`eHcRcAG%p(jY14V^a3A^8h-iff%EFV4)rypVvI~+8vFcP#;N~BreFB) za`!*-hwf?s(tiK@^2id$#1uVW)+|v7z4L9S@LZPc!AULaymCmTYh?)q3%RQ`{CwB z%Thu*`ybBoQyc7gvt)CU4K_T*-_-G6(h_17lB#p7%qSR<=SVLjx4s2?^;ejUeGf=u zPdhs7_UkRMv&5dK&EL4Gb*=0&#X|mBc0R1>?prgc-K1Pg^gJr2(|n;~){;pwM6TV_ z*@$^`&+$6WVM&jROSjmFM5!aE^2jC|Y(^2^zV`9P zmChPIc&l#TdP`&9dhptO_aQ9I$U@-TF`#tg!p73Y(K!m$#nPD!# z5jYtd3R$v3A5Wy5bY$A}4`~=psx*)(+ua~IcZ*&#K?VFd@XY}En}l|x+VK*gh9tb9 z7@Vs&0VhtknJ2Sxh2c1vS zk)a#s_(GZ~BqBW~2?en=jMGvSX({QUnGEJzpA|NvDCDNJuGqKi4n+-Nb&FcTEY<}NV&7{rApWHKFNZI-ABk1ehh&wu#)@cZY_ z|J)7s=VCRu};-A zw_PhOma5wg>n)sC5Oqv~A(-P~JT!Y}a>bJj&k{mB5)%0Vd2~gCW2$c^ z0wmF)n;$^q2m=fqJbHTYkNp>7!3{}Yr}LiI@r_7x)-KuWxbI@la7>GvJ{{tH^QU&= zZNrnt4jaiTEuzXb7J)mt$}-TA*>$mPVp|WMHRI(DWB<(ut&BO)Mnzb()4VcNndNjI zlSHqgX5@6A#9O&18*Vmu10yfPiXHFDd9FHm(TTC{I8Zv_$qY*KdtA_-TXGpOEh zf(k6lgdCcFNHxstxXLK4{2Bu+F@S+^7+W%t!*IgI>~qj6pOjg31|2@7 zkrTUD3y~cfDaobmQkac&8Ie0H(KW#QfzU|ZQB`c$c|%+?4SX;}t(M1~5mf?e_zPlj zv=5Sy{ed^*!CC^`H)esn=lhw*PMablr;xvo_gI6PR$u7yFQ}VRc>)HP@q5C zP;xq2CRUd~=6Z+Oo1{bZ58&MIgv9luFu3%JmuD+4x= zq)3S|-bh@8f87>XM z>m3U8R`sYmqu!f)yk2FHW$qHNvK4<5Xi4ZHh~knivQvq5@4_4ywl+6=nr0Ln8mw3e zwi5;c3T5b8$Z4gN0%D!uG6&m?$pzA@`kM6qNhc(ThAP@1dbcd zXae=1*`h7jTEp4VDFB&1JQ`v@gj5o^iz^D~e>1`QCqPd%LTGhI!)*awbbPkUOO0J) z%G4LmiUdux5DMW9Wfu-S0DfE)tYaV_*A~MB7x8!9Ix%AR>vF3Jg4&D&_KFx-o!-Temg_$m^1u4vrvuu_q2{~uX1>NdJpfaF_6}ucs@Z8M0 zNZypyHb~*#3yrB``>%4EXQhj7g_>|bVk_H#XB}2`(~5!|E42cWrt1{*Lw%CR%>U#=MIawL zR@^tyudRKuuvJz)oELDtFo{qYu=Nl>Z%f}0Z;=!;-MAIdjLtbs=YIeqb+8a@(5%L2 zueXijvZP@30ySQJ31jrv3Sm5P84kf31NijA+MgwlfxRCKvv2PG^Rk$yr(iG!gJvmj zWNsZOkw~H2uXQMaQOVMES9&Tf_e3r39)u?sk<$&dNQ)g^;n@Y+Hwv(eNa_dhvfMyi zWK@$`(jvgUwSJf6rC1?l3eH8(vg6%~^U)dgtaS4E$TmG}iWl3*cGEUE_*nbD<%~+i z&2su``?KOF!#WH!0Lm;&wH&=01ux(di;3*=%}Zr*T0ev&Lyop z+RZj%N(M6V6D0$PZFiU$bueD58?I?94j1xsRHT!vqSSa0ddZ1rv>-Y(!Khg&*Ok?_ z2=*B^5Yf)tRy=jXHE-{tr-NR~h?YE1qql0%^`TwURY;9Zk(tW0pSfylO}WUn3ZTe1$9!3Hp;=q$P4Xd;3{I{2tMnbn5Dwj=Qv zuAyNG+H4?NToclBN$uUGbaH-lUPeH`G~T5wc|?R^{75kdE41T>>|LqvZBX}GL-<4Z zRug&-Sj%A$J)L2ks0?%xh`Ug0Ih*9KF%W?%ZpfwC6;HGDHSeMH1bo`2q^tmHnqXX~ z4aq=Sk*t8l0wgm)AD9Zg!1Ts-KoY8r?m!8+r$Hn2zj0D;UpZCV%dIo5n}2R(J5 z@pLNXXn5leie|4>IC3rOSuJLxCb?ZN{IRfhGkUTdr=!>IXaNqOepbQT*wA6Yu!cH_ zI@_H=MVpEku&YLJUg2)Wz~V&j6yb>ZimA*a?Vm4mpy`;8oczu zOvAAUbKCL#5vks46Ai&lo20x;`0>~+ByiSx-@2zR7eMdMXRmBTa+ILH!M}m{L+#Xf z1u@}fSKFXgRZ&#%7wU`ZN3*^-c&9Rfp3nx_ih$p8(g5X8I*!P1n%2a{MZabh=x`aP zmhu@Y#0k7w3*dEx>Q;Rr*2{Kyw<4p1ZIN^ZD>Wl-q<~SnQ>@3$L-_dTku%~im=z89`JaK`JhvW$(c}s!gsogaJ z6xbkxE+S-2Wv(G`zfndl4ij+N+B?A?8#m{V&RI`}A_=l%vqCUslj$};H%(-xq1}43 zSE8a;|ItK8Lj_5ll-es!NJ}FLP8wApw?qXJC0mz_T^589>qNxx_R1ah+N65w6^D~% z5KwRZMiPA4NNks%jm6auF`vstd0S^P?~MfaIC$_4N=$68cZymm4tfa$*01XnRhP*&cDvac_nP142Q! z_E?yXNUVnX;JpxdHO{dXC~%$rLmN##Wx(tNdrglgSqxAe+;lFl#YwGl zzfJ(WtV*1BWcb&E!!Pb^ZXEsn=;i*i;lbgz&l3n8ZC3ufY;kl^Ewbr>`Q5vL0i~58 z>R+-lKe^yk1e7EU8PU8-vR~kpVVA&Ac7{m@PD_x-x`conXbetuLjcCEzF2<&-a(h! zG>_Q)J&Z%h}klPj~H zzyg9qp0hH%NheEmK+ug6tLs3jqt|s zC(41#`kFx4WCLtG|1pf|=MYr}8nelkNtVJ4jccQiu5;!nsth2YPcC68HifG>&tMkl zH-|B}Brpv|WG<5GVm3M}iy539*ce5p_k7=^#f=TyyNi77`UBkg;l=)Acb>v3egm$4 zqF>|OL#E_*;9IvrD+`?69#nfog9SbDvz( zD%vxJh5oAJ@^w$H(gqv`|1nSt@aD`UThd|1?+$LRDCPTm!jc)@5-$z-9^&rp)f~f^ z|1$i!LvuAId;xka;np|Qj$(^gPx!An42QZ*Df-siN^dG1&TiSe5h8rl=R`}OoPXXz zHS0NSb{HhcNTLJw;`T{zbT7IDFF zwsvj8YZ#1kst=+O-{4^k5^h914!_OcW-0*J$ZS;1Xn1Vsmf2GU=6o}qlV@^Dp&_y{ zc?t_Xh{lwHJG_kenu;%0_Z4xlbc2hUv3fLSh?(RFaRCO^JO?=w{F!B_48b>K{K#)% z-8f_?vd0KJOUAOd^K3-I^BvJsUsJ(689D3-2DZikE{9~}b8PVZwOHWb2n??`U*wP= zU5%n4s;)mn2;N_vxhY2CLxyAOH=(o|^-iO$zIeBO6!&Wi%Fc7LTQ^WaJ;Z=U$=W^~ zQm>JrzW(orlcyw~&bj;Sk(X|}*rs)gm=s4}s>&RnOqKb?QZ zz>-Px1d{=Se12q8Zz+A6&dIKYMXT;bwLHdmdn(SvDQ7~H()PDk*gLx^`b3OsD{8NU zDsF@{+(jHx7JNl9rIxKhq@=z*E6$VY5+Xf7Pg{uC20u?BrzS?|k<6IDIa5_XF|NV5 z3gl?ABg80+Y6Imz&bT5wFpY8PAjeV(yF!~b`ta4^+58BK;+Ub72twouD1LCadhokQ z1L4}bn0I>sfr^3EbF z>M*2HSD!4`qUS}w(Gmqo_jt`RLs-Q{eGGKrVdFvI6LxNMMC)6!gCE$t_IQbrgq}Zt zhRPB=$vFeFc|PZA%mQJ^4?OUT<)X^RO#BGHi!!uv*9aWmOb~}nud*o`m{Vp-^kqw} zP!qtDz?xOkccABSH7;&^tzkuUen;rw;c6IU zj|?6EGJ`-Wv9t`N>UJvx~ZffXqQ6) z?Rwxtip?Bmi*JseBxfn)F@!*ul-?pqk1Nz>n#+C#qL?%}$*Jn&!@n@-;Sd z2EMcAMRszMk8;wcq1H1?M0{gS^-l)#nQ1kGcxR}xB2K<;CB_+m_fJ#7n8m$OG! z35e;o05W5o{|s_*PhGyow{(q6zAw%*Q*yxKIx;RQvuZh6R4`G<3JDox&kMW<*xZwZ zZp#@03+0vg`uTj3PxETQ*(?|4E-V8BRY_ifTP^^v%umlue^axGL0gN-CC#l2VoX!8 zz;yIlY=P9Q2hO>W#O)+cVW(Wk3L^beY_IveEFjM&tU{GcAsrUWW=uJ;`g*@ne;mmO zBa}3TYymrZ{u$GZ@U^ZXF#~*J<6uoz(5U$Nx8EM^zoh%3Cm4ePa3y!g*K6uuF=Mf( z{6nH8PA7ZkUXGsB9e@-$sawiA{U6)zPjNc^V0OMEhwJv5U+W6yQJ3WZUY~Wc{GosC z{NERMcfYT)M+ z2u=lP1R8UB$bnPK%s7h+FyO$GtT5z0H=JxRNJBGD&8NlL)SSo_h|5Q1eoQf~^VvP~ z2cDAThZHg;zStfl-)?`qoxCW<5Hq`XXJ_YbKY2_|eosxB4*$DY0@n<{c_i&u)KHRq zx!b$bzqfnm&erbTyW8BuciAi}!4behH5ez3F0V$*szNQG8`^#eM^;rq*1T6{fX}v* zXXy-R9Sk;kHaarPU`|08>{tEdkpZ;fWB{7;G9BSLDML#q$nK*m5JPpRV|0TzQ^A#5 zpsB2x;K3~Up)5G+g*E#5dT}N#{S{8Tvto|MFXl8fKwW~&0W7|-?;}HQN5HC7N$;;u z9vyvd#x_WfAZ~Z^c#@XsAbI>~AkF=EI$MI?2&Nz)BdaVQ8&OxPj(Nrog&Ja(B}@vi zBY-;@<`tw+!08N6Yfc$WW(Tka){Gn4%qoul;ST}x5iZMdHUriM@E=NNxF-zf{o-F>u|Muo@jrge zfB(RL|M*A0_+7u)A5}30nN3z#&z93;Llj`|0uUi{xX?B*U^^=evrc?Eak}ymGFJ+t zi;maOpg{--Q*mUt7=smG+}T28{Oa!3acHtLcED8RJdxu z&YoQG4tlTuTrPAcZ}{`~pNOCV42x_^`Lb9Pi;FqOGl%?s`3#_)ur$Kb3Q}wtvKKM|bmHF&-bNLS-R3W?9BaD7#QOwNrH6%Zn zjU}u(rhYJZDrjps))SCt>?L6MKYTfS_ULcW5ZXta=b&x}c-YlFIDC3=xIcW%yzeGV zi+Z#9^Xx<|gG>b^nJ%%=5FiarZ(g5`u)G=6-lj8y%7Hx&(`qfp7+(>J!1tCn%CHOwQc7h5h~Q92TSyz|q*EiNvg^M84L z=B2qr_(3%S9yK8Le>_#m6G2Uqh>fB7K)T%d3H_ohB4E4VnzAgZXhmL9;7mhE5Jgcp zZjKHp3lD=<>!I(U=4&uum#m>}$h#kC$icu2;?$RzuCcJF<%KjU0bx3CR8BgtUXBez z*SdYhB{kR1X<*F*W?IGa^vqJCO2k^yIa-e)lfFQ$@N8^&B?-;6Z6mPLhZ)$xV2+py zxVLJtg(+wVYAB0>RIp(0#-KLmt^x6KE~ai)g5k7c^Ky0y-cA@aYPkIBZja~)z6Vk- zF$JPT=%Z3Ck@C101j~2*+8*a$T7*Vcz0eJeyG+}~$ zSJJs<@(sE`fqcQPIgnh7DqE$1&aicAs#FV86)+#YRlsg!AvUomq)^}n;Fig|08yJ_ zQy!aB-`-SNYBIL&L{ydclr?tE*C&5%^zZWAm+%fD14fo&aPFR`b7s8$1;_~)@(!6I z^|OZy(`Q?s1tFMib1zviiaA;)gVxk8iIO_{zQ6+fiMNM%&*1&%`K!a1n@ksHxOpp} zjMfjCoE~OlwrCPkrTBK#G{8Jy^B}##*)*|HW}a7IJ19e{H1L4O!N)+Y;SMaASSU*S z&Lr$$kxWzc>_Ac+>lHK44H(_I8i3&QN)Q{7T(FyhqS|vp6iZz39CNier#n)G*{z&p z)hL~NZE(f<)`jRDQ`9I`*>YTLDXJreYuSVk0Hju$N1^>b1%rLr)m#boHc}CcE4dJg zkWau1-fHAh5EGrFg|xM+27`DY84EOML_2AGmseIXRnqhq=LOdS_Ncl?Klqhd#g}gS zuoCJxsS4;7BLMue%tx?>gKOB0Pp;94tc-zO%eU6H~BPV>7%%KfGIb48V%ZBilW5G*)uFeV^CJ%RxT` zetf9!{Q~*^21^$$%~vP$NlHQUISw`Kx;J?$H1bUrV|V4EWpAsEOsyN8V3}Z+zA3yE zhoIwSKA{8DX(@<&RHG7OoE%>;sC_0)^0=d%3!31&25bFi+4)MYoO}1hCCnbDFVkba z7w}$Wj$in~B*+U^UJgX^_yqLdiswsUw~ay7yT+vHd}@zh&xsaCtaA zv!US!$*I{VaFQEzgj8{F%n+EK*+Ux}BCN!NS4?^g(=;xY;4+*{!EObE4-{Yu1bT#W z3?6OHG6!2(qKh##COgrev7nnrPxi2sz#&@1FzpduK6YK>eXz7nFRE!;_NCb3rFWJ+ z4tGGShV)LA2mRpV@AqLpTEkZ&A4iBSHKcA!K;@8ELkgpog;GsXD?$q#HhI9y~OICdJ%)z)uczf+3@>1L!%ZWp$ z-d628T)83{e1LcswCsEjqN&K8Y)>u|^qSxB5B=R=^apw|%%KxS0nM@~N;!aDWC3!r zs;R>fcc})KBNoc5ndbU}8pk9NE;$66aKV**4uVAF@@W(c4oTw3&`4D7#O=fgVIW^l9tALl+NFNT?yX@qGxh!BVB_v`xr!tgKJ%y%B)XhHdQ#? zE07RPJT?_IX`>)f0wNMKa{&Y2tE?;wxF*g%Ux3tnac093qXkcui@u>px6L0op6h5R0Lt#Gxi=d6goyK`fS4j=*=X^Eih2aX4yA$EHd2;u6>$TSAVglt z=0oWkJOR?KKHD8hxZ>F^p^g+o9^=Q~c~tfa$3Lx7*9!z+)45;G19TbknjETSO~=_J zTVyS)X_H-uBAb8NfAP)pqkWqp2IA)}PI@%`>;F~t{%hX9mk>d{m;BfDZ=bzfes%Zy zAQ6hHUtZ^Gi6*UnEvBvUT0~^kw|~w|o#7DlgTo=tZXjOM_GBpxJ%Io873p7DiU;4Z z9L{59S)SU;ax}}nl44wvpHPkuW5cAJ0vX}J19-+;QQEgJ_VUBdREnDQnOgnLadD8+Mvg=F-+?32YCvN07ooxH&%}}Nd0zAt$SQ(sc!O?UV zPd9c!X*M0{leq|Y;!E+YDm5y}gA)2J3fox7E^ggH3s$Cw@;<*JivVLatvHb9ZHj^E z$?m#VA8utxl-R@^)?}qoDseG`lZ4PhK^r+a-c+8@k7Zrm93Hg(XRvvl$%bG)RBbvOatG0J&& zZ{Ian?#V={AQkU^zn$!`EV4;Da;M+7HmPh?O#zPNhWRqkr2y>XKmOs1FaG#d)&zE$%NlTZ#_a~|W?Td> zG;z6?{2tN&Q~GyF|KG*refj_He!Y8Vx9ge@eBH6_TH?`-vZsA*%Zp(g!a3){f z1tJV3vUxE&v)^og+$uhQ2&XLrknb^87(1nnzxqp4S4zJ7|2WkOn?=>~v$86X4iCnm>D*n0yan({ZIgLC{Bf{t^57sq+kZrTjVSmz}_WO@ZHse%O8; z{-2cFq);CqIzPO4{_=Tm*59Ci|CH@zcHYM9v(JWRdj4$`CoO`RX@0C!L*`C28<-~C z6||U)=T3}cewh+;6&c)pG(RS^@-n)E!zcTHYaCx3$2M-lm~lD$;N%-Ttd+yJ*bGDQ z{7X$&lAsNequQTLgp!*nWdaQxO>Mc+#N0?OQIE={7+W!jxE^V2gij;}@B^~1=%Atw zB28AR$@Pa{W+!E4xZw6!@y-RMzgGjhh+=I0NN2HbB2D4i#mfN3Ly<{Qyk5>jNUN(8 zAe@5+?|E_A!NWk|yg@7VYn;s&XAn7N)9t-C=43H@v#(y^NvB^>)b!zkQokFBVMdC^ z0w_rD~!CJq`zJaA<2E@8*?!5EB2Km2-Q9SJR4pQC;9Q|Xw>p`IXlNRkhKDw zY`FHG7C!djEH=2sB-Y4s0g-+qAKs*s(=tAww*(&j1=7%^$3?jaUt+oy|CLJ+=)a#) zlBe)Brd5r!G0w}y#SmhE)cVXu>BSJjd4*qyfT8{>{Pc5VL>TTlc=rqWtC-U&{8DD) zUyJ(dI)veE0}ShH%CqHNj?smKD*kFC$O1f+cVIMflGj>jAOrT3>}@Q8PhbnVPBNRMc+Gn zm?hxc$o`>GPwdK}3sZCjcO7_*^dlb-i(?Msk;)OO z#b4CzHa6PURjJw*r2--}Tlg5xUm&Jk<*MV}v|NorZ`7SC5d2tsEc~>o7jp0mGj=bSwo|b}a?FcaBL6PrHl~ zSlv$L;JC}Wlngg|<7B!OgsEUbO=P1*+pr6o=(H%d!lD^^4*5UN(%E=|flNp*Nd7`&8h)&D{eUuP-Wbpsce zWn9GeZeYvfISHjB#p%lU4s%;~8Wu_7>Vtx^VINLvr$$5DCK$YCk=GbZWE{B1!-0?x zP!B@3bubRM-D8F4O{M3$xDo*d<+bQAULowk6DO)Fw0c+=_Z9+Yka%mW%L9A~D3RhQ zXz2oVmYhYP9>_IKR|#&wxYW~|`|oKxt`MmPDvgIJ;5=UcU?rQiJxih&57yWUB#e1Hr4Rh9^ik7!t$AG0ByxBhcb( z9ySFcJ6&Q@>kNdosszapgRWSxsEUcqHYjLCxvD73AA^!2mF$4h35skZR0O2Jb~;Wj ztTDH$yaK=jYIJ1zu#C-<8LM(EdC|p07^6`DSS7DJ;=lY39hVvOs@P9&?+L%Diz_ZP zL?)6s>%v+TtxEsQ6Sw7l4rUE9X|DhZsZ{|dpw`_fv9&78B5Pexq}Ywvr)1jA_@qVL zjaxpSS>383rCD{wA%DYx)zp2`FyP`N$qR2I7?sGilP|54&l*q8UZi} z_B-)8jZ7dZ0W7=VX$G2kKac6tlmHb3L2IHm=t4(D&K?=5PrH1LK}d~i#YlhPZT2c# z)QXX8w1GNn&B7{z8CN&763vJLLX}sFhp>XyNg#~dL0m28SvhP0be+KyBZAeB)CUQ7Jbz_8O^ZGmL0lLwN_rQK6|8FUKwQ(qtOw+lNi;xOv^r~m zo2oRyd_{vXz_b3SBjmNVX$EozWmP zGPx`sw7_d6B8c(Ir7(diuSSIxowS}I&ctP1Y^Yn^bUzbnMmXyn3=yjwA18u~K^Hrn z%i=~W{8nOzvjDmjen|Dz7_zdXHXMmz2Ad@{fY@3sH3&r`7EG%y;;m*)trzlULnrUP z?cJ>!?4EQ5?7pcySwVR^-?lqR?J6RsHw_i6Wscr9R@5C^keb9>Or)ABhy_)s2W%P= z+f(T&^L5ExyW^X{nk;FJA5fH)+XqYeYIrN{Z==a+01IF>SMSxOU&%>Re33%rm=1*; zUFy}OBbunS*1Rf7y96P-4Eb6~wALV1aifs>h91@TanoffYKs%25w#XQytn0ISS}`c zeo1Snw}O0hr5Z6}p@gW-xYS9|1)(Nl@Mr%_8H%`TY7rZ_YuY1$oHsp{G#nu{;z6&y z10M}9Y6a=FE?CSLbwoBVyVy3>bnFPCMrmamtUYP_YCEY=)vhdSY8$fqo^M*Vdc#nX zG2yf245G(DNqNSi_Q%Vo@`xMyL*xSH$x+0guFmBckN^{g;P0)66a4GP5DmX2JvPT8 zQH^c0pYG6y-fS|OKyaf9(@lYEGaHe{D!LR14TFf~O?nrM`>bW$XW_U}f)X3{S^Kbc zz51xnx{bOy%?Smb>g?dpu`#K0VJtalw7fP9BxIkez#1qUA--2X+MknrnlFA*#XaWH zRNUItZCU;>{K8yd&^PgV{8GgVrP1RGakS7IQic8wk{ui0%9}dD+!mITh}a$O$U%e` zE&g%b_EBz6{%0Pnlsa}}J^be2@CY)&dDsF#`P)YgHJLRbfJ6^@359 zYmNeB7zJxP@|K4(T85c|5qnhR*A3fMfZ(?t1pPs5Ru*t_UO)2>&)Yk&d2=^ExM^k z>d=Q+4ZfhPyXExNBq>d*(=!XXHJ)v2k=NEAZEFBj>&u1Q_pm@VoLtxh}|4o*Fc z8i|OWre2MZIz*3HuST6PG@QIKSi8Yf9*QRL1m)uB;SSe0j=1l!`r{S#$1CdF$+piV zDAx`YINT#+@e8y7qoKb3+yaWG+GAb2%MmE;*G?wgdi>amR2{cf9k*5$*wgBG%hPa* zMOu+FIywOUXhXC6&K{gwF)}jLQ1Y;v#)W0}8e1S;<*GKQ6t?oN2}%WKE*ZPJEswU2uZRqOax``A>q1>Yj28d{0Kr}nWKK4&fP zIctMY?c-iU)jE7?9~-JV#Z&}9v0}nkHZy8#2b9B*lucZlf#VGqx|Bp&KVqKyMjkL) z#l2t)&Yv(ic_19HvGUsD`wbGgR?~HC5A%;bgYT68wpva}N@faTPCrim6X#`xakYJi zd0L|p_F93)iINYj`6(ue!=jw<48$x>p5-o}9*Fu{_7aF*u!Bc@3MOQ)D3O$C^4pxq ziI*hCvpY#quhw~T3v!*`+|(EDS`#)|k95;MUIh@&QOr;`x1?;E%{X)PG@Fj*7d=_! z5SIo?bhNwOM(c}c>Uaq){iG!}&65tKlI=}$ii~uJ=z&7@h*Zty7eo{tGd5Fd=e$I~ zO(Zilf(#?H*A{re*tj7ey|G}JHjectxi7ZP-nO^0nN@JUS~uK?OsmV#4RGxmgqS6S zN54yisC(bI|68~Ax=BEz;Wxtz@y1^sy*POE&C~tiqZcn8{k>;tHl%E1)UOi>mF?~^ z5K%||RA_+PS{XbAZ)sxPUQCo62as8@Z@NHsiy@+KS8x0X%?aG-p2N=B7 zO8O&$2#SJwoL0(eb3ngESI9rzg&Thof;+I?2-~>fHzJ!;wMAKh9VG01U2bt5R$g7p z_$s_k-JoN+MP8)|x!$SDEZOH)Ya}4ntjL}2--paIKRo(wKLD19E)1slfWOS>Yr>bc zo8SsjiHaV!po0wC{(g_6?yUvkbvkCLWy`A2NFL}NbE&D>y{19OBEPkEhhLXz0Te`? zI2DcAA!BOu<#0m^I5F z(h%|l)Bq%dO$@M;WtEM$ljLdo>xDb;kjG5Ui{)g@sbw)k7BHi{@~Nq3%7YO?UM+xp zz!)%pmSH#EkU13SJzp5N@HNzR|uj38uO)fF6v_%Gq*aID6c3rXx(;yr2jA-ovqczWuq`6E-#HpAsx{&oA@OozDTX zFhppFM%?f9AS2CuQJQC{w3VnArlK4~RLx+7&gr{<_y(w*5mlrBe-42A?J z7@tReT9e2mP6F;q*mO0NmgZ-s7%*ARfQKb0gA2-@aGh@4ux*)QGsr%NIG_p-uw-8? zsrkoK$6`!upex!key&z3T+Q(gsHkEj?M$E79R9&hqids3?Bg;^Uk4P-_6ozBQxHHJ zVu*K5)DY>QQ{-;|PEXUflGA6;$AB=|P2RltTvV1Hf0_}AYI0F7ilym@;7tJrcL_uJ z3X0_)B(Dw*UqY}%^M6SBP46GpGC8g@V;T=HVKA$DIwH5+Iel{}W(ElnMBbDGODrfK z(q6biYFDj0+y?AYxYdqP4xhX+TFruVV5Yv91QYYu(^g^b#i9P4|xY(&W{Aj6&(jRHb=EAUyDZfEi}}*-AgP zkvFo~bnx9Ut^c?SYKco6ZJU<5)egAJoN$h5*;$?0Sa^`*3&4R$Dk;r@v?YyFS*2o$ zqkIO0zu{uCT+`=laL@esSO;PK$<;F(rT%RdhZ4JiV!mE=$91{d`yOlX=(|S;hs~^k z;b&kbrldEjJy{2z$ef|gw{*E&K*Vn?XxKOO4*u>a!Oo~<6)NAl_2#)cGo2NL{Amh=pJV@LGY7Y8r* zn}!6NL=TC#iz1^RYFGfM;oqLW7~=T!*m!9E*v$T~jm^9ZVG5Pu#MPkft2uv=Y4qL< zlEo_E#wK8ai}xw@nN`8-cj6sC5 zM?6XYM(<4@cpU7czsE%xYzK#bdGyr6rU`EF>750exnuU=R*ujxgMQOr4|SWMt5@Dz zV*beM9|h%=rekJcD$}H35G$ucsUu!OK8VjN!?;h(J+nA5FgEAfcxlxqBnVK6#*-=P z+e{f$zMwN4maH~OY;abb8%(c2FFl&ir8qD;+1q&v_G5l_3AjwXcs_jm^t=7T-e{0a z%zR$TrQOrS@Xc0ig*n!m^%K~h8u)sirt$9Y^XQsM7`4GfgkRT`hE3P z4oN!K9Uem2NncftYhX2NjDW#~?Y z(lU5YhT%|nvxYZ}HXhFWuSJ@*P~#mWg_KEKJULC$V+JBOO;`6J!R0vkI$r3frRc!KgohlD(YgSpN@9Ud2!!f0s*kf$4wq{;q z&AQmNn4pO2_JbjZ+UN1qb_28y?l+gm)OIUNfj8~SLN>A8%91LlU5LKTbuhQx%BCSS zR;NOo*RFF}%4fH-IJ?uVC`B&LtBakjp73I_x;siNwz8OHv|Cw_bC5~74^I4SzB`P~ zCXe1ARY{C{VjrUDrAkW`X)T7#qHO5m<+CrRETY--4`-xKG@7fosHs8^5TK~P? zXe7r=)7uqz6ATG+&(wdfu>hDTvHp9F79f9O?dJzO@D$gsb434^%)50eHZ}2iPdOo* z8ptV_Z{{yB&r-_z=8n#aqRPmCf?a4ud&vdsF#yP7sK53Nfrewcrt#1N_JS5D`VN7^ zwx{);yCBv+@Z|#9yCyB`czp|mB7?^12ci7Kh|iI-WtA8YY=E6tNwGxLojKf2v%s|V z+efdSwwktXZUi9{xa||ON??_nmKWTl3^(+j_6!{vL=s~_rsc1h_HJ`#Ud_h05T+no zxabzQ=}-YtZ!yYKXYe4vo#v-t7DnhJt3zXRCR>q!L3~)jexV*7Bsf8O&GU7}MqJ3Z z?vh5tOIrq+g|OnYlT4|Byff0_NNvm!&n)Rf-7#Y#JMT&o=7=88fHWGMh`XqxaBw9&E&@Z^h-o&QZJc46%c=A2$ zsZEsA=Hcfr9Y+894Hs^WrBcGsO>LG1-}G#HR|O5FFf@19-;giy;ew&io1vl&!K7+O zr1fqNYa%bV*`xBB;YZ+AbP5fUn_e%B)?jPQjwa|$5I3H7#=W`;157mMk94Y!KG`$u zuBvCcu3Jh2lJLdmH)LO1=w1ubSgIGfr6~x0aN-9|MTVd-M^nOe*|DpW%@2D6(%IvG zOB}M-9Uc*1buLGk9lKfa)MzPhxW;|&3Cs95xH;nzy2{?=W#v-h1mDHLe#wLH(}B8o z0&ciQ67qiqw=a7kYyh8HEUM|MuC+w6-gNvTAk}JL`ZT5{##yx}i;Lfv>b))%L-lHl z^})y^`ZRE$64D(6K6G|PB$^4Ub08*5#Mz-C5C*`_$$D&jka`>yVRdY80DoWL5d>%H z5V}qWtc!$c4Z^~$6Bth1sS|W4K3SF+;%1fs1{cYh!GEz&laP2a`E)ksirkxcVPjaF zi^Uljpg}{XDT5ou`8E|tNlvHeBP-{s!qr3q)R930&E6q~A65=l592x8*`UG?_WZ+gXPzR_*eHoq1T z`96ja@zVNuUFM*XzDQsAKLa2RA+G|F5;EU^5+|?SYg%M)cX69&0I#Tl?jPKV%zn#X zXOoLwKC|x)XZr7dCt5*@;a*VGCdGXPZGSCh)-EHS{o$(dDX(o-IE~UeN`e?ct4rzk z)^&BDNU>ql4}75Mpc-TFB_OGhyg-oY?>}G(wTL$t)2YA@Du~UVVEGi zQjp#Zj(A9%>xEZRQziy%LppcC-CQ)JcUpdR4NI;oyRZrBTDO_6N4w0{P>iRjrI1G1 z3115rnIcqS;CyV(eytih9M}f>fsAFPQqBRM(K6_=X^wqgxW~A(azbdeMpJ3cvS+IQBnsBpPWf=!un58PtfhWSg-llpS3k@zwM%N7WW8g#o4@gTO8xOC-f!gihBr#H~{CQWDqLm>X#W@4TDRWn37$db?d#-0m~yV9fVv@uUBP zv08!rKLe%JfIoSBrsL+UC|+NJL;V!U^jdsm$SFlaTVp&5M)pm`zI53t{V09~Sht}; zj>xVT0gYMo3tW9`PvsZ&S|V9f(41u`s8Aa^QtE-A4{B6))~epUAV_VqOl(pmU95F? zcOQhu(sh(ae_+JuD|d3Mmf$-Qn^`vS}b>b zoMgrLTXNXdr*UC?pLeiN7YQPR{P-B)JN7XW{IX1E3-I&#zpox0zI^qp$>WiW&r|SV zgxpE1yogR@Z7_}-Rf1pns)|D1it>- zP~+y`C)n)VI)jb}u?lva33%W#8?S^ORcVHw9cvwu>Nw~OP21gjh3DD2@U+8l5Un4Z zlsfm*;N9QX1q{AP%Hm4~@ou|&C0m&J;WxN{eb6g;g9q;?>gCg7?f${d*xStM5oC4o zpA|!gu0D8I>(XXTXs5kAukI%FAMM}pkyqSzThP0=>-ze>7aw#@!O8qnv57g!&3E$T zA?ar4750teNK7xSMKti%(PRqKCkrDfzNQ7&r2&8XTSNXoVlNStuCSG;fse71esk+c z{x&(~kQinfevFJOZYbNZy+1HR<$8SR;;36^WJp2$1xhaU$Qp_f^R&6)AWFN_R$Xt6 z+JasL!8(GUzAtrC8(|ERrkErjFdACjx0?bMjUe2>SXQ?n`LlH!?p@bbT$Xs(ihI|2EST`x8wtPZJ`PH8?T>RN)|B zp?qD0Bh4YeB25MHv>F@r%Mqap<{KNRin-N1URS7VyEEc|+o0fz)NcQg2ImnR_1l{_ z;B2fhgj8EwT%hv?l9n0MHlsvXw8H`8?d zCa=17B&9FY6%Mi^;JEB%x)IX19zFz>E`fd0f;$7&>p60Ebnm?a@ksZsZFm%{cZu0j ze=`n6*WTMqlG0LNmsnEnZQQB1@q6m6%zjyBmHED+KPh*G9w#Zdm#v97ZV|EiB^Hso z?Y4-H03dX^%S+F-+geJeVqLM*OD!vPc-eVgYEiG) z<(f+xpJ#&6(iVo(&5Kz|8H>9Q1oVl+pF0Bl7~kU0yN)(q3INh?C#Z-0j<4QtN6?2o z+iF+nd$cT3W4cz)QfcLe!epkt61f2c;P$xg{Hacd$*(aC2quxOJrJ&SB^&@f0;f-TJ+9ohzc;+TBVe$I-#}HZ^Fx^Ghj1wZQCWrx5+QyI#Iw0YW8nhw~Sp(3wU~ z-XD#$9d{ylD6ZHR5I-PN>oxh?}oFJ_l&dB`};=PHLN0<7Gr(PWs8ZyG%_H17y!g&_j2GA4teV#u=Apj5p}BexsHdq?l?Mk>v#6-+7ANREtM$?yd00}_x9{oW6&+q>}`oEtuTC1(yuSb9Qn&x1-EpLYMX8Y{=htN!ZmY*0>d*ZTo z{rCP$Cf1vuWwUX9vPo2RlTZ@PF3C0<9!@4jQTCYb!XG4h{F^o1WQCb<emtP!q@5C|y<=L39hc7;}LNTRG0mA^R3WElwf zET+!ShHRKFGRQelK>$0`-h7gdGVG?zl8M<=X-V}f^E5h3&Wq(_oXiVk?@XH)$q6iS zwLCe=N4a@v-f+K3`XG4eNzT&Z8jIu7S%3O>a0 zJS+42KK<@6pLBfb`M?Ebz9?}@>3CeCtG~SK>niZo&Cu05?pm)lXRP;-8{m24yZiF4 z%Ub?g&8{$i<(B2sOx{4I_OYgu*B(%8a-WEYm1kF|ER&JS5ex6+yKI(0k`OcW@z${+ zU$2uznPthzayD`qMap9X_gQZsEkhB3m;Z7x@S$U_JXFk3!`F0~tAo3L)uD74h72J9 zPXG5S$L09{v$Q(v%Kfk8|95tG@6`DJoiD!lo&Wy`|7!gI_m7UgzZCcXThHY8f0y}p z$^E}P&ED7k|KiS_ufDAB|1ZD#egA)ifA#(U>bw0<<@V1~*?{BPBvS_X1^gg!Wn|cH z*hXZ>f*K^F+}wb_Hhhw~GQ3{Sd!~xZ)y8+=YaZiC#i1*d7klhyO@a)z|7w72bY2q^ z+u;tv*}-3qIDZoz4oK|HBa3pxI0z6x!1TM%!cKxdmDy=-Zowr%goZ%_@Fs`>^xXul zUHw(3NIoQ4Sps;?AG1z%Fb9YGEsqQT4hmUsH+uZeL!>w|wq~X9vY`Z?x`T9eZ zJ1dANrYM^MhTfaN26f^2kfd0BPX&6*@oBGNn@YWECXPjUh1r8aK!a-zkN_bY8-9@- zrc9AO=K$2}V@r9N z0T4*Ecb+e1Syc_GQG+e>#o2IC49j#TijTh@ygb}LIvT!sbol4t z^v;s=vwU<0`34{zv$>s}Wz+ycfYubQ?;%9{$)6;wp37)D1NSNV&dE)admNf?N1~80 zway7ue_c!&l8+xf{(hfs@cRem?|0(=Q?NC<5>2$)cy_!Aq-7a&cX7gYs1LMS{< zUqj8Z%tng@QhwjYelTMEI2+{^T!w9i41&Gw{s9h3T&W0QYU=?LlhvYyX;>EzG!6l& zs<`Yy#Ap&cNLzNYXg5fUrot{-BIO84a-#y(pq)p*??)xgNjGmXfat$oMw@IkM0(I( zYPMY^nrFfqBU~)h+&P&Xq?K{LpuFPtNY7jm`$d`S$58F zHZMT~q6%978&2gTcqFKUvPmQN?LYwD0N^KO@@cVR1~8 z8T1+_Vu70%thQXM^AkgtzipNuT+oVlzX^8~H|`d)?a%;??>gbRU0j z7HjzZ^XGp)8Xg=TygYdH^xz-+FNEcsmzg~U%;~&{GN2!hCq!xtISv4*Kv%y(=ZWkZ za2Yeh7R(Ejk{bw=DfNTV6orSh@ZIBj_Z@&S46=h`qHY(rb^*6xo$f!7b(0nzDA>aW z*aIWACxs!`n6r!7X%0pwyeV&fIi}R*+rPZajg{FE@5YMDX)mmM;K-dhMn@N=m%1#1 z@LQlYI*WVKtIkSuxR_!0nQc|We#oQNG4m?3BTf!BG+gKP;##G5(r}HaU?ckv=@#Uw=``R;kOwY zePLguq8r^i6-2=wg!o-91)ELZQU_tUeC$$XEdZd?^KkA#Mv1=UHP9`)0 z0@MAB{-{UJhRVY6pl2s;2^yJu9ih>oqMU-~qLNbccrl;N2pyi>^H>4y+Wy2oFI`xo{_AV^8ImFV_6a?vL z+yEi(NKbE9Qcb|EZXgei!ZDv2IBvGm6VTwSfC)Dcg$xY){les(;|j?6%8UoLLAbal z%)sW=0zT83jX0Q&M#Xem03%RgJ5X?zo#R(~+ShQ;g5+U%adyz8^R5PNTlIcJRfEt3 zpjB#^1_aD!9*rZ?f^gs4To33qn7?C-5x1L&({JBRZQxr0g2sL5-7q{bgmZtN*cOr{ z-|9@Ptazw6PQz!&Wxb=tl|CDku|#k-Ae$*)0hOZEGqg_&Eyl!5th0%k8c>6YS!pZV z2*%-b*Sd0YI50Tlj)ey$x{Maur!)-wE!#9deD7K&#UEO-EGBI z8WtIBv#7GEPr-<3y~EV_)a$4v=Hp0zsrf2~(J?!Ik=xT|`*>Ks4k9zXXkN?paoC2h zAwpuHN2yvyi??o#+$&KpwZqbD0{(uo5QV`74f>P4B3PlFkjT0YI`}ixK>3l0@H)VZ zL(z%q7#QopvH^4*oI!|&G=Q8B$({$_zLM(fzUu!TDCoWZyPW@@)avE~*y{gxXZP!` zzO4KInGS#V|N994YW{zphU*`g7^VxdaD*Q(mNS%ak$8Anl5nAzyU+kGNyWy^pXI7M zgz`3s(D1JZhhN;;^bKWRzI@R;&(hb{%o#rKp{REQgwE-4QJPCb&64-Y{x`3_!@|I3 znaZHqhMy=X@Eg-(6G1jLne+($I%6P;hN z-_?`#`vjAT58fp+m)Pqs%d~{3Ff%w~|M98^3PU@jG8_L|;1^cX*)L1*YKnb$J7uud zzM#6tzLv}&_{pVo+}T6?J#caN$c1W|i7e6_cBoAU54GiNh9>PZNF`HZwv z*lj)=zTuaqUEAesG+$QK(GO(-%Fl}P_$D^spA#~$oXR`!7aJVV56=BP)6gaeQmF~N z&x3TpT>f$NRaU;qN2D$V5+I&+aQ{CnKteM~vI(ksE4Igg=6FPDS*91*p6F8Z$pyB( z=Zct}US<%~J(;Ax!go}?W@9knPj0%_e%#rXn1{o3xhVQ>pcux&FVH?4~t@vPN)kAFGc`$v@FqyZo_=t4ZiBS8(^b$8+`bt(_)5#s$^^HVe<3rR~+ZNMAuyG z1^b2GdaXkNARUbpqOJKY5DJko9%t+#Xa`}C?VSXq)6+OGmEnsFrJ5AyW)M)B z`ui)yJJTOQO@xAh3i>%?H^XXP7*c9>#c{qMZ`mp}9bTC8DjSh^r>xk?Bt0Ga$BBVv z&v4~~@FUQD`Uc6^cD(|QM7t7qyb?fKH{8cBypKex)#tnRv!ajg^Os(2rswYSSN+X* zZhINF<_~Up0c=C43e;7pkPp5)+<(%uo~8Xw9sG4;Zo0oy{W@Hv$3GtaaCGpG{iC1E z%KXcE`22A6kKsjDL9)Bkaz025cQS$GV~`jNp88-@{LjseUax0Pu-*Rs`&6=@+)CgL zm8aMA2;Q!`^;yx}xGi|}m_iZaH!Le%d^f%zfj)34ItKboJS1rc6t38vuYI zO2hY$#QBLKpnuK^`E~RdR-`+;UEDhYc?9BDpKAA|7{sUCc~*jhFd>TJhtuK-nd9c^ zcf*$lhkqYFJ2*r?@XgQ8(-Ikn>rc{|p%~!(t_oduV7T+maHp5|H}U~+RQ0xY*K2FH zt*txjwRNYhtuMTyU$hmy>lMA*R`g4+=$CCpzw(NH)mHRtujtopMgQOx{X<*PKYB&~ z*t&>2{vx%n_^w}AEh)0uWob@?>#t_jay~aVA;a+aD@c5Jh?e)0-8!NO_9gPrT>DNP zHKdtD9-3=Mutl0l`)JM{r7-N#voqb_C6&nM59%Yez6onn~oLxpo8@ zrI|z?nrlb!Q<_QSp#@uFC@T#m^3YO0hQ+p}#6H;;RAF;g=I`@Rgf>6>Ecil=Hk^it zRr<$W{o_mh<178+YyIO7`o|yj4*;(EwX1*CPTf$>@$!4fZM^>ky<|7(mVvjx2{VzF zJ0+bq+#?d^#@}QZ(v~hc$Vq;d@KrxKe6Ia&&G9O?i)2#FPWwB3RaO_=m~+PZDw~}2 zyi!Mgi;5j5KD?wDNxFJp7(aP!E=RF?!EpD4JT^^-g%t4_kU*np-1Y7~Zol*R$eCL$ z7#SbjCM38O+g(zZ%r}Gmc0= zwW8LZo@8QQ>zkXCwQTz0J7k;6WPc9A0DXHrOUrR`4mPl(GOfKBxR!A4cG#{B;A~wOTrX!H& zu!M^#LwyxtfwL3?0L;u7a|(oGHXlU9MiIg^T`aP)O0u*xzj!4?OolQP48_fmOlLHR zDx0SziC2&Y&5Cl0ZGwQ`+-%1MfY6k7ZtH9W9UCTw?$Ef3fn#pivV-J0JYNr-6aeE< zIp2W5bUSyUJBH7W9uNPr{{qOtyi8B0shJ1Li+b0$AHR6{|NQOotLp<4&h#DDAMHPW zdGP#D73l{OZIDd${j4UqBYLzYlAHdP=+mAs0}S?xB_#GwhrMi?;juh>^>Y7jal$0h zAlh*WTWf9HobOTegSo-7d4XDnq=$k`!VxgeGq}@dk6^w6R#@u=$sx&N6F*a%C4a4! z6WgdS>miu}NPB7r;H#G}9zEVy^qO8^VC6wF>KnvL{7f+CVA0%R zW%%nENRjdHo|FhiWlHba+RMfE9PE?Q{@#Xu3DRZ=j_tj~fZ8MfCA8VQvE(%6#Yu0u z==W{cG9Yh&w)b}%8`9J}hzK^xk{%q@!D$$PNS#LST~7qU)^9mTG!ThJ0hq;tEW%S8 zL)wY5%M6XzMrKpa%N!!a70?h4KSwDW3Swh&HcKdn=?-n1z1Ct<=0i>Xo9xu=jcHzu z4AD+!8C%JWW903DMomT2Q1s6fB>Y`GQYwQyLqg zG!Qfd_?&H_$b1VmW$c~?g4yMqq`hdP>O`j4#z{KLCm^G}2?yrQYGxAeeqd zG3JQxFu;J|mnSEHEh!swrwgB{$GMs%IKDvE29MFnWYbTLkz{W`-=huX+kAQ4gwRUW z{glvgrlZ~p{0WiPZb)kvZM$5FwH`a=8P3MGy=N({9E3U#wP~E!B73lp8qykBiTMC- zM0Qh??<`RdK(2J>X0NZPq*bBihHrq~U~nk^*Ld)bj^94uc$F`fsUxb;l-ZJI!x&Vb zP$YlY9&gezhcqV48~Rk0jm6WQoElcvex7Z&xYJ7R_Z91l>k~EawF!8zG(?m%Xni;& zn@MM6uCn~}2RsHWWBnFoBx2DH##ySV9k#vXKQ;FNm@-@SDyy=FX<9Q+)d800XqMh^ zoHv5A=%k36bxNM@csBd#&a}1+VFk5NwtqE6o?AFI_F=Pg)*+WIB7SSg-#153KChBV zemnyFqad0FDhtCSrt_hXqy`7=r>!b8n3}&qHFJQOUC=V=!AnKU;6C;yD=(luxrMN< z8wox!t;;eCj-@^$ID^!Uy$Q4)5|bq9yuu!xDDnlMZR~ANbycG}eR3Fa8FZE{s2J6{P z#h*O0Yu%Brx%WEcPmxZRy&GP=fi1hDeRKC#B!SyZ$KyzgyVRD9CTOUEworq33WY89 zW!oo@$sQs_H#;(Y#r#?nh!ps<8H*Bp+mQvOx=O@0#y4*OT82;dLYjpkAoNn=7N}e4 zG9p8UuKN#?`kedXQvxL(@`d+r!$&Xn4nn$`u!Da5FIKz(vwdJC9gU#Z0Qp zpEiNt7m1cND3k*el^ntvq$KeHs}1@t+?tpjyVH!|gqUtU-)ukw8yj11d4yEr8RKZr zLBV?KIcVr7if7!Q-AZpqMg;+7?Wmk*25c0!AvIqXb#HK(u|7()n_ZLbls4jyW1$`Y zDvN8^U}Q#h5_O!D=dE?N^XV1KRp}IKm>`P7poy8;y`~uYR|P~fhAG+^%SRbrI<~e# zgl2CoI~3{IjcsEC+RWYhI^RLGXr~xLP^O`XWkvAwG(UxdKMtE>GR6@*TAQ8MFu)Yz zQQtJ5Ex6=$8#ttW3lV9ctrCr)8I0p>)N^BM1Ge<0N=4u%rJ7J{IeQPtbv4X^Dc3Ql z6XFQ%=sLbPYqwV`sY`&QScX}-?h#O? zX2=-0W({;2zjLFr&aI|MqLb)hrB09R zb_nI2J&k74rcb5#ZC{Gc;$>%HVHFxKGUDin_}h*;dqs++*Giy2HOuKKNc`F9lb=$! zczqpTK>CAu41xKRM!6d3LN0#b-0j#9^Ji;-=tJV~rk_Lu_gt3z;R@7>U4e$0Q`c7DWJ>Eca>Erlg1bcPCS$$gMswNoEY z_8&j``|$AP!Q($44S(2wF?{vIlSeQ2RZtGh;o@zYhtUIp14sGY`UQVNF0>MR@kcn`}bTRMP-xxwvi>WzJ`PM49!UvrTStPrlY!0I|7 zo02Z$M_?j8PZwtc*(9ipKfdsj#E{-Kyin1{I(yW@7(y%t=XLgYcy#b||M2lXYKxu7 zR<9`DTOTI7w1$rJ8YDkM$}1Sbk9W+vo#bV;z>GI|K1c?MCz%<>6x5Tz&4LyK_6wwx zFnfNsWhHzhf}kIpK>~0}AVF^v>2v3&d&#$;J#sD}wJNUkE|<3biS)Ks2ev%z>7Qv& z{|uIq=8xINuqU{|pMPRMBU9MUT(K|cq!F4k$ZbVQ#x4W3ra3q_t|mhCF-{*h70#O_NO)wNr4<{xoM+WSYPi$Gy8l#+M37$r z22x){ok3$XLUQ#fvwVe?aLl3rwxebzQ*M9CN_g%@$HE0pRHtdQ*4}JRSAxw@; z12PCY{+`AlABE}*@FFt@<(N%5Vx*mz6(W9?l>cvn$S- zZT&he&Nh~~E=PEPI(i=xq%jhF%Z3j3L<0`^D=#Nw9RUclL`5j>jIOV>vq}uOYDhIv zpA)qO22{F9z@m#HdTCEny?5KahFNtR<|WPJ`!xFs@5Xa`1YZ4KaP z*7`#p0JBP&3elIjJb6x}V1+pi0%6VTYc}5}4>2X9cVssx8O<675gzT)E&K-;AiSEw ziuU>(cyRdbb3-Q$rhJzzjxMT2Ha(b~6ulc&PBJJo#(!Xn zY>&?m@nZh`1)iA>9G+Ue%D5E%Kr#mOfFf*}6x3KTc zN&#~K_FWM7cGEe`nY!r-+P7Z}XY-!#2xmo|rWSDN74pU#Zk;T|dh+cZA00h^aWs5>`1H`BnZ?4MRD6(Z?ShhLH@TNstEWDF^>otOj23#rq?A3} ze6$a5LVP7dfCh>8@-@6+Gq;E66ZqUcgU0KB=nJ?0u-E3S}doKl=P0Nl8Bc#y&l^PfzUA zzJ2=k^Lw`TclPb~_UXVr{kMJkvweE1YCp4Y5AD-)`}Bi-`oH$+g{pmI-@dd@uk6!b z?9*TE)8AC>zuUL}P;XQF_Sim+?9Le1@1-)va52>K_}T@p{h|xK}H{`wK%T+rW*uQMSwm-_mbkvrwHkYS#eI zC_BcgV}ox0xkLS8LJ4U5n%#vqizATwI5t+Z!(t0L+}^1Kb5( zG*0{l`>)FUfBR;25B_7;WsqQxU?$e54qme(2fliPuiiAy!iIqI{dSxY!}J(bim2M< zAe9`JlB}0?04~_sCm=WnHLKzL9^*D5n_LW=#!k-cVa0b#zLF3Q*8)=||9+4J*^vw< z6^Ybkc&i=GMDp*%&uLk!fG=z1RcJ+3LA|Zjzz~V5EdH*Q;@FF-9KWfRlwgdisJyzz zso|95NJ4=rU+w^IDqfhhsb5K(euTANy{&8H-O#cReB^PuYhWk=(~p-YKmPFO<@aDZ zp4>`ye~NIL1KcY&-|ShJ4Z>$YVQnQYK@X8wO2pJGvD0G5d4+5?kSbh8un}9BJ_&UO ziHGd9cy#y@+yS5LfBWdw)0eiUi%Kda`kg;XCjyO6QySuOZoHPAP)0#_m2mF%&BWZ! zE#dN_>$2)gSdCn<89VmGfJDbMMuqH=(`(mqYkq9rX6-)cABB)|7LpR8+n%nwX zS%b`{Wjgi6Ydk`~OFE{tO3*;zun0m6SMzL?pX4I-hY6V-x&!JzIEnXk`{s)~`~7*} zxkr6eykS8;XLc(YYy@$MHz+bdWjLPk(2olM-!wIVW%rIS62IF_`Z@}T$T(V6sgPpz6Tf}dH40qSt09s&jU?cE2z+?f+x@9`GZIF@%vjn@l zI*3fR{%Pe3>dPA_PTS>Q9|OHqB3^(hLXT@Y*ct@+>@&L-u&}!I-l9*K(pS)gZ6hCS zZ?%2*H`lI#rEV0E2kQgM^ds$>_kOz@_1eN6&vCO`E_e##b@g#4zYd@^C{9)P1Lnu^XMo=xx)>4pc1;Zp(>2sXA`N z$KOACG5mpXXxw*AwquW^NS$BM?Wc#_+x-U{3Ovl;6@Aj!Vu^bV1suL0`!~OU5t(UI z2MV2vAL*54ky~dRZbaf1B~{NLxsHFYvk{|rPkoH3am}scJiVweZAlIW$_ut<1|bY6 zNI(*sLNZElFx#LYnKR5ma_9f;J^gOzV)zcfd-43$54Q{~&}I*6RY`9?SyHeCG8r0= zF{E5N!S61hWmbWm7u;=7ZnIS!Px3c;v6N0HX?Y5!++cIWHt!c;lO1o6E9qugu*7*; z(Kre7x8&3LBtN+j`HHP6^1uP*c_nMK@=Xe%b&}&^IUBRL9L-B5%ocHVm2ni+G@tpZ zml*jqYc@Hh+?r&O+kvLC5bh` zoM~-nR&O+%*~^3yAYUE)|r)Z+Cqt zNSMRVt(YP!?cHNjP2JPS#_si0oAq(4=5NaXj*KnHhyZFvMh)n7uu=MI1Xtsw4y`k6 zktk2WU3ny%Up8==PB&|4p!;#%x#aB(5-LxpF&xDHVH~Hf%wd~W`AJjNcom8#dL91g z!tVk)%yGjRlF^o^h-r%++FD~PiPq-0jA@Ub+8QKPb8CCzWpJGy6tJ&_ThK7+*Yt*q zo<7`++l{P?ukYtXx3%XED>o{iBGvTf?YPS$k$tRouWZljw2d3`d0Ni7iD~YN;67Kv z?j9guvXH*+y>~1Gi#poaliG@(&D)pzYz}LEHg7{4$n4$P3KTHSz@NsgB;pM&4mXFB zTu)lPHH}>VIm6~!_9Rw-Je?2}=DPZ&kaoqkrkv&MLyC3L<|iCgy+q?Q{nf?VRcdbT znZLa1!*z6Zm1PBq+gEq(!rDr3!$soe-)vhUGXeqpb@yW>T#h26M@M6`FC*iZPYoCv zOvU18%a~8c%jsOkj6c>~OZ2>x2Jpw44NS~cM(V?JtUB(GRX3S=nwRxs-rl_SB!iP9 zmi0Hp5P9#_p#AcwBdonr=^B;lul6b|=A`AZ*K$>keIpEterf(hv|*ENSL2=|E|fc^ z6b}4OV90Tl5B}f0Q0M_n4&ikladvX_ z)+}PRPtwYKmT*ITYoIR!LnQ%`aD42kFk&di#~IqW9H&*rZ(?~DoPc#jnZ;yPi3V8j z_FPm{Z)7zaQRerskVU-4{vt`PRD=%Drij$iOLT>Cu8*Rvie)?n_s(X;^7JgJM(N23 z2E#oSNAq(>b9C%|KP?$~p7zC>*b4pnXaRJxg^)pH3%KWc4?zXd+~_aZ7upyT>KzDex8PkBQJs@zNA%;VPBYB zy$%a^m@Bqn)e^W$m|OvH+Ld4wCr|36SMzvAukRn|a2-B+`R(%;F2Wb~cHYj6FN$2SQY>16}Q&d=HIEO8WHEjKQO4KY&yA!ce>clcHowoMbvv z1Z-}$!{Z-bCC408gw0tU_lIE^5p5sx)ywNVp>Ovgn%U-tMA{C@P`ZzhgsU=+%02~+ zRN&~&?bzIsV{@=W7y`7mFyFll3_e*60!Y?#H6Y-Efz?XwIrNJ8oxRA$%Mpxd=0=K| zwWRl7V?Z)7(G5?QcOA|!KWsBasC4nlu&BzJP^o~$kbeZ37YYtU!8{@d9=pWVW_@ed zX1Z{w)^HX{fLgL|;N*?GU=G>Hnp0)S-me-td6G z$^dovwzXP7*Nq65S!NRm4XL8OKmezT5zst1^n!JFw;hf)-nI)u0ET=+TjsWw=&X8P zwYvf!pS>&Gp~M}H@NR244)f@g+(G7Ad#}B&-Ab@xbZ2VJeYZcWrNjQ1(T6yG{1^Q# zaiD~drtug=oYpIsy{C^}e76s>Mydi> z0Z!++J-G&nUu57~l01EG0>aYG^^ofUG3l&fjX^Oj7PYXQ$$UT{b>S)_BV(W=uY00yU-EZ-p0;7Rg>6m;4T<4Z5t!GD+#(6cR`jku2O~E%9%6& zF$}dOCv?P--+>=J(Y&*co{Kz8cA&AF_LTVk6_9f|BowpJG-5U7fJ6=iP2zjr+i#js zwAWdgzgk97REr^ccyQK7)0IPK51Vag#Yio2vsjY&BKX*a*a}?!tLpXdNubi}qPx6Kxvnun3(TLy z^S&&e2CtxjXSj1(**|>#Z2uXP-Wukcp{r?M^QO%JTuO4%ywdk$Ts$p_#{psJ-!M$H zjr%);9Zk#7Pi{S+Wu}p}kbzl%WBY&m;rWY~iNZPa-$yT>zi_W!>>s^+aq#%1d3gEi z#UZ79wOH;>)Lqv@1@?S0}Fuz zen8veSBwvGrQm&BrFWLS@N!^>hx(JI+r_-)dZ8w$(CPzUD&}=AHZqR&Zxr6DuCbh- z{`H{=Up>J2y!XcX5dSpq3_g`N%sFxcy9PMCsSU*3+Dl4m^Ba?1H?BP2FX_h=Y#r=o z0(1ZczC98Y^R3Wk&mA2cC|EYD*PO)83cwrV&WZDOyg5gml@}@jUQtX5v@vGAD^uQn zWZk}*JWF3^<_c7D;6pbh2y8;$d!qiH7w4qKxZ|{84$*tADpO{6qgB0~~^)5q#T=-;b;ZYkrZ#hj& zx!R2+Ks!7o|MAN3Yq%YmMqunE+Q7V2JJD=(jhr`j)|vU%#E2V{XU!#H>YLd@jr`+C95y!TRoPuD`k0*xTWbzRv2(>wIyu*EQmB?5=(% zctMx3bs!Mg>XEw;oHgO~rsdc(HyOyr@OF5qGtWQWWNPmGltzNuR;5aP`O*fQRHyZl6sx^u12#uDjKn|2E!Kep3+M z>Fo{@0^`Vo-we#-*S|QMYD1<|>s+O!8rWSdrw~Su+VIxI6>N+~{V( zRKr?vsQ2Ch3J&;t$!&=KM+Q~TpFbn+hOIa_DqxC$fGjx`2$tDc?ynijp^0@u6Xe_|aL8Hq^yDD?uiX=^7wf1BHwuTS9XMPaud) zeqw6QARzVRL}JPv=L-yLn(_jQ;fV4pRGb=sj{pa3TF?26f$>Y-p@z6cLbYQ-aWSMJLjYNF1&vJsDj_da_~&>j_XsD+MU7yQ>OOT9e2H zGL$lF*VR(FVlp1;LkWJ1ZtpjesNjvbRP3l7qZ(W&DB1a#0+LmO<3BrB5tiuXyZpKZ zq7#(lD>+CgO@UTOTwnz&sN&MIJj;Sx;%5w1ggK95Hm{PWfwOrgSJY zj=*lO`my}&i~ao`7N$OE)3t1ds%}AMS}PW=pa$KnW3i&7><>_k>I7(TZ1UO$zjxsoK^lAbz; z`p%R0TOqJ;5NeJtL$8|Sx|`UF>Ui5(N6Od?&D-uoW1P#`=-qd zD~LSQ9CnD@Q`=_evBKfV>Rd2l^&)Porbe&!CtZa}-cj45%3`FioOh$*2sP%l4LL*- zbxA{v&c;%9m#?swdcXR$d}g8Dzo%4eQMr4lPJ17=P;t_=mT8nFQpyTyO-)TVuEV0Y zr`xfItCjK8S%Me)B;%Q;7spvly2Br!Eq5E^Z;r6pa$OWYv$~!q!iGdy3*aE|6tnCD zS^mY2!Hx%0{h;iuscGNWU60%Pj$Zqq?mXgmL9*!%&K4ik+XKv|%~qU^({hY5+!8`U zh3+0JIO`2CLjaX+CYK+2tB@Vfb zF#HSt{|x?AOo?k z*apdf4X&6n-bY5u2t{NU~%zxeX*Zr^!%K#*>PKyK4iW~CWqmYfyEYt)6| ztU02E?HYYd($QiGP9)IO9Ek83B9dos7iDU`pDxjwL^7gdnpg<5fcTe=$E6|F0|pT$ zjy*=G7BHy%EhZH>Q$W(M89L~ndI-K< zgqSrRWAZQ5+CubuGn#FWZWypVI)V*-GV5;)hv{Nb=EuuLHXNb_U0Nl*&*&=#upi?l zhG`mn90@JU-pu1WJf4%JhqRVjB-c`u_n~NH>He(ZBF$*f8q`vy9o& zC%QV5q&=#|l`Ok=lWxXdf99JMmrk^R>1I+#M_r%GhD`)!3@r_eqV=N;f@>w1G#3sV ziJXyMy*JXe!8rq~h;`3Rb76IcJ*t1)6}m0Ia^J?zlQQy~emI`OJ2HWx?8MJ~JdQGC zBpVf+7~~)){Ci93Ya!xzF^*dJ6K4T-KUgyAkteSdX+lZ*O>x;2IlC3KyU-N)su>XX{ z(D$2kHm9x`b=9`~*-c-{bTckyiuwN|wA9G{vH#*Z49pED3ZR#dA3gqlpTXn8;H)nD z;6eEQ5MlzsrR=7Shby*Oa$R#@=%I2qwV*LKa!-FBHsI=O)pBj*5{<;4q|Kr%VY9}y zuq_?{HwfKKA*vA3joX&A8H%^S+=Q~Zt%okKvz7h=MqEWASnai;cd8dOsOAh_;}8O; z|GLJ@ZbW>DYlrLpv=$hF5x?i0Ne0b}o@cG2#-@n_4}^^FLq}LEUuC%z6u2)guzR6T z=dX*Y-=*g~u2i1fCyRPQpU==CiWea;T=>XK5m@?k@r%-~1rIozc2#qgA|X7&q1}(F z)sd=>sjUuG1PqRctAX(Tf;h}#Rz%LoDK_9vh$_SO#;J|MI=f^kyt>5k+aBj3L_#q@ zf(fDRkB@?T5_d^(Ftkn*lcYc((ZETFRDuW21HRz^OB?S7f%k^M_XbmfnCV)ZwUR)j ztCv}W?b3sq?NST6=!4^C1LAZ*aAjB$4(DMon)+aB=7sNZNZc16UcZg^n@qx5MKNLX z$r@_c6sYDayjw24QD?1zMbEg}lB zy`o7_6D6 z#5~B!gJdV%ToFGIk34J32?E?2P9XDm1Pe&C!uSz$0$H#B77vis-8Y8Nm^T=ivhT+{ z^!%7R2T$BF4U(A#-am8)Y2w#9x)o?iTZc>L>I>#?<59Bc?oy(;5rltCVsN$lVY|D@ z8e?CB)4uFgv2403c6CUpp;29xubeC`N|@t1bX@;HHG2!x4VEc2IVUhv*=4*HixGA&@Ws)AG4 zIOkE|-$8Q;Q>GS16LV^!S;Lf+(ij2#yjV`gj`PPlkQF?u427VI3SKGazvkpTGF+GN z8lqZLtqOGd<9wbkie)vqNGiy6a*{)U>*SG5-#xid3xv4|%&e$^V;YP>oec7hqc(k6 zLM(hXT|Lf5DP-w3>{dS8nomFziy=p-eyNgCk|G~t>>F^I*_9Nbn{@ek?^b$5eH?Rs=HTMW|B#t;$T*0Hq<;7*6>GcP{@$g!@b7n zMwbY06vdi4d(!4Rpbozq9+T3>i^o`o77T9SVAkSZt~!#g?iy`=8kadO--~23;=^YJ zeIM8Me%{scJ|5x@g59pPzK24v^y?6Qk8#j}mfm|jCGh27KEZ{&ZHb&zhvCkpCdNl^ z+eMN%gw6ad@BOoEGSA9Dja_GMI$R{t`m3 zv*;nV`x#aZrsU`@VUfJHi( z;J(eL%c;9D(;blGCvi+B%2*gmP5R^qFam2w(=+9-Es{|>hdg^pnt&t#^U-g+`qsaV ztiGuadjX~CMw_J_=cPyt*eAPMMbXQkbS65e6S6Abmzr2vpU1#iv=v=!& z$K%Lp94=AIaFP(Ed#>MyxiEewX!&Mn zht%tatNi|%QjG-ttX}iZ6D4Y$mWhUEZK|7YK0jST9_EBnB$Bl(ND-8^Jf-^Sazt=W zYmV7w#$sqV?4)U+h5vm1I_(Wl_3BtT8M53u#gp8mtsug!BV>2l0m-{5J{fzFNNl&I z`(-MBMt{s4z7``l-3q6hd$H>%srct`$joQG>kNY+(DvRQ;h~Pc-|_K0mD>)7{Tq;s zu+Dj$H~lJ+&R8pwGN>fJuH)Ac;OlCwL!4HyYr#sL$8 zbM4;HYDpE*IXrU5v^RvoXY1w#w)4XlNiCS*h{5cyXWjoB`rgb()f_Eg&jx9Fc4)zkTjaWux;E z_k_HCt!absW(AKEQ5N~LA@l0`Z@RB&oxl7?^#^vT^ga%jY{ra+gLmGa4yCi!SizNO z{o07=-bOws-SK#Y{3Vj}BAg77*kFr4SWd?i$&Y!&YNtzrBu%KP%o&{&iq0q9O+89a0ca`m6>9q7<%hD%^cijvV7L9ug+OAQo$$UdJ^Cbi zV7oWtk0nFE?s-mqN(TiT+g?3x&W|6N;AIT~c&(dn>-46sw}fvpFA<)^T?}zdxf!`u zP!EV%<{bB1ltl!T@OEOW^uvE7%&n-VMJ_iq_VOEUA({E!0vysf=IRe;rRHCI$Vvs6 zj@X!lN%KYmYGBEJtr*$~xEDGLJwH~~xs{+3uTV1Iycet_j*km$?p4cZnGe^~Hd9X! zTsDUizP5y|w=+aSizsfchq|k+3n?6cvyGqNSlMxBt$ED_awWpMoShefXQ^F0dfy)3 z;|n3QzRJkY&^KAw)rOn_@5x(6MUZ9(TRGCmY1M2MxcuBI)3sj?s+~qwFd;R2Wp&Xc z(^!LfcMAE|WbJ|>BeL1Rb`Q_@zk2uZx{UUqDfLpF6_pH$K*$aihDX@tkA*5|61(iJ z#CW88PGX;3IB)@Q4@&dRMQ$efcvc%E3szp(q^G#%)%oOeG-G zy(V}L@6mu+N}dlnbFBLIMtWT+p3<)NM;T1Suw6Nm81Gn~c+8UBtD*ZH;w_#M35SbytYh*ZH~{xZbm!5LnttsI|Lsre*g+Ybj!s^UIocq{4FxqK z8$S&m`WswdLZX9V&xaQ{jSWy=6b?ol%~7GneWpWV8!jlkg5cIu&-}U-u8{@w(a$VWal(%DrzWC!nf^Jk8ZPG45i^gp#LXz*nHElXMM7q-)8g& zkt%Ydj^j?DZAg5=p8N#CgWIn(hME?7ub5U+lW#LY{J6$IV{+**upd@8+^)tPqr<1M zp+-f(mHRMhd@ePoXVwncPr7LUQ<~^*Hp$JagJ%viwre*XS*2*wM7hm{)L_Z17k#Gy zY*A`r=#h|C!vs^AWKuPWzyXrtBYA4_A?HjM_%FB5Bp!n2Tkc*@n49Bu_FE$emqh7k zPG6cVh;#jTP`_yI-w*_ftqw5ZW8lc!a>YnjpGdkIc++!Q?@fX+-fMG&78Zn-4F2UM z{^h7ZfLK?9gbi1BdhoF}YC7nA?#*ZTu)^komg|N%_+%^?R_&3|LH030e4S>`wSRUr zc~*fXBsbp-m2)T|EoTj4E8+}+xA4f}qF6*OpY9@s0Mb+# znuWBl)Pai+{UEX~gjo{j5H9k(GLv{z4zf?RjZcEJi3fX( zla#Di-rbJAaj{djRjd1{RbOb;)BL48XC5;Nrg3J;cbxVgGfDSJ@~Sl{y+BrcKD`6x zvv08lO?43Ij9=^*Pg>sJtCyG0qwim+yyI_jTyH!?el>cMT2+qSuem<0`x8Cqr$P2d zpR2N>B+;H~Esx!sH_m~}m#E;S;ugQq0(>#@>~38{Yd+UZerIBtH{iPN{B+#*gsx0a ziKxp%q7jEL!mbBy7Pg17YZ?M*B9> z96YAO0fN(MBO~`-v|_77+^QOb`2*#blN@>p=jXSd;|CpJTsxFMY*_h_O=X+#>uo~4 zqX#Q0d(2`3cM^P!_QG6h=Rzf2$7;ruK7e!G&V#fgi;mIqJlmI)6JhxrM>IDfqr98#|x zrI@Y>TRJ_ai~@=pCPsz*9~bKyO#O7I%5u^`m8U00oQQnF{)omr6UKMpjgA_7vqTTR zmluM8t;LkH;>i-}gYig??q{fssdXQW4Uu9G5DqGlzJq7RjS}#>=Cq;nW#iuBHp`wQ zB;p#qn=9v~vr@@U_NUc&Q%*EcPyFEBCa+{b{d2kMpgi|GctlW2Va^4mT!TjNCPf^m zMXHwQHs~3R0s7RPBv4b^D#rW$8wp3weL?<(RJrz2+mSkxU_Ro({;tFF?yu5)7krR^ zPB9rhXn~g?{r3AA0D1wce(1R1sy2iWpGa}=eO+KkFD)Wl+MR^dD;03w)h0$sqwj@H zgo&xodXPKNpXbJX>-%v)5l^2$Mk;XX=a?VpmHvPHJ<7I2-zV7Ql3KX{#brlSgJZpCtEgkTcg;c$Y>m zcncIe#W3hCi|bYl2G~YfphH4{Uk9#)$4kG9*)EaqAusU`)M5GJcsHt^i!A(%A4M{u zD9*nh?%BZ~MFlVlRM&EvPI-SbEU2QJ9Np^@bR-4tm9d~4trY*Rg{z)>-b|pns?Ul* z3p~h&vLArdl<#jPV`G_H%xpd|239GN2Di|U*TwS_9S1}iwM9pioa@Z-N2AV>^^%cuOC_i{SwIrzI|&Hd@uJ8A|;A9vr^&CSWn z%gLkB&du4OZrA!M#d~e_4y-JPwB8|6ZiBs*DAnF;zbPzg zHi9UZ8$~X0ykqM|b?Y8wOu&t27^KQTaV8)y4CE9w6{*1De)13b5+K51VdR-bB1S$g z4qA6~K1?y@64Pi5u?bOgjRmTh@HmmddPxqhCIh7~_%@2hd9QHuy+#wPaR#%<<2U;O z8rqWxq7fQa9kMcI;_(Ncv`z^jjW&EPT2EbDhN!D+s!O!PSgnh9cO+{^!5bg$MmN&z zQ#v8ntE*A)avM3OWTy8x{H$5prealz#9u)+XcK=BZT#Lp5L?%4;ll^RBM7>@4ZRoW1i3FI z_nBfYK>;hMODdDxbHVcM75w+|vp=RB$}wW89N3KG`PSW?T2G-quR)YSMP_{p?i)3i z!Wn>~<|C{JA71P0SV=zF{3h=+&xNQ^2K>dOI&e@+sNz9e#f(^?N`X^?&RZwah;F$dVJ`GCZ-C^_N-H6!r*>X z|1!(jq3|bTiEV@IDdA=GhHC*Ye@$%Ui>~=M@=4>T6p9I z4Ive^?|V^A+uZN8*>Bu7HNAKKdz_|IPbSi@!j^mUZ{Hv};_rj&-OItp_5=Q^HqY-p< z!-1!gdw!f;og6e}YMdO+l@35qxES>e812Np=n9bV%-cp7tqfN<;VS`hmg_@y?U7Nz z=9jGi*bPRoskYozkbAqc?HFkq^OP&uD6p>h^F}oK^qxG&lGKa8b;J4He+ zxajcp6Vm#z?e-^=eGfn^(_|(JOW#ltbG8MHW z1}KmUc`kk)A0Jl^Ha0FapnJQCR|CrMZo5(#B-p&6L%sW3#`Z&dki5^PD9UVi7RYp} z^}G>-KWQ+1v=eb7(hg*4E9i#zjF5is2K9nS1)KZX_RG?^XI< z0?+Q3+xp9(j-PuYCWx8t8X1#jwmokG$t6Ck%+iF=p6f6btQD{(usjelN>{Pm5$QkA#3~lAhC>kb$t@O z#0g}e%3$QEBqy;?oLQ=|NI(T$YfH2xYEs*PI9QLQE3>4k_JQb3_m~NlAi#_}+kONo z_)P}ZZBSeNiPNxlP)(>?87w^3ymg$alKKpcWIs&qm5`*?(W;d_Y1H*GqTacN-U-aq zPfyPu4+NhK)Z9Cf?bHMsksPZ(jM8yb+o7_zz&WiU}|55jLvsefPdD}8EEAy;m&OZ)4lTR7?&|b4UV!_z^J6Ag!2M9g& zUuq2TTwQsCk24fB(%vGh*Ki>_F@5n{`(1@H*}?M&YQqp>{7@-@V5||Qtaz}vn1fs> zU1bNG@0kG@YStlw(98<3w;z)olA=xuUGyUtofDTyf(atO;2R-d8cckEhwZt!X%i( z?{0s;`Y%%REYcKb#mDXX$SZtYY@DoaTJkL=YgYS~fHghCp1vb*jKd}efs2 z$>p$!PGvT9s9z6UUiR$niQr#cCC~hJSw$)kQM ziX(ZKSs%YbJ-p075J14l?Sx>pbsc@#FIG`qe5XiGo**zSD6gwLG649~mR;i+RLKs8cCU@In>#?@L|Zq9{6*`7O#q7t5`n;~#R?IzD5Wwd z(ooGVFbe~9N6grR%&TmlkG0I&dJU-wo59lMPV#sfAqTaOyE&*3bFF=1SV-yk1z~wASW*y z1k5%8%TnE|)fwSs2)K`>-c4dtBo}CPtW}=$X)-fs;%L>{q+fvbNs!# zK7KI*3;$)#^Emshf8&3`FZC(fm7e{L8TxU+pEB;aW^$eYCj9^QIW2Se0}!0sNp(3 zygQk6<&9pG!yfqTM7Pz)RM#0yG**`Zb>?Nai-+I(umA2w2TzZ`q*;^Y7-Oyc3O7gF z{}K3Ejnybz$Avsd!L-cN{Y>z5tM|-zyg?)wj2SWn5r1d-jpXfHqFgB+>dz7mUriLNR0neF*7SlrtUpaZ>d z($}aP`U-6qw|$>>$X)ljn@fWINs{#UbPxL-{_G@#;UM;W&B#YPsRlXtobf;U((sb3 zaiT6M`cpDmjDawRiAx+4oe*<4Auyh7ZEb^r zV6FoxP7h-9Ma+&b=)u8#xYG9JI_+~hkiM{{P{MpRb5WC;WTLPrXaRC`ql(cmbiYR` zhy|k&w+URKm|24mYd_Q6(IAj`3K|;3#O&UX6eDMPF!q@9iui!8{dyrCgeO*85+SF{ zXrY@?Tq7!R@s&Z}RQb{Xdfe4{K#qlS2J~=0Q~vw7Dvq={-PrWWL76jj`dC+tn-j9_ zvFdTt7?Q|ZKW*BuvhiMP3W4l38hS^YcOE@NlFU(_m|{$k1m;P7%#G% z!L6Hg(Jg^`@JPTbvI8*NPBNY$-2-GpL~l4wx+T*n^Yf3Fhn&ZnVE5Aj)|n|+rQQPS z-Ng@L9j$~jR_J!wR zlpDL?re9+{3!qSwKjIHW)BQ;skj})RqA2aYf^~9JO~l?zpnht@LGgqjQb@rYcA7Mc zdCG*(og6hox~je)=yDBAKkto1-I)Tf*MaU`*AaEx2{gh_kb@7P3_HxI)5D?mG$|>S z(1A#=QO+pP8Axj?wHZt?Y-N36;a-8?ZNUo4@-!UXUiSzDO}D?Wd$G7iLgo}#NP>1A zO}3`IKt8P(Pp7T0k2v&S(7`rsYWH5mYaUcF(;Jk&E+7c=#jUAcnm!igqZUi(I zMWRu-gq+|6On5?XpMZq3JedFxI4=BM)4jt((MoDG0{&DajG<|s`()ufMBM`~a7xec zY;8f*f=hriYiEkLOb`2I)MZBtEQz=+O1~fk9onwwJ0=}EZQokrAQh@eWp>Qbb-hI) zO%^Ai?MusZ4$i#f_@Adv>Kk53IFwOtOqWe67eaCqA|>G9Y5Lhj&7#s+ zDJ{8+;bz`~3rU6O21{!Cu$l1rQvEjD6cA=!QY)jaNyWtmgXbXU49` zxD6G<>34|`RRuhHEM6~$$Fb-TE5Uy81neY+M+Yuf%v?sEM^G8X@$n|qlEkXhz+5oJ z>iwh&rjmT_iBwWg45?aD32VwilJE!7Yz+!d{1#4K399o~Vhv8VUeEnWy~}lYCkX1& zBZI!URE1tBLc_q1ZsIVPVwwYIZS7#15j)!=rJ}-47j7ebPtE}&9%o4r4nl_kxP|^@ zH2mDGjt73gSMZ{PbZKfneFWm^z5};tA(rFtcu8!l4Q^Dn% zfwRMQ%0=Z+yN9zt#^s_mit=*$>1*Ux{L^g?Zp1U zyb$ikU`^)=_o}N^ZiJ^L1s1ON3*MYFRnphGu~_~!(>*YnYz}vCtyt!8l%)2w#I- zw$bJRX0U4Z_87F++ox2!r6|(3E>Nt}1(vRnd}eeng+=Oah}11c`ASBG#@@(YA*%VB zdo5DVaGeLR9fWL5CKuOdT6d+wBsnc|YfE7O> zQ`QK}zE~TOy1cKRVD7F*i&@W_D0T;i&qOQ*LJG}V7m%bb?xqwR#&WM+#5tl~B{-Zb z9X@SF#qOqpFgfx!6sxk*TH@s5G~18;8ruEpsb`nP@5kyr(__54{VUxYrvjFfXd(b( z$TRf%*5BWiv10M_kDSRv(S&09jh)IIILUi7r4M-Vvy zSxZT1ayId`S>9#boUbmaA_tTV4V)Yo1ycABb@u?MDU+4yIRv?a&V-UgFp0}6#Dk)H zDKh7f}If>B0VHnY7iIqpGmk>A8(;P&O&31 z#l|E9olzPhy(Cy_;S}|sNx0M;Z=pHPVq?5Y^Z&_uwHjG~qb>e_$}BNty7NDZ znGOy4U*f5wHD;qRX0tYC`*-=@vY|90Y_cG3G9mmwG}YZc-&CTzg65^7b*o}Cpw>dF zw}R%QqV=j`bD-8zqQ@Gls*SN_J~HeejW-73C`~jb>?ln(7VIcZwPxL*T{VUViA}4; zv}V1a-DKi4gEq^WJWHgL>s_v>i?%4oWRtkgv-v5kV2J}lCajQ!`CaYAGi;=Gxt_lX z2u1M-mGg_@76K)1rRZ5`*hnQivxt{>E?=uBS@;N*?9gk}a$)>?Xx8a-`khnqRc16E z+8k0in?U_4T+45HQhXG#X*TO|fznqRVof@t}?h;@_FcB)rtR zR2{3JoWbIutehri?i&p@m@s~;wuwDb=Q7{X)0H^xBUswa-5O<^5$P~-Kk_jkA(efO zTI&iZ-nlbTPEM1U8ZQmwB=VQZVND(^%kYoDvlG1aYcHSqk&csWjo}Wjn3#hBzdc*Q zP={0jAxcGCdp?l447uJ%MhB&PUEWlJ!Lx8wS3YIaZ^BzT&|NAyz@1E3beR*wzaB97 z&L$HhU-eD1sAQq-xV_5REvlwM_Z54LCh#l=v!|Nzrmtvvm&JwzoHYFUxn)$0Y zD%rpeq%U(3ZgbC1Z7+;I!Xg!iH}dFjU%gTn^F1YTWU(X;XV&JW(Jnu3V*eC^4}KHT zB0e17^k_+weW>)Q^dVa3YQ*V7v_K=L=$;yt0rfMwUG4NiVKVhVV@AWRC|e$S1c7Av zW}fTaA{z0a;D-+9=xi^v{hUZLyhM7NvfAyP;Q*?!H`vAOSkG0=UYaOHB6>hCV$?CO zAO&Z;C+^&@7PASuNxxke=h^Oodr=UMx6z>h;_e)7`uD?f__g|U1IF7(=kP)Hi4#ST zgn0sZ18YzeI-!%j806qmQsfQBO#1!Y7Rr`=w+g7JIrl$Wij~`)pNmXR*sIWjD2P0h zQ|0>PB0$uhz}86LQqsRPhqnAP{usGYyFAg- z5+)w6ukR{}PC_-A3L_-3pmHhGllUYr<)u8&2_ zgKl%p6K58b*+a$xp3T55kO5bn=dmmu)7LVDl$6@cpWJAS;{3JfifbU+8wO)*Y1|~6 zwdh}G*iD(xMP6h|l->7CCT3X(u})lqOHXfg!c^UK znH|sZ637SvH%2h-2g%7nrBV_*;JQ<&XD zXqkZHA$&Y^l^h;gIChw?8Sddv*(*WYSb-KWxn%~&7WS{O?r~hdLM|4jB?48xtIDST zct{q^?g)*-eyp{C>-x`O!L{(d$4n+qRTXP8$1>TJ}|;b2d5((*i6{kF#XpW zU3?V$u)gk)6&V?vc6El<{0wF=w@AM?=RrMv!ppeJCqzVR;Ey;?tE1?AWy1bi`l=+{ z1+S4OJ3ChFoswx~;U}%Q1zNkgQUP`h<ZUN<9;^xL634du!%=#b`DEE9oRLBXo;&n(X5_Q~oNeY>8 zu_H0|w#GEWDGByLAin2qyOYjhD&p%mj+Z2GoKy`j82k@-{?8`2%n>r8J3!I4qcfL^!Y)hHXG z1(zKw4%Z(S7g;J9+N{`V+SIl$WIM4_=Ycrl#P*aH_ZB<>=rNezs!d$AXo}HWGzlRn zYHa!5`gGQyB6*-=1QCS7O`}E*0=>|$D@5tK@eArhP0 zaZG2tW7lP1rcs!1iS6v!AUoYNPu=GxTY@QwW|*7|44ODx!eWVZ6=QiqVu zkWDh1B~bjBak9BE4mH|O6ci5-G)ohJG+9T2#8;SeslsmU~RB%F2YsEGl z#6f2yL&_X8G$G{^ke&rCgLgs)Hp9TCgdg@fM*6s}E9Jwp-7JtX5K~tfW!K@uRHMr% zSLetLOgxO}@8p)GR9Qq~F-#}XFG27Gbkb<#a)e$R#+CrLGnIIO*f3;FJHKQ(A#WT0 z8mSZ`ub_q{ifxd>GP^YQA=r7Ja<~y9r^tT8sz@33exyCr zCPs&mJD0OST~0}o>>8hKp736Shv^we(K#%bp%JZmWF?Uf-AF}F(|?E2m-mGpB}#eu z%Vm|oX>KzYmKbg9gPn}V9Bm7Ma9+ed=3u#DoOMf1J^k|3@XCx`8CI*2pyNAIGTF9{g+Arrzvx5I^P%}cgX>v1xm zqF6}*mWPf$A6mL7=$3X$Det$IqL1JRn68*vSRFiMRI%tYaJy0 zQ8~w%Tq&_T#$yL1eto9B0mYrb1rg@kh91D(q@}9G^#DHI#mTngys`+V*(r;ny5o}*1H}V5y210qi^X>^}(Jh?tFtM-x#iaz%3QdB-_>q{BSd9 zq8sg>zX<6u+U7YnAt0pUk&>R0N+(mlSWOB4Ie$XV%>>C+j2^zjtI22lfEi4s(h{mf z89<<5`W*7ad~gPbwYmZcRkC9iwYI60Tab7JQLfHy>QoINbfOU^s7hy?B_N{sj#1qLoKNa5WBx+Sy0_Dj zLV<qK2=ss=2VU569|12&&SAiT_D8Wkle0p^0U2&Nia);_>W^q# zsB>SwX{_r+9%z6}eq~W1C8jRM^!tacEMQxYYFmXgX5?F<CL^{MGts2$}HdzXwgRqqIecw3|eVdW{Mmq#q@a^zP4>G$TZ(tmu~?$-ovt4 ztid%wkm=R8gfukqASj3Vx3sr+#)Vxu!VTNNYy*wtWC1>(8+;{)@$R`SnhOPd6u(%HChdC253Xr>-6 z?nv=D%02KniZbuT*_dMdBx)MFyp_dn&T-GK-nH9r*8Guh&tbCvtfiE-Cd|ovRE4ET zJUcqF3b!5WU4{^WtBVvs?(`V=jQHp~PDcjmlno3`M;S~mNHdy~(GZ%8J2saZ0)%BS zvrsr5)iC6Od2z@Gwhv~AMW)fe_kS)=A9GFem=?yHH8zZUcG%}4ER7CAY%xGW71B|) zBx)9!E9#`va)RB~VwtR17sq;>}|qf7JVBIjv_Aw_(c^r=!6n%&VcmHX>?E*x{GB z@u+kOoEsuTl<(@PyFkXq$X4ID{Y==Ye}X!;_H|deD96EKJ`OX^dHsj*Z-0#qf>#10 z$pRCnaFV5cAFx9GY$j)i;+ zh>n=I&NuJIn|z2W>S(Oo@SPkqWkz!IV9q^UvJK8biEGq zmD93tFcIo>2q6b$BZCW~6?H25vYN^>WfsUVV{QB2lA%+wSvriT9f>$hOZ#OjR7}qr z#CiT^v0i`u%hYCsN3qW29+G5V>Dn1dLVx*hXu}gAl*Gnfanpk$WPv}lIAJ3rhUX9+ zLvaRi&800)!EtdrMWN%@b{xK|{E76;ef;YG#7`VDtxLSc2U`a0vpDsz<4|V<3_$=F zD9)8@NSJ1?8lK%h?sPV-iwi~2V+r!b=I|9J=6{InT-y;crRh{)DCaL&-}oUVWB zfSMeH`Z8IaBf4gfRt|NFZgM-vjmLty3sq}8E{9T{j!&D2l)N`lSy`NkjR-f*RI6B`a-*|PzfMa@ zgjV%2Sp7Q!dWMoSzrHvNYW~n|&luhwUS+`I{As&&)s|eysZ-8*J-4mFVr7fofEf+>rj%BAVxmuUazW*X&sh*Q5H$B+r?iq5->h`+{i7k!NTX1(0K6x`I=VC0PK1@^3Vb|=vmCN1 zI1zG(mBmOs^N^Su4+$`l)ZUx)C(UcuXH{nqQp-6x#J=>ltd6EB*CVnuuAVkJ{eA@} zC_{|2WP}Gw6gWxx+5d>!gi8x^twhX|=sKT@g|jM?n!R|D9ztL&?KXLCz|Ri>$xpzi zfIt{K;xqt@UF0ctxE<6k1;JnR^vM`SYm(83H|T%%W%7J~h`#@k~hvF5C1RjXV0-Umz30qWLfvfq^c-H zOFD^1xqSI3tN-gLLH_(Wb?4l zsUaaiD3B_PC*48B$0E0;?0q4wDhYux2wH|WFKsi0$s2XdToN4MVhqe{T#ol_y-bXb zSIQ>*5n-w4CiCv`u~?pi6MeF33yPvu^{J|-={Kojf~Cm$E{X!g_|AyY%!aqb|W#f=GR@Zh3Qdh zB+lLg+tlDa!4*GRo3i3wx}+eRQt>jYTEJkD$t|b)lJ`n@*X{%zsv3mRR&+zQ1L$^$ zhm|B27$dlR1Fu$+(Iy%|HFxuLvo9y7#d6bOjU{TCuIw;?QvXFiZb(UYKNZXeuKu*0 zq;?9-z>mnD+g#cn@Yw^1Y>_M}OtG%b=x7`#5=6JZC*T&Ks zhUdcgT_uHUM>4c06+>N*d7e+K-I$Q{sM9iSiUlhvwxmguvH3}(ha-_s3F)9*4DQPD zoaQ>oV2xV;F#R0-{X1e3Ior5YdSt;9j49(!h?&$jU^>`iB+#v^hM;=I4aZ#WLTm%4 z-I$d7k`%@aPr0fTL-q=dsPc-kCK z!&J75Zc{Wlw*@fhOUBlfS8O)ghh|OEtTQ*q@`mQSc@0N^rH-Vn4j<{Pu-RtQr6F6- zkx3@NL=iD&n8Zn`;+V==rC-)NyDdIf+nNs_B=PJGn&ao@8y?XRcm{Efq0PP{k!Jc6 z1~=a8PnFd+?%}dad5x0@MI3pFP!E>~IJupHA^;fPnf?&YSbj_Uk(aa_n*em$tXgM5 zuwV$F3c56qLmnK#+Kb)yZ7{?&m`)&Z<`IZfQUFIdo#qwPy`x-&BjiYp)vg|iEsB{9 zs8V!y1K%|kIh0Hk?@0dM0)jbp6diAh3-Zijg5auyVq|_IIH|RAG+4P;P3~3%#X%rP z)HpeEPnUv42!lR*J*RAS4g3)dQI6&8kdl6fbxQ`obg3j``*w#3AP1N$siRo5 zwnM)C6sKe??OC=30D;;<+Mul1k3>JyE4lEkk)8~3mdr8p@Ise&M8pwFAgC0v!i!!#m>!o6?;DuTkZRv}s0O81p#RK)d@Y43)HjP1ZwW{;i6nCwag zQFYqgNG#~Snt(T*T`S$=Lh$rs$NR3{WqY;0eEp9Uyka8?mC4Z!Nn$MbJcaYML|Ggm zVLkVkQJCzdp(VUx5~B)Yl+$z^5h@7xtVwX($gO7nk*$YUHm6T_OGiZ`$!$4G+C+V? zu_l)SeY4>mh2@A-Xu7Tf=cLZ;qoAcY1B~i0FC$oAuq=i@t_x2z2zKogDT}CVLa`sqQvXDB#Jp2V3Ef77a3MS^AEoh!cung z(w`2Tq3CURO7Q`dQpCh{M5SR&!YQfD0q~|SDQHL0-XfL_*NyHwUrxsLn}LI0N;usf#@MWfNsa96^4st{HTbM1vz z>$|F@f_8oBiv26QZz9oBH36Yw9oCSc3?yGg@FyH9Rmj$IEO*blK-NcBZ+N)@&nVbb zGu$nX1~WxvY_h<}uCw^{VCR{75cQRZdoS@dulKcqfyLcx@;u?I_swU-^dsX$P*X`1q#Mlx^+iR|Y#plfXG&%}d0BEVe;?WjWazvsfeG)f$3bY{rp4z0yW6gu?v>s4IRmo5 zl&%1E0yJJ9?y~#X-=05zdXW4mMbo1xuqP`p>Ns5Rb6Oc5JB#b5 zYW34bnv-Ak{P{RK7T$oP#K0u*Fu4KZ%$YQY%&XKHMGuw09C#z4OR9c{1TQWPLKIiC zC--~xWeZpuFl-Zz+b38;splODHIn5;Hx;fnL0Pjw5wRVTv9;cr#-2aZKDlmKQv&=h zCDnZ8OPk*|xJ75Adu*BymG4?JTReN#%(Yd8pD^(Z%+WW8YhUg66b_^(7t<`*u0l9ns<0DE$|wN*7nM><&Ucn4jj2+4P6Hk zV@&Mn(7Sj8qNcXun`-3{N|T5MugJ`KS-UumnnRGGV z2-MLDinC?cU840m!?8u2mG$cAUnQuC3srz_Fxv3Pw*wzX3AQQ173=Wx0aAbb0(iZh z9i1KCPqp`c0}o{pc*u^1TeV*=M0nYfJarj;1^UT(U^pWha@^FxyC!EC2g#}RMY`Ny zEOUy}EYym-I~(-wg(9g~b9U@g!`B%*%8N~~VFFPD93@Z5duH z>PXAdd%f6r-~N>wHad^4G>XFMdBvujwVSi`rBz@KDayv3I=fl2r9>aOf>O|%=a{!; zC)mcr&qGikiT*vg%3JH{?l6=V@JQw21-*f(a#|WR>}X!HRsyY7q1D+WQgY?#&LLF! zM!uV+8!m%8T7IDBb_Fdqr9-cXp?Eh!NQ?F zCZ&tNNt4&dAXf!wI^LDgfIxgqIeX#=`Rs1`b+)xbA*;fTq>eYwjqRiy^HrM4)Hn!r zpJ)69|CFG#c`jm1*=x5=?`qP;tzQ z$?11|ny-^b_NOShSA8KNYyd-4@*)qx*>`^2}qYuSHP2Wr}u8Y3HO( zcq%{xp_wy7v;b z^-J_BABbq^gNA4h9pmzzlXqBm(5hjbF8HI#5-?^jlBdHVBdq{pV@0FSl?-hsjzcNO zUHM|i+#GemZHq?&7?1Wh&%1MB&s#s|Fa0(#_j*T`2xpW-6fh3&7B>&?0>cpzOQ51f z`rgeVmPQnLKF-!EEXm1{7i_+6tgfOf#jsnBfV`VtY;XU(RohNm9G%_*qEThy0fKJh z8A_DQLk-IPZn}LfiMP|K5*-36yP|@pTn!h&Ac3j})c)55UlGLgXT?l1b>7T}2xa@a z%O;1^5E|u~7OIoY^lIMp+jEqeK?+u%9GL)`RG}jUYf*JrX&vJxt#}ZLLCW)JNLcq9IA!(mj_sXDSV zl(XuQh|(2TKpnn1`nh$$YoD?A4VHRJfkb>2B2g=VggAqx0_^1N&wk+eQRqbbaZiYcz)hG+dKIeP7el2j|NE%n$76Gabv9<9`PurcP}sMulFy0&gzoozPpv) zyF4tk^GqVaPx2^yuy?U{o`r%LEhdxeWV9U*ERz_R&vWoNIsEwohr4xobdWd3S*&M< zZOW$Qvz%rrKK1MIa=||gt3MU+dpeybxU-2r;xL97H7#>HHVWZebN*8a|uQ>wJ6D~CTFLa(N+Pf z8I2iaqo;Xvqeij`w~&&#nCcxLy?njDuhheR+M=iIO>XByGYp?$`|?&|_CS)7z-RXMMd%f8i7RTdS{cRR30M)YjJ4nj35DHF#cc z)HfP`tA6?J|Lt!R%w-KtjzJI3D>v6$>oeQC!1L4iy&kH$lzkpnPw(Pu;`wJk7qhWME z@*$B*M2by@f!g!Yjw zm?eSJ!IG+<%Fi+>ur;mHX4{S~CT2^@#7oR+Iq{yRm3E$kP6d@o!51c5Sov||tdWunrRVqK83`?nS>ju2oRhJ}m+d7ibJk|_p_xBQsx}8m zPr4}{ETRE!yz!k4l!F%c@eI_69DRS_N8;#ho3$J7uH$)K&|up4cv}Q+wc);9G&VJt zR3>;TX#B#7QO3OG+2oU5Cq_ezmLeE<8qR(tq~D5YW1Ax;(wIC;=-H+bQ_Xn|YE7In z=28Wn^`q}GtgYW)&Fa3qYjEm%WwXDn3`qqmuB$v-PLwKha9wZgd5-IZeY^n+@dOs} z`yu)xuo$B&cr{h!D1a~{ut$7=Ou!9=%st(q7Y5%)f@RVeER$q5$2{N7V;Q#lvLt0n z&p>txb}1E5Ay~7VVF_Y<=s1I1D=_#p8mlK#*2v``ebIQDT>#tRq-qp>rNs}*)=AL! zKX`qPA$>myKD3Y%3`u5c+|rH5_?k zX{g8uNpWeYK;uT%{O^Aqfyg%?#pPoTJD^>}Cwh3#(K&|}+5Yj~`MC&wAk9?X4i8)Zygd1rS?80bho;Yi*wvqNh~ZCZW^l%-$f@S9 zxz1!;z8OnO(uwd*o|Tj*DyL9miYl3Hwdx#BpB`+FV?O`?7=!YQAGvwmwTBnj zRe0-N$WI=@Zc}voa!0KD5vk;ctY70h3QH7OZDAqIBDY} zeQ+{Sf`|;`E*f~j*TkjvYMaN{>{=Q`yd0W=rZ%=Ca$`n#(cqY|7mha-7g6K52uWun z^3w+Cr}QvcL|~J}H;M9D9q?Cb`sGuhGG{<#bLkjI-xwccQIpWg9?33G@0czrkd2K? zW^vs;F&v+mnf8Z+(If&RcR2ho8R6AtecHWki+x)#a9IMsvwxL&`Ip66lUjzXT;F> zdq-L({MN__B0;242`@B(;uS~rSqDYiLaT(W8Ga}eW{l$YYDvV2_tYY5oO`U!>UokS z8|303OPXEJ>$&&|!>244!CFZy7L7-=Pm`%?5PE4mw+w&ztYbPNR zR1(>EmdjXKnio8aPEHT_q|(ve@zH;dPF@R>Hyrus*u+19N0%F^lY@J94mm%B^$EY- zJ0g|bh%Zb>EllmxadbM}{0ksgZe2fu-bQ{uww1?M7vqt*$kWFO2^)^y7Z?$P$&0h; zHP|5|49~NREFe*E31xavq+*O9%&RI55S8D}B#*rlBcPk>b!qynRp%aLV~GO~(x_Vy zc=K&biIEhu$&zt;hGjCK;8~`|wL9)kVMJRiPi5kYWZ5UFy%sH&ryxdRBnPczMffv6 zwohw-FF;s%K@Br@f3)rOXZXx#|Kq$XZ2nKP|EslHJz@V>*B|ZwuklIP|L1RyPUbTI z|LC4Pe&+eiIsfT`$_MfP)vAqp%KWddJ;wijjZbp^U+jVHI+y=X!rH$$ZoNLdCeXE5^=s&{R~3I0HTxE0xz1w^Kf zc5lfq8z@Nt?R%_DHMwFgaq>y?rfE>;eVjmlE&AZ&COv-t;AHy}@z)2KYMTP8(G^XZ zQb?Az=oU-k0t_bH0>A92R3X(b8I??SI=9H1i)}v;m@(B#%WR!=+GRmTJ8558#+7v1 z!;$;j#4~OVcpnaB7Nfp9zLGcUTu-h{Bg7|zS8&b9y+UK1%7NK;<$@i=fAK8n&5q$T z2oP1^W;9==_*vqFJF5t02~QunS1LZIoK}#4U`YW@LYIDO?1FW&wj=Hyl~e(0sE&)& zBELrBe@h7S*QIUe7m?Axuq};U>WFRxveP8wH^c29^le9fXy+KWnK2iGIAz}DR{?`v zQgy{+;cUjQV-J9bQOIv7Q`J=%uYiw%L=L`lM)03AK`UQC!(G^Tm7v$3gx8(`8+f-= zpFd*{f5>bP266tvR1iIkSBdwPoM+_m!ib5gWO5}9ocHMK!@j{+>G@Kr7vh`1-6$o{-Vu-*b%G?yXxL1oiX=-#6Qv@CxS~eYuE~8QcapvCEV1$^=3I3Z z`vrff8%uIcdO|s0;(&+62$cS1}87>f|=DReJ1^$eZo0u zpOuvrJ*v9rrjUW|LT_;tmbj;Syp(9lsYqkABxwZS`5H4Ezx;}l8Dy)9Hw=MTaHPb{ z(PJgJByF^cLF~pMc0UuMoU96>cpxDrNj;4_1=~xA0?rd-KGZch96Q8(V=F}}bbQVPMATe};%6e6vW%{ahIr<%DY|3L?vIS$~ zs;w>zujCZ(&Qx5mBo+TS$%89-ECE^d$C!c(#xkaU8|=Set!+BDQX01wq-ZMA#H^{b zNH5NOnBK(&tx}l3xe9mT)7%UoNK8xe3_~Z2f1IS3`}AFCjgX!fYDCQ6u=eu~A}#c| z>AqB1lBvxX%+!hYk|ox2_r~@6ydDgIOg_+}+rdIKQ#nNwfqOwuC5LgqC2kj; z;k`s%FY-E|q;1pw65a5IXNPaSvFGq6R3OWT-pHlfriHQRV5XzcaY5Y!UKZ(9WnmSV z89LV#myO0o;IzHy*7Jf@=P2L=oE=w0N$}PEAr=vE*k!LUfdfFmo_c@_6b1$`cf_qndD*}PGSvVEa4Kas6HtnaEXwVf2_RdIO4Du#Qen0LnD zfPCkScBvF`r%;zt{VCZ#57WzH`BsW~I;hEqHJ}y9hqIs+$eB&v0*)CuXn#`Y4E8UX9}7gIFg@${d+Z6aS_(9 zGfe0B8Y~ylhh0B$3Hfs_n%AA{N$_FDWs{c4*wSe(n+4a(Ks5eGu9}I`0w*f3@O#*G zR)GENl@_^b_I;4)D=o|eiDL1UR@9h(tz`guxs?E?uehRQa?Mq1nM7cQ1=0Z8O$XZj zETI1f*In+tz4BV4UU>1eZC8p*3bkBYtF)xZ11{9*+x36cs{EivtX_@9BF$@NYLG^3z2+kUD5cFNyCFUa zb;i1`KaTEOgGoQCI9;#f-nSxuV5AjchsB8FTi^?ouQ&sA(Q@(}*N66Wg#kRF1h=w? zdZ5;|9pm2hU^Jm_0Q+=rd^~wh4-9~GU_QMvzfznhes=Aj#(E{Yjg(7=$gdPc@w03H zOhQEZrvs!n3LgLJTE8^386#t)h>i(ZIr1!2pU9sNtqDoEBr9oitfH81Znek_1^Ws> z2s#NlzyHAEqU-Q2HsKzm=KvNv{p%7e1iS<0z@xeQ40WZ-$9X@!j@R{q4(Nx%1d-75 z`)y5TL4nu=5A|aD04^sJHm{gnriS$2B+ z^8oLvA@Q|M&`6{?)E2VWcFl4LmuiX_!L8@fxj|CP?(hpaa)+SgtrfwQ{lun%j4A639Jc+5O1t*h$)J=hu=DL^;$#z*h ziK)h-lMg#vsSRYQ-&uCf2L=GCv9M-oY@Ft0{hT1OifvzfP74`vXEBrIb6QA^Kr@># z;>qS}>OIej1iE!DUfEARsvE?TE)!nFCwj|jV~vyV&n8hq?I*zUqo72 zgZgEpRcx&P7-IgFQ4~L@t=4_C>^kIhVUCVfqB&*ny`eq)nA>4c! zbT&N}&N8_H3y5^yND%^0?y5Ybn~z~yVu>HkJy*DTE*Xp#^1Cm$m;?dj&xa*}glZ8% zKAf8j5|U_aGhx6A=cJ34z}s?8S7Gf z;kLn>YxjJsk_^BO?CiqXt z|KPu!QTUP44V#%kuxTXd#C&LOD~IL_Ei6L&xqgAI=ohiAT4Ou>#v6zJ5NF8~Ai{%e z>hk=PkmDP=(2AaIg+Mw>KQbt_ZObqyRVyE2S~YLI+&?{eb@aOR)7jC*Ve94T>2d3y z=cgx_UZqpUO(@KgVsvX^K|a(}5m1-DrQ3dQm~~`gmjWn@f~GWD0$@ud=R(no_6j-rW1?Kgtdn4zJKEnCt(F8W!dB#L7}eK zo?I9v}sVdo&nptK$OM7&Nj!r_VP&1Md1Rn%fyALZhHqrG%{xOA4c(_Xt zC#y)`{diAzQY#{UE`4MT7=qaB5+%jSKBL^+3!E5d1C(r#XJUFw*#sL*!1@?XAsJqy zRG>(!@3*fyoq{DN!L1*3rxd2uXG$49FgdMIe;Sxcgy&$V6q0wNOeuCJ4cn#9W-x8a zFMc}-K%_e9ZL;_sjv`C;p1!>>3K}w330;qv1+$zjVPz7rGBFiNktc04Hx-BNf|UMs zaUm(S@nIoxOa*5U$3_Jqm$f{`S8^`RjfUfh-Y=c zM-1t%l9~y-MU6#sV*qf=V?c0k|Caa?pAz0~8F zqz`#w%oo)6wBf%zIX`-Ra(IxU)YcqW~*<+xe5{=ayTl(7ETC24IGrIxY#!T_618t?zGch-74^OPX-`w80ZUy<6Z z%4@UwixJ$k8Adm0OWbE0xX_qo-Ogj~oK1oiikt;t7&9JvQRD|#;kG{X7gKTF_HZ0Y z^*phv^sidv3QHCcTtbob@ppPXXOD7bZ4si!Zzc86!C4>>JX`$OvF(C1@|d_C_s%VU zD%Q9WsGQxn8FDS!G#{x36SR1rN@g(*=EZ~YA~cNRuFH(KAHDG~P`lERByq8kPo5n zxr$fC4a{aPWV&tU*1ZoMZ{SB9STc0n(CH6@EBtE~?hFSWym7;b;LxSRq1&Sit0qBb z7<7FUWx$B{&JZtc=ImmFf$LmjYOxZ?1itCU4V-KD2E-i!g6b)i;r}RBZ>eh6leau$ zVlZ+AO*qw9S>E{^>%DamKb6ma(ff+V)B_$uTM9G=@&wvscDMZ=0kKT7ly1aNS3MEh*F$ZH<{YOY6@Ae> zGhATZr2DhN8}n>UvC}Vv)SK)ROU?*pArWQU(Z{3ZIVyR63UM}1I)8Blj3YXHtT;Y^ z)~TrnAH9r|QpJbP&^n5@ei_h3e+GqON209#1GZFzbWED6ANWdH1Q_Q4q<8^;)a>%idMXq3ORjj z`)tNJDSN6LP3iC46W+b^2QY%HbFEMlT~3SA0j(j7QAC}=v%51rU5&4jk`zr!Gta^# zuM$uPqbWo}rS-_lJ8fsPkvPSd&AA4R!6lbU&iFBV7Xg}?1Sfg(C1=Cew_eVUbEKn! zrsiq)6Z3iAT*FHUBFWgfNCY~S`4f!3h^2g=7n49`pQh-`MqLCaU=!0&GP21(AZ;Yn zUzZXRYA%>6_Q}uBtvm;dzN;winJulNcSOWm+Q@&SiZX1QIP;s4hJFpG#r#W zG|e2J&ws-3evR*L&OhUDh9xFgY9uhhZC!i)kvD$WKwytEY0!^JVEvqy3`}gtE{ogg zbkwhL;z+v5WrR1LNC^F|hn~-5k}t!U7B`IQ3DvQaoN!=nthjm5;E92SeO$q;punf6 zP%(`($_bnVWOZQW;ZLh@8=rY*VyTmoNyWUD{bNa# zD9-Yk@9YS-xLSK>XM6vq z<0Y7)*pGa-58@IiEGn~wAQl!MPo^vw-Q?`;fl*Q1Xp}g+f_DZ@1O}R2BZke^;jG3nqxCPYFC+(HzBu0f8V|+J}OkuqjS4$N*)yyjAIl@vQR2rKbMwj4Sw%DH(#5gKrF`Q>ut?xs_&mn#uLo|ARr2rN@tNtDT@Z};YB;&>-KMw?nb6%%k* zo+_eV$d_tHr@y8nzBV{M?Q1`yJ168L6?=2$^*{x=cEsr)KcKtt-VwY33LuI^n3gsA%~Mr$Md=&^@gM$4Jr_;h9iw5YMe|poEpFiC<~1(RH|rh zo6-aJh~U~Dd#(?_n~l`(FY5)z#IqTKGUAos5H+gEFPLR~mQQ_fO3L}4Q}kLC6=ET^WY_Dt zlYX?lqowv|1uKht5~KuQlFakjvCFMi%X#Tt0q04OB+&d2;!+1%RL+F0+bSKC!@-D~!mZex9|v$0mI);H?S zX4mby8}74k+rv=I}y{b%w?OFmh>URqpyw)~)sXIK=RC zZ-9y~=r-4=;Brw6Uyqz?(9;9&2GD4GUf^I1Ct;H4ifnA;h7q^&1KzL=jex#>)xUT8 ze((X7z&^-p2Tyo>K-HaY`Nq5us#SqZXtts|BKy&zirmiI@er^14rQ_5D}+Ra%X~5c z2|gQ4BJU2MNnnHMZs!XyoERI)8X+}aio$MyTTb{_nvATzkD@V}gNs5nc1mo~8}%v< zl&P>*Rpv}TWSxo@V4QH}cgI~UEgGo^y3kLK;-d|>kA8qffQ#@hw_Q_zOj!$g=o5{DnchvrF?N;41V(*HiCxO;#_k?8=M&@V0decGuW>A&nHe z*By+gS9s$yXg=(PE<259O=-Xl+wX}9+8)%WC-|4)4BYXDRf6zLSUjTF&z+N(Mc>@Ld9!yN9W{*Wlv54vk30%NXVy#0j+q@;H~aP&>60L1yv(?(5GtmDuEjMw$qVbSz!%3j_j^Iyf{lzCMk89 zT1>t}tDYAMY&o?67`#%9ILlIyu$bhOlv11+LW|`R{G=~Q0mH7Yr0A$Pw$u?p0CJ%u zhe^mTF~$c3A@*9d$EApBYGz7o;uFJ+Bl; zN-MD`C8rdPV{browTEs;5*n73okeSx zDj;O=iaf99jmLCTnBh}ovr}960GU!j#U<;GdI19ESMUp3{ITcd{U0t*{?*z$I5?}n+&e!k37d9V2ny0w0O=a925Oja zge8Gs+T!{i;29h)aOFj%C)5UX93C5W1#5@}TY*`5VK3Q>3u$3@Myq06jN6dpK7*VS z%&IUI@k?3GN3}Ml&!IAJ9`XrM3=XrQu!}H%{#nNXSf#_HU~zPw2&vlnKtSW3licnK z#{0(Pc@Y6>&jRJ_qsDl{i`mT!#U_;bM(6aEY#YVi?C6yu{z`WrtN_W$@?j!mmXI)A zm@w7hb%O8UKe4eSI*sLwxwf!Sn6*)l-I3fulX0gb>!&61WCQG0R7TuZ z7UjN>@q)mXQFQ=g?d{&{L-swI7!TYLOpYkz*%5_cXF0nHmYh8F98pEiLIb5FvJMD@ z$|hjWQdC7p*dAYcr9w6;e4aXm^1G5bdF;C=lny zcOBR1`aNI>pt!7{1sZ87q$&p<>uUh2!|@&cds&__2kqxPa>HvwwWbr(TrHwambNGB z0N^g}dth=qgUt=2x%4B!+m*D{TdYYiHK>nJl;GaVcWT`x>CU|*rd471nMiwbqY5E>1 zV)WMBubVCzOq62!Rs8gay;B9!)9y}E#Wc~A;b#KFxb`WRTCH{Fs-3*ROi?U3o+N#b zEnD?l5TuBSG@ivE3^i`q8lMfLPk)5K2Cg8J>YXKF3jxiSwQ70&-=a)P)5GL4Ie3=B z#y4yfQrS#crq=c)l(l?CO=@VeP60hlDrmCGpFvN{5g##Qlx99jsa$Jus;XxXq15(- z0RD^XeAC5soX}H*RgOHFLqz2ZsLxjgW1ahpD}&~dE$Ln)=2p5$%paTd58BEd?<%FQ zXQ=V`Ac_J1A#cvXtOGgg29exQTg~CIAceJl>$lK&4!m2HkPSEgZo>yT^Fo>p^=NG=_CD;IhyuC|i^511l24T}u4 zbeU5W6N$Z&IY44q=HZDXGvKDr%-k^ScZHg%T}AL17~SBLJf72Ck)X;uu^Mhh1=R?N z2#uutims~I&Tp!j$Bg`q+%e`G8c2;Q>wr*%5ex^IbdMKUgU@7@5UTJ-G=(l;R4B+x z!Rm6WLn?fqir991G@$Q)@cQ>9WgF1jatciOTj159&A3!&bUBM?x=!Zq77;Pd6Fd4+ z8u(M;h>g~;*9&pODi&KxiGg+2{@{MQ|OrL12evHB%{i0Fl zXlt0ept1np`N5Uddg#_CB_)(_PbH zL=hKI%iLp68D^!^vVue$7NVKJc`kLtPB?NqFyEa$6$lV%Q0bYBvwfTOJPp#_DF6SC8UcQHX(T-;1XP$e8#9=L-kLHOx`AGbXP9Y8doLtcHYwsMCZC)zG>EzSz*GTi(bi;*B#+x#mQj%ok6UAe zVB%_Gn9?1s@(B4cGlQp+OBeeELyoZA?86_&KFtD8;GYy|GxsPq8>}NPiGGT>J)%BN z%aqw1pUi~UeDp9@gcf{{EG?J%pR21@&*C>6y0;MsQ&w3r}Ur8Qmu@)FC>bx=w#i5T};4fmhrvtak2;iwD^BR+L+Ztojfk;DIaYUMWOhNV2(Z$K(`FZPX@8n-X z896VTfJ zc1;sCW3E}FnWT?an78KnNTK~sGmmKewT+~Fo|v8-2MLF2*GYBAs?--#e5T4hkpYD? zP<;exDmIHb(4-n#7^nQ!4Aefqr+YukZ9a3Druv+DB`fGFOT{XN&hg$E$9d0=hW7f3 zovgSg0e#TJ@rR^*HT@^ji=>YeTec}rdlOP}tx5PmM#C_|@}N3l*v~^CkOFZQL^{`g zzx$AQXn^Tx{L!b`A3w8x=FI;<8)y%c|Djr~)tbrt4;zig{10E@lg$5cwEwTev$^s= z{Ly`R{LKEDbN+|jqLcq`Q|JH2`g$V&OKqdR@i_m#!Y4WZ&kz2L$zNj0OPoOLG=)2{ zL?LL1i+7;($Wp(Uy*obKJGp#&j_NQTr!^{KCgxJbsqpSu#lhl7Cr1}Yd&furIXo+W ziHsxS7{7r9e=Ql|O0gUxRZ&6(-q=w?W||SJb5x1jyB%Yovq)azR>eE{G_`&(RNH$D z0}O7ikG?w!UG%youKO+Mn9Nz-l431620xq~?x|rYqi;bLZT)m~(x{iQI^8kKzZVsR z!=K-to?R?{ix;l;7{Qjrkniy@F{GVA@~Vp)S~Izz;<#5XB~P)PFp2mc#dT!*;Qp|H z?7uTIYIcXb?}sqWxX&rz^kEteRR(oh?I^DH-<8Dim{={2`jiN$trwZO&$$If-UCj; z0yJWRq=`Y7Qwa=GKZMy%iLRZ}a*tD}mIVW`uS)d?`j$azL|37&%44T`@J2mA&$F^> z!fra_EogTlIX?JT8QxM1pUC?Nk9h--EgQ;th(+m9Yhos_-R9`v*iN;Mo{iNN;g;Kr zQiz`7efLp$L45FX_u$)a%}sIWz9x*iNtbuq1(ss(o}0icJl$wGzLz`lVk@3Ec+=5R zb+#An(+*(I-uS~wNEdkWu?BL?PC8K9Nbg#4Zas&#x@B@0MhwqFBwJBuSB7+hvW6(m zO56@7eL5MzDObnd3T^XX5>k-|2(<}^Pu}qT=G`Gz6{Aq0-S;kyiX*mG;SsloD7W2? zkt`^KEMpIrG#8euNDeeNvKPb=hp+nB&p@P3?Qzx^i)NyOig3`2Gl~(&r5A@1P%lK> zU1nE4hbv^tPGmw`KC{O_Oc!&qqhGqJMW>!#IG^<7X1AUUI_65L;l4aEu@FTCQrifQ z1LVS1gzIgqeUAs4Mk-823oPYZq^_kbN!WHxA!Qpm`$c)+RmdZt|m;=MLXcJ(D+B3D6(i{=`TU9soDCcx(X?6eX z;{W&a@xPZUczt~Zsa%%hq6Yf>aR1`y^u#D#)?5HXeO5yB%8SU13nk`8XPNWSnYfhG zn34`a6k$x@AB&KRe@LqxSGiD4)7@rFa^#v@r9qJGYn9V+Vsw=*7!6tlNOROneM zd-AFX1Z33^cMOE`9suC;Wsxzwyt;^zLmE!Pegq2*O=oRB#ZOhzngYejqR_94)2<=F zOME0klx&u1U zg^8}NisT_rLyLOMgLU8ptT6-X=TKLv2bUK|$A2F8<12Y($Y?x_hSB{9co}^-!7y0y z13pN{=7GFiB(%JhxbiIAj-vra>0EiG*zGdb%`zC+C&_VHKX1Lfe06^GpTmVxsdkBx z0Hx#CEqrkLs&#a7@n-Mm)`j!bX_P5iq%2!RSPRROdPpay2JRg!VRuHX6$XOH+dk3L z*P-761CftX+^-q-y`VKRusSfYszI{`nlS{rfPmRPHGnASwzAtfE#A&B-cjnA+mBF* z8Wd^lXS1<-Zew>-8k>!(U%-ED_;-h~`A8UUtT=P^8_!VW|QSLA{1qHVi`_#UZR0!*Lf2 z4IP&dMmT(U7K`|OKf1@YW{|koMkGY<`|XbODfyuelp{b;?A6%wUY;N9ID;Xc{)aJx z>cJJ_8F__T4;Ky)sb**nZk{v5MtayeF+IN6H>c|x!i*U+fqockX zF{^|_STRSG%!Pbx0B91RZHH$}z{ciFi#+HJI-9WKgeikENKpkX(-Fv0zBK999S6`* zD1-S#CMiNkK;2d7s$r@)=NAWuXJ@TfN5_XJr+^wBjUEG+mO|V|_?DyS1Tc|mU&G@8 zoCbn)R?x&_etedXrDbt#lIGj=>$yUnHK=C%#QPMEJ04DgZj$RsNg*2Xan`^C(gNwR zZlbjk-a~{o9ggg9!e?f2MP?nv^f_^NlT(CmT8RTz@#3%~CR=3%KP>A=MPpX#m!D*x zuaKva4{wku9G6xhnAAZR&*%cdz43TBZjDP~Hnm=#T(-_nFVFT59r9ZC#7)7&MLMZV zZ~%|Jgb$a$@@1phQ z@cex5^8FYYw5M9BkpeaTuP@AG(!oAtX5rjwbs-AJs8Qqt-pjmbSzW%~LS8BMI zL3dDD?y;(#OY%`30@k!qi;iSNs2L3>@W^7tG5V$bPUHlXnv@^vrvQ$g>l-W9skjpjxbXE~tDUopp9z!|29YUpJYLBsTYcbBgSr5`Vx z#yZoJ!eB=f1?WXBAVDxJ9Uh(hxX1a%I9KZ8ytDX7HO6xVoa!%Ht}_-{=*Vw9tQr+hs` zz}G+_es0sqtg)>YcrXM!lsUYbiByXyoiyE_A-#!l+zxJ}fIu=*o{3m=wf>_A%>(?U z(+%2Z)(r755G@Lm0dQx@TfJJXI7^OGU2<;3t$pxo+0zS4-1PD${9iXvdFEQJJy>`ky|^RA%0d6 z?--&+yl!7;Hv{In(wcf~p)S5{s&89nr9JidrTVt7z8$DAXN)097*?fM zc^KSyW8pv5;Dp#gMtwnyiYpoXc)p9|r^}aVB47r`k+Z_A{M>1mdHKX+8lvi+S@l15 z)mjw{9j?EZRo^x1f6T1!iu&S)Mzx%bxKDH&1k-DImsS({4uc_=8X^)tdrox(Hv@%E zfj&Ku;_?<^W05%9J9&LrHbx@0N@R|S7(g*JAYjXCq0y#YxwKMJ)@vDYQl~;ld;~g= z*vSxuk?@z{%}=P_i|dv?e_4wko1-&lw2Ip&5fhk6)TFo#om-FP<_&F9NRloKXwZ9c z1Fl%c#fgLTG9z2FQCDLa#xfX3^ZFp-mU8lRKL*ekRywA;#B(Pc8Y46>DIiJFQDGmo zD$ZgNowMN6B`yoH=2Ep*Z#37|H#WDHez|kLbM!T=wJN7FxpSWFT927jV$$BxL^<67L5k)sDwI0~s9=sLR!tpK)*{o0^9gk$!Ieo4#Pqp_ zaq@(&noXG&b9ly76dUQ_BF_gahb20lSEk?#JY}7 zVz!G)r!j5tp`JEEA*Ib%7MnUBL=1!=(sPBL~y8&GKk zp4TPF;TW29o7t3;z-WP_^gLRl%vZB_r~dFhr7_Cu z!;9A5+3VKH<(tE^qy5s2i#cj7#{X5XDfGf%dtMm+{{6Nw6K?2gb(rJs_^RZN$XTV) z5sO2FePeKDQMBb3+qRt!I<{@wwmZ%jbZmE=bULdl*}x^>UF zf9~C9AKW@;ue}!f&pRV!XU>WZp1>G;h{Q}nJiyxvcY%~Yz3N9sf+}eP*e53u9Yg~^ zcLTX5WXF?Qcw$u<-Dnr{Pkj|@6+uct<` zz&D@&VwS3>k>g`adOS{k-9>O}lKRh)h5Ooa-~NeK$<-b6^LHFTyS5G8%tPHHBMAd< z6C#2)cVwg8MaNZ3to(*78(Xr&0)bp0^>*BAa_tcE{=MgO`~!T~NnzcJNKr>|ZiI${ zpz1u>Krh!F_EJc_VSkg!KjEJ))H3PFK>}XfmeUygd4m-X5(Q&ORNIlNA3- zqpT99BTpd*q1_8klj-7A$noHw&AOcRvh`t~PbNG9s7^StPcHlqB2vhJHSP0(`WgRMC3nnh6MVd68r6^D4VV7mI#&e6P%iy^q@)=JF*Id-cpg{5i! z9|Kd3xr4|-{<@Y>y+BipiuXU^!v@4g4!dY0%EyxKMJ z=q@!u^2PpPgbm_%#u&nHl!gXyjqG>~@vKQ${Ek&;2G&IL5HoyEJ^X9sqsWkMTu!@T z1j|#P)1;R!d%&CFq??AmueC;h-ntoVRBs}NZ&ev(K`S>SU&R>m9NyJdT zCrM)hqneHchjrLxi5^pd!x5^k(RZR1N_LA_s*9_rOXJHB&)VN<6Q(U7nNpK!^B-2| z$H}DQEEfN!BvTjm)zPA=M6wGgxpvxeuOg%l?-64nGF1W4fXXavFx46+j`0q zD!K9;>%$JdIKJxP#^P=AyUnS#fpZU?jv!@l$nFdoH2c(4$m8sBL z4#xvJz?C?EW^W?x&aY>Lxktu2fkWaPy}OCE6SL;k2-QQ`w3#=$V5=?n}G@)2nf7gGH(IW;?<6$g+k zs4235=Wy^?Kdr0H7aj!vB46n2jf>303?7!(xARH3q|0o<*vMRiJQ;n%tr7U6Y=fnY zzx#c6j>lI@B+*Kyt)2gyNYLyrOu)b}q}l;eW<{JC)Mgs1;Up6FkiswbYjQ9OG|0^d zk;C1~EZmIZnZ@(mzo8ewm~!3XYfASFWG?p_EnB2yiQ8b+=@ zA=L}CVFM4T`#F1jbjWNL%SM8yEBz0R1^V%q+rK-ZyBrWy(+Rv+tucBl-OG+`)0yu6 z`;}zZqu1vXgP*0KHJzFp-RaZ8_KjVFA56+8_zWAjocjemq2e%6yg4M(PAkttp)mn_ zVrK;_LL?PM*U#R+ZHF~hxVwQbIGQy^Uq8QY+k&6LsosDi1S5SK?EJ(62u1f>_DIX~ zsqG$QJtyRyMn%P^$0wAT898+`awh%vlr`i?I$qW9W@70yFyS?EipVHY&FofuL%OH2 z0w~P?b}?a~HoQE3czO9jUUm-hv)^EUT2lwSPkv-5eeS%f0NARH;Unm7fewWklb{Ym zC&6)0&9zK+Z+Z*fqz7ys3J#mwbrsyL)FESZO>+=x>#u8tH2e-QYJ)x@d)iT)QXg`} z_le&!ojTq@s9&O_?`y2Y6o~wR0N~^ZEP#!+pM*^(_3Tp!h53&t{YJ4oOJjZ6XvrZ1 z(%v7;#-ytW&H3>|UlXEI9AiXW_B+g_q28RJw>T9yI+Ipc!=kwu9 zC+4ABl&x}?o612k*41fGlhY0vZ#daI<5?VgVez2ibP$KO>a~V|+jVWTJY`UZQ+Ka$ z5t7<%hKL=;^uI;27IZ5GvD|8+@!oB(^zBj=(ww;%@FIT6@xR%3cJhBu0F@W4htg9s zgrhvxiL#L?;YY@q`QIfMAh{Nt%g-ynx1;&yZTnOzzs?TB`#)a)X1kg|xv`q1N#@04 zo+OJ8*(%Z~M!m7Z|E%VUs=ll)>XLI>qfHPzlng364{M6SKevm>$8mgEGcLwS=+Kr>b#|3QV!@Nv+Jb@h_!El6M+bj=}{cLjQzIUO?r z+aal!kI+d#-n^+I;VxQmJ3g-8`|zcnM}1A46p$XH4Er?}qROT+??92U%l6sebBTCz zAy2?X7dBnHGqeR~j{(+56F{>462#X%q7ZXKDa>dr7<6f7$;&SQ@?PRuc}vtLkHMbm z#QZ)S>r7y(%P<_t#mmBlp&CZQ>w1UEhkQrOo7IJbhXID<&Tld z#waJg`&C$25#?*hl)egeZu;BRSmU-wKC&78oQ}ZyXYdbJmDYclACW?X)_)dp?xnU^ zjPIxImwPWEV2u)?!29Mc-J@N*LaMc+Hm@OWa^`@qSZ1U3V2Q+ZyfIzKZNc)!rqAjl zb(TYEQ&m1WMOhFrAJ`KaWQ9aq+w?4e#yC%pLDurW~iDzEG|AUsJPRV(@591V&8z` z$ivKG_i<_Z(ihejC$LCeiY2jbO%7y=Ev`0E9p4kD#6oGhWlHGFl6j|f+ZD2yr2gfx zO+;_$0@hb_a{=K;AzhHEYojI=t>Cs)&}@OgS~;kMRxc6%f{knt0$~+ndj?&A%lMw9 zg@QqJymEMr9U~LH7y9(aiyi!@$y!93s7533s+3hU=NihAQ`f+#2r-%5hBB;68wa4= z{+<67C)nY%6^%Z{&rZQzV9!3eCHDTO2 zZ%y6a6#BYb)A#M9dBp>583-`+;j*SaL+_3V96;Hn(^5GoE*x+AAQMYX$D>ZBHJ>YX z#%N)zVVN-bXamnt4S6w2IU7YpbP5W0;pq(md{N@(0xG6LatWrO5>6f0EOx!fuqw{v zDgwJPufmP*$BL7jw2KoJ1sh7?#f#l|+H1OUcrltPL*LJKNqBa>G-Tyfpxwplf*iip zKq|b;GY%mGRhIVQt0;PG^;7YOyyityXOw;PWq=AL(aY1lf%Yf^i?8_$M+y*g6N@i@ z7B_#lX3{&&@#;1yNF`_lXQ6Ma2fi4>!5TqDOkhdqsD*hH zWswm|o$Y5+{NTZ5TDa&lpw{-zDI`ATf67J)@?^O6e|l@jr{W*Hmw=NBRdn_I69_310`d zjR#JzrlikfCl9dlcA@qkZr{@+>O#d?w*}vIzGQk6;O}s(YWAUrNne0DG30wnxzCitk+BdY zTm>IquT`y23To7{uHMnQ>M7HQ8vOtlk*uY7g>^qV93J`_aHf==?QK ziM9$f<8)`ra6j9c27nbwQe=a^@DcM(mO1-u*tk1iGE~BaP;vP;%M>vK0a>Tp?$kzjl5h z@s-#9HTW~xqUC|C7ynNNM@Ug_ZL9~ke?4L=KYVUg#o2jQ1u3}$mHw=!_;?WSnH|cM zfG|5aXNo00{Uxg%c(?EniE3bwZWH$v*@H5la2w`mI-OWTY^0^tJiJ58R;MxHJ_Z@g z@)iQnHXs$sia14I430V{>pCSyw&Ug&2Vphwq0Qr(!_9mz`$q9JW4q7!M7C|PJFIP$ z!eVygM5+vB>Zu%NjkkiLdinQV@bU@`Mxj$xs0Wp1a=t~$_CcbXM`i|U zS4b<$?deIMlMs#Xc7^W9n6UG<(L?s!B3}$5Tzl&L1X96!REp-@i*jieWZjFl=8uy< z3#bUwp47)YXV6kNa_h*y%VQR%?{kkOJ=oHzEHBPc>fh-|LN!Re%KWa~z6$JDs(%vQ z7stwOIJc(?SJZ~Iy&k+$|GlBx!vp!4CI0u9DTUoI@zo&D%*kT;GO4-42jzCIj#8OG zeWp)>R`eEKlGcrsp^fgQN6GG2sXc6=^PH)r%Dfh}d^UFPBU-M_@uFjB&?Q<*SCf0V z#P+l9AJOM{MG78qzQ^Cm_BAl@A$m6|4NfXrV|wP|FgWW7LNH~3><{HYuoyFl?*(`a zDQUiEiF6NU=jn~s-Vitg(SLbsQ5j_9awKi~Zw+p@w?ZV}n(n%Qqd@_zmG3I@=i2OPcKaiY(_cw{os6N21Z!aJ z;J-akc@DnfGEPq>oBCrX5z@k#hpzdAUQqp`!hjNi!(uJ+8_+={`|+{;TF`PvXfyN8 z1{%^5FfJdFNx|#i%^ud=#&u2ZX#&%&1#RZsC;oX9-H{BLqxi~v+`Kn|cyK`QR ztHfC|C#4}PAVZV+8JIfN@!gC1Khp|A**osNnkhMRXIyiZ*@S$z`w>Ch817X&Z3I=R z_UdBELNhzq~{;Pw8B*j6LC-964zU2eK2CU?)aY&(zO-hH@+Ta%|$cj?y35_mVK!>>-Ar(UaZ$u zY`Cf4Wg{fqP6Lg4$RIyM=oS26CT6>s>i-7ECvO&xt$PZ+_6lA49tQk!-A}rwaAo`% zT}B>~LJ{YkVm#~CjY)2i7L#JF-vg=UHuaJYs-KKw+f0vxQw+r}4I*l(d6w2+nSQ&8 zT65__!HZ_ceU^_)%Dk&|Sey7kj)}=9uE!>7Zg@~mef1|byZr21cVlFfFRn0P;IesF$PE|C-AFE>B`HY`)&ZJcPhJ#6&*qExv&b;#(OlmV4pG>!AcrrKka|x(S z{X(Nr+Laz?LrL~xwq~F#H^QlaPqy3slQnsVUGLG@*rv*Np?iTJpADLJ%b@1R&{uo^ z&-gr0w%hwq*_^2#%gFP}%JK^F2#CQ?(Cr-oE`(>er*7F^Fb<^k%s=>?*W=?>xBo~1 zeBf=0eZK=WNj7ABA%~VLwMb^ZILzYt`9gCU8fSXHKRI1rBb*2#BUh^FIaG4iDxdO3 zNp8&KZf$Q@BX6Gga9s38yd9W$U7~EJj+Jld_zU?wf{-+KFB`aU261}n46Ym>c=7N1 zC;4bWS#z-jeyzV^l8KU}ZC?hcNPJcSP3l)ojyejO7Rx7|k z_S{arKDmfFZY%L&SjS&E2+UeO2mJ$J0b*Dnm`E0}c0|8*j_7|{IRf`6WQ((jQny+f z$ySCq?h|z=nn=8@=Ef}4u}9`G&*FdZdd(+ILO&q$D-e`~2JjF&0%okT%Jy5$P-LVp)V+$gDDgAL`qHk=8WoH{jXd=q~6j;e66N8CK_5wj>C~3Z+B1pKweiT zQ+s{%yI>AqA1BV$=ROv?2vV{4$Rx}8ba(X%6JYJQ?Bs{Ee=;N{F-=}P4k>IH_&~VG z{%l4WgcOfwKs;13%l6A18(&Rr+nv~y!KNDbJ~GcB z?--*5vLB~2&oIX^F)<_Otdx|%umg_6t%Qg#Npi2aM@o*xdnfM1f@b@+hlmRR&VfIb zp}TuJ`Gv?$iND;Cn`6-3#my@~2LbaAnF4PAl>Z5)@^KI8%jA7}eSfh7OMEIyq2?0s zx`&#ZM^+H&<13x!mcRcQs)L9R^*id{X#gKhcujn~;lW=i?L$nFFPvO}c32QU?+Ccm zPrx+{yPphZFBs(e;A~gDPN%5ZC1IEABXR(LzhMM^FVV47a85+bn^1(yt{C@BS~?Nr z3yQ7{Qq>rm+bPK;`EV`$AUOI4%IetV z3$(#KQi-1+L+lqt#hk%l_-}J23Q9}6t1oHhhY;B#OTmG} z1>2;}-2sm1AE|3dJdeB>$US`)4)dNav1=hWA9Lr*T1LcOKafpyuX0c&LA?{?FmVW= ziX|?sb+5HxzPQ8hB5(ALc?akwV9HuuVWE9svH)=btfYPF!seuZ0W%@7@h4)n7aJ(9B?_MxG%LY1l$(6 z8x_>CT&G|KLVBJ$|63s0i5M}Z2V%}4(}|)Ego=Gb2G=8wXviK$Z;}q8`9cq84L3U* zm-Vv#s|_|$2I7Rs%9=O{XJGKn5h<$9N4g{%)3{9rRUY8KYwhgODX^iH*K`VN_hfT3 zD1L0&((>vkbyk+8-~rLbagyCCpr1hP**N zVTomVMwyWk6l;k57}zt;6}aq8UkNze& zmt|Xv0thOCSASyXyg$uJ^y~wAo|wEV#t#X7vn1QRvAkons9IkU3F=TvT^w57vDUhx zbH-bYEb|*n8JX9SghY79FjR4tBJ%m_gv=6m{`>}1KbISehRPu)-3B!!H+L4z-to*k z^)%1VHz~f8p6lbjM_`0^{~}R7-J1}=U!V@uP}ls}3cAA*jDHjQub0se(8#(YV0hiv zjd;gdS!rzdMN*gF$(TOnVQgrHCjG4n+q!Zm>RxCXJq3Cw?==(;QP4`;#`=m!hUIHw$stYT`i^6=uqhyDc-n!S-Gh}At7<;!?WT=<&p3&kv#_AMi!0W zpx-6LF`C_V@{2ym zr4Xi?{nvaPVrbr8gO395u`NDIMvVC7a}LV1`A^R;K@T`KlV7rnV)LIJ8G#^ zNq>;{Ojt^;9)ZYQp9ln3~Ppq;Zi^|S%_lF4%al4doD+N|&n_;{%Tm)3wf zyh4&I)0TLRg|t6)@x$BIvewBo!>=r7PR<IkNPdw8zs)Hv>*=2E!7Iw>o#;Rk5+|2sU}n|*}XiJipxsq61g$VpJj?Su5FIjNK++Yajl$B z`KgzQ!+%j{Ht|@{mP(HsFD_T;sfC3^MUmv=$u;%a z3pEfr?=$nBeZzvYxFxw1csU0WuYzcf;nZ-TM+&r42)e3x14$dndV-H@r8$|I_%GZmFfL{mXBiimWWFE}6!$rFim~BV3ZZ;k zi8b8d4B3-lo)VPQtKT?(T!&-BHMH*?aO_x#CCJD#YxT)$I?hExGK{v&-H)g?u^`hiE&gPRd~uhV{;7B7Uo)A?N18!h~8*!B6#3cn21$ zR`sUKR%zO3F4LcQAm@9N$x> zV|!@un2RXCZ(p7Ary2Tm$^2FGqC-Mq-9_lZG0h-^U9}gPmM#h(ETn}v1c<0d#Wf&) zi;^LMdpZH6HHK1sFP3?9s&T4b%w&j7jZGbH#$0Al<*t!7=_C)mQVpvRXQVv+ObTxX zjSb+~nU#O+$Y+C$oYd{&AOB-hvJUzx3qL^CAff+NeYgb9rhlIV2AJuA2!>)o923tq zL$SbFDwU!O>KLHkcI3vhb$myc;lMdZ(lF3B)?*=@{qyce)%n?0;SI?8ea7b9s!*@Y zp;YpS^FzTbA>T>6IREFT^)rYp3|QBfXZeVOf1nc+knp@gV1HXvwb)z~VVFqnEfKYg zMd!+<5TMH+B`(e1_`%#STgh2`1>(YZ1U9sNT7T`4X8uoG|I=)volhEnx&a8%oi)Kg z-6~sOux3VP;i+5jMLV@kz_m>XXGRQ|b&c=k_Ofn2l5mz%S8kRwL||BZoe2-J;11HP zP0bQp$O>0ym9ACc+X?jZz!tzBYKb7VlW56%3s4-{q++v(PK~|h7jGEY6N8hF+9H1g zj%gOv^BW`Z=O-uZOX~WvtV5BZm{MTEJcBBIjWk^+d68o=jW8p*PIus!?_XeJwuar` zge*Y7!k7O?JEXqa@s3ktx(w`c)B|PlECU^)9`XWE1IW%v-$j>2o}ms%-?iYuOF4M; zwqGsp=3aq~pCnZvsdEr8@*(~^kih(1)&V01gpUt0z`ADsc^ie<&!Bo@i4W@hc{l{d zU;DxACZ_Em-vUZL24&gBKlS*>{$G8%^=ai#w*{8JJ@g&0XjTHSO@rwbgfbq6|Kkl1 z4%zGlp%ViNbc=le!XKN`J^-ICCSqxa5^sP{)*5l()HkPwc9kvJI{^FomXB{&MquQn z2H*AXBLN@4rx!}^0c)TX&VSrtZoz6-cc5A0AAx326kyFo*=f{0wI3qLHPUTae<2el zMzy}OYRY)c9ORooYSW+RVC;M_6Tb>kfdOoj1!f776G5f}ba*d4K^w|8 zX~3?QIh&RqUGPB5stval5)8mQ93yC?3nVoFa=8MTTm+y#Z#40Sf7HNbkx<=7rHnGD zg4ak#X4%{=?BBp-gO2&QOdY=Hi-m?k!2iy5U){L%iQ{DY*wgi3RCBZnB6xWOVLQ-1 zBz+ZhS$1PNAn3pOaa%aEDNj)LjcT5sTiH3lmX_t!a#L*3pN}2rqS~z!hvGHRd5&&X zTid7CEokl&*a-UKNCr_o3cg^Q(Z@;wCX$xF%tuSe#~&` ztUF9%fk3}Npb9_YOOVvt|9-)LRza_iLT7S#$bg&iM*utikH;R42?%2tkdqE^;QbqD z|B*lE^Qx(9kON5gwgX+YASVe33|p75cp#$yY=&*vSc(I@aulmncAQBB0Kit5-5P>7 zkQZnNbn^@ny7~Z8zO{%f*Hl)qYc?N#0Igr;_P&;*n<&igjA?+Obhqp~$cMw^17{7S zv<`ZB{hHJqg8yY9`kE&mZzT6c?&o<%tIa^hXQ7$<8jf2BAR?WE{hy0ZGcTLMmN^AU z&=V&o=XxEELOToURZ%st=>*hNHT7WVae1=|D!c?)zX`2>g3j%M3GW*ZsrdMxP0-q( zs_He;bDiW8Up95=qzQc^e^CLT*n^y^SF$1@01Ho`D>-B@{2@df@Zkl&%X|P%JIbP#4S;&tXZAu3gamQI zzNUKc!_jRj9PYtN1lm(HW{R&$ouw-p%dXep?^d_eEYJmm&&U&24p-g`5ov7& z5DP-)Zt-@Y!fuCf9+H!H7fYDs)7gtl$Wky zx*$Hxz9SQAyGUYCKM0}h1a9(gIb9MBF!FyTG!uqgJL^_~sXylQ{bt=N35fN=&=bv?muLk`e+(@ULBl@K$zvc%O2N z&}-n&yc2P;FXvgZ$+_h>k9Ct|a)lQdm{H@66tl5PJ{>vksUpwRqE}+}X?NrV)yw{I zci(OHJ)J;=87bwNUSSVk&u0=qvy+f%5Z*=JrJUU%hy)HnNyUM{J&4%lnIp3TIfqRb z&s~UmOu~m_E@@&LgrD4Qe*Vk#qth?BqqSZ+z$!o;NK#3VQpix`*c2WU3IN!locvow z7IxUto6R=m17Y)uH9H zh@zC0cp5I-LWGbASh>FUkCQ2eJYSTBY|4CbLpe>j3tFA z(H~;-&jNRYzQV0>%Y2Oy5d#*R z02wWSW^|J&a-bW0KnI4fd!(qvAm?|ECy39(fBxtbGAO`xMd)vxPqoIpdjhDwyZRli z#u9TMqX=3xT13aCR^3v#|5Fe|`F@yfT(g!?Nc`t6**_}zeBr|FDD9v?lJluAxky-& zUmWI>2WxZYT{2F4=R|%SPg(${y2a3qMG{gY0&F&6`6g@e&$1&m!>904n>``)Z(p$s zyTZD~E2Xr`63_R0N1GdxTGZ2{bPp|?&~T`U4mUm((6-IuSdayI7&5!)7#)<@Y@)Lg zH71q0Qs_I`$O%hGvQH zQ{^j88!?$2IIYwuRgoNvcLbQ(en={QILDqZSnp68fhYcRhoIKlD&Yv*RV8SMLRIeg zw{cqMNYLbHWwbwSTk!{e=8fISKg1E!1Y9 zMUOBak1|9!F?SW1@LHr>)}<$6&Cr`-5k*_PL|Y%`=2Lz@h22QfJ53vC3(& zWm&W0FOgKcSBKz6@uhFcCrJ3`N2ysS@ZMqnBZ_V2!E3zFQ z;+NX0a49xl-QxLbL7Ge1yV>eQg$~uSL5m44i{r8{DHR50AeioS*pKE2*&+*y)U=2N z`Gsy$salXpl?X2^Y?VnKA*vcQ8MQ*r&`ypzcC`a%);YnHrAJ|%%9gHPMb(NiAkHd7 z2M-qhYT#Sh4;I&F&3}1{W(9Qng4+a32Ykmpy(aZLe|?W%+^l`5c@pQc7Ppqlq7|us zhe1>0jlS3;?y(2VHGeZ4^dVRMQv-5v*NI+plF4a}!FHc=qhpNN0ThcL1$TPx0ASaatIWFoOYZnrI z40=VTK`y)h@XWDYqy64b1hl=Y9{d*ck78$EMda8MVp0?~smJY`vA|b8Nz+6$n>p=RUfp*1hfVNGCXu&&V8Z7=8VWgnH$n@ z(!am`G+PnxQ9D6;touB@MM&$No}w)RF`EptsQNuu?z}aUDmo~KC7^G1b7G@FxWdpL z%^IzbUPrU>PhiuU{or4aq{AgE=xXKJk3n+YThXz@`QbJ6hX(P-ajnOkPs7vQeoNfF z)Ok3O<<^yOuv5Cefz`6+}8IX5TwN@5^rr;cEb4kcjq?WUG$o-8GY zMk(7VX$+OI*HP%!i60}kH;i;n1f!vq3-`H|jp!F_;LvRX$$e-++jErSY6w!uB0K}K z^gl92Y1%s}peUwz2{S-u27gf(8Dmw$p@Mvye95*mk^Z9E1TG}~n#F^#f zpWru8CtYl&5`T61m^j(Kwf1NZg>AW}k~n~sgZE*=lZ*A%WV`}QU#R?K-I`uNFy&;~D3 zfepH$lH@1l{F#C~X`dDvWVBJ~9D{A$Bgx-ZJeB&5j8I{YA=Jq9`SEzo|FQHkwD$K? zVt8U8kNFGN^WCsMdEXB{O;`%Jt3ZNvcz9YAK!8I_u_u3FfER)o$pwvteuYR*xh;e5 zC*uvyhDLoQ|NCc`66Rla=a7o-1h742iTa>XXNb)PiCV8*a`(=q3XKAfkonm94z9;e zRqrL2$5&;ImYCXNj?e9AUDEd;yMM}!W5rX`sPM@Ny={sodpA*|$;037J4aNsKeZ7$ zs|niTk`zp;?Sn-G_=Q$XZ8-OmjY7Vy);6fVCuhPI2}_9I5F#hfIAC}AYS_TQPPUvAXhzu&zB#D8kU?Ob;r}8sTZBo38e*v< z-1WIl-&o-6{*uD+Bh{=5NOw6puQWb7SqE27R4<|_~CN0`e&WR79-ebmW7-l!&McQcz>L>w) zOahmC>oFmGc{6&4wNK>JJ|}u;L9l(Thi2kBaipm}&yU4hkjES*t%YTnxOwWbJWQS2T%v zZAq=8WU0|zU$u3xx9jTwL*uC-My4n3>h`JpC=wb{j_y)o-9cfuy8%O?3LX8$StyMI zuOOUH1#k4jG6+EA+tA*l>k_AlqM1YT9MHfA2f9ra6Z#(aya(#`g^Ms znGh9dW!#g!l6j>`R6kE0#NALDXZ7#0$E8_T@9*Wql%*l}oN1CbHqb~PRmdmBL&|YB z?O!&IB9_?)wvI`MTP`2LHGxt;^L@5KNwe-&P|dHXBjg@fj>|$FWO$}4@>M9tXLXHN zNRUY&dY6bX0}Fl;ajrRNFaSJ2e`r5?5qi-*mkfVm@Xm*=^_V|&gaNlaA{-MiZLp@} z@{iWfrJ}iQX;t5j{cP#zRdT?4JkzmEJs6&paJhN;EOv_i1EEPsmCCET4hlRSRF!%< z1iv!a@__WQ#&~XGUfKl0NQ*HY|5uTQ&B&l%H*+8*+kOl)&-7iLi$|gD?fG`wLaEb2 zTb?+{;G>igVfQ9Rc;Iy19P08bYRui`>(RQbzZ*rml2x)fG)hN~%+P7uFYtEohC-oA z?k^enf`v&etC$f#21Nomn85e#xN>gz039HV&;uE|8GX#6It%rPqy3MITL@$Qv0t|W zq(krLRf`F4Xv{bHK|Nq%V`m64eDC*Pw3F3CGYrkonhQzC(9%NLzbSIPW&*c11FYtc zZ=gSruO*8RciTm1!N2E|axoQCCCPyB53mJ5$z`du@LLhW9^d@0_ZT%AC| zfY^;BWbmt=mi!|3j+7C=rM_b$+B>t(M|ZReqGuZpO*6&M^2yacQkvB_`RJP zYvZ}lF=^536#C~weX%uvkYbKnD(XkR0>}m<(YA23*C=oTo?2u-?`ko>x2DN+nrpJ# zRDBp6RNM=Ujb=7hwIi(j5kUcQJ#6X)i{AZrZs^0|VC8!nMn1=ghzIM%1h*WXqU)i} zi3LZWe%}1+PRs*{=n2eMagl})&htzM@0ev;B2tH#AM6@$rIuS2;d$jV7N-|?tMvm; z=3n4Xp6fX6(zgLUeuQu^Rb#@oTh?-;);^PLqY}3WBD22yU#Kyx#mWClkDg8Kh^Uxg z+(7mFFkMSa@mGUk2vJ+aimzxScI*#X4>LasR@K!tJuBeN{L*Wo%EqMrin zznz4jKsk}7Rh`AnJ3YLY_V)G48afj`S6=^ha)^1s3WPc~>PXd%%IcLBz@*dc%+8M( zC~%ULXEh^(3P%kQw4kCom8zxlbAkt?njkh94{TKd{EAr1!uIC6=H@l+jm^p>F1sa; z8l5B2>DMz!e}4xv2xEKoimc}yU9nDV)Lh=FrKTz$2fL~|C{05*JGa7R{f$``+lts& zNeBOmy1H&?SdJCDCSEQwQs;uI*!6X6T7@M^tnsBxg~6r3xl#_hua9TxHN{R@nwb^0 z9W70OgX08u3vVEuIQW~{b=7ruBj-E`fAV7tK^iP;24N~F&v0^btq6Nq2W#lY&J}gT zD+hYn5yUnjo#0*fto~RlX9X{zP+GpInG)* zG}SFZE_LbFaGV?Cuh=dj{i_#A_vkGQbKkLcTr$Z96iH(7G8waLoyX#zgf7L}IuNml z%rAFkD&Uw4R6${QlI?sz>(ezN?9=NG|IC#~iCI#(zF(c)DU*IC;HMJsl~+LBWv#!> zM1LO~3zAMZab+(WDm68Y&NOxmYzw}uPG*5j8-7;1SqCU+)e!HdvbyQWAW2F;E3+EZj(Yc0jjZP>}BJHxi zfs;A`5}?&kPcPhXrN!8hGHOX!zxRGic-qDh%$7xsKw<2LdSXC1_-`Gwy}F(grwMr< zW;s_^Y8AInZ@BS>I#-sSo|g>7>RE_8MCHZ-%#hfNdhl*xH zBcm@CW!EZXH)$+2jQ(u2B!w;YG<^*C}+kcdrddvLpcT!(51Ed0oSRW%&>Qlx#toHZ+FvGme~ z?ASZOVBiHR8<2Eb4#6jtT&iVJ*$B1yBpeAoq{_>j$?8x?+EQtdWDsJ86&wWTdUWz_ z$g4rWY^%8lSagB}jc1oDY~N;gzve|?DU%4nMUfsp-Vyml;Rj^ z(M^;&CN8=#(5r)~&D}wb5dY)~?$*ziv0)>4tDR=Uk@V5LIR~v`&1cwU6bM^I+rtbW z_Lya!XI2QFYCb?(EtoVvC0V%*5Tn}geN*})etWkM1l`kV3-V3a&0YrGs(#b@a~63` z3v~|b-fGHg-INcm3(|+WFT#<;ItDZ>-R<)nPA4w79CPK&LCtx+^qp ziz_tNwKan$8A&Elt5qkb0PIxg?A2^|q?`R(HvEjj8iumL43iuS)n)4R6$)FANr&MrL zXK3&f@4|e}?5F9%>t{c_G!uKNGpR(w3D;E;Dx{3dbrSCE z6mzDIZxjT#qWss`#qikmH|aO()z}hvSWi5@2gC?OZX49{WUnN`$tvCq(cj#uHCNL6 zLCch{KmDSs>$tCE&+rD%S-1{bSIWF0@ok(?H^#4Hei6{HWy~+(>&T|qd6G~R^!kj`^0Rv;Lj?-W?`G~1IFD%oXG7v2|77-!AnJvl|Tz}GkzI%W=$Gl{xVM>!8)@<`x zT762ArSRb&0uLwW)+KAdn|1@CzlkBi#p$g&PjkQcDjJ1iCm-p-I8 zj7HtZiLU*Hr=WHvxHqd%(TZ3DWI`HGmHrO^m_TR0v%bMUYM?KRO33y2s=lQd-9l-UT=&?BCgQ}C0qt489f zfQE`|2k-t^Nm*&PieAzVuimr@6Q0e_)GcXCY?`j~`Fqiz{ut*^h4uuHxTDJ`i7W9s z@R&!HAQL@v(kDDf{Ks#3(H~Rezt+G?O~ijRn)S!{kFW5dL-vE;T7er3!@lQ@N=i!( z=$t-J#dETaKudx@E}fRflVM=b{5J7mZpJJpnk!?kk46}-l^D8?2?Ophu{5Qobi(_f zb3Gmg;-I!u;wbmAhe?3JbfcIpK5OQO;lvB^SO^P*M0(#4Cdf#eXt7j!(78c))e7Yo zP!_0I{Ppi2JB+5WVOe|2rFZ54TQ5aMRSb0%r zv)p@oL>VltY>o_w4uwHYlud{Z2%mU*4eYb-NsDjzP?-m@jWNiD1FtqeFaNk)sQN zwT2@XU{U7Cz~lU%NG#v&0KzvcA@v8Rh> zCFCQwSDL3G!O;48@f4=UaoaKU3D zaTMWOsLDvL0&N0$N=Hm9gk-ojJY}?rhs}rFhi|_no>ujUsf@UCU4xVf#F#2P8T_{L z0`V5H+`k%rE;5L8EIxE7@NT&V_QH6DkEjQv@av{jgb$vlG(*n~d_=cMc&2oVxXSL9 zEM(;eF}`BwsG9DaLN%pjJqXFxd8!v^USC(P2Qm#W#3l`|1xY?d${nlTB*?6(6h5q# zPh^W1Rc02m!$NQs=y41dd^2*A6znuqf-VDN_55)dC0I)`E0p?X*EzX7KDJm|%&^wj z8w_teT|09U@9@bMN#T{1)Pp8P=}ofu^Qip*q&8HH46Y*@Cq@|8oGc=1Fe!btp$8#4 zM+_+y^QIUmS(;H`jPW!Se-#!`0>xiVdtaj{%L8uD9o*nrR{sc$eid%417SBHN)Db; ztKiB8Yu>=Ub84Ur;cW<%_=ajAz~`#Fze>LrJbn>xM-hq%WWC&3$V<#KgKR`C!D63E zoac}zNq#c9BX4^0a!OJk0Gabh7r;FYSW3vlYs|nzd2^L``gF&}AbYm`m68KJi2a90 zEpytKV*k}@wQ9otTdS=<+J9f+!{!}~AekQ^RWRc|Ce}iej;}rxNlQry7RRO0mIDQv zcFpMAVRr(hIY69Fl7^t6fEFE10#MmG(MUwQfOtGdAGim-Fap)o9gW1PV?2jE7IEfi zMbg$H9diav`~!MshFN(-@c62UpL6TU9PeF!>|vsKLQNz|5oy`u3UVwf(8B)N(Z$jJ z-f`>va37P7u)ZuZ03;6woSwHXPhRaEoweZCzslvsg<%MiqUZW!5?KpV>tm`PtOwtK zD93gk480L)g}9Tb!ppbhD)?Zrl%lVDeop5{Zivhe?`JA{gse>*2_Pd zj~_qteI7#o^ZEJgHuB|vqqeb;^#5(t9{qn`<0IV+d*|neXBVxPm#;9Z+d_S9J#oIe zGZ>X3d6ofwRGh%OYSEX`*snONtE=+FHK<4tXi~U8;TrY^JZ#g$w)v0`We7vZDJzHaXSuRqWDLN=>sg5cmdt{s@4~XP z^ey1OR*^fSw%N0^l+S*8nHS zwg|Waot0yo1Q&7`1x4Uy zj`kyQ6xioQ!4@IH3k5Zu2no13lia#kQNoQ)`u;TL z?dMGUoCcH_DAGVCd*n1A1w$eDb96w8iX!kc872q56dr}(&p{t4PLlBB4g5Amo;W!| z<+o{k|JxLy;^at{BEWMci<2#03c)UnxGiIfz|NI1PPU*a1iLUzY-y8(od8}`+T@9x z-==Z=Z$(8;p4=(YN-?>UBYX-qGH+L;1S--FN;VR2<$b;>GH~x& zQHCgbKW%2b&$mk~*fB{|1a_V_Yr(b!Q6bPp(YEDK5zu*ht_9i_Lxn&WC5SDBlAsf? z^Q6!;E}6#X@AHJuG&U&!JX_{W;gce;3nLz5lmf7`CCn5~DFVANO=7GvRi1pzktZSF zrIo=bAE$Bn#~g7I107?NBA_$bBnH}Ml0uLRB5kus5y+V=5`%0rNFm4tsbRB65^@4` zjwH$Fj%i%|F-MH#Ge-f)+02p08$|#YLpjD81psHWMjmGr0bGm}F~*oGH}LNIJVgX= z9ZhGAS8pz$@)SQGRL@gIOz1Xy6osCrjF`|Z7AXw8AnF#I6a}8AkeI+NRw)d;AXzMS zNd=zp@#QhhWE#gzrXfGcg|#xav=$WQV(CJsI+XNrK&l{0xfQw(}h5Q zS=H&~6rmQzL;Mu43f6-HC_YP=dk7fwugZTR-fHNES%IB=V-)5c?s?;oQ%3N!S(sF_ zCXiE}`TJ9FSs0JEj1TiSS>ycI8fI$%t+9>|wPXlul>c(WoOXMbyp=4m@=O~>RVHs3 z7xYzO1%`1!UVEm8G3&N7J<+BOfFjDgM8~>Qi5{mkpRqc^`~ZIFc)?`gKraGHCeRs< z$6h!Za`rLtSRPX1lNBLerX>zeg+m0TE@tLKO!5p9LH>PlfzZQ?g(qQa1jCC5+$;9u z@Pj|XGoG=>*e7^guPoZna^Ed&dfHI#cVYT=`=fChm)SNR$Q_lZZ8_neT-{wZ`@%-0U z_yE)Iwebip-CP3m_uiBDVB;xMT$I#_S-zInJRU1${||d72gip{r;AGhA5@j^JGxqU6jrrp^E6Mq>W2RU41z|G&oPna=HoqgbK2Pj_nuPTLDQ*OWGT>|sJS z5C*I1hvpXo%-rkY48%N>`d4iSHAeW+_4<)Z`XbL;Yz$Y)F)W7qHOzi|p=P%zH5{cE zlAc*n5YX<3#cvu+$}wOAMX!nZyIm2>WqZq`LmfF{zV6ZK0=R3co(RFVOPxJm8wiK<$0o`fIe zIqqt?BJS|4I7^4)F)Xj_%`t$edlik|edjcB>!ZQh#|6FvzfHILnGaw+(&)o)%=j#R zaUsgaY}$V?SeTBh6aIpi)&0KI^+WWjVP`NW0j&7<_?0t>;g3%UK~WT|iUBb3m1(a< z+p3ZBGw}o)UoNL%gQ@%#P7*wI6Gj#DntPqdgHagcjZ!7WaR2?Ufy*vX+WEUvb+(-`TFfE@DH2+=2Q36I%O%mOZAigA6ej}m@X?`+%)lun;TwDaWivt%5k-x@D=}jIa z77r=JAUI!G{y)I_@Ahe2S_Ef~^&jsRjIaOe&H7{hpRe&Ly#DY1aC!1Cr+U|IKX>X4wexWrIN^O5d4tg5EZ-eB_#W0SZ;W}LaS4pV3j3Hi zb$AD_x}nVyc_F!TMLzOa()6?IJ^~k z;*5=o#SuXEjLDp>CBWK3g_mqOuM`7XWJq+!n$o3lH3$3fqG{u zJbnIBsAhlbYfpe5F89w8r7l$&Y)-vS){~ zUxX>S7yRG`fOj>f&UX|Ou6~!5gg3E{DK$p*=Z<8<9d6Zlg}G0K4rTW^ugiFahUoSj z{hqbs@(vTTb$MBvUPj_PFdPo@MxR|477u`W?i{~v;g0O-tJd3#Gt`9RL460N2Eh_fxkHMy*tSa)S zWy2>UBGNQUU>Yp`RfVU! zCW0?sI3{X0+S_!D6)=)D%%a(i)`;XE& zd)+kK4?k1wzx8^;|GT!)c(nh%!e^@eCoCU!K#{{2zV`ep3UNtu`kr6xT|AWAm)w8j zvE{mzn|_X41(N~P^e+7E*337{?g*a2zwmasOW(Vrn(6kf;9aDs9>=`Dw7{b*D6z9q z*>mjaTvfboIex1#kFU7Ul>)F346X8qCl@4~j5sNNy0d7C-lgxt5(v<^FRvtos*cB! zYg0645fA^CS?JhZ8Wrx-$m-PS&(J^Et1ReWlik5s-D^r7KN=5jd|+~yuQa#dRyVFaigJinT+;o}8CMy9dCUj~s-r9-J`+Qij$({l&G2dz=Zv`n1 zys-z$r5ALdU+I&N@9h)>_I{ffgiKbD<(Fcl#3Lp&(?_c5w_=1?VS?t{_Zb+7w((jw z4y|UAPlQE~rf0@_jwc%NqSOhith)buvK{Gvq!?+yO0)^ zm|9W-c%$!ke7ti*k25>KNbyVxqttQDl0Z|qq?^tq*)xASpHP|Z6h4Vf!XlhvRLS8L zp7jNJC0!K6xy9y&uE7mvFWrad2Xjo*cGFC8x*c;lm`a**E>KB{5Z%hz_B3UD4Cq3g z4v!kBJz*rV5jLJWaiz-|Dd*mRJjbe3bNc=t8bK|F$!3t&1ylCY;R{#b%cdED4`Bawt|!5VEZYx1`SxGEzP7%Ry#Hfu?a}`G3ZKIE9}aTsdN}+L z!W0{hLAKD3os0tgu6%C=3w4JrjuF7%5ik#T5^ufEa4_(rcs*Y@$aWHsyBW_VoJ4sq z>kT<>l8-|W$SI|1`%*altBc<{e1Ev-T%7IgAGUrtJ^fdyH;_k91XaapGkR_x&Lt~3 zABYt=4@NUnoHa^*6nM8Sx7!`7a8>PXvpu=giRF*pcvvAzoPRT2Do!$XiY1?c6zc1; zwcYPV+y?svV{SrfW%s+VqEX{~DN@zGL^s04&5CL3aM`Bu{e7uB4sv zcB}|5#aD_Sm4H?(%u8J<8y$xT=?L-(*R&Z<^NDZc5+l|%hm0P@#!Glv5E`tng~x+H z#Rfw`NvC%F#H4|ZQMVWwWAmEN2CMmwMxcF1ZyKcqY0}ZCY+|PD zI^VFPO%4hg)(Ot_{x6bQF|z1=8Y{<#LQ!hA!I0I84TlaydDyY|i~;x(^jAxBo{VN% z6r^^7lC;E6rzHJx?VoL91$`Kq&8__Z*?aTGIIiRDH~)&CqD|Yn00ol0FJ|bNq97S> zC{iyMJC3a24uPRK5(q#u1BsHu``K^(s!sQ*S+EeKC^;b#i@;3x>CLbrTDOJVt)3QAHqdwcuzJ2=Er-per zUYwHfDEa(t%+ByeqoL(Dn<7KioiRl=sHZ-B7&`658Hew(DXtBuC|`lu^Ke@XC{gy? z!AQE1hJUlI!JKahMpaazMjRWnC)Q@`K?3+!tMZ=U%p1c*TXgP z*DV+;AxN>ehZ+grn*`%8ZhI^YM3CuaOapBewu4HN&=$+_|YfO}oGFAY#F zru`n24@}hZM^9Hj+FSM+4tWwkM^eKoc5!1f!K;5VH7AhHQ*!%Mz0kyiGxpaZ(M)1nhQYdA-Y>5q$)Bf;}F%u*{;Iwx37aAHi@p7;CFWO#j zIWGx;l~FL|?gITAFSh8!$Vwl_7jD%=PvYg!LQ|$>o!^ehti%hjg?3C!IKLegc6~eH zkA+<{Ug(dCgU`luJ2DKEwRb?{K@@n2cjk5<Yjt#je(>7QUmc!{ zJU?Ii{QMHn&(}|na_^X!x!LdJo3;=4Mzg)q{@CWqYbtS(51(H8l_DYq5<50>a*5#^(EI5nE_QQ-XvAcyR3#DEZX4y>BqIdNh0qI-GtyMS2$_p5z!xK)% zaocimW&7mnYQlsc(XF)y*TPN1Bl_cMd*N`px(M|cuI>+^?S&&~IVEveGOJMtw;B6g z7*@gPa6dVEDv$eqY^~nZW68GV`KvgXVL#_0UcX+~ez?3~y}@|eB7XYMzKEZ0u32Aa zte6#RYTW?X{H+cGpSyujLe<4R5GQA011&AP7o+*!^Q))65^(jEey)4ZUai_*P^w;I zXRMdXalhDP5br)7P06LZ9!xZpjsOyf(QjU67^mcqX6Kubxp@t2m$dqP4Amk3>tPbl zj!vcv3QTf5o+kT8Cr=M1l+&9XB_q2zjK#B)!F1+mg3?p{Yx+2%^uu3SGMU+}U!RPo zqr>?al7S;0EE|44l~au@LVZgmo-}mF4ZY`P$RhJQ+wbbjZtJxN&SKS^rS+J(wb*~s zC!)EBxiN9a_3H@VwX5dU8u_V9!Zh`jRdB9|>~-Kbm5;x)I66f>pEi@d-QlJ(e6(u! zt)I$Y!wbLO>t?I3o4xMqz6B|JnlM~1`92?gE82kcA~`%dyuKvrObCtl!(#XAx_ldF z{%TJRlr#pNI$rgVy}lF3t%fTLBni$dwCM4WgdWkE@>OyRxboqZ1oEMy z={&@2cJDs8|LN_k-^d#Hg9Rn8Je&M_*7y6Lvwz;g{trsZ=?LJn_kRZ1cV+(<>HGcv zE`BbG|Lr5lSI8tkME9EQ!W+B5Ot$x8f03ObR23es#)gqS&uF(U*sy4;mbruYY!RbD>RpeQi;HT{mn4aHWUC zg+G@E&=UccUoHG?k%4k)kS`bz&@L=8;9bn|f95zVGhR*zz?}XU?bjNfyffCpSsk^O zzU+JX`=3s}TG!XD#O&A87lxZeR)r;tEza5E0*O<7T1+iMY2MRh{X8^TNAj}I4vv2C zJBrzRF?z{dhhJq;E>3IQ`~sdC6z3v~uP%p*7ik%$1jVSiJqrr+5xZ7(7;A zb@PJHUTH~bNCtkKbuWJowbt4tW0)-Y_SOJ$n@?v3Fk7Uj#F0()k21(x%s&%fwHd(yIVAN}Os@|O1BUHoyH zF?H5ne>$2?_B;WWNQg)Oxt7jyku}<=rG8RkDf!4q2p|2Y{(d_-?3d$DED2@w^F+>v zd_GlptAopaaJc5)#n8{;cth`9Qg`}W7NSVr^Ssj5lG}MruxM#Vy$t^5+v(=u`=9-! zO->DY6&=&dy_TIYr3d`)>cg=}E{*2Rcmz}n&qs#?rSa9JdqrL7M@w!VUi9W{cx7$t z#bg$}`Kv_4?`|+wAH6!O{mTB&?S~KE;sl@8?B!o?p~e19!~KUBo&ED?aC-g}uI1Iz z9rp%gcW1KKp2mn5--Z9VG{ApCX1t#kc=3K(%Zm5YdQQBb-Z~@R>HEY^fXEM<{P&ff zTW)2|uis3=^;eg6(&^N8dUdZ(1ZKOd3lnZ@AUySWNeg=8{x9xY)h=ScUi?gNv9y_; zPV>Pj0zE$E9Ko)Abe&r2Zr{h{&aFJ#i<33edfLJ8RvtfSEJHg3f4n1*x25o*+a1@< zl@spL64XXZq9!OQ#QgogSe=W9x%CFGepdKB92mTMKn6E|a!81m1jZ zV)*Pe^HyE9*+pw>T_DqDHTbip+SDl#4xeNbZT}&%*1so~*7EpWI>@#-K{kZe!Bg>e zi#4bGIIeg0Z7e=RJ!Gf2eAQphy1UNx`yOgJ6%|@mMqaW;Cc|M&ilB#Yc*S?F> zJ_p({Xqk8acK6=)Bh>DwM{XtGxGNt0=c6RA($qy|M5&Zdzq+-AfBnxV8(A(YoXGw% z{Ihvwk<1`ltpD@6JO(^E{c06CxJI|N7jw#lUh$1VZ@=LRUQDrSvoF87o_!U;xZe!= z$9Ho1G(M`o^rLLRrmTCB?^MTHmNL`D_fg%gSJ19s|6_0Y2cVoi$UoAar+@nq_-?r? z24S=77P>riD)JYhMbD2&H0XJA{t)V_FF%(_WD%)fC5+|u>x|?d8FKmk>>+y>1Lj`W zo;6#HbcKi#=^Z};QBJ>E{zm!r{> z;q!-g?tiwO#oh-0)~9}_QF{5^SP{!*TKWDEYSom z?uKKHE~mg8f7)Dix-wHkoi%%i{4NWC8;5bVyKkNQBzME9PvqK#C$2ezbmGoBaE8JC z=$kdedx|hUON!d7fAHNYl97G+9sBZ_vh!TU@A7`Z>^@^yzFQiW!DFJsbJoC|->?m` zZl>$~XkTE4%>iIO1bvh5;+uroUkAng?j!4Re4XKOeFu8@Hlz4mz{jlMbH^l1D?kq}IZT22rd0GAV){2(52F1R4dBx9rBh9lIVmHAB zX^$B`i>}9|{d7jhPFGAWqL*kp9tytFWB`EE*xEHWK)-v~w?6g^(6@hPxcoc+@O~w} zIztVyQ8;tG9DHp{Ms15z&bz^M=r6vtkoV>n)4k8addjPX&;KHyesV`nHn+}l)>+?V z@E@PD%=%_G`pY-}ds$k((aP(a{ph1x!%OySuYY#@rWfa@uf7ZP_ste$-|p?-vm}S7 zTxM0q85x?n@ETq0#&=6SS$_y=b!jq)a2)r8+^>%yzRx`_Qq5{uqt356CSD~3lP=!y zLVqsyh_l-58(4o7$$_)}dF%Re&U;=|D)bHd&-xX6b`1I~@&u18xHyf%8+#8H=+js5 zU~vpC#>?Q9v&rjFGq@XHkDcL+Cti%8!8LF}j)pTHc@9m(S!eC*Ff}aq;1XmFD+~OR zah@f|nh)q$G0dd^kN;rh@LUKI{j=fz`k%KD|C#Nd6975~|Cgs}wwnJ_ypR99i=PYQ z|4s+`<;$tk#ThIMkvGMi1$Eh%rgwB2TS@=%>jN(qQnlY_Y5yv-wBKI7^Z;r&{ADs- zXG^KBeO~e!#+3L<7ehL{y6j2!zbS|H{mQ zy+8lo!Ow-y|ND>s{BZluCy$aW9V*bzpNk~HtsTtW(e&#@{0G8ncPJ1bQtsM-`Viw- zF87w?`a++*@5unq119p~=}U&Ln;Z_wON)s<1MW1$QC7;yY#5YqK=#{mzoFNiS}Btd z$`W<{;#dsmz{QU@n9<}00oOZ7S6}iq9?^RO zlW&e>{TZ}NpUdAH!$XH>^p|;0?@x-8>c3gZ!nHZk2e=sFrM1X~)%#vu{2%!9_VmB` z-e~Xnc-P27_Pru_j{aBWX}P5T6>>xVUjKUsKNrRS8h%LOM)K>k$=6}NliTF}5Loy> z+kp_qYIt8!Au>G~hx8u))QCx+(!&S0&Q&Gj3Fv~52797X)_8^UfUnLuv8?Ae!b7#v1twZB1S*;qkn4NmzfmlBFDA%QUit`CH7R$=f;MMSs zQMu}$+#jOa^#I&@(~e)d-wX4(zTbXgeqFu3*kE5-TqN&1{j;tQwKWI5oF;xvE9QU7 zdt7u8o$?mHmSy!n_on} zx~B0pCcAnEtYNc`qHZ$Wbb3H)cJegd8_o7xp&e&pdh;>~EiagZ!>8w1Zxnw#ISd3E zFHR2TlVfTOOkRwY8DBJQdRr&3!uBbJbW+GydE-_&~z&f?IEX?R%T@m;QMR z_TS4l%l|CWqA6DMKePAzfA8YwT>H-`e4fn4$tanPk4KR3n8#8@O03JPL_RYu2zhik z4tKTkV%dpKO8hYBNRzS!QH>IJ*?X3a^5Z#!ZpqG#GsHqNcs z#c~njH%t6mmk(n3U9LKGD1zXhw2!$Kem&N-x8(ZuKMv_{eOA|*FtX*r zk@=a9R)|=F6GNSfhB#hNBsQU1j`Uh z9kz69eQJPMrk{=HL8d|jI}SFd-4#*>+{mktzFa~<-;p;sRkXv8rT_Wc`_F(KE8_Vt4!~4Hq|LHnU1D*Rt z9{N#s>BpYh^g?38WnbwvbQoR8rtPiEdHJ8Y!235H7*`K1AT>KW_+IW6V@zJ>#W6;O z4$XyD!*y8=`CDEM!|Kl7banTyESqcZxxIGZZ=FB#qijqry!3yjm;FfEEq=CLbv=Lk z9e1r~Z!lG_ZJz0#ShrjC-|VgKT5EFk%j_54(x)9q{SaLCO!`II9i}e1p2X34gA-K= z7j&_lda|wWMn9u{Ka2v_Hu4)B z-2H&)@ctY-y?;R3KV$gl!fSLv1>7IwQJjsFA13qXnti-p30TvMbhQ_6Yp3)&?j;-l zrH1eD=YtshdpZkq}UIFvYgz7me+?sjKy$ToJ zS5>&n&aP9g`jh;81&fDVO25(`tMHKj|tc$|+tXjS^S6%A+jDqs0( zP@0OBuZqhnP2oL%Hk~Q7zqH$npIKx2dd1w&R{C(M!*H?TxOgtz_AfJXuOrLcz9?ji zhd(dbr?VcCFFqvySsv2w@bzzUsT2LQHDNpay8GnKm&jXQ*q2g{R()x2v`jDN`o3J_ zzs%5`qZ_U3$AWKS&BA;=|K2~zOTQJ7@{R=Pd5Y1xJrVUHd(#!Q=+a)>b&AnK?X;=r zLA%C;xf@Eqolkz;vmGyy7$5JQ$85ZnOaXF0r82atD6W8V)9yUmxSAx#w|_5>-n;$# z-@a6WVYnFoV06QK`hjAHwXwj>vd1sSdtau+>(8c7R_c(wKpnCdU}GzNkVk(hPrY%& z$`HwgefPJLp4qtXz(oJ-yI@SdN*8i=tA9US)9$OEgP8ilQ)Q`KKErFWz`oPBZ>d}c zZ`UCXoX=;Cv{Jp;jUh>4$d#H-mBnY>aw2IN>*MaTpI%H3 zhf$HgMlWB(3`qWaf#+rsO=T=MgWA82P`VjC8Pei@GBaTz?C{eYE7s@HH@{5Nfw(cb z`ni>81dVTl{RjD?r#nT=1XR_{Yx3u{mYw+m=Y*fuyJmcEof4ah2+TJxO_w>~ z!+rYB0xsHr;jzFANkB^ttiju?wy(bX)sJ-sRzK!lm&rU=uZ{Dv`9AF^{Jh2dpV{+M zo^cNULz?ByD*tEmp8xY*{QUMeH=j-pZ_b{JV0e~%nOuGOyMI_3rAq%-S4G@heVJWN zzDjP1(lS20lAx~~P3J*F-M#zZ{-?LEesgm8?H}~`xO(N;6j{YVwWlKmcvEVwCRgka`a69cy=TUQ!V0y$=>*IHvWU$#f?dF_vrY?>E!F@^JHUh zGs$InZOMO{FVOo|1a{BZ;vu zW?SaL5aXABk51;+&Woe{$+L;2v2LmN;#gjKGRHtYo*sQa*;g93(R^tP_KptsCzks< zvdmX$Ydnu*!q2s(Zt7Tu{_P#@kCPX&+{r#NB17_HMFp3y!f?ix5==T7c1Q|jZ5&Jh zP(NGf#o_)^7v;%f$xL31r#H^(l04dtQ{N?d_x%%`9(}KTUxp68ey@Bpv4cr?|NW!A z6G)Lq{iNQM`8v`Hbh3zLFNj7nOp9+y=T=*A@3!#T_s5e^$!?sGSk*7!ZYT2l7Y`r& zW#^OaPm+)SCSPtRcOQK5w}(5Q{rPe7=Lh#b*?#yaxpV&$`SkwdhdUoXdHmqvBWvjD zokw!}RsHtP{l6vK|MA7c?MIK22M?2-&%e00BQ44!9^SeCcxU_3RyHD=jd6<0h4l(>}2~d5PUuQ zvna)jFKsP7`0M@chrEe@?d0Qa>Fu45?``u)edkYh9&X=#%*(~!?#f_FNAGP}Q;)vb zzPlrT$@px`tKE6{x2@3dqwW9uM6Q%ClTYq^e&;iJ(~XOal1$j$Cl9wj=M7|N9zFT^ z(c_)RPabb4pFMc+iB7($o3aA_DkCMmy(9O3qND!cK5yQyrR@g~|3>qSi_Xke^4C9a z%jXXnXdS&f49g=K#k-IFW%5uN)bM)oy_5UfpWWN}Z2SJ*Z7z90Q-9rgw7n@4x$}q% zcC-VsKIFkqbOe|{>0g+-PZyW?mdox|D-2vV)~PQ@N_yEKT96%eEx~-;)%#3*{7e5M7Vt}@?m^FA;u*EXA z7WX#>mcx&CKEt23b4M2ShaV=t=`V<%*X2+cBk`XmSxPlv_wW7f?A|5G;;Vjk|H)l>N#%h2tra@Edi@h?#}opX zm+dr?S=Yn%`X|24tF#%qVuh{j?!JFFuU)^?k2IU*?e(XVxoyY&(R`H5e>@&5+sU1S zxvb(nkv@;Ma}HcF9uD_M)BWVG+<3VsvR+S{9u*mVf3z#2Y{4)5Ra){RQSGi#K zaqBXnq&Uz)@tkeqS7$`GNua;_FJnU^jmZ2V;l5dt zLPm=8R+7?*>8wlb$z=q5ejLxQ1Z8(}DEjy4VDeLlRSpk|pX*>=cwb5jpYDiW^x3@! zAB#Hk7dflZlSETo>de;CK+tvGC%N#3l{10L)M-wBnm@|CMk+Rww2f~dH+k%|5C0WA9;^QkMD>Qntb}? z{#~@FM{8b9o<1B+JNNI2>a}~fpK2u}nk&`=Nwe{rwX#O#_tJN|tvA=StP#ek9PZEN zvOG7!8oee)#MZE44efCa7=%h8E|xBw(%8YJ8Vj53ls3N}AMQ>mcSF*cZa`^bZ~pT1 zUc@`tGX9>ez9-l8tI!R|5Y{y|EP-byhkMz^@aOtRFI=53UhPmOVb$%%#VDe0>An0> zvTz13oX+9IK5Z0cKa7qyPJi8AvYmcxao(@0R3kY(Roy!6{8)e6Y&OpAHhZk>)#$U5 z0aMd`HhFP!pt3Hh%OC?03$jB+AAc;8#6}`94l=Zw{*xD@z30*s)6oN&bx%K zhY0E<`7a7}y$a{gYu|3S1_kDfx15=B(jOnDpX;6=a~}Z}6_XpiTI~;;uSHUREvn2R zW5NH=>~>52->{~xCGBP+cg${_)f1k!IK=c#Z~2od(Fv{vu9AO`Pv+(aac#LPgI8wp zxII#3(0b>twLa>*>TlnV4o=4X99}}moj3DmiM+Fflv6C~QprHKlKH__awPvA9f-)+ z`1Et|aU1{O>hjMN?c(9c(i?25A44)sYyEi@S-i&+o<|7^~Pp2+ioONsX zh=!&0Q@;Cey-^;rAxiu8WaWyJdFY4qt`}*W8%Nq+kuP4AEndaAxNr02HlD9*qgZO= z`B@wB7nsfBtQpDAp3Us6d((F}o7shCGd*)Qiwn(WzJ4}eTQ(kypS>52UNoLB zsfqOQGG-d2i~DFK|JO?r`DEV5rq=ll+-*-UciZ0OZrgvo+kRHPZ~tg_7~^3dO`S~` ze|;K%-LpEd{h#~&#tI(sro-mh-;l+_ju9wOsZs=f359UCYJLa?!V3xR#5hmW#gS!na(wmd_Ei z|0CB(_Oq>#{AXJu#m}}z$_uTL1@V13p|n=$1IV16DgrG^ay%QK>>uI6I(pI5t&2uG zGw;Uh`pLptRA0ryLcBK3w&Xu~SdUXs=^x4+*Z*jKYB^l<;H#eItKVheyOpAe^4%8? z9%V73YhhElRw$Hs#-GVY+KzlWwJg%9vJ_}P>GcQ8y_$p`y^8$`U3y!6kypEanf|;w zMFohf;bbaDiUjE$i3rR`SiSVfktoUu)@}vQGMjUU$u&-huGV%38f#2Xq}= zzfeE=2ip_hT0g$AS@}jflLN*=+oz^(@=fSaeE)Cs4A6$YAxkRs{j7J>_u3lT`q^J@ zeQ&*tSm%>J*Z;rp*`f#kGLeDR(;`MPwg%m0dD9HzsYChFy`5?u^720}e#XxHVBwm@ z@3W=v^QG^LrSDa5dAazR>IFZp4DWfi*eHIg=)B;x8KR)0-l-$fln-~G{AQ{QBNCzvR1C zb)B|FS7v9u?)hO~->-|s==?4@l*7;H1Xho#;kAA^ntr?V(Y0e*ES8R@wBIpn_gQJ1 zj1+X9`d_z`(8!k|Ahr3GD7hhIJy|~CvYJ-@XpQ~xkMep7d|R-yhy47{=c> zuZ5ucgMS~;0a`~#-=D_V`!;P}><;$-Pd>hPexU!9ok#Jh<-@NBXSe-HJ-#38a_OP*%x~$R1LlxCuS01`VV;b>mU@uJ=|4 z9qwpqI-IY8L8j_t-=1g*r;A&3=aWZ|AD$&{;!C{{A@brR=$P_o(@|CQFt>ky%Izqt zCS{78JnzuZ4KG;EY3Ph5ZjwB!*|*Y$U8C8oa^aeHyDv-Z=XkTaeB(D;Y2vgulMC0q zSzYRziL&9LcWYm?SMUv*-@kSc?rRK#8&?9HXt!7T!PzYwAOk{hZ)B#)_!$Hc3z3r> zYmSmNPt#W)#A^RUna?#6d$??R32AHz_8a=O14RX21j5I@+Lz<%HcLl^&VpfX0FMub z>xOf~qzVT~FQfWvKKiJ)Vr(F$VL>LJg;|kPJd8&l`jTV(%k;*LgUO4@eDlUB!?UaW zF^hKCaGBk@GH8&vXkte{BAR+4FY)qO_<154@rM8N_o1y|yFCXsy)m2>ng~BkSQs5f zlJEZ2+8@L_pZ)%}wHL(n)9HBhtrN9`Q zRz}Z&#l?|QX?}8eFrLx%C4Eh?)?bqyG=3=-4R(3rchc$aH|!*h7{yL4@1)lQC@n4_ zwUl0O^Tp!xL*Lx?)xWI;8NLvTu@5)bw`Nw>`8#^yTiL9?7c*2#28PxDraS)ihf%G| zWdY%x9LwK93l_eQAKgy!1&QL~bkfKWzME!#=m*hMm$CJmgNx<#)~CaeUruZ-t;N&C zmR`0ZwrsJSCbs_l|2}cDs{h$Ugq`-)g~gfdnx8|Qz2lEb~O?w#(a%0&zm0p^tSBX9wWIcDU?ex}=>9^SB)7-(qlT4}AvsVL2 zgyts>2LDpWbrOz`CLXH3I2eC_np~Cskh5@Ala0xQJos@d&iBIHX7y(8a8R_5rWPG3 z;o9Mz7(jDNFn$&s&HB9k{V8W2dhs&gM|!w(xC9QOg`wHG>pw^2RBoP*ravx?oGRY) zg%!U}_Z-Geec;?5-QbBevi`M8vx2|YBH~k9sItDs-<^yObmbl;8@MGzxXK$M!0~#N zLoW|W4wa#PK?alzcB1BmyxDB>a(DQU7wl`M)1;UMNthemV@mmSBl&!^mppj%AIbjF zct)Y#=cDiSmtk7%j@|MNAJb;&myju7sH#2>;IXudYyH1X<40LN>=CpLq+wXA?VcZ zpIubvXNl^%yo{*MPZ8CZu*x)`cFJ?|%|-KPjr5Nra~?2c6f_YB10Z*Le|cAokxNSo zUXWJyi2kOVSNaKE5TG#}!-f594s5mh&T%2Qz{e?ey|b4W*Z#(b$6I~bVH>~A;ct7w zrgPu-p35iq?s=z~bOE!!@6l%H;RPIei{XHDlYk*_yc{8rE>%fZV%#Fh2Y`aJqzF zrH#JjH4DC15aNQr)?aPXEH?TYyI?E%X3KI}m-fRnUUV?}I{xuuvLF9YxgnE_(*IAk zpXg98mg~IGhmgiA>n#KkUnMw{7B6hwb^1VPM4AMH?9+!(=e?1j@Pd*%TL)5 z7PQ?UYO5C+ZjwLU7K_o`(?(^oW>hu+`|VpoSpFYCvszZC!6d9B&2f1m@_ zO9vo4Dwt2XJB-_M8|zLDK4+Bl*1l(Ja7)9`{S%DRyBimW2DtCx0R7P9Ylh~diw(`% z%PuoC|J4u8_177i|GMPR{QO!%FbfVRQDIyYYK`hWCk}^9ucohgQizE>n~uk!OYgaz zFZ}Zc{GU(fdgu8Rpy%>G*ZDI4XHjM4d;ZUN@N*tDmTlN;SHffuFZ&XU8?;n8eN2?mnE?;j-xM?Z|G$+g3y>9yqK_*i~_dUT?MvdN5Q6#OuG zp)|5zpNyuX!}${BxlX7*(tZCp{VZ%aWFm%S8eq^(GOllgutcd|4sJ){N=?2z&ZOruZnVI z|2J9oe*eFNpL6$rj~U;+`|$DZqc3*u-+l1p{^NmSfAQm_lvf|{ids%uR>>~@`VD@W zC)ea3-N#G&R9RAX2EYE@sr!@GpAHa>ho3;?9j)>5y4J;Y*` z(-R}Bg=}_Xg}?v7qunR>KfSZ_a995NckUdTmA$w4?e5O~oyTW2MGuynNC#1KJbykN zkM?(8oXp2BM|0}2Cj4_N6r*1KUI*+$ehne_)%RM{>TEoRliY6HO1^mf=ZD*OKH2^J z$>Z(+*!^Vt(>qV@J?@hV7VZwubK~ltZP~9YPurIjGCle+bo17l9==-F$>rO^Vi|7M z8iJcctVq`n@uIWW&xL3C+&Nq~XR*`65G_EnhK};hISYLo1kBR%UT*iIEB0u%dvf?} zG@0sy?dcb6s$a44n{2Fma{0Y=cdrh`x`(Wc!^z?-|^2Iod0@6yaoHeN}DzIe^tLf|KGvSx#xe+xe?Ux53JC14@gP}f_f|M zEWe-3cIWP**_Zws{Yc9pq*_ZneRWm?_7{wv?z>|#0EU}9*MQxlrPgjISFg(HI_&XJ zA3Xg0&g0!L9`4+K{OLxvmE@ZytgPX0C&wGUB}g&B!TS85wiDOP9am1`lIt$x1DKY{?%F2 z_C|4Eee}_pBWb&2VN3AOt>tH_Y85NNpR#Yi(x)xW=I>?5em|2r-AabXY$a#Ul0g90 zZu#NuUmPE7U^+mk86L@qgr-HGkYD4X)iu1h4m91>)ZE1;OqIQP4T#n?YL<&old{BzBhw&;?6Z0i>3yzWznwaCgQ8(On z?9Y-l`*7)eId!cp96WmF*ri@7U|7nR>yDZWc6#@4K_L7B@1XYW#b07URxcmr$5Sz8 zUT!TMlP|`z+34$VT!dF$cWR6H4#K-{rN8q+C-g6^x4!>*J3nt?|2^M-EB@cCSdIVY zb@d+q`7VAg;{SC{8?!5(@9*wuTI&YCB0it*Z-wlp4UtMmdz%~f*Jl5*ocJnT@AY2E zN00A3e7yVc!IQ^3_qUVj(aD@*qqmcd^;c}tsz@dtC0E_m$3IqAOb}~gO@>w|tNyiq z&__woDD2LFCchHwSQV*Px3vQ4&@9bXrnyM%N2rB1aN0a%d zN7F}g93PAyJpD#Aw++2`=l-Yok+)S7`|F*Z$GZ(x0TkEB8z9w_n@Oo5=ski}Ca#C%}2} zzqF}V{C|1=KK}PEelCRnzoJtebX$JkA0Ny|Dk1p7ww|b<@<#|I>F3w~ILF0@ND5z` z9PhKSwvq{5D(s_UXd6&|Jvhq4i0?Duh43*-8!>rJL81zdk=Y z7%Lgu2ZwS|SX#5gV2*q!{ZgKE>)-VWR>{>*KYx5ums@o3W@5uRA=mFPoZ}NRdiytW z%)4~z6t3Sx%k7h^^`dLwMnV@q8ShP`U$5868&`d=w4G0xXC+Uep}_4%=6|@&Xx`!w zP8AYE8s0}4`apk~rnYX#yBjx-{F8-oNPaKBUa{Wo-ArMtac1e<7 zqpUf0r^6pc(-$Yl{*%dEu84Ff8`o&X49m?$6=lNNy3Fn$TI`D%e4u?$pXHK{s+gpe zyLtNQ_Xgc_>NKytqZh}cslDrLOlg%~J=U(Z#n((Ai zAU*s2?Dp`*iTr)_N>GuOKGz#=XMZrKxm^^x9H!ap%YAn@!?#b@efzYJ&ctr>+WTSL zk8Q$YxYXcP_EO4r(8q02K%wkot%~9~Rj$FKV|fY$HlrU#KN>rN-Mf)(ZvIBq)UZ;9 zMY54yVu75oK0eT=9-sCSFPGk8xjR40nqBDA4d1=Ix|R3=(5`4tw(ozkcCz~waq39D^@+RC8l1iRe$8CR zvY~Iw)5xeYiTjtGL>d@o$!0Hzh5e-|9G@Qtx@k}Ws zbHPG+GZEy+bV(fblP%ZIR+h|bNv6YJcperJE@#fe;cL&sEN|=<`{$DO zznX`$=Qk#Y7kI`w_J5gG%_{!8c_06I7eBxK&CREi!<)0`SFR|I^-rUxlRqVs!z;+D zz2`^9QcNYd={))5Z}&dg{q)YgdmrDq`|rE=c0PW1=i%RWzqs@G&$qAs>FDV6XARG?AbyqcKz@E!N0HmaLRqxHv8Ylv(es_E59|k<|rZRVqELe*}*w+ z_lt+ypYHs}pZY%^-2XHIlojey9vQ>=Y;QU_o{P13B<)S+Y9>7$Q`C}p^Zvn)SmhLZ zJmk+%LK(7aS?vkgRg=T7XE&~d_i)eJm0xfF?jL9N=aoG3Zfri#;qFhqpX{HE4)huS z&&)Tf^x;bn=^x(9cIof44P(uddW8#!$0#X@%F1PuO`<=%iqyl{Qj%t7Ck!j z6Lse6FoXU{TWcDZ#)cX|muYi-lg}m!T#&(>PRFxj5|4F;J#O#D6=q+N4!6@Q04VME z>>u@m{kwR3a{X|eq{;Q8K z0T4^WE&I&g%YIL;-WCPvcmJ5b5=H6u)&6nU$H~<{&8B<0TliK4@Y2NnHhFY(GTpPq zHG4ijI0$QA7Ir^aiext2u&!%)tx+;Nd1{x-nhPz30${_1S{4Ald5SE|R-Yhk$`(=;Zuszcdh~3#fk94IU%l+y^gB)7Y(+rbKKFsI`#4>#vZPK93{Z6YcmF0I=o#|6f4I+I z*Wqr>r)EAylIsDIf1E5kpHqah!? zNe;q9VdQes(gMlf3Za#c2BD9}>%7S!na7%w&A(tF6S6R~DYLa*buK zEIfPZnyX@gsO ziW`NNd3rvdAOGRz%^!aF;f8X*&GsfYo=>Jnv*(i+qw6O(Mte8DIllS-y|^yl-u(Qr zu8*5y;*4M3cs_q|u-TUc`(ZS*vO*LD6J_v3E|~^BbNXV|TX(UX*TH0Od^j8bA(W~~ zMD;)ZaXR_>`8?U!+e~sV&wYoj zr6-3IEgUuf@kVk-bV+S>hGH|GejlsjB+0|^{)8l}rze)2Crw#`pIDQsa`TCp(sCYa zM(oV0cTu(}RPnNQdSc9m?&qwp6F@RucqA^Qu3T=(&aZFTG zS?Z>aW#}KukCm#Lq6b?EEvsg_*AHiGDIrZ&hJDK{XU%?+wtO!R_m{dTPu@EiDJuA^ zF3F?aIQ3nUci%q=mW%ev_hsnd>-Wkx6Fb-!B-=mQ(~@zR*(*%aA>11>; z8>YoKrE{w-xOZE4?fc`2wxcD+$iNNDdRTyN?a6D!3z;W0~1&SwiRuRD9+cDA`gj-JhbUDJn^5u4N_rVu`d${x2pC2cG zesJ%T?T3$&JNG}4Pwzi|xbyLo#}6JpvWBkSc_g=A)o<_I|68*CA7Z&ZdXzkPnCyK1 z#l0PAQ6BN|&i%(b+mFbnxO?x(Cp-5)+e)OBEl=LO&)xj ze7^ng?w{q)J0I`d+j;yqedwnR~>45oDS-j+4>=!@;UJMx!|&$hhUorizh3JpKn{=ZM;O8GMR zecrrZOWO|~{*C4t7oC}{}UsMeaM5K=m;=@(!VfupDr%(EuHIR z=hNiQCx6-DEy5kLIv(wW<)p)S_s?M-jNI=0T&qB;^|6vpJ=*!tZN-77Yrd$m+5>X3v+;-)oK6dt{GIm_~peW0fLS(?`L}800=_A$yezt|OrLB?3r$z&bz4@2e3g`Wk>zEc(p;XEMVqx% zOLIjl8fPZ==r>*4q(z;k1uf)tQ+7>TXQ@86kjC;VE4fY{nif@6H2hH&rRcL&&4aSK ztt;tKUeQ8QmE4whxpuGZ(yS=*j6aGdmHYCFZ|Gc_<@jd8cntWJM`6(bPH@@?F}|M}2Ho zHF;O&+CtrxrSvc3n?eVo%36JkuFHAV!seo^<$b%N)K`}|m(Ew6-UI}+E;H+0D|SJd z<%O-NuB&9kZ7zyRdMiUyW}$_uka^QJ*;Qqm*L6mRWbpG;mT#_osN`LnE-UmcGMU!4 zma(a{<8>!XHmx#U0*qs0!_TCovXbmUg}jBdR_Ea@TC1}uySd0@z;!ldNM(iAx)loP zvj=r1W_8`9x^$CP_Ia0=mF&iR?)ULtVzY>i793~ zOGI)fz16u8VJ^Z@mXwJl5s|q($;6FF7&%#Vt7nbuq>kO7Er@KE21Nwwnw00up`)@| zgia+}vngzhGZ7Xt8agNRcs$!)pBI!?2{nnWY@`IR<^c( z@>cGvv^f#&X(f}R`(7R^bKZ1efpYlfmF_I0yG&r#gsmV;S=OFz$D&eknA`A^{!xdG z%3H{yus(2jNvG@uW!I&$iYpt|S`K?TRdlN-m3$%Nf>04DS2NK?m)e;n7u8*tnPQ+i zOV{YIR&rv97|CxXOtx`b27SM0Sa^rczEebjPR{MeoJ@rbn$1}$ zFW8wZOGKKBkQ2Q{msi(`G;50Remu7fG72##^P!tA5j{d=Zmye27GZ-Hlj`7!Ad{65 z6q-~X&l z#ALx?pj%xwMV;rm#HA0i0hu+E5AsUVxXwi)XJLW5V_QZ>OP*`GPJ!!HRDj%;fE@N| z-4$V9$aYa>LQmdK0k zGK135Ce=emq?b&M>}NaGTiFw3Q1oTti3QL#CLg3VIq*&S5}ifX123!+Sq{3c3L8~c zr)Y*%Z4Y9j<~rc)EHuGFCAW-M)5wZSbuMJn%004;RrX{Y95QEm7KxHq%8oNhD-(lW z(3o%)(I;!qG!@xnqDspEnhchmlm-(369sW ztB?^U@g z#5v>{Rh>-{Y6v6Mm!)R~ilMG388tc8GHB%((aUpVNb%OYGGeG8S- zvH(<9MU&=?Hmw+GFB4Uo4kOdggH3yu!$$U9ott>>(4KPJ_woXAU^n&_GGnr!M7h(Q zg~cTcO4Sh&&(htru@hBh0u`yTQ(d%OQM60j7a|B#>=0d1qL<6ls=^KrPCOChIu30n zr%Yx_6~|(RW@h@N^jVbga7-Zv%G9P-=H5&uo!U|wm+oeEs>{OV5VMgKy+CH7YV1j( z=gHjI+sYWQ=X8vDx~@?ZL^2N*!ZOGx(FR2MvKPcSkd>^v8%0-CNYmvi20vV+2Rltwv+`M>?|3cjv29W5TPmm3+JXBf+7P| zG@?n!IWB8f2L+Lyn?M$YODNt~)E~ zq(M0cvmm2Vbh*wJk*FhW9Zn`0Vpe=RHg?mvI!6u(S;=os(hCt#>ujRRywoFvOB9Y41n^olO)AJa{@NBBNv@ zn8uCvFO#H`E&`4LYI3s2^5YzpiMYg_aru>?DFbco7bGGQYA%oQhM2LWH!CP(CO)o*QTW%G%&*Yi+pX{_^b;>dcIrIOis!B(%s z2`#b?b0bv=ljGBoRb>ji>}zSE2|5h!CmGDvM2E;`*(Rpb)@VU?ojewKT<6vyk>Vn- ziZVCLT|}J7a+5G@9#JE8P-JBwTuqk~(^1sP)@D=eAbGjA4ks__j4Xi8d;~R4xwZ@5 zBrzjp26T7Jh@%AAMwQcy<15JO76DP3BucPjtKLy4o3^4vu+Bw;1}~PHZB*GkBBbiz zMG?u>w051WaJHbWe$lU0V>UesOB+9?c2yk2X{ZUavD3fl?NLWYtkV`DO12BhuG^Ut{^ zvzzG=kr5F<`7+0s=skH|*;^znA%e2%g3>|!!CVe9F*PE%HDg1L+q?=kPKE!_JQK+3 zRD@J&cDF2KBuo)hNI7NjmD|ZHhZd@}>D5TkrZtUJ=7L^Vl^ytaA5t?hF~~UDnjppH z4zZGK&$T!*d(FI(w^N6;jks)?!hGv^2jn16GxM*A$g9!$bPLN~Z(7w&(4%Ch%E=Ji zr-aS@JF`mZPjJCY-(>ik*1W78c1>Fb1t~+C zR%Xe_aVnPvYghyuWMHZJx=&C#sm2Vm`KZZsNKr8}n1>%9W#|35Tdy zhruf#Gn*ma&DN8(A!mPPv#G0J<|6b#9)i(h+e~DV*p$JOEk|3GnjV$Of@(RUgU6c} zOwFH*GErf}1~nGJ)0&G7lR2&0E;acelD6(Ob6MZ=8e%r9P${vjT3vb4uSNt!VQ(Q_ z!F^VkVJk|D45V2HoT+%V&3KV@h!Z}Xda_MLqUzKpo&4Jr+Hrg>qQ6P!%|Dq`Sf8C~ zt__h!8FJlboU~#g?b` z&0Gwg8at;jeG+36ucVE5UZObZ+C}DZWLQ)#M|p0{b`cdt1_TF$tw~u>GVEq=ilJZQ zATeVDPgiXoZxNcD1F4;6Vg<>0tx-qT1A<7@=hDs6{A+^ER-b!t<-N+M;!=#rxc z``@B4sQn@?O7r;0VVWTXwGX2Ii%e>4amd!eQDNt%$dek+t*wWSfM^>I84=@1{}2Qd z!Nq1Z^-a$AN;+<5GVZN3_&`LA%jOz%7*TI!Zp_>l>536;i=$05HhOK7E?t!^6g2Y` zcSCE=&rIZtOkjxj$nZ#unMGekgOoorU6Xi@M8mBtQY5c4&=Xwv^5QmL7p%DJqrTH6;Dk)uXk&(eY@QL>O*_0BV$q6%6lFOj{+p=$nD zHY|P}I|pQ2$U6pa5)`{c6AC-dP%g~IAt)u&lD8JZmyOy4ACF9zSWxoDL61@ZsJW<1 z)Kfy0rc{ZjL5EW*j!#E+z3NefL&T^Lds7ydh+u3el|*vhH!@B7SUJ;0yD3eNN^pW8 zAazyaCl}o!=#vED5cxXdIeJ}ZeU){H1J`_ma(KvgO6^!_@N|ewDZ*ToxthRFOmmf; zs$ZcxrK;(fJFsQTw!x*$=(U+?VkH(>g+3il|2zvGZ>;`~D7z`RjdWbjBeh+m$1-s0 zI7UXH0?7exPVN?Wiy4T#M1gPJ1a~EeW@ge07lwLt&1&wjCuFEqB#3T_9Ai&8kDnyqD4WuA?o*Uet&UHDMF9_G3 zD8H@!Q8yY2wI!a)ktYJjMzRFHQrZg=*_W$2Q$od9ltZ&NA8<#w6C8ytaTL#>1u^TQ z^>x9k28y5y#(J&zmB!BQ3^$XEgL$$u6&7P+A;c&)aA#)`AtD)ZgAXNLhkeo7 zT*x94Ez67&bu-DK(3*f})!=8dBQPW8Zcb@Ysc`95UGR9fVr(=P#}TtHqmM;6afq{P zB#6-yX=X7Zc^m1fjEe0|W?i(wBKR?76*YyeDA`7GS@=T)yzI!_7C+IsD$mVgWg;e?Wd zIGDMz59H!H9M?p_a&tY43WKO^`j~qG8p{@&oKsaAT+bPPDLHNpQ=p;d%nlEPdxj{o z(;uXhsQPUX9VoGtd2%vBL9*h^n_e3XAjB^)7J?oqeBiXS zxke_a3x4~Ir66awi5pR`Qx3f*L?>~w7Ist7;UBS+2~7)RMQf4+&s1i{b&g|L?Pl|V zh;G6GX(3>-L3$h&zVNauEwqapMb?VB*3lm`oZe*%t-6L;y`-g@S9-^{(i&++v1` zw2@csf+mLC4-y9O9s0XD&|C0rvZ2iXP>O~p>%fLf&XmT+*t`e1E=0Xf#7Ggmtci$O zjDPI|ryBlEGskdyXxzi}Y69V+&D(bDT6L#}y(!DT(=%SiLC%*p3!oQ~s@2YhQlLq; z9%LyuRS>(fdzb*57*XY9-Ku#OqrNTj*n)_Q`qvJ+%HG-}ZUCNf1 zEE}TSY42q%RJ55siCjZB(CZqlf?Ncfmta%?mDY*@Xl!rF5hugiG{NnWikXautCEOJ6~fB}4o*>uOf*V+*v7UdqIL-Z?RDjrPQ#x`U2@=i_cf>A4DD^QlnX}ZKpK=q!+8TO6 zq&wr1TS&73y;qvrDQh#wawtM{(weZd8wt-5^2AJNV!7tY;T4+J!ee5LH8PrQZTdr@ zD=*7~HrS}8U`iFJ&(4sQfa)vo9p+k>0}o%2>5~eyEP@jld=@ZX-GL%uFo3OhI???% z*lD7mw07#%gc498Z1syxpPAT2xK<3ZnF@f!I@47X1#}cm7o@$+nsmtw0%Gjg(>nYq zifj|StSu{`HADbC<;1a*w*omf22S1)GUB5frrW}?{OfwTUtYTalQhN zupk)NYjPKC_^}5X!~7>eVbu242O%Tt5YH0XR{%}1Z7VVt(PVvA)sDe@&8wz%$tbC+Ezr(3|0${PS(3+Ts+)86(4a05}fES{Akya-?fUl48! z^TTN1S@u?tauv?RUIW40A;!shfEeu5*hra3-g>7cp0n@o#MJIY$7%Z*d&|HT2V`e zdd9pXURs#FDPu1SSw!BISGUf}<9Yhy?r1t4{kSfVN4;QJ;^FOqjs-wi#UO(!(+ytL z7~H18$t&VptIfjWu$ImTmnZoXt*J4%Vnj?8Rfz7&S!sc7f*UeBUDkFX1_jQl4qnMr z5n$?T5^VtVa27H;L<13lZxJ#urd5{~Ar=QZ5<}I55r}(>djM{by^&k=x|DMr2_3+d zG6^D*Z3aY?;rwrVe?M6+7SbXu1vtOCtPw=AE-ct3GOn%yW&v1ZiB%Ust2(fTE0E=` z=@7b>IqB-o)OPkCy0!V3*)}-8)EP=ThRm;tYvTNx1!K!sT~LPcEikjBQ&q6|TSeXH z0sbb3Y=}Xj)ROvPk>ga=HUsn(idY2_9-96?s{ zD8kB<1)hcU2fTxT!>t7wkSa4HtOlZ(hA0-1i_`}4rT%Z4=IxtM>+95o#$yAOx~pH-$li4VRr0imAIYBqAsTtTqQDQVH2%hmM>x zvNUXBWTIs?G#1^J!7t^I2skvdhl-$elRqF=*{&t(FOni_1HLv_WTe?=6^_K(lAsVa zq#fEGCZ~tw6!Y71&Q<2j(LiyY1;l}fP#>A+nwt!Uiod zXL?O*K>m>(sn?lPD?rLv){Ui$AThxb=Vl;av|8I(HjGO%y+%T1kw z!Pkxx-Xhz2T?(q+EFi^a*R_QgWqFa&)>;x9j+I@?UC= zz>;^1W6bbU7U-2F6msl*8XIM=5eS&obe9;^!_;+P0ijai}Ocs0OiK=rz zWKy%XMH*C`D|t{!NM0>%)@y(`r2bh8vYwa^6M3>!>&`O6SR900ZO-s70&obq0*%7M z?7#;Hc8p)E0n^q~@73RHRD^%)pjcK?XqZYm!BZQe>7cW?*iiLPO$U z6Pc++Sga1{Hx$OEYeU>aHWHp%3yEN`XvVxgRxSbkWIn}KG>M@2D*#o7tOjZtQM4nF2o$*S;DDp=Z7_2P@@1uAKV>bDCD~ezeyveS^M|u^=|eznW~6%+g@G%1 z)4INFGLIoahYHfMOwC;2#hRj)U@l`K2R&NrY(YfM$)>>W}j5Ai67SShD~eP zIyl}{>oD*Dhc0l5CU~iHTtjB`5QI#-kV(h(ZX3(F%)ttPGN?=#5*#XfjGGK4hrhj` z93$xcmWl#^Rkn?OgXaq)Q>|?=)-}DL2>T)fE@MXtLPY+|!Zyp$T*D$_(t&|n3oWN* z$Q9_o{EJc(V)dk!SwM=|uxvtuKrp(M=HaR!!rm`y!wV#uc2iigR6$lL0>grcWT$~1 zvnAf)SgQ@5L?A#mO7J?!4?}KyljE^9=T{KImb~=}lB6B7e>hJHTj51Epy&wJQsY97}&%5|gtKVFo4D zn6C*x2P0udiKsnAKq6*nFk+wuT~*{{j^owjq*{)aFN`?I5W*H=A?}h{-X|ayR39^V zYEl)eG+0Tp6+~XtA&zWb?mz*d(I8VUW!iy+mIi zu{rOv{8IpV=By(9g20%a{tbTQ)*_D`s$$ay#15Ke-3HtQHiHNlbMq9bMrT7_7f_=j zz~35`N@?(*f>OzXvEHH2=OGP_JXRUnBE*LoVDrssUL zBdXG80*N751iM>#O!BmkUAB0jEjOK=0VRlKb=UAGRHi=!f1|lmWg1DOGS-v?s8~g? z)z&0w>RlCl8yGHK8F08jpJX}MtJgR{O$$OGLCF!|I~}VqFi&JyuQE&ijp zbpnLuy(IJ!{40uq6?NHs$0Asz(_P(16g%QXwrx3W(09yzMvn+{>v3H}S5a7V`1H%N zO+)@G7>~5G77C5#roo*`5}OE?+Lm}OHdHV+aJ^*UcP;+IF%s-OX01Zp z3?7t>qE7=M*|*KJ&K zd}@O%BZ6ciPbZ9{#0~|xW16B2Q7o7+veq1Km2!Ke!M>^Re_5f5oV(}Cl_b89Fcat6yL#61XUrFqbbOCYc?H`pQq#UBnj+7szx5k*;D z@HHAvA0KmB^!^9qam)ik1A{9MR+0vTM6nJL5sYxdRb}e&$%9@EmKBd!J|ylIy-r(^ zxF$!NtyWDPHP<~UKBy#?x1n(lvHe3rJK9lWOCVQFS%^W&u2<7NB)iFBRzy2X_GV3# zM%M$;j5f1BnpBPa06`|6S5@I?t8s$_e}Ar8Y;7@p0OjP+nBoI80jn;!_=yP^o(R0E zf=RmpFMvX5z94x)2up0dTKunuuN4&&Vv@2a+hH(^ke!A_K(IETufeOCs}Umxd|BYE z>4-&?M&OVWfj1r{^y-!XuYoNRH0L64o)q8r3@-4swoLbJF1s z19xwTD<$vGj}1C&MrekRL@K=9U?zFu$_v0?9(BEJGoU zs#6;*sK~6>F5nFj6-54f?1t=v%outyv}1@P%y-rTbv0q3xy>RI%taPYc6^klaWvT^>?1@d@-ND?n7JI}A@>xm z!vYQo>H?;R84)5}6`*HYF(zej`svY|1&=q&tH3*9U@SPT2)gQeDDYPnLCTSwj*~2e zzCg&EseoOJ!@)A8vBluOFfgjjf1YPqK$lB%O#ms1WsAeDv{*ek7pa{80hFux!6B73 zQ*V@=yrd4xU$*HI1)^qYfPx@+o77$jr&CL!v7Uz+P~Nf)S`fi15i233O?_lpKbrun z0`^lmqfA)ffRO!I1e+I}C2E?TymFgBhi&#@Qcu>2T=cbhA8{iu9(ihY6?SgR;l6&Z;^{P?moe za)~n8wH3CDg$p3rkX>!5NkC3nZ8l7d( zs?fxvVSana7^*bnGXb{(fDj-y#Gk4v!2Yp#+dw4)R>{gNP~q^lTBF#qKA&< z7E;=9WnFfbo1k&w{dVvw2FU2zh1$XmnbSDF7vbFu*g^(kL? zcAJo@R*>PC*8x4M$c;2WV#sSPO3Ug_Wc2`3K&-!`g4zqhga_2y)~*Ov#H8to1ajJt zgG;h-)urYG0VY!B!Sy0L0r3@*W;vNyYvvZkRx8ar0yh~s;{kyJUIFoAd69)WoK3Ki zVU%UT-juI6RO*mf4jELQg%&z|Q-P5n!v$X%PE})cb)N*h;+LLK2nI1pp%+tgQc*X39S3-gw9R}PjwOBJt zt%3K%AZ0CDm2rrwc)}E{f_zY3CtH(f6Ph0LM75<^$E!QAO&!Dw(5&BoIVCd7Wh;OrH?Gkk|Fef>ThzAuo-%bMLt^&zI z6W{|dJs0B39WpO>kD0iYqZ`k25Y!!|9LvJkKrb0>~p73qhTS zQwilx=PV`7whsGV4zI4Ri#}k4N*fRq9f|jqi6t;zIp@8FL(p$zs~FY~)L+}wfmR_U z=olNLGUm(XN3;a5Hsa8lDj!Lqt0_;MLM#>@st{mBg#dHrSiQSxB+cA%f#cQ~^UEfz2afatdqN3N5j zDUhGSz?WI@JL95~czCe8p|L6rga)yI2wgkyNoA=BHtLFEG4x);60byF1}q8yLHXMt z$Q_8v&U^$Sku;(h2oB_Iga#*s3GsU6WzVNV!oT5%$Qt2T4s;ywDd%}`TEO$w1dm;< zByzQ#I7n{PP4gzvQeHJ73x2osrmX`e2R&aAF(wU(`4)C~fDa{ZYiBYA zig4tslrv7|ETI0BUCan;HN~65O@}{2k+3}sT(Q#h>R=_w3eOtz{3k8mveJ|^C9f~S z5rN7;yw7}tq=f(#9tQ{-(ZZ9{-q zYf55Z_a}7D1R7vC$FJH1-ChNok$G~mWt<;2F#r$6cxXZ&2p^cY8Q>01(tylUIDjF$ zSsF0pSPDj_7=n!o*^hLvp})b z;lv7A@K}^x(VCnD8YtqyjzG|DW!1B9DzRZv21*@3q-k!%>?mkuAa5j~qIomsu>-l< z)lJAvhxsHstxi=$D2O=T()2&%#I0^wt)I zNg#H`iD-GP1j3DwK)s78Qat2b7Mep7QoOO~(k38k8e;WkuTz^OFsXw;E&`bb_EXjGTD%$X37BUBf|4%qZ((la(6#ikTJ&Fo zLK+=P?@cGOJuQNEgC4Hg9@>Wj9CzD}ab~8r zl2t%lkO%A{`?<4M*Su>~g%COGAh$FAPUJRriJ64TQ`l#9!nxS?qJp-JP7n9Xyuu0k{lBCCh$g(huUz;sO^F;AvMwz)y&Dv>jtI{!dqsaYa(kJ zut_QDO=%al8F;xa*u3z?lx@hukwa5yO?tdNSd>7fkJdCKLbN4*_Ppo!WpjquVy<9_ z(vUaGZ?2xSCu$9o(%W_+$pY|c)0hrJiUxlDz#E}lEon$#*5tX6F;1dXWwj_0a!?7Y zRqOM3>T;zXk9@S8$Mf(1_sNE|iW#cem;i2M7SqkGgs+GQL9bIQuWg}V(#ULIn>~Yf5Da-9tXy289YnX0GHb3a9R$25JMIcO)??m>AIgZIl|o z`9}1)OO=tV=x>wFE&>bUXVSHZ|5n4(G|GS~1H#VI)!TMfF9vReUQ4eteo{(uiP6TP0wT*X-Y|)DRPTJ6;3g7uj%IqtKK_>pUn6 zaN6I-iLaOzj!dPbDC0;1TEc;euN8n=)eR%5ae&k?k(t&Dh)+XOYIGGHxl^{i{x;QD zQ{v1W*lSuDp2ism`w$9NrXGZ0x|Oe*=Yw`uirisM%9BO%waNriDtSuY4AV`Hj@IcR zMgx&U5!WQpFrbW?6?>6K51q`FzFqpS$RDD#?l!zjDVTDICK25INRrivKL^MTXRvfD z4hg=@mds9Lf0DzXzpa9NIn}^&i}5PDQydZq%3CFzfp`ii&#-YsrfHzARVqQ^6vNX} zuvuDH3FWQ^T{7H;v%PAyW(#DmS(Unx6s1IV0Vgg(NQR-m4HJMYLSi5&2(lyH)5tVI zk|6)_LwCb%06j>u?*I-pc#fkiX;7v!QbCn}za4}E%Z(_oY&i&JdH?usrX}aVVxi(M z+W0gu_R6%Cf}+OA&*N>7Pav{Mjw3-~SS0|3l?tVWQnqGix{<7u{)Uv&66V%=SjsSQ zFt#1>4{C+Tc^ap?2AH8>SSZy1?G77+rG?qT?-xBEjDpmk-LrQ%NLIHGki7o)*ipYD_ zxiyL0M`O#B4n|hyFmJGaL0g`|RYSNw$`TY>j71RU9B6ejY@CYxl9U*8W?Z~o>PAwL z#BgDCil2=WKybLOls$w*T`x<39K+!b4p6-wA`^#^EVOomq5~*Dmbx`boVSAN9J(gd zuQCceB1LiwR%NH=WSs5{LzM~&nG%l3ybiY|joK~XZsdMWvUqe9%9*Dp4r!M7HHI?) zKRA;L#Z~9d>{i6jXx4xrd`Bb#f94okn>{$wKLH#tu)TPInG@ zA4Uj(>L~C48ENLA3`+*0CL20guz0Mv>Rt>(0wF3YfwE$Qvx*xB zL5H0y9YuV>iV)w86q%0X0cGB-ygDQL4gL|J%SeYN$`Xwr=1LOM0=*I|JT9)fQ%T?9 zVh5(u_P4?LtXw>hZ6P08cjS}gXBAa+lwms=PE+!=<*p7VKKTBo@)KYnGm@SPzAA^^ zO>T?wWxxVKY&41>tA(Nw~cF(+mT}_ zy7n+6;Kk(ZAU!VC^y(;rcr*C-Q-vnCeg~AQ?2|8}bfti4VkZ^fE&3;&B!Jcpodoej zF+bJ^6=qVmaTGw_P_v#>f|ywvMUbIh8bar4K<91((13_50B0i%J_vFU9J zXc}b>EwMl=fY)_=H_9=pGpwd+5;HtJ4TY=Flpu8uqOnDFM$W<(H!(&WBv;W|WF2O? zp5K~yChf;glG0Zx_5i;@8CC3z@SKvBvgT6aO-@xSwT5l3iYk7fE*fH}BLu)JHLOW^ zikdl+2Un{SP7(PWL=Y(^5tb>U{f75mYayT-YK`2uxCnGZvqEGMHMeczyAhs4x6|zZ-1!s|SVEL#6&mtR zOPdU9l4KHv)WQy~+$_#mLBq@e{o``(oChN#+eKs4upNpeR+nxEkUXrr80);w(c`7GbUFUA29ojMq-C1p0X zK2Wn@h?P`s1Imbh0runJ5GFcEW+AhNBy!4Ar8|*LsE(i12|R`ue@Lo!GA=sj3t}u_ z^i&o^L7L0fA2trD zyD+^#AP7sZQ+i1|A!%6iwW)X-kHr$<14k-2e>khN_%vWPDSYL%)|LW%5YGT&OfXPW zR0|kGd>U!7TClU$AieZ6E-nr9Yswbr3{bQF-I(&yDjh5MU=XkP|H0j-kk0Gm#&;_# z4P?Q8mRk<(!jNQ`4iuWz>Ra&)C%rLm7 zYiD;;(m_|bsHv&0us9?XJ5&@cvJ$x#jTI`yR%%$Z;8=^iagqyw4^|x*rIvKn;mD^P z2pOiNwnC#>xf2KOAV31Vq1NUemNL}aS}F{*2d>vB@HjA)RRtyQ)F zvk*IpgH&x5nA$1PS`WgU(gL{_FTz340wa%dl9ag`*cqf3Y@9)uH^3NiaA4XM6uxnq zISK#@Aj(Ur(qou_Qfmsdnh-UxV$qY{d|JH?k7>avWUCwKxlZ7^EEa#eAa;Ee`}3Dy97rE|ui z;8(d8Y$=uK*-0)acHtBx`Ud+*?Oe8e(J1*SVGy~^`Aaff<}KKb21j2V9ad~Z3IpRb zP+UXXx{;*vkGh`JZIh)@!y`0Bu)fu_bbyV{TEv*8YO>No5`~W587xyRDc`|>n@0N$ zHJEG*&;YH6dn7y)`G5ax{-F?MyZS{tJQ?@v0@Fy^Nqw%1`{i~Q9ecI z_`rt>5e=#%hkuL*;BKv9wx#?H+n|jis8EtqP;#33O7WHGXoSAIMLEPaloKSLI8+%c zYPH!r3T?IRFmI&OqQ#}S2?+X@aRO4sk>hOCsxg_y3i>kugDBUMeZV~m=ba(?gtEoK z0|cyXoSTKLlT>}HU~%ixMZ~CXNLnp;2L(KdDk2l2rhuRaH>Pxjcy6k=yHh-jDHXPA6wS;{HwCE+1@HhKR^zsdJAf(_ z>_RP~ihs;afL1+9arUU+HjNG|>|#CAJ5BH@oiJAy5%gLxW3X0Y^r28PEG=VKC{)k6 zBCs<^?U4=IlKzuM(}-{uPITzFfXS53t*KOrqAAzvdqx#)Vkhxgww-#%V1#x)b7<|P z--_SKbzKflRJ0pXDdk)#&@=OR8*xXhZi40HHm3eIm{*_`C5BUJsh_w_Q6U;|5nB2Q z_R?rkHT0dz?<_76Epa4?p`+mgIfYcp#X6v`Y_wb$L?2o&IvSbkOM`kDvVHbJ)Pl$m zNU>Of?Nb=s$;k%aF9XfHbtdS;)0F9zFfM)=(gB?P23nBfLDY#-7~61mi>{d(f}a?I zC@oG4f(!?%s!5gC4nXqoG)RzeF3DKnI+T?X^+{Mn(ZNf#$$>@=+cYQNM(Kg6T$p=( z64WHX(~fX3>_jd^QbPI0qJ=+|+SLBUK{hHyDuH(@fDenSW4jRoSMLS@Bd3`+jmG~J zOApg-3?o3-Tx*U`1!$o0=0i(X3gyb_EGZut-^C}*Tv=7x& zRcIQJm~t^h!U0A?G45}T*v3v~7*iz)pX6_~K#Yb}QfM?nSwsrDnYrCf*+R0J;2A3L z9t=YQ`W0;$ByguC(Q_v!K?7Ehhk(`R)EagG1-y|!49YRVkgY0KcWQ-;KnC$Au3N+JPhn0Tt00P)fwp? z`S_g0TIyYJK)*!y8NC0j>}t_+F6n zAiR@ZsTTh*Z{Q-RGJ-TDCfqq!gtD=fYQUwL&eE{oa@DJ4C4+yUGKP=R7V1SZ;-H#l zo!OTXy@E}v)vTGgxEf$+Sc)yYddllKm_|@q0ZLXI54^mOK$G5HXnjOT>B=a~DbEf6 zQO;?f5)gCNVo_;+bIGZRZW)jK1go>sq8C-IpzA1t$SQ=loH}}1oS=v&PNj@KS~iJd z#C78o!&EC8LX`v6E*RGs$AM=&gJ6fq051M`n}!E#O?-t%tsde59E(_(*kjT~tt$|B zH^sU#^`dan5iN;30AdEHE?~VNRueZ4w0nroR8TeXk6%B@K&cxt)_ zxH|{%skO4W)o_wMiJIhm3V5BdUNNMBNDQ(BTqh`T9CbibTc={vYiL$rj%PCJELv1l zNy30Dl?1#{_g;VEWGK5LV>qK>v52Rs!gLDIHXsy9M)G(rFbGijK^e-y?+&U;4=7lT zGmTj~$3BF{6_67MJ4)vqfX;-yP2tf@f!@yZM1C0sp9`WwMy56BT$Gmp5S(*xD(={= z5=!s27|V*ZsN3iTG^I?0oAY4cS4BYK=fVpW_nQ^7!T?K2XOuPckY4X2ktf&!r=@gnl;i4_h@IaOgzG?03ToD%J$!k%TT~?smS{ zjIyw>zrfN`k^m9o0rUd5aiLikbG4cur%*r6-XTUnn2g;sB8wa>7 z0cy-);vmjD(?P<59ab5~a~EyXN|8dTlEdKw&zx6{R3+uNrMNTfkP!$8oh+&YrvpBw z${oTK-6A>&+X1YY`LF|kYAN#~oDXjTb3J3Dk(s*bJ_(~{F5>K3xA_E#W zsqnuP(P7oXbtBi7%5sA6%bYOp6qtcB2SH2`=M51ehdFqOj^bhtr=V2+DRucctN<&; zXrET;V%mTU!kFxkI;C>O;J0U|k}?wgCFC*W#4$I1!Q$>!zOxF%kBXC#B z@JM6~f!{a<19MbU4oEdB6c8|U5)W1iRc8P^g;v)VXrv-uQ#Bbl^D2J3ZaXWx) zHIi6ma^c&nT>vz-0s^HLRi{GlFx`z(>H~GsOh&x^Zr-42A)^`qCdmL%PL>?q3OlfB zvEGY~vr42YC8rAoen(I>5EiyLH6DoBxR`LM=Lu-{B30CKgxqAH_b5C@RT%X4(peqA zc{LY`g`_pKhv_a=BBgN4)!1WIoNm%6w7_|z=+G=82gITRq@w$tY8eaYF^-Y~cMtw+ zxC*0&M+I=AMimriINeIGD-&YwGUfD8pp^rPpc*ws!A}Z@LlARIQz{)lm>Q}83?^+& z=8;Kkz@BVq&{+0qoN zW<9j=-9Y7084p~ah&;IbhBc|RAC$eFlraL6ag~7j$qXsM#&2R|C83OYi>D9=80@<) zB#D;}iY@pcNZZQt7)UEw!W1lkCl2tn^I}8v0mPhqix!PELfptmf!L|EitAnHbS^D> z28$#tD8QnkX5QfK!_9&r3PVV|4O%Dkee!}-MbF~|fY?*V0xoNEE8Qkz;nYxh24{lWB{;*nBC2XN5onTJFf&1CWp2O0yaatZTaGYMbVn2P zZ!D_@H&T&$I1y>UgtRJ6k**?i16oH4L80vf8*Lq27D5U2Kq^}|0M`h;N2_3=s^tyz z$Eh>Wx@3y{O6>40q z5&w7y1HP5Rn}|JJWzMlrQ6t$Bg`$ixPNVWf$Os4+Iuy|o2ZAx$RYEZoWG#r=IdCpj zhIF6-b`Z9Y?}op&fG|uX1|&PB^G~LRWyeDSo3J|;sT@ztw+Z||S?5n26hGo#gdK_U zI+raG^&I&`YH8t;7!YT;{gEfs>l3Jl@;bK&rH7sF>#c_f+gRV?ep;VFWJq8@4*^^{RO%pXNW<=j>VfoGNih%%i2=3FByWPtzLA z$`_79hh*irae%%wxRo-JGqDrmiNiP%2*J}xHhvTB0CMe>^Rt0Q55X7TjmliG#Sn9; zu?x#6OQ2>Y$XtGpmQ`@Hov;&p?9ldu!+G^kYtNDiy8sm_o`5i6u&bFhCHTBBvdr4tjsl~ z12boLYkpV_Suk3?`rVx{2iZ+jf&{Fpsok-tu|bUls%_`s4WwGv2vMp;)RbY3BT3pE zu>6#vCaM&3AMgk&B}T+LE%ulKK+5^b zZ5)sZoknEHR;$uzzjbQJ;5#KB6N$Ot9;3cEP&KSxob6Fgg5E*EAe!^cIlmJMVKQ?m z5m3M<4y~z0)?t=!u^=c+8*LmNbIecZ3tDKZqFzGoA^itDM-?i62zS^j1cvd`RF%tF zQOF@S07tr3{)qEN`YQazg_WevqBBl`>7qA6>zcAV8mHkYUqar2jUWV_M)QHCtgacz zf1tr}<^vSonASM#6hrDFVs)jw=14xgQ>hCQ<{b49(i4A!6+5~jO0dO9Sc<>bnmFi= z4n`1#q*EKS_INI>Zz#t)$0Pn7=jWmjGp?*!vl8HEjLtaV5I|tSMQLVQe78=kJ(B}K zKoB!0x+3rfHV|y3s0luhmTMJ4Lf|BdlF1ZtL9m&S4%lhLhTJZ%E6c_Q6QmkaRXRhA zeNUFZx;`~*6L$b12PT|SS(;%G<8ON6aL_2zHwam0t`lRzi$Yy0jlB=6L^BxZ2iy_R zI=Kk|4TN7x9WdnLM}Y@O4U$#Xz6Mr2yg_RNqZ;CI!8<5zZC}G zsohR)Tm+1R$RcYFV^WK%J3w6tPyr_&{uKfx?UZW@FpcL=6j2RLW| zf?-MF$;vD_F$@WL^BiyNZM0$cZg5Q3Fk0h}R}5K<V|~^?7N7HV9@~1$U!I|Rh)ykQSdyg08tLRiVN<- zg)KrB2t_1Um#{N6rr4p!W=NDa(6K*)Mayt+ATa63ZzSEN1 zL?(iW3zz+aa}qRoq5f(%+#nRdX@Vjl=M`R2=Ny0|sM4bnUuoejVH|OQ(4i8g#7COB z(==DmL)s~IK*5F^78m6a07wzd&$Ps=Q)kG|fPaKs%F=Lbp_92*{OW4Olm{npFS$@9 z0S^G7&NXpv&`Ss`TaB_tHNXfIVdp~|)@fdB1wleYMWCMI5+N2+6ApBSm6De9gQH{EBlrGeq09#{Hxq_@ZW?Xl$ zHYpq@S3s--FM<4T=qFB6QlK0&1w3cii<;{~m0g#l2WG!w4G>Xc{lz3bpqd?`7gDMk zE*y8_w3^jHrX$h$Hew8sXj-WlN`VIGu`?eC72(3j(=0IIMdb-3KD_q8mUYOYxeiAp z?k&tW@IBGf%PSGH?x4&jnaW8nm|!Z!eFCHc-|L_oWcp^U7DnwLL2!TrIJc;()llOS zCcT(4QBvH^NGPaMo;$}!Qfq=0$=V%IkUECR6U9(z9yzH$b?RgZLZ2GqFpFp@aUT!R zbf_||)!9~;F4u_U2b5p>vqI)b+swJ3O(O!WtxiErJm1C;qD5=zp9+X}(?Ud$> zM0=cTJ}0xpQ%>3_JhDiu2mXI?1@BpgX#pP;*OtArv*xT+ywOFg3q z-aT4VW|mUPqT-swv4n17<);eo98fHEPUYnhWc z&_>5TVLu9iESk5>o0R=nUt73iS zy{SS0`A3zxvzxFBe3g*>fsb%8;Z*B{Rjmfm*pM-0B87C7$Wg6C@443}NwLR;OkAQ- zKESwfpo`UN1+_*cT~MAJH9!-1gf=zjn;fzKEc&S`!CKP^8LJ3V3<2ca0k@W}86guVF z14XPtZlQ7+wK4{-se@yQ-#FB?E{HRYGNMgjSV=z4yDYkFD=J=!wiad&5mPu=`H=wUm4e&(5&lI>nmRO5jU=tAnKgxve#WwN_$r=Zu3+O2j*K z5h-w5r(i(1CtEnz%Gs%A!8rw^q67sQW&mxw-1%Oq8V5NbAO+}ltLSP+RiJZlw*YZV z9pC^+fWpafNP*Oj=bNS{QN9+Yb=5eXdrT*~g#DyYkG9vja446WU!SYj&}AjVYfvej z5TSKEdc&G*HF;gLW^sU|F4%;RD>e46TERwcDWh0Xy`jS%0ru})e+ojNpeF$PvaO<= zOiJ}B0U)L(SK-5Ua4TwX2Il!vLq^^Xsy_1mab`653%rw)=cPQu8}qcBh>DoLTAsE&|35P%X-MARX`;XJwjMDeRXN! z8fmQ{$%Li9XtU^wP)uckF-&|yYfi+atmHnNG=^!c9K;R3AO|J~EyR#6WdeSFVjQW~ z6T}@Bg*ko%n4hqIuD@Cr?5C)%XYu?21x&>4+}gwkAdOd=8$ z7GQt3d?e6`=qh2nP~HUD8mk9#_C4sRjg_7$Ql?R8yu& z_6{ID7d0ZlN`M*bANXF32SE2$8Wj>Q-Q1_Tz_OfE(7jS85njc?5F?ZaG#XSjkU;Ny zO-Vq^s6-C%Ca)vUlTu6I@=<0*gp-S#Vpg|}Vli7HV-XogdA_z#a1&LvQ!nr!wSvP6 z*d?pk0qT^n4vHmtwHAPhLub=pKW#6+joIpb)+usRNahq90`9Q?dny#Rb<9u~~e15I{IG zDim@!0C^_-4vl4B;jaDYdO2|>Svu7 zLGm||rc#-NI)`ef>M%bQh0psgYb&#-ou*>48S6FeohS^E$}A)zE3H z{GTMII&Ub^xsq`L!&ffw&M039Iq11^6-Gl0F~fl@sh(&c1o zo!?G@9~50`eU>B|2HGhkk_3Pw`yOz`4qX_P6PSt1e@jRe)iQjd)@Vt^h-#-AN(*Km z&6^OChm=xG28aUXxtq?#s6@9X$DCxmOx**e^I(+Fb%3X%(y{v8qU(=JO*QnA7Ls&M z9r7l%{;t+`#0ZV=mUmjEg&K{GR?cz;H2bMmE^9oFNX20XC@XnYC_}v#d~#kjJq!uF zz;sjOF29?JQgEAkL&4H|Y$$10P+d^ii`?Z%SHq?vgsU(mf^beMtIkqWNQhk2+VSC0 z_pVS{){;;gHhG-x4sfSf2A!sYI%-EssDmg?)g^%}$(%ZqlBc3PHI+GgsE5M@5F$Y_siRH_=Lsq$ute{yysMVT{k7swdO&tndrBG$UD9+O>-?R$2SN}3` zAeZ})cUGOpc)LoM*+lh982C7*)!>S}GjJ2Y%0;dnz!0A{2ek?&bqPc@h2YW|&`L}U zbhT4|bVQ;iBuG<`6as6d{c}WHya2H#tqemBice<73IK1Tx^EFF0ux>b>@BQDIGmWDOe&Ov3EqtVB z$iz|}j<@j?Z67q|us2PuJ;2%+Y=^Yr4tMwx6WNK+TyXWo!?OV(z}a0CC1N4|(< zoMxJXLSVwtR>DmINPvY$y}PMQUw)IeTpyvOz04emDZI9 z0}WELafp3LzyT7a$|6XyFsGNKV6s?^AU1RF{HBOKr+N-3L4E&GDC8t6P8t%IB=2G}W@ zCBjlV;(4j&MMLvec$OPd&MaJ3km1{8(K$fatx=s9=wahha1$U_FqMJZgQg@x+@$|q zrr=@Do{l((I~GYq#fGkw9W5^%8yIIAu|b%}0Z2Pz1wFdNivj}~;egmlVvq<04g!tE zIYRlQLkXy0xB~BjiZ`rDG($42aZXa^)9nD_*@Zr?h2Oo6R6GQ(QFj&QlXaQ~MycU@;woesp)N=026{|N)P;6|9XZbv=z53@kzLrCkzxWjsnR)arGJXLr&m%vrTpjP=Tqn}QXhC^)s&G_4{AWGE8>&Qpo#RHi!) z;*K*BnU(@F%%f&b+M$9*(c~=OBCbi9DZnQMq5KvJ7m4i}POehxnl=Q!YbRcHTx&H^ zD=jS=F@~rVR2{Iv)DmTJx=Be)tym!Cl^w+d?i_}>q4uI?u{y;N#0*6QwSx-v9_B4o ziZG&6)TnQ}#+$TRqqbjbWnrU=O+F51CKdl+Qz8r2Jx#-(M9gq3m^u*V4ac4?08}}0 z&!RBb+!;zHfG8(tBr^DDZaKa-+)HXoI@(I|T&M#DR-d@OL)n5#gs8Mo1-5jcYU)ct zf=rGvNgteZ?tJUeBXFEUB-wfQ7#R^*15_;l-se!ZtYQ>b(TuECl(Pc}F_Rlz5*Sdb zfM_jp;Gr*Ijey6C?*@$@2@feaq?B2YE?Y1`wR)nsHvk*fsOYl(?3RhM?zRR>=u!xkkwGys3sq@ZIoutqzuN=tI zTaTUTQYEp4;RkYr$cs(L1R>O0gfSOeWI%w2aO6`u+$ZU^6m&(qWmsw=ACXZ+i&kgAb5q^I&Z%R;iVg}Az^Y{$ zy=r(CwE$M8$r}{Qa|U$5Ay(M}sj58EA8JrnRJ#`U8SmhY6)PzMf`M2#M0$}-`Fubus-)T>JK- zhO@hMBnnbfh`5sdk>2&ntX*If_;g)j)N)*xGNl=OV zlvsfVgmDjuCf7OJ2n(Ifq3S>mgG=dm4girxN%}zvJ41k!rwW&Dn=4aW#7c4WDf$?D z8I9NhUt0|)X`&%qRaFyfbmf+ih1~#8UF*nf6Fo0TLu`l+DWBUg-vwqqi5Z0Lo4j$L zC%E~ck_Szt*^5KsqLzN`l!XZ89LvKciFJ*cS!BbIrWHq$-6*=`OwW{yp@9t#{D6}|xTrCt zIjb{QA__rCMvdx#D3x+aVZ1XZ!}YoFiurP4TEig5RoTAGn3Yt^Zkj9hC!vaGD&XH zT7mP%O1a?(0ygEqGsI^pS*0XHvPvQfMx-k20Ga?)w1L{bKZP~B8Em-jt;*-|;Dc^7 z!I>o8+!tEmT`NL;rQuiW+*_GcC{m{xQS!ozASyxNn8(inr9tF$!O5We9(P*{Ou*$A ziwS@$UiBderen63l0Q?Tfjo3V-Vv&kLF!6gNvU=;Oj!Ncz~0G!kG_aX zJy`0|tTivt)eZ$s2@x2gN$Ce%0Iq{73r{45nAU-C9%A+Hbs(L!?vbwu#)fLu6ax$5 z*>OZ6r)Cl9b|utnt&f1#O5sL93-BqKXMaOeCSnbNRR-22D$A@9@FzGxl(N7Rt10-)Q`STYc(Gy=;Cz~+UjkR`N4dNb&eH^GNsjl zjSWvPb^82i3KuX5Dz)O&BRZEv)CzqK5R3U1jVM;;NllA~*9(u2vw^G@o)k9CLO~kp=x#WD%x~arg zHMKb&i*>09A|ehU2dx~Nu)F|@zKjXz6UrL+Fm0n?4u!Q7@$*1)oUWCw3b~Qggnr-+ouXLd z#=*3g#g=IiJ1Eq>X0GKaD8&E~NbS&qnTCpBDkYs6N>pc2QFR%DaWa=$MSZw1Ejhh# zQ)o?~#^)$iq?!X=;5^0e9x>doBQ;8PNkpC^1kOVYM=cwbxhCszcsDF6u-U+#s6O3Q zzRCe7fTs5mm~XPk?+d39`JJf}BC+ICAIM2@uy+tl!w2gqj|sLwRDf$kvv!;_ND*Cl zyNGj=rR38QKqj#X31~K*=Tb#au9Xx#1@!3Cl5ySA`40h!_86>&iRAx8jZL^xv05j9vK)*@Te}4B z3LX?(;8-j0E=CR|5XyLC(F}<)`h-(R0t+gwFkXZ@UD@f9$;vpt(DFJDdg8>$G>X=# zE4z;N8~nB{uIvoMolINjR>MO?{HcUsUAerY%dms~fXawDj3YjiP0K#wEL2`N`0FC< zk4!pst>W@beU&=RMnM*nncvu^jGiYPHc(;0u|XoPOB9A=7&NLphG2QD;9%@;WufE_ zm6d{vw~-tTBEQMBG=pMyK|a32Nc=75Nkrz;$tKr ziBuQX#FjFT@aR;_;G&EbtdP!8FJ-!Z$iFDe#gs?)7{LtmG|k|H+`?rsP^t&Q+Eg=u zw5rV@UUAiE=}8Ea*_2M5sZ^E3A4hE1lYVnd<;G$NaJDO)2r9_gA~>5jMvxQFH!2(` z5CMkumYAhe3~N=HaHc~ifh>yCtuZvn#W<&MT163rQl43%)k;a?qh10yS2HJ(@jwOK zkkl9m5a$*Eq%)V4i8-6YD_E!I>hivJQ8Q7($dairU1PLKv+)~9(#XxvtiDXIb2aK| zwn{dHG$Z*~t?ba01ttJpK%&1|pqvWc@u76KY)6)nt7#4{T`3sI@T8iAPa|Qy){3D8 zm`QTNAWIqqFCiIKsJ$y9QCLQWQ$f}O+=(u|H8TbfG*r^48b_BxC>~7$uo^TCdFEUZ z6mlYTS1ZuG^ofgFeVJTk0D6F>qt0c#(8|}8Dhs?@B`sJ^Y6ii2N79|E&qA>&gU@&5 zXZo|d)%0&-VJ(`r3#|Yb&~mQoHY7k1^%63JAlyJ|6Zozih8vC=)J)>tkO(?3Hl7~h zm=LVuD0MI-GNx0?I8eG0lih_@pbk@_7m#&Y*~x*PDmZRf$Qe<4Uy@DKL_)h*%kXui zjU3zR3w0Wk4g^2Wh7#Zt#S`K-r37iCdBnt$b7wwekOY&l zj35``lzM;(QXi+K(%UN|ebh_f#3H>CuX%-YC&D7&5H2b0+G$@>hu_P!E)mbvY7Ip@ zKpOEVX!;)RTAW;tdWln(wTj0_V>7wqucyd%rU@kYrgHx@&>rj^5LT_$H;dXUkd;b{ z;li?6E>st+;ASAgj<*GZVplzvfEF1aok8>+`qFaf2y2T<6di7IMepQTdZcd7&+U7JP& zI#W_Lg=crt&3vH_F)7PDgD5Pua2@MIsjLpb*U+#EAA**{2Qa*jJxKM+X|HlLC`M1` z9mJp`*(c3Iz2cm}xTGDvfr7_`RDowefbHQdz;Q^vrwjw!z^=*x#~VIf79g_FXR@Kk zwCdN#k5nt+P9N(4`hs$Ys~T0wTU2Oasg{P+lX z)Ry^8s+c1%sWQvR%vH1Rx-OO)N}!FUJ|D_FC@aTCMKlE)AD4Q^0T^lA;6j49R}0u< z7W|sWfzmu$$IuQH>6=SN*qj8f;KX@LXt&7pu*!)4IX=E9M^+v zClXpm$wdHN3lV9!9u6^QQ31(lGNo_hg#4GW`nSY+jGr$Ul z+iKE{@iYU-!FB7ReejbK|A9OgK(KTD=}IV-9D}MzwNvP^dvKWHM9Gz)JX(vK>9X?D zObg+_&*}~;t#}W~FmKU1;!qgvRBctX9Vkyn8rN#>41B@V$pQ2k1s>~(Kr%=rCGv{5 zkrh!Xr#TUGm=1?`g93(h11$wb5~k;?!RX^;N2?)WDjJ!1;6aUw&C5Yud1i7t(Q5ub z7$5IuA^VM*Cq<6FkKO3Op&A=h)DRBy%F}_7hsQeQUrHc4TB(ZcN$d-X?vhg;edd}1 z4I?(Z_f#JBG~8?@hp*;wu!l90GSSUs>^606aPNgGxESY2u&4Ry#;iqObh(Qj>;Wld7 zs;5e$c?!T9T++ zte$J97tzlqbxs}5fDTTF8c#~5uHdLVL3&34D{NLg$WEUmms|lKSl=W;I0DL=ggMyn z3Fud~U&^FcBJ^-oHxwBiMiNJfk*EME;Fxd>GJsOZ;!)f*`8tuBqEw3{)5_oMI?wU0 zJm9(33M0@{N4I9Ji0M|11Fb9NDgo3Kp8)Dawc$+dyy#WKKS>?p9H39`OIMakTZt>K z)5ENdEGq@rbHqZa^gj?SxorxYLP1g>qC9sej%HqDO0WX)X|=!;+iJO9ttdk6U{7^c zvZ?@mBkKSWI2yNFi#LF%pt4z(yPUud6iG{Lh>^;DhCSLJs};wgGIBBT->e zQp9}OL|6&|lRPz1Z{APO{svc41GfW&0F{#^qMhahIf#mpUg|iMh>YN9 z#u%>(%@1^SQ7Lbt*em52aJ3AR68%h*?U4B*o}5>$*33U#U#+tK`!$&v3jkT`1aqn; zF69EBD|$Ic3I?MyGHc~8V2yw@G4Fk^;j!pCt#H-CS?I##c#J9Wj2|VpoXx1cW}1pu zA)vsnI94MyEvUDEN2O5Ks$t`FT6;tJni^w=bhar+0_0XeXNo+Hb~kmG$y_Ov_q8ZJ zrwaiZMfDX*8kU#tG}YP$q@gkcpebEsMXBP!d1-2kp8(PKfT^&g)rIXTj~gg~F~eG| z?OavyHpO9!2(ipRbWOK$$P$E;1s(_0U1BK)Ad*zhEl4ke48u7-VEZL@Lhgno7z|d2 zhB9h3xU~d_v#6-m0~DuI3_Zn4qxq1jdsnp+02rOSO<`4dY~dedZF!Op6%wR(!}yA& z=hmbW>%t(w*_}rIdQq_zRf1IkUl}9heG1TamkM|!F)D&!(0$12q9!MzJau(-;6y=m z5o&ykH8Bet`S=Z3II+rceSZUU;n&6$I zrVueec^nOV;pggnNU~F-8lF!Wr&@=w0vDEPwG`b$zVr!9crbgx`Jy#NTro-V?xyZO+>DzrqwwW8?A}#;Zo^#Yr~-T)*{#|tpU}6HS`(wiqcolH>@@wb6k*+L zGEH%C3I=*QOET4DBz#5D&qeL0T77o}-wAa(rw*I}>WfB)C|ZZq(W)V>W&+4_OOL~& zOdzrRT(xqP^x@rF<;4c0L{ScnF1l?>jtEt;Nbk`S0)w1{ErCdX-D!<4&o@e78M1th z2g8jR-D4ndD8!9(57&O_)+G2@1e zsLEJtVY{;R1%(q>DH^S~0Yyb=PLUvjGztQ#S+0@f%uzxzRqDLAaGAV#)VXr4pQkPk zz^n@;*;35|;eo6L1WQ~c=(21mf*@_}7-Te8r@`>Y#76?f>bdCaL^5DNATXq1kbafa zO5g$e3^rFClh<*9p*1PxuI4;?m@~&>r#!=CePwktA1Y;j&a`HPauB&8X;kB&P#WOZ z&Jt6?NRs82TWQl)k&P~$9Hbn0aZ;$Q9NHi!VZpCK`PcBC_{Ei}s-~1Rw3skne2vA;pC7okhkgFEM(w- z4A+{-rLV6Eni{7h`?n#9-aB!guuRz$8CfX>kX5aYl_!A_RrJ4>Bk zxzhrmXgth0D;uXQmt||)Va@n&_o_c z^jfM=W)bz5fH|%;6A;o9@}!-HhelYbF2tL4F>8zX8!Nrjla?&QZb1z#G!?DGVb`L) z43E;^YfhHBF>Y?qNz6T59%i2xD==##|_+bCbDvfNY0`=zH6#2c|CKyd}6Z z$RCDqfGR*9i=8SF5`bC~GjpdaCrV6bWlTFmyt^ZxDqEV63iUDzJ<1YAp%qxXHX`oG z&k4tq0iCNeAdi+XUSq1X;U|Uu(UaNY0VTXl5n4hTZ8VMWwNq|N>n0THrij8ELOH9! zP;1^$lw9~~3*~pxRGr3?v!r0uGz}YfwL@Wbl>^rByX+kAvQ@ICQUx>YE-~Mf7%Y4u^0qa8 zo38T&U7V!~AgpzB7m-#9W(nd4ojTK@p^ZzKGT<$lPs-5O5G0Blhx3jQ0<~WVxH9yw`=vOZ1~gX}`cIwJvRr zlNnesJ|{9!eLW=Q_Gy(hm69Tn3MSDiJrHM57fB~-p)hTwLL9qSy^)cRt*+qWC?T=8PW`i!C#Vq{c6F?kQtz%6zBX_ms+FQHTVW%_1sKL9FomJ_^?rC;iz)~ z#8($PP{|TiVc6Y-B*`{!T1A<7#7CivUpaU>P4aN{qd{g6gk$sPnrG~M3M_xrc?bd^ z+mD7A3LGwfOf5uS?W43nSV`MATuSQL-{45aDuRSgljefSB&~PWoi0%;99e{%X zJQ1sF*rsR@gr7+BCeH6%5y}afsoxFz!Lw0;29k>*wIyy^q zC0STl0LE)2|A`~XZU?5yz~FI-%s-ai56irg@htF-l8ixKkuML|k0+?-YEa-ac? zDIYJaErm83LnO+}qD4jEQgu~)$R;n?HX6z&jV*Uh9l~oU`O4By{*$wDO64e_kSexj z>MEvi+`uzW0bnd?pMI5Ux&*ceXKwqwVvVTZ z6_i>ioKBx4uPG<|OYDd!R|K7Dv1XtH4U4IB0nmi9?}f(el)YJo+)=?m1wj@o1U~oy z90r37V61Ks9Hmxy9EOBokj9wNR=O^Ck}KtiQpQD5(qNE{beF2^f>!}v1^u*x-m9ak zwKB0VCb&ApiPo~|v;aq0DsyubuVgWjqt_|GRSr~_Gz8lZP)w*>%DQxV362I1KIK}e zG%eV9XKH02rY$Ibyc$YWrB-|;32)fUgPbf&O&uXOS-GOzNK&FheVvU~VyNm!;e-F0 zI4#*tTKd?dHK}uyku;uWHA;F?rAMC~d%=lTQgDAF7+R;&3`E{srX zqC0dW09d=2*;E)Ti7A55Ca)YUg;>36j6f1{WdMq2H^50MBw3Ase4(H)0V5Q1tz|`> z@!_@(e;e?)hL}ZJJG&d(huMOyN73~|t`VuR{Bc;t2_M3&?NyF+OU5pz?1J=1laI|`pt7C65>o+gj*aF(Fq#aXDXz)=^vr!6_d*GxZ z#7(&iaRtHmU20iaSGp4MLXck($ey<{(*p`l?M^F!G|u;m9-k>+PNDoWnNtjl5Dcy< z`C^Q(y%0sUQ^T-`wD+}99amb0uTh&mDi~TNFUL(s z;sLJDVVkN1LTza%pJjQ@{cT0A0sK-kuGJSGg*nm??*({x5LG_gF30er5CCaKRTK4+ zLhEW|%Jv1AwQ|JR3>{eQjwm`@XE7Z_&8$ZA!U1r!MeY|D5>k_@iLv2b7&HZ{4}n~T zd`ST9wW-a6zq1KAzA#qvU<5xJ%01Yw3_)pA^kg>D4K+N8H=3$yTGEB;?R$8tu0P z2OS=Pf+~oq7J+x#D;RO*=dqf%u`H+7nQWA^AB#TswR#!FG}pR1_%KSB0}UA=^gJ8{ z?;Ab` zYG45>k0dJ;u_hlM+ylj)3tu~wG)?uRH%5gN;Y4`*Tdl^1Tbz1nE z)_l1F0wx5lOU2|@E=?8qHRm1`96CMyoU0urliE-KTwBeGA2dAnb6shDKoHxR)9@0G zWq62DzS$xX2q}`V%rp~T>DQY)#=8?OV4^iP2+o20h%hk53OKUT*^-~{>$7ywRPpv5 z%PEUQTbzf5Mv$p5i)!x)WI%gSmK^d#vI`tBwz@Smkp(j{bEXmLx7a;6`IMj7MIY#f zl{6ALk8iqqqZ28vSkz(yfr z^{nfKRsn%(i}JyREA7Qbl%+yx%T)15P78vtgnbKZN{WdT(|GtS!#+dt(2_*X(Wr@1 znu!Eeuk7LoJO*(CqXQmk7MM$l1?Wd7)s;f-Tueg8Bc(M{l${H3vePF4mZabhGvy)c zqN=7OL0T|UsS~mVP935d#O%;r%gjfhwHhx;{e&0`s(m=C1BXMWx#pNu2)5x2ARPsO zfR={>iskty$ti^#&S<)Fm)C)>jC6#$(t7W$FO;dsF*D^M#P1@)isM98Z}0()R*+v1 zuM9Ide!zxR^kOTgU8cmkct(l^@@OKTg!#c?pcuz;JY}KuHnm2kG0Yt*0>&&twSYFE zu`8obA!#+Yl|=GFOQ*WDa6rl_W?IoI*IDY|MA$!&65*6p8ZkEk=(Xyggw>xN5Mzs) zKt)^~b0)P?fc-Xwd6pD=d3EKHj8Ndp@Zi*9Gal01g(S&}gAfGP1PCUgTN5CQLW;Ev zF0t z1L3B2t0&pVIG`jr@ERS+c;eJ)t%bXDlxEaUCnwPo$jzXxXQdg%(eO{aeBr&-{9Z6l{#Bymope*1@ zqA||k=t3d_ZPkQTjl#j^oZniamEA!Mnp$wf1V9Q<0S3|YgIIMu12j~l+$~^?p{0#J zJk|*Ug_^?Htu7{v>yP??c=zQjO`}^AMuZH)zLIc1^(Gd`14t=_{!wdG#^5f9L;$R{ z)q&TXyQrCKeh4{@h1RO7VokhK%L^-yq{8o8XTNFTQuJ#}W1~*_FeJFfjH{XiCSz*r zJ6S5r4M|Msm38DE!_jD-7pTCX*mDHe`0rr5AOoCYb#Zrds$&NP&RLeErz!xJRj8`N zf-R%c3-3Wo_=ifqn8^#ZX9_~oFo(cl*LJRG6KP>7r(Q`l4@zFdrJUe`#y|xO(`1pf zFCl8(8gf)4IUc?znm?3H8OE&E%Zyw|gg{~eMFxHf1JQ^z*1`$aV zxB!$H!99&5fZS24zo7_5tQvfGDDIdukPcSfVFmX&)7mMFu!~80sO0e0kp#n`(z?nU z*!`7OMMO$cp*~ZZE*IuFu7_MVCvu)>^$g9NqcYt6T-#E z*7*|Sk+ISlWnlFNHrP)XR-QeZJflM@5k&rI4 ziRY+Qr)6f!Htufq)#MOjRb=;O9_EZ;Tq?$dBrVhvcbY1s6;qky2&+{!9Qi;#3i7?6 z?#mpi9e_@%+`iCK;Is3^f0T8trXXEi2j0!_ZWQ_?ua5{kC9)kjmog0aV@Xmih$^|z z7P-*`-^s6pUE0|=Fl$v>jJP2iq;aTrjq8w^Pics{TqqCHsv`?8FPH<+IF3ZD0jdwM zTQRJ)BO+166}ddAG77QQhUsP(850T64Uu1Nl^}d_EuAGIugao|4ZsIyzCu*Vc5EEr zfEr`qL*QAEMMj1A80$bGBumUlKs;O)STu$tG%SVsRpK^f`+<+IU=Y<~$@#ft<&eG# z!8-X?5nsZx(n2I`gjD&l;x>gDuE2LgNVaUE4oR z9YJm1Nt}!%C=^=dv-TS&gSm>rM7UisB^~g%pf3skO3Sy`}(;TAqZy zPVS+OpO>J=S09Y4lLL zk-nK`0+7=aJ4p;ARYpHxUTN;Up-_U9wO4m7A$&Ztb&*?hE*Yz`O@3EtvYiCFdTH`B-kDQ-ms9%s0HH6lN}K<<7#I&kxAjW;T9paZ(tVpFNo zh4UrTNO^Q2Q7NndJ5?)7G$RmYiJXzGRoF}Mv^yqGYIQ&+0D-dTIpp9vEl9**M)nRH z3tw+!Uc$qwQBK5mk!L=a7H>nUClR9q9xOV)sjY|a2Jj!*{w|Y{>={j3R94Kc z9~1*`(B@&EaRE^5QiaXI=FyP*9vQ~9F&sfKgHqhnF~hSZQp0^KKluLRp}VBz>))GBRT|*-qps2!2<)>wqb{4{{nIadTp?u1l%rqW8sKpJ4n6h<$_FJPCrG#;ECt%@gV z3^a`|x(hQXVRe}D0Qk88Xh|I~JjxW1rT|}5dbO0Z);TN`aq3b6!7KuN(_pde%QRA7 zQFwzzt_3OprF5;<=(_TF`_bdwkGDVDxxX)Ht@4Jt7{&V}Q*@7#Tj zd*#+-;@g-Ox%@eM_F^u7T-%$!l&*g;n$5=3`R@Mi!Fc2A@qG8i*Kn@xTi7x{*~ zeSD;hl34G_)jM2~a78jdI!b0QMh6F1d8o9S+_w5;{U~fG=w7~iHaR%h4KK1G7jE@$ z<)3iog}ZAmJ)FcR$+erj)S10FnkE|)`8vIoOp z@Pt>ZgJbWXyOFs>_jb?g;O?1ST)5qJbZh;2J>BZJAy`=b_U!2}EN8BK`C8Zg z)lWZv{IV{uhUFjrWDkT}{ku#rzExkZtLJP}y@~lYz1$*kDXq%weIR+qpU~`?aIHr|M}_40sjmy3~K= zXZc?s*#ANT44e;Do&5{E=~Z3)OZs!aJU=J1`DiMSoE{y`&${ls@0VZhpXBGalf%7( zll^gWRp+%|4!ity!;^!9 zjpME4>XXC%7>PG*CgbV!XsXBVDBZUYk6%yL$y0Vizd5+P=#%cg7glHdywU!jjt@t> z)A4LPKT{B#v;P68E$@GFLf-HHckpxW{+HpIPxg|N!jL|E|m(X);IDkm}a(Xi;j-dtLKvA>mU zn4sDGP@39GKKyAbx&P$ey{&{eQ`+nCPOkJf|NZLkXMYzx^u~=Fo5{1$Hqk<{2#>jHX|YU#G=`>B%`wK0kTA7N74u zKRNt%xlNlRZBc(+H@92iDzOf1p2MHZv)upsV)FFM@~efv55|YG0LrD6VBepz4h}~z z#`7b&*A|5QBJ04K+6)UI-CDN>;*Do6h7agw{*~HT`?oIMuctrKw06gObLL__tE1M^ zmwhjP|I^7=>-xH-<9^oa^^-{3I+YuXleV}*;#~LZLzeK3^ah__wFjQ|1YOFPc>+uN zVvk^HU*-w6#MeDzMVpC77!Ch+e=KI#_=Ed@e)8G&2c?r_cA+%1m6%wNe{}@pU%Dy( zh`KY|Qg=q^?d+$1Y24}+rfXaj93RgPMqkescT2%;na}nQac=N0v#lMSeLFecojluM zPd=M`z5D&>;OnV$C;9M0kp~BpZ^s9?1%}TzHn;L$|)1AT_N! zFE^NGoiXn;S$P%jGFf@2mm4DcL;(Kwr5Ak*4X<(J*toCjwG&aU*TaPUO1iJBWlis+ z^jp{YVYU4tZ(&kO9<0LZ*?7M9e4}56e;H3F&wflswi+j*L!7eOR2Ie;554{AWcTa& z#^n2%Epr~QmGr%q9v#faLH7$z+~GON?D^5j!G7|5^!+&b?qoduaU{Z57yQ+}+tR;> zYOrXR28|@_MfrR4=IbdXCIjWH(L?0wvVwA!k{TOX*vqPrWXx*che70BSQ9aM>;HR> z!fb7<%9Pc~Ob(BdgCh|p(`5W|GMm5d3FP*`|@s5uMGTeaT>fW z`+xTQ^ky$jud|I&yJ^E*cGw3_@ z!9O~OW&@&c>T32m)ezadHvmab|!8$y6=}2+F%vvY8h;VxQsS? zU00iVdUSL^TZFNn>IZ@?#dpbQj93P#`5}MconsH*#j5Skt+S5n-{^48kkuRkq{i7K3jrrc@ zQh&rcUHJ5+@B3A{vt^@Vs0iuXOZ#ohy~bo~t5 zz5FO%|K}P-dDk}_McFyTlA!c5;BRdD<{p-{|#DZCG{4B!*wI(X56>qD<76Y<3=a38HWbjrHG( zRq~sSd!H$!?ZKzJUp#)8+`gR@c(>!HJNF;!r<=}OzHzlKuRop4!#YZij>pr%gE5yG zKE{d}@8d!E?f76ezT$L<<*qLL#uoG*s7TtQ*|AF4q+bvjdZ z`gZd1&g0MTe6joao&S*E9yw2Oc);ZF*^#vSlz)k3^fA@Y{=#Jn`ub2W?UBdf(XO}Q z0v{N`U!^|wHX(dEi~ia)_VBCtFs~aeeWmp zg;CV7=bH}IIYXn+@9@BH_@KXGo?Nrv2tCBl`nY8e`(87qo4y%+r2W2?41>-E`exsl zP4$h~R1^89f0toWGzic?{_F8_Q(MV5n~Fdm=BSShJRKv^#}*PW?72k+tbrboexZp8 zc!2-i^P;012V~7e+;Uk2x|zb&&xkZGJu0RR+&MU)vokkwaoR3BV>e{Q_R zHD$x47cLIk@Zi;98v~pg9qxy_zPBK}C?4~~jmjo|y}jq-ncfqdyJ5F2j`t+F&NA%B z{Q7kn^F*d`V{+q0`c=64Lc8^?&XO(?`LmxC85cKI{ye`>-~8mAt4O47{!Y;&PMy5- zSNw%0@Zjj^+gWlvn(56i*3MkEZl|0j7MbavVX7846z}S0YH1GNVA{^_t1jg@b*Fci zJ{~Z(ad*Gt!{Rku2u>Jpt$vIsUh}bZWzDBL7VEBlw&F&36Z?Pf+vgWb06W|MR}P>R z`#&w-$A90&&uiQNhvOf1_s0k0xtag-*?zz(lQ#YBbNNI5d&>ZVh7a`czy;CF<1;j~3A+1yG= zUL7*vmp|yQxUq7{jjKdZ7a#CqGMjPs-Hw;3bA58CvX^m7=75QAT)nlZ8vhDb#{wjg z3v|dgbZ#70GJdn!p8-LHD^B+5iM-J~d?Zhj53g<||M_HN0J|?W-6Jb^bZXb5CwmAg zaK=8??8kl6nGQ5>%ft`WtjAP_U%cDvlsBGylnt)0OTDyfVD+7Q^z9>E%5VPKasnM4 znld!xqRQ_J%U_S=|k`G0ZO8zbR z@Y@^5$sZI*s|^j`=KAgT3*U0>?Rf3NUn>jTHulESoYBSak2lw0wB1c^E%VsfLYG5Z z(yzr49Qrj|c!%>YcSv$sI(^OpLpJ=bAf!_iD{7^;i`{ioGYX{I;IdjgM4R`*pU$bY+ z;Be)+3rh^3!3esvybkwX>Vc=o_ed35wDUhz&O zb4B!5M}Nf1*sW?L$6tQ)mHpRQ*Goeb?po~cdB>(T{GrVAhtt!W9whwQro+~@roIb8 zAx!D8q0hKwI_&=SH!>MP3m~WbR#z!y4FL2^4?z>~HO{-@0Y}ZFHL|5!&&W-+l#L?7QL9#RaKD@vZTk^+UQdAHQIeC8GndkU#2p ziOutE@N!E3?Jmw0nXvxTr*ieT;Scq4+J9fIkKu(a*~h*XU+90~Ed0Z(2j{O^ zqrv|3WRJdy|9^kHH~R4c5rDJ(|2b7`SN#7){oeooE`Bb`|0K5HQg-nYwbi8f*nbYH3R61!mE`7f`m zbdzh=$b`iUHa#beZzuF$8yWg_jeo7Vg*W)x)S;x}U@*X;+IPsjn0f3jN(lr`{9$T!L_k=|OM z`dgr)fj;K=1QLqh+)nft>EQm@`l33Sf3_$Q{!b0aI|Do%G3)GFf7=@OU7w;M4Qd=- zFHh`pQw?VqrF1%#Kxp8#31bsEZ33T-o_*_h6&7@kvmUv~iZndXX5q3=yrSTR3(wOi=A0=qS2d?PPyS+_YS0!X!eJD@B_S$?8 z%t@lk&S=jPDmKP1_r}NbYpF!Z^{0N2Fdz|>cix16-e+J?`mup@1m#Hv%w#}cd2O!cV$7i zTySRgeu4nr}h%?;dt~$FMj+_`PQFY;;r2qr|-Pes~W|%A;{oV3Zp~}1c`X;jJ7PQ7~|QN zHlq}EL8Xw%IhgEC=7Z`m-VYkV2NpsdGWKJD!XB=uxtY-?ez5RSibGBp#>!Vpu{CJ9 zi{B|9Xcisjt5IKn`DSu+S-*@fGE+YnTAzG1ps0rsO%IU1$O`zt={m;wwuT3B#jSm| z0<;_@`rtzA>VvaoTKm*-<3c(e>`XV7u;;`Y*6E9 z8Dd@UiR0T3?5iH5=FT@en4r(S`Rw3i_I%Oou;eTx^if zT5tN~s|$>#eV5m+z~%MalZu{3fBH*Lg36Y2h0|F$s)g=(1l1Rhk}Slzjcu|#uxD>2$D@;(B@yk(wcqMf`@81BojF`Q zm0lc87cPe`Gx_qx{#V0RpB&2L=990V&&m8cmep`&Aw6t7ER*Tv>#xVtnZ5$x``$Bp z%3-m1B)k~CToLWj@qMNnI<~MzxXY)#Twi2$zlihugPwc+Fbqd*6xf5OmrfG+cp@A2 z>%GERp7cwfO+NU&=Be^LU2In`LA`u_h(LGRQq#kG>6fAPUhMS*OBZ{!1r4CT^|Zba z5u4ozL!I6@>|cB6PmlurNG}d_Si0Is@^49+{2|%C|6u$6o)i8e*jJlL|c{Qma$ zz6(G%`bbhIJB~OLi|hspg+igKP$(2$y?nZRya&tOzHif*7S^=2)xed^Xv;bc0Dem2 zwUq?y6(nq`h`o}GbyJjax{ytnD@md@jl=xRem?J-9%f_c9JLnqPREM2R? zd1q;*Sz2Gq5cbVCDD7-MX7Q zU&rSv^Ixw0lq*C>b125tlORjxf2~@No&TuS%6I2KZsW6d{wKMWQjfeOfu=PY9N*Bl zOULfPwQ97PCk|GQHyFy-o#husTh}wjcYlLD9hIQZbH?wyDI>0;aqS-PI`t%MZ7|g+ zsryiMvc5>FK5obu?(X|}pe2mHbM84E_1%KwZ_`}n7G3{lZre`u>3NN7>FDK^@$(Ls zxWh7JEXVISRo#du7_j()icFCT)`4b*ib_Y-I>|tF6U^xJ)TI&3MO4|Fw+xb z%hb1Fo6YAa_UX6~_gh3|Q=Q}<^gfXQwZl~dlHUC)7+xl&&pH_uzN(Z!4 zc}@~hRek@Y{p)I29yqQw?RCFnqR4WBZ1nWP?_xd_V|6xMr zjsLvLRU!WwE2017H^VW z{r5hg+ZRtmFQ@TK5V5fdgx;&-iX|g_sQMTLKoUd(a48j$OsAeqsbnnTbLodAc=$re zLMRKBM3QK3(bZzK2plVc2n^~eJT5`2aI4GcHuP*!q)3=RuT)$!aEnNy)rQqVM8^i& z)5{ke_8e35A~OXqram`l)N`j#9b8?0dc(#64>I{TC`PBl( zv~DsKMk(j%eHcuq-q3pon~b#i_h8f?zn`f)3}Y7&uCb#cGxcjzZg-!hpR4G9X286< z{&(GaHGcnlx$NHQ|6BMxyuZoAPG^S1!kJd9;) zr{AA?bn+4$wEDO==!lk*BgS;v3P|bJcf1q(-C#yFgTBwl(+^nO496;}Y!^&17&*I> z$Zi)5rw+!5l(SPs6KNK*Ye3&xy-+EmwaSrQ3X=}3242p&K z!W`)uTX|`R=OzeHUkhN~%E*itb2&RP9mt`&f};#fVq#to2N-zOhztSgA%hIiiy4=T zK)8UFDfA!zdeXniqB|VS&I&wB4(QzluPvVOp^#nJ@*$Vu-cXPOkO4X%oepqR+>3@L zB<@_3YsM$<&oJbn{tN`G6DU|T>`Ar$800HNdH-Y#nfqSSjf|nS5ku7 z9K%eHz*;jQCu1SamP?IePu+4e_PAc<haccl9<5jfA0N{T;;w|uGI~lxMm5{M{UhA!+Hvh z)Me#OroBTUNrb1U&2=%WbU%QznGht#W;aJb4sb=ue__&6KC~&e z)P?O$USY2473})?jN%|eXIKc}=XWBT59Z!?NwBsMjc%5pTi4UmAi+46b@* z5GuuR(gidMr7#x$<~q$j2k?D`0m}w5q4cSMe+sVtQz+w0u~^i<_^n*4ebRg70%+z1 zd0nC##iVJl1%lR8l&zstUhM-XOoEJAAnSOad@v|vwN1l9GlD9pXg4Sjhi!FN11Zzv z@fbz0-v;-6$}p`iKtx8&TF!rRGA5ail0^6ejy}J?VNbd+LIoCE0ju?qGLv`-QmWXE zAeAa4f}9MB{gegHa93E()6NIw*IG{~vYR$HM7SVf-kM7nbQkqJQNypp5l99pV|ul8 z_!9^n8vJBD0)mH;SsxdB)KhWMV}7l;WM~A+T_xN&=LHgxydphRv$Nit_INRSn$YLPh^9QMQQ7^ldN~z+# z+Hp)5GX_3XwOp?^%cWi?yQ&$dP}N?qUM_d*CqY&1Gh7ktZt0}f@Vv?;uwnl z2J3pIS?zY}EgNg`S!P)Yq1K$*|&#(er%w*ES6FpAR}v)Gx2}sF}(MHk5BPeXI0raEi`{fsX1&2Bn06z(9B^8+9o)bm8oKfox&823%!{ zF9)m`bXCB*^v-3y>lTd)NsrZfDe_!5db3XXtN4FkGiLDW{@+TyRE_(8@8bV&<+E!3 zcL@}2t3zjP4t!hEU2&x1!p;L~Ud|r+_R~*? zdwcEwes%CqKi=xOz95zusZ>*JnyUcb`SFnV&R{he6O9_CvvFHH*7yoDw|L& z1pJenBC(@~XO<%j(jdr3s75n2^R+qoFk9i^xfV|9q=jifU}N|hv#Xe8N1vqj@gr}< zD=EGpyO@%(aD5SS%vD)%zUzD!#M{5*M(cPn!*mHmRY;6szVyb^>0*L6L6TH`eUxr5 zltxB3rC)8k*VUvCFx>2?P(DljMrXDBXlKC*O($tNa!x=a9+Ld(UnBZ;4-UNdS;$ie zS;176wAm7uo=Ny9fk`^Q0Vw2mhKODS<2(B(_mP@VG1OLQhb<@V?Taw)GzKO;<4?f9 z{!p!ukJ-K)?H?g&<^1pe+IC+;u0}19p(w@ePwJ^W1R;_I<8Qkv6MEkdO{C4WP`eCb z3Zr%9bcVFmXZtT9E=|ook! zqoZwiiFY8teXWMVt&pKr{~;^oKDxrUMQae)%Rx#ALRL1WG4(rqB7&<+bDIi?rVR zGDqsGSs1+C-lk==b`LsIwocn2#aVh{{=*62i-Gz=;LlQ`^L^vk%QY)s{v8^|Bs%&37;4UG1w3oal2z4gqNFNy$ zWRjW-yv$B!;A&C~hTBXWXQ1K6ue;Gapo560k+g1#>!XMyvfVFzZ!YlAX#eAQ#Ef!c zUIEZf;*^XP!=uTt)AJaImbRQMsKz=cbEbtcDS$XN7&gNsHyE%@!l0y9&8TIMk-qjwvg^w- z{bV?PueNcsX|COA%8*oK^_M{IHLZSN^lnRL{L)3YyTowIz}C!`r)ZdWG^?VHF6aaW z{ae+99L!qqg;684n!9zvT!z)ujJf1CO<-jfqEA&m;pkJj<|wgfSn+yA+RkCp!k%f;WEt&9KjC#F_+=!t&$w1vsGb>VI~e>gjm z(0-FS6yk}ZI$b<^Bx0%X(VLF-SIy^-C>aWG7aazp5AeHGAoIxMt<96c-Jzq;^||W& zS7t``F7UGXpK$)SS}on}|K7$Y*#A|#`FDxk{$8JJn*W(rz*7D1mSXe2R&U(R|6BNc zUi~l3b7ug2#?O`KKYa>Ig1|1L|I6jO`F|UqJN^In{#?)e&vpV_V*OvOmaBL3{}w)< zSO1fF?$rP1{9I-J4^P`FZ|ss7@MZJAQm@D6f2n>q|8L{7cK&BG=)T zviBiYy2Z=7!>e@pvlkJBKfA~M%lcf){9m#Mux$QUO9}RW_0Io)8y`#mf8#utIOoMw zXZ;yZ=0!8cj?5kg9!MsilaoPjAc^Ii{gE?~8_|1AD#Pp~F>VYxyt<4iP4uH0n48MwQ98*yZ@omsNCiMx{c4;`d<%v+hcai znSW{^bfuwxav=3GztOloTlCJ_ygnfH#xeiWN(mY;b@L_}u~zu1lpJ#zjYr$w#b74c z=Lve2ai8EjZ#t8MFQIm~N=5vSF|_mP&>Q9S?|fdPvG;QC@OdEv8@^}qw9b6)olhj= z)BEXQE?L^8AG}O{dpsI`Q0XEwFkJUWGkKKXc1gPZgMaNlTS_`9C8QgSCJPm&=^vrK zzIzfCVh^qpvG5fersvBLjvyJm_+f42#*29}+2Y7O_+pUDOpqVB)x)0V)+H@&?2;DA zL6dM@%fr}x9oZL>LjsR~L~+BF02zqKLc>%9*M5S?G2Z(XBvVkNmzen(2c4vS_X&%`@-zNs^peabF7G zYY^~X1imW>I_6#Jw`Br`uCRHeZ6wQndh0@_fHa{|lx5e}g~rBiw5v~*2}FIQKUG| z9{Hx21tMuOuzy)I0o6Y_`XY)_9^9KqV`czrkGsQxoXA!Oc*r10g~g|sU4&bi*(HXZ zk007Yxt%<#nD&m4B!y5W*VGD&(RpW*4RkUcpDZuY9uB&vz20gC7o+#kyxC>tg7EAj za~X&wWzLp#Z$4bo%3?n4^p>;&wmO|I?b3xBAHR^vUY1aHC6O?mNXE7eIX|Z7NN4!I z^I_(n0!x+^FJ2z!<^_7t&D^>MX~A@~RzkhVoDJO+zZ2BZ`7$5-0%41O{jF~Vw?xsR zWQq|mrM$s1DzOw~D@C3ul&(=VGGeM4EW%>x`eh2of&*AkdkXj?BN*c$mQ$lpaLfil zW?_5&7o0wit7eooY5QPBE4{l0Y`KOz8*vNs{$M))&_0*3S*1#Fv3F*9MMIGq?Fr^$ zHuP`@UJEVFiN#d12N5C}g%ut;vSevF2BR!3WF(OKn^DjgF#1bIQ6oFm-(}pUO7To^ zW(^IRA{YN;6j3^b|E|Jb)Ebj1E5ute#MCLOtgP-vwV;r_+iFw(=fYDd=t zVr2)5f{?FFYNxvM&Hd9X6uNv`wfU{(B3f&CYpqPG>#apZu3yJeBz6%syA;1Eh7*pq1*9Dptq}nd%IFn z2-45bfM+l-NN(Wn2dpL4z3!@`VyozG`2k5Kje>cA$|}C#*Y@#~-6y~7wGUn#?3uPA zVHFDQwz_6XW5K8g<;|NpkWD-sZAj^^Ov+UEYY(!=#IxX=G0L&s()q zy^P%_Vw{U#<*j@*nP~(_DbHK44RNNPBnXtsvtr_12L?S`99h=C!8lFUhucgf@ZUQKo$M+V@2$tXRv+wU+cvHD>T(igiQ?NttLvLs@>#WkdtWNZK z1=RRM*iWgkj`b@_A*~yhYo%!4{?cRe({M54-Di{ehqW0k3-=SkO>-FQZfM*S2V%9Z zm~^97hn@&)QLR~GarHh;bjrGOdsc>JQBI+sx5qCmL)9QUKP)`OYk&f@-In;ZY!=?iegxOpp31-3{q@w=W`EUL? zQ*r89I)5<(J+DN@w%6((=^KwI*`n9;X0v2ubKyWXURzbk_Sb!0{6);ixeGl>8r0a(P-3v9@Qd#q zAEjh^B=EC>^Ef&gLO=bLoRz9IGAYv3QYs@yI5`SYGjhrRgw?S%3@eFof2XW(^1?2i z^+{S06`kbnEmw+0I0ZXLA<1NKC<#B<4+Oj5!8Hja0k~Ws1?`4Hh@vuYrHjUi`rHbxYY6l}#)J^I;fJYhq>ABKt!VVbg+s>NTP7B){;Vz&WKu>puU52)?X9g!EYwW}xGiB) zY5nRErJv6_&)Ocd`girMF?B{EFyxa_SQ0G05@{3muvS(spN)~E^{<$bO#l84j5M2B zcY&**2RviMLr*R7f}dE^C3BZuVJU}Q*Lf9xux6K!%$1`{TJc-GWL}s4V8%=@{jHUU zEKpb>`Xwxp?Y0UrBI4nt!mnraPsyoQ;Jh>au$2H!-z+P;xTr-9*z@|{sJGTJ zo|Il?bz3k?;+-o?kM)Mdw$WBKV=ql@i4jhvbyQHa=i_;2*go~T({d_ z6L&7vBeS-jq{8d3VU_5YpT`v7V&-4Fhx@xfKHF>W9v<%gj}BS0`r^B+)$XJYY*4=4 z`+ne)4!X|lf(6v-hTW9C_E~Rby~^Ta_7`@R&SKuYEH~-qWy!d3EsU&>4RS<(Mu zoTSd<1W4EHB#lw}tMZXXd$+Pg`fGVeXJ3|sboSTrk6sh1zU{U5bvZ}FlseXNq0yX2 zMgH$AJbv}uKckUWf#)j<&dc4O_fm^m$F$V(hl3yY4~|wTKAbLCel(O@{k+*KbGJFr;K5w_9~<3UCg~vvIDB!^XG1+H&#fjRhq7tzLFb zklC*4ZY(nI>N$7T?WMXs_R>zv#OO;+I^NtJZYJKBQ8X}9RdtvPZZT7L+d4w5nLeZy zYA}f-wj<}oM_jJ&r{3&WiH<-iEi-@5*{^Uei}P_KlZb-FV? z6I)w9*2QSg@kH4PisU-Zw$sf!--qjL1d4FF5T&c@avw|Tq0SGEf{@QU|E0d>I@{g+ zL%e!q|L&;z3Z8-Uy^XlYDbszEx^wdcDt|nMSUq*bweK)DOa%4@N|`k@)#uV^hG z{knF#s|$7F>>}4)sy`jro*|ZV7+>;32H3ckYemk4wdJEiXOSL#+%;&7D4p z&j+(!+ZeaMG2{l+FcSzEyTx*|St^%nMjtF3CeRjxK5#F~6q zE>+6SQmf`xD{i?}t29v`SVcbtXB$sj&2qzS0!Z0yx|One>Xu85X1QEx)GKJP<`M!6 zHzE36w+!`Csx~40*;)cz0;o0{Zo{oMTjfftUULP7amN5*aYOP87)rw`eg@UbS zz10FMv?}F-Q!AogtzItGS_-!Y@UNp&XuQ^Jkp*h#9{XPhx3tPN3}3nA@_%Kt-zb(q z{wB(l8kJ_P0(RlUTBS;ssMH&^W}{M;`psggjB-tM06<$!Fjv887Ha^B-qu|h1yu}W zqu{iPrAoyGn#30kRw_&g+A4zNMeBD1p_qz&Lx{P`ERu)PwS;B#uWX; zxRh(9W}PGG7SW+<2|U;Wdt?jS{liTDVgSl!YjC%vJv?*A$ikLYd7U)qf zS1UE}6+yI$EzBeEe+z)zCTAy@AG9x{b`5;rD7keY&!?DZ^%BVlW~)-xRLK)~+#+Sl z;8xD;mPn9txmW|y>SQR45yquO5EaZ-(51|=t6*ZlY9xqCu~{#J4$Z2IaYRpQm{(;G z88W0qE3;Jv`7k}A3zs7Z2+fKJH#~0EL2yh9w+X=n)&pOZ!SEGI72rlvx!@c; zZD1Nyi-6|z1gfQA|f&~boR;-W<@sk7Ff-E5xkg(OJ7T8s)Th3HG zg~V$?cDJCXnvI5{MYRDL0Kp8DfngovNOD%2(0nd+ZjIE$%yhx_Rj^jA)Pm?m=OFj$ z%_iqatwf0mdVxq_L$Gume1WG;P_hc5gC#1k04Pyw5QkX2K?ISpAjVq?1V}{T2wMTN zuhyd7fWpvUEy_raah=kc42P#sG8N2RkOLE?R1@)Aqh6xw#u&Pky%+@)u3+7$d$lBW>RH3}`T1{t79 zsa0!1J^_z9EF7>B)}1709fkx18z>3JfLvg!cuJ||LW)6dL2I;HO7PV=&B{=fE_fD3 z31l%kf~O<}BoKxZctb;BsOuF-R45#x1;i>fv`P?=Ptf%hiY}~`Nh_dSC%3>PgKEOW z#5@C|;%S|l8#q8*3Clt*5(ITGo{&xr=sAoN3u8iISr5Itws z!r-DKc#3&03<{}?u><~iikSnsPc;jkgPy{CBM3~LnlxW+LP%lmkd+(Y3-kf=w_LAM zD#H@OQ}6|#Vl2U(5aQr+ir)sLGUhm_#;FNjZ*fo?O_{%C2ph^S6rhg7(-vqCE4Ngx z)0}{jK@hEClZpXG2qqYLp@zxcg!sk0$GCytAdD)aB@n#jFI`DzvehRakxv$QeusFjpC4CLn4>-j7x%&w&nTvWeM% zr!Y{;v|K?SLZ*q*guaJ4*eFptG|_jAB0*4I)M>#|BS8R^IL(?=MJ1sam$Z3^|RC*0Wr1At(S8f{S6=9sEt ztwDBT(ozWkK9~cl0EHnL2&3GhXvZi5O+2lV^iZp?&P*_?dQ3|cl8M3x z8XqR9KmqeNW#V8cq0lfn1qy^BsShy;eap#7vmDHZzN-lY2;{F)MWCSf5DzeIA*7+D zX=}1Lx{tX{Ca3v{WmidnQ%TUA#2C=LE-BRr4uUX;IRRv8_?B6ZhA@U5!$nMB0YWj< z#4>d`?&KG-&%tEieX*u7@(tRXpa4iu?MQnQW-sL&m;sGKEmf3p3bPlp2Q#cevl$W# zCWkduE@fSzF05QIaFg?a z`o2_esUAV@VX1&QQsOF$+$AVN2Y@9|5Tt^ASAy6P;~pzw8XTY{Y*-CAnZl zHtB{gQ!C-0W?9W6Sd0xW(dt#I2Cz#*Ryh@}Twt=c=t?eeN!yZzS*u9?X;3u5wu2gO zxv~bMSzQq;rpXls3|&Hjd5A?%ODr=?0`Q;AM%aho*fJD5_!n~y5>;@4DF}XPHXw0o zG&`zlHo_K!iOnf3>;=U|djw336)a6xXdFyB$Qv;Q0I(?bBy~F%6cy1tn1aPBBqYcM z2sN&-X!*Nh_`x!(a|MgZ&3PbDAj)eLeH4}8G4Ks>snM`;DHLgFa8XAI2u=o%at({Q zSL0fwrN#(m9_+yyty|Cm=0H_e*|6xy$Pnb@#unu(hYl7Umo$({kVLQw08%6-Y&yC^ zAbGh+q8e$*5*NxFQU|7ukkT3$Pl`2yn% zjkf6jLI=bf#UX`th9Se$oj`!nCh5|DX=f4BTPUUBiLRP zly&;Bnsg~tN`wOAiz})Y-BMJqT-x$6co)hE767J2sllb6^oZsc7YbaFK|+zq1PUf8 zMJ|`zSjmGHf(r~V2zA=(buRI%TwMti%u?ui$h2w=@(~mh#nYtwi#!JvP3qT`+@-Nq zQVd?F;f{$RGXUn6(7GZ{Fwh+~9*aZqpeHcv0En(dal%1p6}hH{JcqdlsY&mmI9Ob& zuLj)*;@N@NAQYHfkX`gLLG)Cq8f22fxN6YxiA56(DXwyr%Zla|mq1P4u}iU5mTA|b z+@-NpqE>KaWyZ&lyY%J&OqmuWmy-geL4^!OA!aj|G~z#ng$1?6cJva9NXH z1{QA?Av+)ufgE%`4SBk9Xfsv0=!cD2Q}Rc=HZ+>3c0unFz1Fp6L;PJ}Qm9h&4EWIo zQJyW>Oy~^e23I*Sw4l6{9}G-N#qCnRayi)G7?tTO*-WQA0w zY$$P0L%0rB5M(4NDH?#TYC?e(MDGN~sVd9Pj3TV! zYP3=svb2U0!C+L$;q*O1mNdjq2y2InIQ&J6E{zP)uvm<8mV%GavaC*9T$Gk+?m)tV zwP|&O{`8sPF=ri=948T_7mQrx^u)A-C_;lE6|^KB%Jh1|uEV&IsN8#Kpn0MQvj!#? zEL*HJ=oGIumFpCC4c(rgBv*_ejF?_n&A{}e%?Xa?{!XbTGXQ1{TEbjG*;Z4j4uMiD zLXbf!(NL?>Hl-#3#zii>Dq>@ao1-QuHn{5qnGU%EsY;8gBv2qR=^&va0epbTSeHg& zz(KNba|a|NFA5vccLmAVq({4c=Qdp)s?3Ws?Yc3GzJS&&g41oPY4>I}>{Z*rvRU{_t zmj+!c^y)wta}S1+0rm?wR$vF$=!~j!E(sK@vLGH%x7CDHYKRO(@p_T#Et**L0)PRz zvVjpprw%4Mw|=+=X~;SNJp;cq=*ZyC4($$MQWz>g5QuXCS%7Xbr%XGgS>q<6%N;gI zw5G_q2KU!WqW^(3)@%*#Bw-#GsYkG8fKubiL)_HR*0j`J&ReX>ICP??VJ~6D2bMyI zT&hg&i?KL;`Os-FCb)OWg#&v8I|c8XvYk_-*Bo^PO0!5OE(U{U8pcoU>cJqPCMi)l z(k|i79o_k*B0ZqgNfmCtP>K;slkRwz`-c!5^zgVcMl?vcqgdwR1I7r<3E32Ytj7SL z*LZ`G<3^`IsX}^}#b3g;I;MA-Lkfe0g)zbFVA}>)A7p8If1pU|7OGXntuN~%m;^;G zc)5v-l|`9bTT&c7!fLuE1~`VOPUo@%SpeOL`4633qpibvDSHvFSgLfN)A9sV*;Rq% zL01RpLSe#nxCjhb9$Xkha?yO{qN${y(6a~GU8i+Z<|0&3r(*}U9@GOj)}VcBw1UdC zJm?vuL$pZ;dP_+uS{@KC#1k5s+d)E?7JEeT4|4~cQKe^w`z7cRmnT@FHmCtL??NkZ z*+6xVIt>m}OSV_ISqoWJgC2#=fZrTC=rMXm@rQE}@(xSImO$Ycarw!Ow>rm=?1kxH zWbXk7Q>j)KO7L-=?k-LxZhm3bV^9SOxUNcH8zgZQpN z=UY61FwS6l(KkgXG%>{INjDhq#9SrnFfnRe{ZahWc%(Gt;|k>nR?!gnFi}9!k}FU+ zkGK|Ya3g|FJlYHu@xQp@$gFV3tt8IA3QP%_f1p>1uD}X6vIGhw0~iRTqQyg7kltD# zTr6`_k0vu6)hIzIFeR{F00TAXpQaDDCc8f3XsOcn;zE;RO!mmjSbk&Vz~bEgt;@C& zAH&|Iw~zapu<94*bJ;iQ)xj zMQ-wwVhws;>1r1!O>St?!vcMWAw`$jqYBm}%6r>@NM_JNg+1c_8CHLohu~UTMz9`b ze~|VbO_Z9r@~Fj$y6lx<5{YP_y06g;hFGOfl?zAO9pKceINK}S7vs8yvypBtK$k|b z%A$%TCg4-L5jk|+0)rso?lpHdz}kv;VKcxAfQ;qVflJ?s>@>n=fK23`3mRxJ>VSqC zv^7vfo8rvmLWYZVK&f%7pTFoFhe*OJ_DFmlSjch32e~Zsj*q!+L>Fa?o>Cg$1xk&Q zp+yU=DT7Vjfa2n?yPWHowh*hXJf$I8f7=;doOniMdoFkK-xrCajEGoPl&by9$I$jQ}~0iOiiB z#th&o#kWi65Z3=Nu1czV^exlAZ!wCC@e>7!8EbJ52?Df2gPtu*fElNEox+z}2`!Nw zeCl%ZhW=PCxhmpw!(+%f?$a{Tg^tjwD$I?^GGmL>YJ{_rPfLtsk*JVaOxKyfAi!p38>9jnozgbd!6pj5-hyV*jEGC0BM{bYb=xavSW?qWhdy zCZD<_F~~@DLPv)JK_3W2H^nGD=sMVro10K9u=u%UZzu$thN_ir5{f~Qba)Jci)*XF zbym&9>p>u5HILPZ3tb81RQF)?(2vguBWm!bCMVZRbw&na%?~E8tHO-c5q}SviD41y zX-%SzTHJXC0mUP`Esk0gr%$LtXJ<`=;F&*oTjj!<4jcud%=$2X zVAp`tTMb>9BD~7&&8qCDC@f&<(An0IVFvj$EnFDoTti{DLSwlVMSFTY>1<|vAJk`A zKP}S%&)r|T94oR+#0GJ@sbjk83tQd^}`snR8VCYS(zTup1}f^qG2LbB&ae%uc~T4OkB4l zL7fTXkThgcKQ_KjA}b_u4eQ5Q%!zR$c{Ikj;*N=Nc}q}@Oo)nfDl&Jf3jgp#6AyKC zPIB+oNt(W_AD`<^qF6|hAgmu3H%_8tN)lqMACY(!;h%_btV;ce9Bt^J+K|w#mZhr^ z`CJ$N3yX3^za+WuB&vucZNmC-)#D^;sU(HQ`f=^thR5iyw30k}t&VsNlMq zb*w-QW>q6_(+S&wy|Y0%H+h_H2|`JrO6=1=dHg3tq41*w8tcdr^#^}H<%^pcCM4T! z_fM4Zm#Ihn|^3AI0MPJXrxo9>TT0|!eA6ePhgsI*R77y z5)szqQiwatW$}vFQJkI(a3mc`FynG=Oh8c$NN?CPT9& z1_!!~+kDVtC5C}Z6iboebB(6hV+~i#aF<@u23T3w3@Aonf#s9Awc$N-g^?C(j<7G)wg>Ugk2mhLSf z6=}uISAA*)6rioyl+z{z6lxaiS)68$dTB7%@|Ux8+`z3fcDuqeCdw$_0SvC& zxZ+^6V^uw--?+jRfv zaS00*Ux9QismB%B+o(1xT)CFeTg%L0Tq1X0v4DZ_tIB$uU56^=xuzCth_wOLN*M>` zEc*-+ngeCYO#}#JD*CG01}ZUfhTeQ0GpjQUOCB-~yvcPC*RB%4AnR$K9%F19y&71k zmy^$P$|6;ELKxA&N3xy<8Muh!dKQb%R#S`-T8vyb!J>s($CG@@>U62i8C+WC-nt6u zq>$&~2gZumn)Kugua>EsMTKFc8;r@Pc~X;@Tb@*6kdUhhSYxO(`3p-|Qc(;!7|soz zLTidKq^QcXOjPWiSwNY0Dba^(IQ8ZL6;pT9yN=wVLu?&^^V|f}C0~Q(JlkB`0*n zGg4O$jj9{~0gG|XQxS8N0W4^uBB!S%N>mg=lOa?PKnzTlwG_`Og3%@Tmf>Y$Lo;v( zas-4$N9v3)B*kb@F&M2Z_|pJya5}lP(LnPmT}+B%Bs457vR+f2geG;GjE-(GjIXW& zdF7l47?e&LAX;u}sm}mGMt^WqTWJVPD-5Aaejo$Nnwfsi5)X&e85>hqY@#MT`yo-? zq7Un&1pT5|zS8N$XPjC}TllP; z|Lfog0&DN^XrmH5dVcit(Z+@=9p2oWf0%g8+%I- zJ!i_~H->}Jn;qI1cy#KGa>^Z0U?uryUhn}zh@)^muAoq$+nEhiAQbDM%8qG^zhpZTT4lJ=acydyVitPS1pD^ z)|^j2@CG!gs+COeG;g4=Y3Kc!H*)4@p7VC$Ej(v13m{AHCJTYXO9E%7OOSL!NcP5~ zIn1xZid+CY3GgzgV8nq`3hPzvL{gbxH|J;l zHgEeHyz{0^?>L?79I>70X}}}*K@J#p>U$oq(vzZTgEeO+1*{I447PRZbK8CVK^@&0 zL3Ja<2;;eChQv;fJYa3Q&IGpVE{nd*(+gFpr1m&{+kwSG8R0BK%)C+mddVPNPwP2x zRjo|;mA3P_M%`SDY39D}MK6RxN${&M4GKMbWfWP_gIvcPlAcA-Jt+eJY^(_7A$>8}j2q zd2?*K*o9WpfT%p}^t{B*D2$cn!@p2_nnjkWJ%6+Tp}6vhv+#v-9mWZxTTawxf}5|* zB3g_iwJJbSRX#MM<-*r20LOw~n`WU5nKA4SLCfu_H|(&VVvWPJqo-~$xuSQ-pK683 zLm9qwugz8D|7gaX?#op`w)}UQ0V*c{YmGbke;c2b<^PZS&z|kSIM{f#e{fu{wx8r& zKN>w@1+G{_rm@Hl{d?}<3k8DpKKCttFzKR(R<*BmU7#u6m!gV~W>PpGDh=T=nlT_j zIlPA-9GZ4psX%64FMxu|_`OE$kzHN2ol-x`gKl-dyo^3&pFnK!uh6aDc=93EE8adc zn|BjC*RgvSWsR<07Fxi~bE&(OBHc8S0Mx)+g(%%rsstcbqRaWDGxe@ank*9ZE&O$R z29D_o-BZVSKeIhlZx@|m?0TdGIDV5$;3RaVtGiNO4_((NPd>JqIu-Rkg;wv3gnnzH zzik8VT7ZY%Z01AlEa^;jz0Pc>^(NHlx?M@@Nt8rcopB3}7V*xu<9;neRMEoYFEv6d zh>}jfe=}kvfFCBzPBMX+IHy@(SN?29H>uIwfCjMMq)r)7-H8aBUT1G=HuN>+)N0*Y zVQgoM?i^No+;FG;a?7RYtBqcsJcWw5hjgqYF%~wu8Rwee&`iK zYB-{5dtX`ilC;gbeQaG9+n$d{IRAp_`|rW%4dd#Do;RVDJ(xMe@#s|D@kLL! zj~W=v8})aBN16rHbKbDPq^>GQLf$H77!%{5ifSi#2spl!tU5mbl+@_iyVDY?nACT3 zHu;t6wzl>A-kbw11euP9!|iUT_oldILrjx+s%>-0u>#I@w;v!do;zoqcOKRg-pDy0 z_Xj7&t?}uy?qrL$h2+QOgy*DJ;$!Jd!W)c4!c#NJl3F&iHtC(3ZV7KvU-)w+;hmlr z>8|sy;&T=M|AZ`j<@-N*RZk;+|7Yp${?FU^EcO4ZQ-nu@kDk*Tc9aWb&d{@t(pqD= zl$=3nybM|2dym;BG11ir&}gSnY%^!VAg0Ag9Tog$aN<1sx&3nY=RMxp=vMu0hNJ&% zV|P##n3~C0syW0`IfI>6!Ch zTrTIG>6@bC9QnQ&ydHX;+1x2Rv$1rO0DJlbdmw|DBT)6FjA?5Ho|a{={Yq;_yu zy1majLg`sSZMWLDsLS)HT^+mko`piU-YaOqZTC*eHn`(Ar-uWTqT8tg&f2%uJvbk9 zf2+zdjo`r$I?P8Nzy0$csz}}{%|C1{scZga-+mu~La|h29duOP{VkpORyCS>C%lH# zjYOIO^)e5k8oY$?AW^+cEx3e}TZ5xwTV4C`+yd7AmdcPSe9_LtI6)&IY0bh6z-C- z7PT20?Eqp<>D35KXlQ7>c6ZQ8<|W{<8u(O9^3Fpig(dT$CVlm(IH1jwSl~pX@&cJ; zcvi}_^&kdJ9Uf7?&@xr3n%fG680o4}eX55QL%KJe%GK6R7)ai}Slh-R9N4XT+Maqd zZ|?Uq-aUKPl8q}jB1bvfh{$E88i3?uv(2?|*mO)N!i``;y&jDiuuBOx-5Yo8Q0 z0=mhpXUVaUzC?v&{TZqdnn6+}c}_(u#f#ERR#s;Op(r9KC{XO3g5Poj87qr2 zdFO%ino~zkKzvdrAJ#}4DEQOV!*T}$(p}8u$d6j2apB^1dS}F2D7ltR?S7X>&=&%A~c71`Th6!($8) zpKb7T)nK&nq=Y%RGr0d1@{z-N`InZWZn!WlfmdT7%hK2-hrTc=S>GE zAMjqF$QW{rRBf*D7bD|N{KCgZ7GEn6|7N5E6wsuQ%^ZnhLyuN}FzZddNoUmi5SIYI zj{+IsPkMj&ohL)DGjbLaB~+6Pfw0|hDQ~bSyRY04`ZxVt+5bB`TiV8w^FPe<8T0>E z8r3`h?=5`3`Nu=7U?0xTHaEXj=%Z2rU;BgU{6m}D zI*+#9Lf`9kKD4?2f?r#A;^xLbSDXJ>jjyl)v~2!2Dy8`Ruiwr8+xRTq|A{Yr>4G_1 zoT`w4?sz<;)A)RVm7cN2ScDdtdJdC|o8@>dFK9*!3~rIA6**l<1?O8;+;XslS6-|X zc7FwM@{UIr%2qr)A;8x_5QkO5`rj&UVM1Jq^nq3{*;3g$I%x1n6YMDvGb zEko@}^Z)ER^1ry%S}iI6%iaF(ZG4u_|Hwhv(YWumXX=kS2x~w7GuYOA@pP~KeD~-5 zCr;^t*|$3<4bR#LI`3e~N|A6j$|+!ZN2Pv|GW9CTyzxFrCK%)E4X~0oT1&QTzPi3e*jqL2d67v=(sk|zBkW+16Pt$wTvvOl_r_F>wX&4CdbXOq?X@C=F>aD+&NC?@T9hIns#Iah~tw(HZ+eH=ox#hTE}{-$zUC zgbo7ZZ^*f96TB@Www&zFpMeBkV5oU2cKZjZTyO%ekPxxHVuR2LELEkuLKSP%z%T{y z>FV78!?%+&pT9UTWZKDDD6^0})_>*&-4&HvNdNM?pAR4XR>YZLEawrz@NdE2ig5bg zWPTRAgJZ*#9(pJ9^YKj5vJLwM=ROT@Fi6r?x`6VJCF|KLD4gWBepebR^7LML>!;_( zi+Xj7&IG~HF#6_3xNR!uvr&5}KO_%LoE&C;L~A})N1`{)RQ!^!VBrq(|mD|JXDf~T3VbfLVO;TE#`MHoworV9QQEZ0`Y1MJ|L_0(KYz&Z-5#=pT}Wx>U1vCu z5YZ$l5q`!<$|Q{%$?fg&c(e8Af!v;TRT^j_NUZ@Yo5D zXml8K_r45~c6}4hhwFd!8ASh1n{)cLo^&Zx^aZZr4$CU>Ss-C{iNKA_u9y&x&!f^# zl4!n!yiMs&MELqCt?uNnC4WQSo%4b45#nL7!jakB{7jVy`!U6wQ9u@$vSZEIc|lj&n=s(Z18wGI6Ymc4aFNS%oN~d9(F!1CyjN{AXmVQ;2WPdbM$eZ7w)6=o zz^*$#2qCt&eF|h?yzQ=m^BX~PMLH$oTYh9sRautNobj!%<%1c~jd5*i37s1m2DJtx z)}a-k)mms3sMdeF09#;HU)&iw(*GvC=nR$C@LN+<+|Xnquw?3fa)iE>Yo>2Z-mFL1 zmFLuFA!@?3Fw3NyVxASo(N^&r6#P@h4to7ug2c_Re-njnL?c|<{8oWd8%zQk*l08= z@ba6*i2h+^JD6Xz-!8ocrVWg*ppq^Xvbr9JLZV^gH#ky)JjLan{c`5phya?JqLAY}wx;W7vIjn3TuG^#G*@&Uw~2B95LoF}ewl`<4~ys<6+qSv!Xlj9TTBsx66 z%+yOqBDUX^H2>?>@G(zuv0TT*2Vh}2kv}eCdfXg_y!vDYa2fj` zvqQysz{8_Q`~TT{@l*Td@nL>t^}a4JJpvyWwB=#T8m~5r_71`}yv#Hzuew7g};^87a)hV1UiaGR&7< zMwl&;VX;l3s4>?bjYok5(>X5UCZF0p%ubIMhrI$|x+vCzOXa&9XK1&zz;cc)g$njt zpAdk&9Bcv^lGV=&i%6|u%=VsK)4F@`^WKKJfWUtxkK0GbdoMRyjbMlHaPOtuQfT(y z%@i;1DA>;Tjt|s4dhpl@psA@GtZF%1d>sghq%yw%qDECaFY!jXFZtzrNAQpzz8f>> zqY(}Es-)mVg@V09g?htzZt54>dcm{2C+er={7TjQS@!#vM02&ExK^60K1OIR2Baou zgw`m39kfn2vAz^~8;kGSsBQWernTt3O!kJ9S(W06;%Wtja0zX&OckuJ2Ubu5>uG@H z;(vYFzeMn7N&R))=jpnEII#Isg~TR|G&y-cmDsI|Ez!i-$szE@j#CC9<^WY9k$i$4J#1q>8s~2 z+xrK{dx!tpeFo(eTqSkfe)8hg0sj4E@5w)dBDK)L6XCA@;(XE`PrOlkz`N3V-ej(H zDBeywr&tuWz3J4%Q1{L{Q|JE4088P{x#yn%I_XR7B_opwD!28p@4b7-{Yb1aUbO#x z_~PK%|5yvuKG&xweYp!H6rLxwBX_bIadr~V%jDeORKz@)j?bA6;(P%pZD#&uep@fd z(Zt*3B&|yjXg1f2{jkvN&IMx5Etop@aw_X8~bTMD&S@6OeY z(h@3K@PFDKqJe%lkoelR-d5&nIaE2CPsBvc#kyvN&;?mpvWe`|RuUiD=~q;KXcO}k zVpr?nS8}#5a%Yj;xs{S)%Is@LGPcPvRVLIKdJd~Ipj*Pz2oMjAZDS#pGF=2k=YRiK zFQEqyTnSf>kxci(&|54q%q;Ue7usT{-R0#%WAh+4*j6K#55_pyw)a>1L+}Py`G)W+ zqJXz$O})-ccP!w6!H$iUd1~>HNpiCMmdXJM8JtGug&gl_q%OOflJ9|Z*2R?qpILO; zgOl7pAZ1Plr)|XkclW~t8v0VAeO{=bs^uPS=o0qgT6~}y`AsM&&xhDHB zGYibJ{10U}ZvWNlwLAOo7Cu(~7wDVcoUM!h^QR}-fJMxG+QO{dx^TCgKb#$&ZXRtq zPEt}A7JfuASO~Zx^qbP`kt{ix^+g`eI_Hz2_o%eBc{2FB&%@94xrX`AkE0}te{`?^$jPvxc*iY`r+Z8QuFxf_1xJ>X0TNvzA*}^H zO>E%a%z4)tF1#=}g^`B&DTBnbbP}WWcF?#yLQ6c(QrIN)&qR3tk&ldR)9Q!$OODIK z_tU{VJU*M|Ok&m`g=w|YKqbIU87NB#a!RxZ&}qJl!haJ+AvQQNWcC25L+7En7FkH7 zLk~Djur^+o6ohk~yhMEezL{1+b*WQu+HHbvxB2ba{=r_monKCq*=jTq3dD+#gO;#z zgi^CLD7C3$@+0l7K|i)8IY+WcII~FDU4x4AGxB`~gC!J*({wHB37;vj#V`?A%!I@U#0GEoBH2`1iOg83OA3m*YB*=ueD9Rz=)i>M{ji8TfvzQsBZ$P%~68b`oJX zd_8U?S9oHQ@NkOzY!nmvo257-&bO@yYeb!&0jneono$g7kJK?un~m{(M4zFo?ZZ^Z zTAHrXJB-T&irLvrmB#|8;H&^{X25}z8Y+)}TC=y{{3AZ4)sfI7s;-Kg-`e`f{o^LF zi?+8KdWO<5>}~|P6~{Hq1kq7l2Qv+0Q=JT+&BriW(E^QCeLDDB7%|Klm)>U&rsACk z^zZ(;F8@!K19++bw^WPye|RSF&i`}spBwW3Y5}HY7jvZC`F_91&z0x@DYw$D7XMSj zyIA~B!-8 z+ndK%c+f)8_UZflndaG7{B)`#n|v1X>nT5fIF;tNn)&TE*5j=HVAfH|Ev@DQ6vJp| zj>q%{$3x|fN*oY7>woHi=xO{H7!R=k(}&|9tB=R*%bOq32tf1uZRGbf@_S~SW|vxa zpMIGkN7rQkT`B&@ZN%e$>h9hC|80D3EdIxe_X+q!O30G$i<$&~( zlmEBy`4Zy4{RbjY-gSn0Tj0(}y!p?S=f7MweHHu9tz)qlxBsel=Ra=av#R~~b)3lE zWH6g0kaCs{r!@XeSz}p-*j5NnwL!4SfXA?)4Dz(-J%sRUdv76dFseuSoy@KQkb+^m z7{kzQKVw34{ZcE_gx32erlX_vrr;Pc>&&o)+l>!lf7}iPPcD%ZLZtNWO3m;Y)LJ^cFbKI1 zCC&pYwHq0=I1=1&Ih}eQuWEO?-u%6X_1f79fAbyVsjNiAmOx)WsFO8oK zRF}wMI`S=gxVLN_BVKEcMn>`K^KWH>Uxwl-4*gjYn}z@Xv}3yfH`5{K z=Cl5YS90)g!AY zAA3WuGg?d{=MZA}$=xN9XNnh>D!+&=>IV@oKedp2SmBn>Cb%{tZh%PCd1w(O@? z6KI-H%Jwyex8t)ICfrB=#nuvYI6JM&Ws|ztZptM{?p;ue{f?Qsw@vI!Qbi_GK&sibGk+3C8zGV6&c4j-kIR&#vjS^o z^vb!fnEThQaNVM?r)RVs5Pu8>wLWiD>{ zDVqWVIFx7Vi%WndjjNHnKcQm#RtAu`CJv&$pNnN9*K?UUhF2!-*T#l|>EAdeH(aS_ zD`#2D+qtts`jsvD1h-GM z37Aa3B=Uu+w{otoz`$DKdf2&!{9VyRbr??azDY5*%s|UbnHCb$K?BJves9~m7|di& z3I@&$WL+4kmNDBssI^=Rd%i_JWA+saF4EimIC zL&OhpO#(}!VFc6hY-W3Ye4An)0HOS2vfx6AmF0pUL6t+;V%`(-AIm55T2m)= zny9m#-aPQ&DQO%Y#ZvN=BBt@=l{;3>?T-a#q)r8-wv^ZuOYf?|z|c@XYh~f&pk;e! z&ier#bv*(*AA@1@8W`obkWuDzbkTU}RpVvwf9Xv@v2$m(=)shooh*jVg@LWBDpzqd zd1q+qUIcGR&5~NW$hJjn(q5#aZjb@jrK2+vd`2cT_~U~p-C`(>*1$^laRu@kV? zI@%E0w5!Hf7x=QiTM@6P;wxt}Y~|9NkI zkv;vF&;M#89{*FnJO6tdpX;ChceDMk^tt-{SD79d-jx;wxMcp<%Wf?Gw^qBG|F`nF zRQ^|M%!9;ycEF!qV4bv=8)-n6q$-URP2F*_b2_o`R>(~ig0UpvZJl(o@m3SEB^rch zSZ@R67&eI{%|IsFXhI^|d`7ZBGqwroK@&&+5@{I)z-$Totd=Zw)xl1W_zN`|=c>!5 ztX*;JGf5jlZH>{JEgiktve8TQ-rEqsRD-^MPfgnNUGAYv<30&*N$uU%*?=~Qao#+< zFGCC7m@MY{-qX={jIR-I<$JHMoD3G2U<>0`SZMF3-B-_!GeQv6J*bXJbo@Jb?e{JE zOVaqrZa~xEyWdCOt=xN%@b{6T;a5e3qro{~bVlBIF_XzC$)mjty3mi7t$xD#XPE+8 zm6+NQn5~DB39PH8koo1_;q$CG$y74?xAG>8e3;bYQB0kk(a^M5;6c^C!18GCcw?hn zyXGhW8A}VZ-`qOA-CJ9+NM&vK%tn(bl*>tO87CJEeH^)~SfgqPUUc5n^uh!o9aqLnw6?b_|Eb)# zyZ`rAK1=<7TvRX4yWUi;U#CszjpoyVH{(?e&Tr*nv0VQn9PhJtaD2GGcjUA-H|HNF z9xVzL6XgsCw4v0W`Huhg?NEZEbLx3sz2_4a+6>o)C57VN@Z2grdSfj80oG}scEWBa zF+9BU$^1i`RdN<|=RW_6g{Yum*%veX>_8YvQ8#R#45k1ibtBQOLv^Hw)|oUZXxmX{ zHke^YLlKY8n|xA`eJa4sje)*4mk5oag?nc?*o+oD6^i`efwCA)OgxthgV>qm6&sp* z@4V^EyU_*{ta}bQ%LGHE2atb=;#;{{D)+67$&R+xtwA?0?DeS4Bur7=)|{O)yx?=X z&k!U1D7*w6B&`5UQELUDpTHO|4!B_buJawEh`AC_Jgxc=6w_oCa(+tUYE?3YuM5tK z9a+lzJq}@*@3(&#@cuK6YKcAK(F8rH=-&BYcHWuy&TQs4aH5PK24-;Eb_J?A^qdTO z)oEa3Fb}skSc1II(d*8Hj5hr~4*XtbcGE^8jnlSWd$o!8GAS+$r4#uolP(Y3C6xJ; zDAtnNw8nI)unQekvH|mw=qTWl4=)W=Krc6SW@>6BHLxzI|)X2PHd{r`d-Y~dXnlz@bR z%(`x7miGUG%zPQTU4^ehGbPU378$Rfx7T9p<{!bc`<;Swzl;BR_zx)E_we5d{yW8g zXV*Do9Ukop6e6B67kJA`wvd{gEQt11GX?%XuTiIJdMJR(;pKHB2i?; zhnp!n;3Qzt7=eN!-IYr9LZw1P#oM#Yx2MZn01};a7-+n|5|-XtvUDl)UowNISx^?j z&#VzjdZV~G^2QQ|NpBMCXrc1?H$AT8|C?Tm|F2w%-~Ur?xOe$KZ{u^b{yzp{Oojt2 zg+_hvLe+4KMc+UQ)jN9mY@cTmpYHwY)MBnfHr<}7KbBWeKc-i3no}>AdjvyzR#ozS zBrAA!5-gXu({$HOhTA9Et3zTlKlHt(_Uf-tuC^JE2a2zZM6C zoRhabd#{-c!+0U3VXnyfM@EScf9(*_E*HN;cGeuXED$4a%fBphKn*+x`-!e1^L# zH+b+m@7(hnF;2`SUQBDo-RQ*5!rZj-V6gpq+eIe~+M&K&XD3#k$tjd}lPH}@=xd&# z%jkF+1e2$qy{K3+49j4qb@fF8D}`3spN*p;YyFkx&+Td}Tjt+} z#J>Txg;|lT2(DY@UQJtFQ7hcM+Pa)N&9vKQT;zU9Yc-YA?R&>sQ=439ani_Rc0X#{ zS+#v$TgGgkU}Vp{)V4xxY=_Xh=Y#ioC|uydYm+#B}QIDW}i2i+VB%bvH0J+`+sifvx@)!pPn}n zr`v~7PhCnN9{Yj#MDg6>gV|ZgrT>3l?H=wP9P>Ew(f)t-Ha41WtGu}>hEq^WGI33V z>us#76zz285n%Vr*L;y+l3n2ANW}b&ivMhE{O47!T&oB9j1L0ro^C`@eR%Yryy;c| zvj3Z1>4EIh>l3ENFibpib}|?a?Fc!QY0jjgJNl10ZwA{{8T7W3HqJRN?Fms0E@x<@K%)|r4 zjQ$Wa#B)VjGeM1U#DTsZ?{C)1eF{}HTB z|8MX-OhAds7(3vv;fEb7MfHFzi%Gyg;7QJhyl=ppQ;HVB%(J8aBrVj`<+UI_2?G14 z|8Y#V9#5x>35PneJ=mFgqfSCGJemwUJy~ev{4}HmePfriP*>ZlE5$haDpQ4iJ3cvK zT0v(4%dW|s#2@B5c|`_d{*4!zvdL6bDH!DN`iychts4wYN_&@ibwwa~vZ@2=wD%cU zWz@3iZeKRUA)lTM$M1C>)=WB@t7vEVo>N51uX0g_If6S|Wme68+jcpZN*CR(rC?@B zH`S(NYXtXGG)#f5iaNTW6BG>Jy~N$gFN_*9n*g^?n9H!5;{VI8#6oH*{?wnSU(Pb> zn(Y58<^QZy%6Iqwex{Fg{`Z@84EoOx3aSVd72~k96|T*Nf2` zb+A}Z9`z$Y<Cb|7(=v`G3m|zQ5_t^YeF~|I60@W`R5H z&PVMo%&<4@GjDKuHqUk14ACn%9ZYVIiHe|5yRdMlx^HA?A)JrC++T8 zd%m-=p)-?7HZ-m>%qyBC0xG}#1O9E3@&&y1!E!O8lA1Yw9erWRjt$juERcf#&^wu{ ztaee{P02i#HUgSEDF2X3kUOi?5q!9L;UoIsfsf9{r;SjD{h#>#NoU>3mGZHu$I-a& zwP)&&74@he2QG<&7f<)v&v$>`e*zP$)ht&!Ck-sIx!97d6Q;BK3Ba|)mz zDams6D$2d_K7_(0Y5Yjp-uXnQ=u=hYbVK+-S5}2g%m@3(@=q*)f%i7o+5SPWC^!MK zup)o#^c8lYd%IMX?g~}J8WWb;{sGuKKR8|al6SWKXWyG=J8E2|hJU{ieHvW1KV3|Q z19~-Pa;Sg?rcMw>-6!`*PCZXf7l2=Tp3@nrik)e9FrRj&ADrd$k3zlXnVk2 zeE=QOwQ;Wf8OrSfeCP{M&H&SiT#iFQlY5@;30%4%Wt+{OI8E&S|I5H82@AZwR}ACHt~qPR90t}{mRy=g@~lrlQYU$)iyWKg(CS& zZO!tRuqBfU`uv&(e&%(iy))H7!TFZlc@xcyZmFHg%IzfhXUvZ3rhuAy^t&0>bnf%7 zb#mR-L9xeuZ!$j%hj6QzRSFpIzGwmdu%1l`PCryag5N16Bx{zMq1IH|-0-2qluLY; z49eKkO zMVPQ&BAAs_6JR_;H=Ws{OV$pEoJcxy>VwBlXcG7Z3lYY*`5hQD=pq1-v)-}=yQ)F# zkU!#>CDSqn;uh0cT1uFys-%S4DuDj!=CpdbuS8E+Y$$&fGZ%)X!hL$)?R=3e zV#xy~&U}Gz`SDj5dUuVy*Y^im;2lj?T_EU0L^GjXtClKa9AY+ryrT$+UM-TOyoJ+f zDN>^=V>MbIt;tffCRavlvLaf!`-8j^sqqw)iP}r^Wr!&g&J>wMB7Pm+7nP+m!r@jT z$U$g|43;Beq7Tm>qjbJRDi1y(R+dmNI0ee?L<)88asYFz=h8WDiZTfkCD_SKsong=_ zp4@g5lGg}HgAHuCEiw~kXl7mxc7k$LtbS$nsF2Clk;<~x3%HAUbb3W6yq3ZS_G-Fb zcgGe;x7aO-Fuulai6BIw`E>k&eYiT-7JHq}bGoEr@u!~h7OCBK;;S~>&w1ybloR$e zmGax$iTlhDIMdFkKR&lZU=C>Y969A$@Z`tcpZTB^gvspfJ$0&O2&j{D$%nzewzH|* zIsHum`eP@>Ng$Y6g+igz1?Nq4-iKvB8)T|whUlJpBk4zuXeFLMU_Fm0pQw&RAI#3*$Z!}+82AR01a0H?e_H%A&5bOjxvVTX;l0`PGNq6!|7eh%h? z&XCbZ9Ck?uVy#9u;?5z1>l=mZ8{qmHe|(@DiecXTk?gv#3n)(qJND)e9Vi|qe+ZN8 zR+U)SaopMiVj2>GfX^}osOtF7a^)`m;}$-v-v2mNh6R-X^Hg(bEU?W>o!BS?bB>g};Eu=g8yXsp$>3*gFKqQbqx*hBH&TpwyQ1QM2+@MU} zSd)6dlMzW9UMT*=!DXOC-Uq+JAWw8grQrt$E$W*=SL((8&yF%F`Hgn3w@$zK1Vxx(7jUxfo>E7fIaS;Ufv9@-3qMvBnB4$82FAS(X65aVaCD9pw}4*CpqL(u-W5#>IfD7eIx%` zl?W6?AK&|C>!;_(7cY+ww+hZ_>>(M!ucREVKHYUcgXO|yNo)Y1+CTbx{3L_f>j;q^=*#AtzeWTz_NWpygT$doIv4aS|RpIuPfp&sQ=R9L=FMvfMyR&i|Au@%-Pll6#l`<2F7^{XbFrWX1w*4n2*9u!S<} z!WJzkL&aQ`UT{p=;OY6GtFD&_eh)n!%hmP=e&?Ns=9emzx6)6jjLe?O#|(5FLCQ*F zLEkuy+4-3_m=XbbH&Yi*pwi&vg8-5nBy?H`VdK66GMLFzlSi1}4G^8aVKa37yP<#; z?i0S1Th1HLib9|*d!t8DOs;qjU(g*@Rp!?AkYSV17BAwRk3m7dQ8ZC370oE1b-YBA zis;#``M^TjF?RnLJ7L*~9Nw~j$F)r-e<%|#FYkUJ+Jc=z&LWgSU(re%e>TOE&=-J- zt_PJfB>-xRvi8YvJf8Y515@)zxQvXTi>jUGVA;`uiMl9?5q*yK_E+wdB6ho3qkL!(!Dz*VxbyIRmx?qQRDNY-Pr=R*J zLO?O9EFo_~s12`6su?&yD#A9-e+Auh9yl*|f8KK*1&>%+ddf3N88T2})If@8v9N@) z*Yr`rrs!!z*fW`%D~K?Tp-xy8r1O`~ zZr1imh8tF7^P#JsR_Q1~(=<$gOo$CkfWt<(w~rO08h$qq)0tjRs^z-$d^{S@$D=_n z(PmCjn}SSSXQnGXU#(eMO*EPm4|U^DH6;o5lfHG$$MK7@a=#1(77T?xDP!|TDaoDM zJ6nw22#@^mgY)d?Hu)s3ebQBqCDjLX#2OGcDYJ)OPNQK@W+6~ie$ZhoW#a-)bs}ta zM!`yvh(td1M4Tiv9+x9OTu<|fz1K5N*NOGqpC*CKx9TYj)t|7&uOvhA;tGb4f0S`^V?`gtYKvQi&6Sq7Hg>^^)bvxG@!3pSoUXYx4G)W1 zpsu_=4b#yG^Dxll=sa7h2wxeVv5<-K47xlR@m6Mi&m`Z=R~f`mVL8JTV`8DFsidh_ zmLGAPjislHmI_V?x;!uQkt)L7b1C|LfR^*W|Ep+e)%-r%D^=~mgD`wmFV;xt5U*s; zi8UImZdE?wQC|}Kb%u7rWbf@_@UAmtrd8F5A2{MKZ51YxQ#yC=n~xV0YY@GAGiS>0-sq9Mv@qe%d_w#4v7P-rn*j(T_`Q9j z&{JVEkNRS+-SFAKLjXA}F|qkzwLXl@Fj{SW+T2_sE#flTX!zA-hMhn_^(v~DlNmUo zwEUJ=pO_)9ee!4}^Pbh3f7E+^Ti1qR9p>|& zedJiC+Aq3T)O_LN^6aC$hUp@uzV~ri&2VAKWVxxN}EAX5~Su`!o)Yj0(GqMTdIw%a&cvB51!qD4D@`?h!xA`+5EVf7Bb+kZiGaEiWy?l z?ZK-3Sj7kl3xTk(H5zUeF|ge1h;nS2!LiD`TViUY@tWPCE1Lt!>~(YS)1hC|fGPKn5>4C4(LyQc@Ma~AYb-sMRr^1$axae$pYH$kQ~UYu zulbv7iiO+9(#)0Hh_V32Hi2fPu(=rqT#~4Q#b~ye@M;7dew4sosGCHkYXcsl%Mz$* zWblPxTu7)PeHo&6AetS}A`2XQO(z|91&O9`C)(@m;|=D>xSg=R@B4eRJas^X};RixFkI zHam4S!_09XmWz(_uY{Q3lfeaAn2gmm7n7+s!>V4Q1HYRE4Gn0@$?cY`Ia0A#v>N?X zsZM_Y$^wYoOsU#C^iDfdDPqbN9OmS4=Db2lfOzJb^O-6GT>9hpBU+-wZsc{QOcHBj zDTlh`)(VsmAU>Msoey+Rj89IS?gz#+pD)lF{szO;KH$Gc&SE(6Z>|L4qajZ^!ZudV zzwMlL9z2fn&EOAhl=9@|KmYHmoIVR|!TIs0m$1e2Ksy(M4xJ_Ypd+~(^TOFqK)^CZ zbRJ&AHd-O=8)`5&rbL&3>hHXf)9FbLIdh%Ie7YE^n?XY+fzFpJKs(^|0G@Zz^Sr(= zlhj~nr8>*3PD!YVHko2LPI6oKc;p(;wuIE`ZbMdea>K#-VD8-O=YJph(KxEN(rT>B z%R>>ZQHVq(nz_Ybo%UJQq_QT`2?(RE2%A|!M;H|~wF=Il=oOv8eCEuC;|T-l1q$y+ z8G+^JgLk|HcAv))I{?)4ocH)YCkNPi(wTDid&-57%mb#N)JgsX%#1uM^K&xrdft19 zEYE00`t7|yFlvr#4N~Re(rLKhrvhfOr6HG;L&ssHyJ~f`SA2rMZuH{&p3t)KjVeG zGQM(ugu1lGUA7Fh8ka#xVk_O>zBiP-yQ{F68qgWopf?|W0B`j$urTHHWxE%TOU0Tx z$RXFt3167PeJ@(rEtY)ouFiA}nIaQq-+P@mp1Q_YGlz&{+k{hWtR*?Exg{GL-@i#7ccruj3- z8DAPiI9JNpqZu4glJOG8@d*cc7mOUmL#bF|5a|W~k@53=uIbz!rj21C`oa8SX-u6- z*o0X^QvZTn|kRgDaZWe42vIuSC;JM`*?jw?gvkK)xlvR}$oJ zx-z?+&z07HFnluuKlLYj{a2|b=KrhS-T!zSpY_jwbI{wcJkLAR54N-Uevii6JL{M? zgDwRc6OrPpye>eo&g5D=8qAZ4*QaR=9qWe?K6dnVb~oDP_Gt zF^n2N?-8FyK1}l_9c&8Jc`i01QxP3{vzY?sk7znLP3jeZ>0w%7#VS(gY;md<6lzc> zJJ!;|n~ZyBf$Z_gVwV}6n@m!D^MzJE8kpHqYfXVP5$}V#&|aNE!RlN)?ZY?GgtEL( z;@n0E`K8CEb6e8aEyKp@U!T$CrJq(+e)-$6sWI=V=j~RLX)3_IROA3HlW>XW*w>i# zq0<3+WY@w^b3y2&fM$%V;t{WuQ&+s480eh3Tm@Tzz$mz1Y-Tdb8bC0fjn6B0jq z=@gsZ2BLmpA3~lpWgu2kote%PfiGrIqM(O1Xp26Hz$3CZAhL%jq z)*FhG(zHQV6KcGcx4;3l8t_jTXTWBal~j#kOw(V1*tfe9d8Qzj()E-K5yXu!Lu6=I zI)VMy)`qJXp{Yi`cT+lWWJV*9`Fi$V6|0q6*AgP4eYMm`I%Ia;7%0hPx38XDx&KQv zo+zn#&Ja7$ce_RxjYpo6n?{$MbznD-d%eYUs;%U>S_;WqTPewl@S-53MKvAbwzne7 z=KQSRmWxs}oge_he!tj7s5(b~d)WE-p$!S~CO8{PVBA0zjlds0K_vl(AbH>SExs2V zzo~+MBMTqB9uG!2DG)T3ts*%rSqgHJ;+%--$ikAf!aD1N5JmDk4ljkqZ->9Rx%gg2 zs6Xyw;OCOQM;xJeRa7}Rz7P#Mxs39>!YW7iN|B^^GF;4}Qa-Yb*Q20mXb;0G=IUQd zw+O4-OD(;lm+4UoOSCMldri7VVfmRCTpEebyW-MF1WT^Scm?ZI%!xEYvj;pI4}B zB>uia{ckMM{k@3m7wCU|DZYwd>T49>cl#foegET5RBBOq72(Ewoofg;I;bSKrg*AO zXX_cQ_slBpSsTVtJ-b;f*JQ?M8s{L3^OBvS>zfX79OTipnsS$(1rgkgF1}ea>Wj8~u6L0b zwj7O+xt!f-(Ns;NaE^b2ur`OJiG)2uzRuVpj6u_rIvpMhWB~k>KvJ26DSMcbgWiJH z^>~`jUlk=2bwz-nt)MpIX`L|4EBR;H{xXYqJx8pzkNv)_9as1T8n4fG4}ackKi@yN z!EL`U*jN-U`TW+RkH%;Aei}5sXnWCMd8aCGPgQDYpI@x&#MoCA>>+9HCl&u11p3$G zs9oC?s=97V^QiPPTyRC%wI(+G6m0s}gUuD3+dhuLEQq0LPmmkk^ZF}^RhuX&vQS=M zT^g=)OxHfhSBSlk;NIBNa4dTIdy*{mt%@a1dz;y?TqKnW*V5+cx*&gdoY?wY`TVa+ zV3K(XIQ#r>ty-?uV)>uyHTUlP?=5_mp8uVHn0U<7{<(M1KK`GVd+mc4 z2YZFklfA>k7l)BYFZRO^bbz(>}YfKdH`vZ(Ow_ zZ%7TOd4#t4Op!-F1P-#^7{(&-RYQ_xEJ74PnpV|6LYQy=phsRASju1!TVrGPelYKy zIk{Meq-c*xM5sFlFX4uP<5Yqvb?QJW9xc^>Oit|2dXv=(CgLYh(M596t?~ zMWBKc5JpU}E--k5FPLl{n3O9v=eH1o9d0h#0_a4#y*hYxwD;7RX(qt{V1V9F2XilX zbo>;3ZvV9Z468u{?c2y8U~c>t@Wmei1{gTiN(BGkM_cjp7n<#5O2nstJj{GyoC9Ck5HlN_dL9?fFX>=?~d>O=COb04D9&Uv9P2b)jvCC7JKb~^VJ z%|aMIHW>d`5bcWk4;)w9tVswvkDO^z=j>s4;4Fu;qbL}{Bbe+=Zs%zp^Uiyd54ms2 zjaDOINA#~?TFS@RgUTYz9S@8c(``uhtDO|;3$KHy$%AiiL)hcTlF8aw~?1^u% z*d?sr#!&HpDkH`G{AKs>DEza%cl2cUbYH1ad{`)R=J<0g4Io()WP#R=U7kW#d>FyT%-eZSuOv%%1Fa;o$HFuhk)QS)4+QqQ+h zl(w;&mPDMsx)(sYB6YV#P3qB&fRF(|WyP3xTJ+nRzXuLLePgkV3@rP*DZ}(w5l4EVYErGR&ih!!}4m zd)t&>4dXDWm2lljL;B!RxQIwCx~3>CgYGGx)=5;{Jt z*nNHJ_*&5Mb(`bsEGMLk!g}T}N$cgH5FW z;3Se@cWrTiwdYqyge+k&OQXWqX4tSx8FcVc=@2uqzXC$+-KmGqp&k-UX65Vp97gkT z1`kY=7tV)Y*;F__TzlbgeE7Ak9MGe+6k|Mj8&EwH3Uw!IUWQpp>58M{o$M!VW# z|H=$j)u{~PmwYnb+?JT{PK=Fc&;Qou+Y|rWQ~%qZUFoNc!G8Kx zj`q{L;Z7OuuXMcc20V4Zzt$1I8}ii@{$G5!l~sO;)k_c@__>xZ610=er3r(07rAq@ z3}w?dV{nSe6j8yVH$W0~cqe;@V~s72bWUAm)uIOeGYi-l-~YXbM0N9Xk4a?THwcyf z%A|zVv(HWP=rr)6CP6fS_}7t~@Oi4Pv^gr50l5{ZoC4ME*M zqXl#w!PTOOdi+UF&Sf2QGl#t8PrEanc0R;Uy6vRb7jg>st&NJ1iMt(Bz|y3hbmam?jT~oqH#3K$4f+oQPX50oWvtkxSGZ4DK0EYYs-I zTjt=WX_*j%W0v`pL!g)94}a5kX?syI`zd88Yz|&B6j`JfFtcUHij&$*CXavKj!#sP zOLciADkqTHY%woyG+Cdm(&v>_VM46NBdf1w0*7RJoDSK8mm(`KHF2zX7@0b|gq<*V ze!IQPSiF!n5AQq2nrKr;P*5vHf}W|YOB-8%I=^oz zDg67^Bj;D=ljPEgJRbcz9HzWk{F?e^m+dD?$JKxI&u#O? zEfvcZ>#J9_e{6?-J+Oc6g?@wz?wNwy7WC~9`0X(K?QjFzp%%8oO>BqT_Gt^7eY*FPBXvTYB|Ig1|6OM~7%yhd$hu`apg6~; zfcjm0|IID@hQ~d8Eq^Lh{`iA0`uK&1Kkf=u$0E~$I!5=iLe5I|vSdVPCUqMMeJ1wm|Fp5^^z4A*-6U@UL z%hIcmQRSJ`&&tK@gC<2n+TQ?9geb3^*`n9;W;2K6lQteX_n4?%VPbVlvV(maSV^DK zX?|sjUxD7TJh>fmhd{4NZzW?dL2WzJuw-qM>~`&5SEqj-PdaZGUUJ5?sD&A1N8_`+ zsv0)OLmAm5|CqJ{#aJRaa;wtoQ760!_;J)G1q2 z!mso?lo)DYt&K~M;tNIhky?ZLmjedL#{Jeqm(+waj_X|G-o+eaxE0NMFQ{+o4Twl` z@#KW@;W-oF`pH?o%^=iyt*G-l5G>@0G=cKkP%kbjtcNMW!en=}xH4zvmqiIvPk8h7 zA0c8ckp@}fKmmOG`_{b~Rc9D{ia;={`i%7BsoBS^rA{H1vD=!GciFmc_H&i|-}6E5 zjW^9q{VkvD{NJTYqg;>X|E|<(^}GDvxA0k-|2wKQ)u78CPVjFaNCzWtS+1xM{9Q9>K?o6qa>DMSI?V?L4xu||Xv z4AeEcF7eZ! z*g(3}ZUmQ0>_x!$y~)sH8p*h|o`z{6oB&NhFm%~P7G{dTsbHtx%$sXV?VH7j3DQ-K z%)5Rp%x58HOlQ7j+_t$gvlLP}#J=?<#tgGqqlth>XB$oiI;GVxiH>z5B1yx&A{whu zWp!jE$?(l1v^KBx#*+`rSgixAt{Z2yjxE$IaCij+5AU7r5Yf;HSyaN~*ZFyM`N{m#6juNv_;9M!|# z(CdsAleTx^^%j^vC}Ucj^<}eemXm@74J2>BpG?Q|@hqp`1x1uM%5Fffmpju_S%#{D z&VBx+=gIueqky?;>>r5;hAthupj;3j1;-+YZ`SNiPZ#Ijh|7Ld5L%2K4beGSjCzvD zTkk}lcj!_Lw=aU`xR3IF>`cdy zNHhBPobj|zNBGzQb_>{X^Rv!eV3=Y5Zbp>7>8LXl#HJo~*Pt^TeC!D7XX0L)63DA#|07b|P7o$}3hX2b;O!ZQQ7Lht3@islNWv zx(w0Pmmn^?1u^Xs#Et09Y|8Cu@h=@)(@~7W%^V}aBu~7gji^*>^+waK&hk;_FF1a! z&{QF}#b%0CxDc9;`Os|~&i&2#S-&lz63G*(A55aWT)jwI&DCrQF4o+P&l|z*6r(79 zFB=~nt+B5dPS_9S%w*A;GQy6Qu@MZU%E`a-O_tG1B}2USZQp{G-V6Uu!`&0PXArof zb|Iw`Y8lDN=e+OC9KG^E%i+60)0<1Cyusm+YbkDQITQ>^?uPE;9*rIEAXLb9ZqU+_0f0x@&_V%CUAWdg-57l&#clu-JPezjd zZf!OW@NWFRydwdrd^2}0^bh%Iebtion{NQ|n{On_6E)jCZ!lC3a|t_q{zbtNU@7qc z^E}MlrtumZ9G177LrpjDeE+>OpyHKZQk8;?Pt&kcEqqGuv5#c=nvk@xq?;sP@Nu+W zrK7q6rb*;vQx>+DLH0okvbn(nH=l~|5`5XDtT%)OxwBnUUqf`2q8Bt|t!7zj+szAN zJI;mk{bLI$#n1-X3m=+BVtL;$&{tJ9+*_6__;8{-1ghf&=i>3bEzCE3(KPUh5T?f` zN5{#R@^SkrKxt90D#If(warcH1qUSx&QH&eSu)#dv!GVWsZhD=1E`gLjAZ% z1ggRe3eLv`Smom)YGg&>`8({>`FL^OxxmVx?Tz{q9il?d1Bs7igz?t7lP@L zeZ4Tz3O1A+K^K5PrL#6jR`th%)}S*L3%epp@fs_eMrNT;31L(?vn z%rxBkD$I$L>2_3fArRDGP^4n5+dudrI zTrEeCE>Y5{k*k*T28Gp%ZKo8IA>Izkm9oH1EvfdZXs7R;*ffq?og$TQmT{OX3H<8`ntQM*oh(?|8?te)M}>OPd`h!pO=*@T?oYG z0876L$mI|}1`Jh+m#g^Y{05D9cE!2eBJ0lY>yhCfJ55?R$AG`-=^2{~+c}!THvQQgGA@ z8ZH7M$^44jU#e2rhKa|m$YbZ83R+iDlLax6oO}J?dh!HNVh)*BG|mw`HA?6pk&5*v zZlWcHdiyU|z%UsN`Vxg?z6UXE2b*O{1gne0C?`F7X}By!CKhU%W&n?rf0KUO5x2$q zcC^NiuYP*Acd+5s^fleQ1Xn#Hm+GKppDlw+zIQ@ykeuIF4$)%dH&t+o#iH*H`F%6a zvl_dc+~Lhjew}vvU%Tzy!=I1ZZM~|qIT8(wdgl{WKD}8$+ID5>K>$(Ot8+~b(0{w= z3tFz5YG^!b7W^nIukZ zj`$@Ut4J*}j}nU9mm;+!NHTwPvF#L4F-Y5PN+~B?thS`sc0xJ3pe}~Vn1+`rp#W8~ zk=LcRD~Tj^0&umo7Q=b*T9m95iaKsQzA%v6oR6OooRQvy?&cz+xIT`w{*cL z0kpPypo@HyT!wO3ml#U{KCod}1-?lvVHH@km2j%sXja8s4r(J)rr2ZhG{-p;Z;08Xx@JARzKebE#%4fSjJ~1VKO)F_& z$=~mK-&wDeNu=M;c5z)&G>SHztkz!@epz9FI=(ij2u*2o;Q$3qc&L z3UQDDF*HQOu~o8^1#*!EGCF9(3mdFIBw3?3{54huZ_}x3Q_WdUk$CY;O4}4%1$wM_ zHq~viJIiTr6F&ufYy|8faMl?Idmx+@N8{~UgAp&CH5PBz8j5)7%#nDz)ne;nRk zav0uTViaC$y862vS%iue&KiT)gLNQB3ulhN>v5!X)&RU-ECU^-|Lcn=_?5{2`q#v2 zQR-W$%{4tI^J#x@aC>g|j&3<6;TmD8qcbhyY)SUq1etR7R0HqadOE`a<%HN;&z($A$eoefK-7 zRMNYck1eCnU1Z-E{kcl~-}}LQ9i`TUut9qe>O0&*; zt4BMrcUH6Uml3?;1+2z*LZ>b^6R0VtaeAnsQ;qmV1`MJ0d*saun-sv|Lcx2bV#59M zPZE7f#=<6U8X8G0SoM}Q5go&Inu*k1)|5TuYZBGGo_0oW=w9X@nKiS)JFgHNe3K?* zGa@p~3AoV!FmQY0^GUqJokDyxyQ=&4A=S?I4@9Wobj_Eoe;f3}2NAgUBgk)sLiVBP zxxoZDYGbmiz9_YwhHiW6z4Mq}=Ch%_osD;NQ^8`NjQ}$s+vWL>S${-4`B%;@%SQH_ z!K6Jn$^8S}IT@T{JiCiiL4n0Mq^xz~&3k9+41@uzyUB?*#P1x-(S(e=7ygi!`m9*e z`}TY+Y^TfR!mOBgU#Yr5Wp&iNLl-T5Y{6-k2y}n;Xvp7o>_Wsn@rACPUNfO{?iD(B z*}CNv!gN6xdacW=6Q-L+nC>#dBo(j;Q@#RWE?c*J1)^4BL|u%Qamts#2v_ZIN_V@T zd)8-dqT0ebLFhq97o7fWiM|Z+6vFefi>WkmaS23kNo>Sfa`iq&jCPD`{LM~M@x+pW zRkf2~Y3kFNQ38Vtw|a2`8}ninLd%9vDA`W9@$k;MhQ^bzZ>%GaNdsjAmP4R1xO*!? z4G&Upr9oPif_>o8Ur*KdZ;E358=_c$Ew+v;r8NZ9GA386#Z@G)V&Ts4T1?K*I%Th2 z=tmhf%{E(IfX2*>n_%zGRoVMm45v#NAveR`y~UIZ;3$7zp1G4eIu@ES$4Xko^g1zy zkJpTe*{^EKEobaY&zM;K2yK&rppYYlZhG#;D#y`_K}bOEn$%1(x5nL>YfjviX0Afq z?|=XLCUC~&O`f>R$q=0`DU!iFbp%1yhvc8D*g`3k^vwkA&I$?1-!7ur z-{iEBy0*eJAh3Cr#rKbO4|mig4De%z;+29cDox2Bq9^HOGi>%->5qaVR0z(_y+nE5 zQZ)SFbaaNtzSI3M_ndnl7Z35DGnhHO@pQVFFuj;x@jtI5TaIp#M2R-0Uv0bB*QgIN z+;DJ@#z2T9Z}bV%kJbUw2!ew@7|&e_&^qwmXJIt@ZtjpPw@(Jcp^esn=qM5pDsYhg z`lxYeE`kx8eSF|b{tcBj4OR7JCSlH-#%hG9oWgvGha!`+NPbdDD*GV@G|^>9RYz|MJ?H(O%yG!;Kq@iJFdg`s!P`DM+O<-_7 z^@gC`Dy^O^)@(G~1RL$0EkD=Lbe(|LJ + +This directory contains a vendored commit from the jemalloc stable-4 branch. +You can bump the version by downloading +https://github.com/jemalloc/jemalloc/archive/{{ commit }}.tar.gz From 3fb14914abfe89797f18a6cc5f62ec5cbe0ab2be Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Tue, 21 Nov 2017 18:24:49 -0500 Subject: [PATCH 070/177] ARROW-1268: [SITE][FOLLOWUP] Update Spark Post to Reflect Conf Change The Spark conf to enable arrow has changed from "spark.sql.execution.arrow.enable" to "spark.sql.execution.arrow.enabled" Author: Bryan Cutler Closes #1344 from BryanCutler/site-Spark-blog-conf-update-ARROW-1268 and squashes the following commits: 5e784351 [Bryan Cutler] update spark conf to enable arrow --- site/_posts/2017-07-26-spark-arrow.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/_posts/2017-07-26-spark-arrow.md b/site/_posts/2017-07-26-spark-arrow.md index c4b16c0738c0c..211e5a481b482 100644 --- a/site/_posts/2017-07-26-spark-arrow.md +++ b/site/_posts/2017-07-26-spark-arrow.md @@ -57,7 +57,7 @@ the conversion to Arrow data can be done on the JVM and pushed back for the Spar executors to perform in parallel, drastically reducing the load on the driver. As of the merging of [SPARK-13534][5], the use of Arrow when calling `toPandas()` -needs to be enabled by setting the SQLConf "spark.sql.execution.arrow.enable" to +needs to be enabled by setting the SQLConf "spark.sql.execution.arrow.enabled" to "true". Let's look at a simple usage example. ``` @@ -84,7 +84,7 @@ In [2]: %time pdf = df.toPandas() CPU times: user 17.4 s, sys: 792 ms, total: 18.1 s Wall time: 20.7 s -In [3]: spark.conf.set("spark.sql.execution.arrow.enable", "true") +In [3]: spark.conf.set("spark.sql.execution.arrow.enabled", "true") In [4]: %time pdf = df.toPandas() CPU times: user 40 ms, sys: 32 ms, total: 72 ms @@ -118,7 +118,7 @@ It is planned to add pyarrow as a pyspark dependency so that Currently, the controlling SQLConf is disabled by default. This can be enabled programmatically as in the example above or by adding the line -"spark.sql.execution.arrow.enable=true" to `SPARK_HOME/conf/spark-defaults.conf`. +"spark.sql.execution.arrow.enabled=true" to `SPARK_HOME/conf/spark-defaults.conf`. Also, not all Spark data types are currently supported and limited to primitive types. Expanded type support is in the works and expected to also be in the Spark From fc4e2c36d2c56a8bd5d1ab17eeb406826924d3e5 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 21 Nov 2017 19:01:48 -0500 Subject: [PATCH 071/177] ARROW-1808: [C++] Make RecordBatch, Table virtual interfaces for column access While this will cause some minor API breakage in parquet-cpp and some other downstream users, this is reasonably long overdue. It will permit implementations of the RecordBatch or Table interface that do lazy IO / data loading or lazy materialization of columns. I will write a patch to fix up parquet-cpp, and will look to see if glib is easy to fix. There's no good way to go about merging this patch since a green build is not possible, so once we're happy with the patch, I can merge this patch and then work on getting a green build in parquet-cpp so we don't have a broken build there for too long Author: Wes McKinney Author: Kouhei Sutou Closes #1337 from wesm/ARROW-1808 and squashes the following commits: 55ce663d [Wes McKinney] clang-format 57923708 [Wes McKinney] Revert to debug build for now 06625301 [Wes McKinney] Fix test case that wasn't being run in debug builds c9a7cb83 [Kouhei Sutou] Apply patch to fix Glib test suite 6b02e438 [Wes McKinney] Move DCHECK in SimpleRecordBatch ctor to Validate 06bced1a [Wes McKinney] Deprecate arrow::MakeTable 498bb082 [Wes McKinney] Fix glib compilation a4e403c3 [Wes McKinney] Set build warning level via environment variable 9a61beb5 [Wes McKinney] Move boxed_columns_ to SimpleTable. Remove DecimalType backwards compat define 7691a5f1 [Wes McKinney] Make Table virtual also, refactor bcc0cd15 [Wes McKinney] Some table refactoring to be virtual 267dd218 [Wes McKinney] Finish RecordBatch refactoring, get tests passing again. Add option to set build type in Travis CI ef00e5b9 [Wes McKinney] Split off record batch implementation into separate files, progress refactoring 33f341df [Wes McKinney] Start making things virtual --- .travis.yml | 2 + c_glib/arrow-glib/record-batch.cpp | 5 +- c_glib/arrow-glib/table.cpp | 3 +- c_glib/test/test-file-writer.rb | 16 +- c_glib/test/test-gio-input-stream.rb | 18 +- c_glib/test/test-gio-output-stream.rb | 18 +- c_glib/test/test-stream-writer.rb | 18 +- ci/travis_before_script_cpp.sh | 6 +- ci/travis_env_common.sh | 3 + ci/travis_script_python.sh | 3 + cpp/src/arrow/CMakeLists.txt | 2 + cpp/src/arrow/api.h | 1 + cpp/src/arrow/array.h | 2 + cpp/src/arrow/builder.cc | 1 - cpp/src/arrow/builder.h | 1 - cpp/src/arrow/column-benchmark.cc | 1 + cpp/src/arrow/compute/kernel.h | 1 + cpp/src/arrow/gpu/cuda_arrow_ipc.cc | 2 +- cpp/src/arrow/ipc/feather-test.cc | 5 +- cpp/src/arrow/ipc/feather.cc | 1 + cpp/src/arrow/ipc/ipc-json-test.cc | 4 +- cpp/src/arrow/ipc/ipc-read-write-benchmark.cc | 2 +- cpp/src/arrow/ipc/ipc-read-write-test.cc | 19 +- cpp/src/arrow/ipc/json-integration-test.cc | 2 +- cpp/src/arrow/ipc/json-internal.cc | 8 +- cpp/src/arrow/ipc/json.cc | 2 +- cpp/src/arrow/ipc/reader.cc | 4 +- cpp/src/arrow/ipc/reader.h | 3 +- cpp/src/arrow/ipc/test-common.h | 44 ++- cpp/src/arrow/ipc/writer.cc | 10 +- cpp/src/arrow/pretty_print.cc | 2 +- cpp/src/arrow/python/python-test.cc | 5 +- cpp/src/arrow/python/python_to_arrow.cc | 8 +- cpp/src/arrow/record_batch.cc | 206 ++++++++++++ cpp/src/arrow/record_batch.h | 154 +++++++++ cpp/src/arrow/table-test.cc | 177 +++++----- cpp/src/arrow/table.cc | 315 ++++++------------ cpp/src/arrow/table.h | 165 ++------- cpp/src/arrow/table_builder-test.cc | 5 +- cpp/src/arrow/table_builder.cc | 3 +- cpp/src/arrow/test-common.h | 1 - cpp/src/arrow/test-util.h | 3 +- cpp/src/arrow/type.h | 11 +- python/pyarrow/includes/libarrow.pxd | 11 +- python/pyarrow/table.pxi | 8 +- 45 files changed, 725 insertions(+), 556 deletions(-) create mode 100644 cpp/src/arrow/record_batch.cc create mode 100644 cpp/src/arrow/record_batch.h diff --git a/.travis.yml b/.travis.yml index 9c714a68948e5..ddadf739aab25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,7 @@ matrix: - export ARROW_TRAVIS_VALGRIND=1 - export ARROW_TRAVIS_PLASMA=1 - export ARROW_TRAVIS_CLANG_FORMAT=1 + - export ARROW_BUILD_WARNING_LEVEL=CHECKIN - export CC="clang-4.0" - export CXX="clang++-4.0" - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh @@ -74,6 +75,7 @@ matrix: before_script: - export ARROW_TRAVIS_USE_TOOLCHAIN=1 - export ARROW_TRAVIS_PLASMA=1 + - export ARROW_BUILD_WARNING_LEVEL=CHECKIN - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp index f381af0a2c2c7..f23a0cf7582e2 100644 --- a/c_glib/arrow-glib/record-batch.cpp +++ b/c_glib/arrow-glib/record-batch.cpp @@ -150,9 +150,8 @@ garrow_record_batch_new(GArrowSchema *schema, } auto arrow_record_batch = - std::make_shared(garrow_schema_get_raw(schema), - n_rows, - arrow_columns); + arrow::RecordBatch::Make(garrow_schema_get_raw(schema), + n_rows, arrow_columns); return garrow_record_batch_new_raw(&arrow_record_batch); } diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp index 779f2ef62b8f5..e086396f8f9e0 100644 --- a/c_glib/arrow-glib/table.cpp +++ b/c_glib/arrow-glib/table.cpp @@ -143,8 +143,7 @@ garrow_table_new(GArrowSchema *schema, } auto arrow_table = - std::make_shared(garrow_schema_get_raw(schema), - arrow_columns); + arrow::Table::Make(garrow_schema_get_raw(schema), arrow_columns); return garrow_table_new_raw(&arrow_table); } diff --git a/c_glib/test/test-file-writer.rb b/c_glib/test/test-file-writer.rb index 3de8e5cf34b51..67aed85f73b48 100644 --- a/c_glib/test/test-file-writer.rb +++ b/c_glib/test/test-file-writer.rb @@ -19,14 +19,18 @@ class TestFileWriter < Test::Unit::TestCase include Helper::Buildable def test_write_record_batch + data = [true] + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + tempfile = Tempfile.open("arrow-ipc-file-writer") output = Arrow::FileOutputStream.new(tempfile.path, false) begin - field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) - schema = Arrow::Schema.new([field]) file_writer = Arrow::RecordBatchFileWriter.new(output, schema) begin - record_batch = Arrow::RecordBatch.new(schema, 0, []) + record_batch = Arrow::RecordBatch.new(schema, + data.size, + [build_boolean_array(data)]) file_writer.write_record_batch(record_batch) ensure file_writer.close @@ -38,8 +42,12 @@ def test_write_record_batch input = Arrow::MemoryMappedInputStream.new(tempfile.path) begin file_reader = Arrow::RecordBatchFileReader.new(input) - assert_equal(["enabled"], + assert_equal([field.name], file_reader.schema.fields.collect(&:name)) + assert_equal(Arrow::RecordBatch.new(schema, + data.size, + [build_boolean_array(data)]), + file_reader.read_record_batch(0)) ensure input.close end diff --git a/c_glib/test/test-gio-input-stream.rb b/c_glib/test/test-gio-input-stream.rb index a71a370430e6d..2adf25b3af51f 100644 --- a/c_glib/test/test-gio-input-stream.rb +++ b/c_glib/test/test-gio-input-stream.rb @@ -16,15 +16,21 @@ # under the License. class TestGIOInputStream < Test::Unit::TestCase + include Helper::Buildable + def test_reader_backend + data = [true] + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + tempfile = Tempfile.open("arrow-gio-input-stream") output = Arrow::FileOutputStream.new(tempfile.path, false) begin - field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) - schema = Arrow::Schema.new([field]) file_writer = Arrow::RecordBatchFileWriter.new(output, schema) begin - record_batch = Arrow::RecordBatch.new(schema, 0, []) + record_batch = Arrow::RecordBatch.new(schema, + data.size, + [build_boolean_array(data)]) file_writer.write_record_batch(record_batch) ensure file_writer.close @@ -38,8 +44,12 @@ def test_reader_backend input = Arrow::GIOInputStream.new(input_stream) begin file_reader = Arrow::RecordBatchFileReader.new(input) - assert_equal(["enabled"], + assert_equal([field.name], file_reader.schema.fields.collect(&:name)) + assert_equal(Arrow::RecordBatch.new(schema, + data.size, + [build_boolean_array(data)]), + file_reader.read_record_batch(0)) ensure input.close end diff --git a/c_glib/test/test-gio-output-stream.rb b/c_glib/test/test-gio-output-stream.rb index adaa8c1b7b2b0..c77598ed110d5 100644 --- a/c_glib/test/test-gio-output-stream.rb +++ b/c_glib/test/test-gio-output-stream.rb @@ -16,17 +16,23 @@ # under the License. class TestGIOOutputStream < Test::Unit::TestCase + include Helper::Buildable + def test_writer_backend + data = [true] + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + tempfile = Tempfile.open("arrow-gio-output-stream") file = Gio::File.new_for_path(tempfile.path) output_stream = file.append_to(:none) output = Arrow::GIOOutputStream.new(output_stream) begin - field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) - schema = Arrow::Schema.new([field]) file_writer = Arrow::RecordBatchFileWriter.new(output, schema) begin - record_batch = Arrow::RecordBatch.new(schema, 0, []) + record_batch = Arrow::RecordBatch.new(schema, + data.size, + [build_boolean_array(data)]) file_writer.write_record_batch(record_batch) ensure file_writer.close @@ -38,8 +44,12 @@ def test_writer_backend input = Arrow::MemoryMappedInputStream.new(tempfile.path) begin file_reader = Arrow::RecordBatchFileReader.new(input) - assert_equal(["enabled"], + assert_equal([field.name], file_reader.schema.fields.collect(&:name)) + assert_equal(Arrow::RecordBatch.new(schema, + data.size, + [build_boolean_array(data)]), + file_reader.read_record_batch(0)) ensure input.close end diff --git a/c_glib/test/test-stream-writer.rb b/c_glib/test/test-stream-writer.rb index c3d0e1490cef6..32754e20838b4 100644 --- a/c_glib/test/test-stream-writer.rb +++ b/c_glib/test/test-stream-writer.rb @@ -19,17 +19,19 @@ class TestStreamWriter < Test::Unit::TestCase include Helper::Buildable def test_write_record_batch + data = [true] + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + tempfile = Tempfile.open("arrow-ipc-stream-writer") output = Arrow::FileOutputStream.new(tempfile.path, false) begin - field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) - schema = Arrow::Schema.new([field]) stream_writer = Arrow::RecordBatchStreamWriter.new(output, schema) begin columns = [ - build_boolean_array([true]), + build_boolean_array(data), ] - record_batch = Arrow::RecordBatch.new(schema, 1, columns) + record_batch = Arrow::RecordBatch.new(schema, data.size, columns) stream_writer.write_record_batch(record_batch) ensure stream_writer.close @@ -41,10 +43,12 @@ def test_write_record_batch input = Arrow::MemoryMappedInputStream.new(tempfile.path) begin stream_reader = Arrow::RecordBatchStreamReader.new(input) - assert_equal(["enabled"], + assert_equal([field.name], stream_reader.schema.fields.collect(&:name)) - assert_equal(true, - stream_reader.read_next.get_column(0).get_value(0)) + assert_equal(Arrow::RecordBatch.new(schema, + data.size, + [build_boolean_array(data)]), + stream_reader.read_next) assert_nil(stream_reader.read_next) ensure input.close diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index 4998f190f9891..664f7ce5fed2d 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -91,12 +91,14 @@ fi if [ $TRAVIS_OS_NAME == "linux" ]; then cmake $CMAKE_COMMON_FLAGS \ $CMAKE_LINUX_FLAGS \ - -DBUILD_WARNING_LEVEL=CHECKIN \ + -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ + -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \ $ARROW_CPP_DIR else cmake $CMAKE_COMMON_FLAGS \ $CMAKE_OSX_FLAGS \ - -DBUILD_WARNING_LEVEL=CHECKIN \ + -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ + -DBUILD_WARNING_LEVEL=$ARROW_BUILD_WARNING_LEVEL \ $ARROW_CPP_DIR fi diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh index 52c7da4e0178e..21b6e266ea678 100755 --- a/ci/travis_env_common.sh +++ b/ci/travis_env_common.sh @@ -38,6 +38,9 @@ export ARROW_PYTHON_PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env export CMAKE_EXPORT_COMPILE_COMMANDS=1 +export ARROW_BUILD_TYPE=${ARROW_BUILD_TYPE:=debug} +export ARROW_BUILD_WARNING_LEVEL=${ARROW_BUILD_WARNING_LEVEL:=Production} + if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then # C++ toolchain export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 603201bcc166b..5f7b0a9a1afed 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -63,6 +63,7 @@ cmake -GNinja \ -DARROW_BUILD_UTILITIES=off \ -DARROW_PLASMA=on \ -DARROW_PYTHON=on \ + -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ $ARROW_CPP_DIR @@ -78,6 +79,8 @@ if [ "$PYTHON_VERSION" == "2.7" ]; then pip install futures fi +export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE + pip install -r requirements.txt python setup.py build_ext --with-parquet --with-plasma \ install --single-version-externally-managed --record=record.text diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 496e0da9d621e..94705781fa42d 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -22,6 +22,7 @@ set(ARROW_SRCS compare.cc memory_pool.cc pretty_print.cc + record_batch.cc status.cc table.cc table_builder.cc @@ -144,6 +145,7 @@ install(FILES compare.h memory_pool.h pretty_print.h + record_batch.h status.h table.h table_builder.h diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h index 5d2e859f3a4be..7cae8414a774b 100644 --- a/cpp/src/arrow/api.h +++ b/cpp/src/arrow/api.h @@ -26,6 +26,7 @@ #include "arrow/compare.h" #include "arrow/memory_pool.h" #include "arrow/pretty_print.h" +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/table_builder.h" diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index 28756a6abdafe..dda9dd38be4a7 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -279,6 +279,8 @@ class ARROW_EXPORT Array { ARROW_DISALLOW_COPY_AND_ASSIGN(Array); }; +using ArrayVector = std::vector>; + static inline std::ostream& operator<<(std::ostream& os, const Array& x) { os << x.ToString(); return os; diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index 3e213fcd5ca70..a42f9024545ec 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -28,7 +28,6 @@ #include "arrow/buffer.h" #include "arrow/compare.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit-util.h" diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index 32741b53ac438..e59e166580ae4 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -29,7 +29,6 @@ #include "arrow/buffer.h" #include "arrow/memory_pool.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit-util.h" diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc index e50ddf6d7032d..af2c368c32975 100644 --- a/cpp/src/arrow/column-benchmark.cc +++ b/cpp/src/arrow/column-benchmark.cc @@ -19,6 +19,7 @@ #include "arrow/array.h" #include "arrow/memory_pool.h" +#include "arrow/table.h" #include "arrow/test-util.h" namespace arrow { diff --git a/cpp/src/arrow/compute/kernel.h b/cpp/src/arrow/compute/kernel.h index 7ff506ca0cf2b..0bfa55cfee198 100644 --- a/cpp/src/arrow/compute/kernel.h +++ b/cpp/src/arrow/compute/kernel.h @@ -22,6 +22,7 @@ #include #include "arrow/array.h" +#include "arrow/record_batch.h" #include "arrow/table.h" #include "arrow/util/macros.h" #include "arrow/util/variant.h" diff --git a/cpp/src/arrow/gpu/cuda_arrow_ipc.cc b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc index 022268e034758..a7262c8b4d4ba 100644 --- a/cpp/src/arrow/gpu/cuda_arrow_ipc.cc +++ b/cpp/src/arrow/gpu/cuda_arrow_ipc.cc @@ -27,8 +27,8 @@ #include "arrow/ipc/message.h" #include "arrow/ipc/reader.h" #include "arrow/ipc/writer.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/util/visibility.h" #include "arrow/gpu/cuda_context.h" diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc index 6bd16462df94d..e3de17f1f75cd 100644 --- a/cpp/src/arrow/ipc/feather-test.cc +++ b/cpp/src/arrow/ipc/feather-test.cc @@ -29,6 +29,7 @@ #include "arrow/ipc/feather.h" #include "arrow/ipc/test-common.h" #include "arrow/pretty_print.h" +#include "arrow/table.h" #include "arrow/test-util.h" namespace arrow { @@ -376,8 +377,8 @@ TEST_F(TestTableWriter, TimeTypes) { schema->field(i)->type(), values->length(), buffers, values->null_count(), 0)); } - RecordBatch batch(schema, values->length(), std::move(arrays)); - CheckBatch(batch); + auto batch = RecordBatch::Make(schema, values->length(), std::move(arrays)); + CheckBatch(*batch); } TEST_F(TestTableWriter, VLenPrimitiveRoundTrip) { diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc index cea720bd01bc7..077dc39305b09 100644 --- a/cpp/src/arrow/ipc/feather.cc +++ b/cpp/src/arrow/ipc/feather.cc @@ -32,6 +32,7 @@ #include "arrow/ipc/feather-internal.h" #include "arrow/ipc/feather_generated.h" #include "arrow/ipc/util.h" // IWYU pragma: keep +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/type.h" diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc index a560f09d6fdb1..e496826f96b9b 100644 --- a/cpp/src/arrow/ipc/ipc-json-test.cc +++ b/cpp/src/arrow/ipc/ipc-json-test.cc @@ -31,8 +31,8 @@ #include "arrow/ipc/json.h" #include "arrow/ipc/test-common.h" #include "arrow/memory_pool.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/test-util.h" #include "arrow/type.h" #include "arrow/type_traits.h" @@ -269,7 +269,7 @@ TEST(TestJsonFileReadWrite, BasicRoundTrip) { std::vector> arrays; MakeBatchArrays(schema, num_rows, &arrays); - auto batch = std::make_shared(schema, num_rows, arrays); + auto batch = RecordBatch::Make(schema, num_rows, arrays); batches.push_back(batch); ASSERT_OK(writer->WriteRecordBatch(*batch)); } diff --git a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc index 9ed0abde651f6..8561fb8603707 100644 --- a/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc +++ b/cpp/src/arrow/ipc/ipc-read-write-benchmark.cc @@ -63,7 +63,7 @@ std::shared_ptr MakeRecordBatch(int64_t total_size, int64_t num_fie } auto schema = std::make_shared(fields); - return std::make_shared(schema, length, arrays); + return RecordBatch::Make(schema, length, arrays); } static void BM_WriteRecordBatch(benchmark::State& state) { // NOLINT non-const reference diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc index 40cd3f0eef0b8..1fcbdac5ebc73 100644 --- a/cpp/src/arrow/ipc/ipc-read-write-test.cc +++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc @@ -197,8 +197,8 @@ class IpcTestFixture : public io::MemoryMapFixture { std::vector> fields = {f0}; auto schema = std::make_shared(fields); - RecordBatch batch(schema, 0, {array}); - CheckRoundtrip(batch, buffer_size); + auto batch = RecordBatch::Make(schema, 0, {array}); + CheckRoundtrip(*batch, buffer_size); } protected: @@ -292,13 +292,13 @@ TEST_F(TestWriteRecordBatch, SliceTruncatesBuffers) { auto CheckArray = [this](const std::shared_ptr& array) { auto f0 = field("f0", array->type()); auto schema = ::arrow::schema({f0}); - RecordBatch batch(schema, array->length(), {array}); - auto sliced_batch = batch.Slice(0, 5); + auto batch = RecordBatch::Make(schema, array->length(), {array}); + auto sliced_batch = batch->Slice(0, 5); int64_t full_size; int64_t sliced_size; - ASSERT_OK(GetRecordBatchSize(batch, &full_size)); + ASSERT_OK(GetRecordBatchSize(*batch, &full_size)); ASSERT_OK(GetRecordBatchSize(*sliced_batch, &sliced_size)); ASSERT_TRUE(sliced_size < full_size) << sliced_size << " " << full_size; @@ -411,8 +411,7 @@ class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture { *schema = ::arrow::schema({f0}); - std::vector> arrays = {array}; - *batch = std::make_shared(*schema, batch_length, arrays); + *batch = RecordBatch::Make(*schema, batch_length, {array}); std::stringstream ss; ss << "test-write-past-max-recursion-" << g_file_number++; @@ -632,7 +631,7 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) { std::vector> fields = {f0}; auto schema = std::make_shared(fields); - RecordBatch batch(schema, length, {array}); + auto batch = RecordBatch::Make(schema, length, {array}); std::string path = "test-write-large-record_batch"; @@ -641,8 +640,8 @@ TEST_F(TestIpcRoundTrip, LargeRecordBatch) { ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_)); std::shared_ptr result; - ASSERT_OK(DoLargeRoundTrip(batch, false, &result)); - CheckReadResult(*result, batch); + ASSERT_OK(DoLargeRoundTrip(*batch, false, &result)); + CheckReadResult(*result, *batch); ASSERT_EQ(length, result->num_rows()); } diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc index c7530a467b398..f487487dfda9f 100644 --- a/cpp/src/arrow/ipc/json-integration-test.cc +++ b/cpp/src/arrow/ipc/json-integration-test.cc @@ -34,8 +34,8 @@ #include "arrow/ipc/reader.h" #include "arrow/ipc/writer.h" #include "arrow/pretty_print.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/test-util.h" #include "arrow/type.h" diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index bdf1ef52b40eb..bfb3d282d8772 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -28,8 +28,8 @@ #include "arrow/array.h" #include "arrow/builder.h" #include "arrow/ipc/dictionary.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit-util.h" @@ -125,8 +125,8 @@ class SchemaWriter { // Make a dummy record batch. A bit tedious as we have to make a schema auto schema = ::arrow::schema({arrow::field("dictionary", dictionary->type())}); - RecordBatch batch(schema, dictionary->length(), {dictionary}); - RETURN_NOT_OK(WriteRecordBatch(batch, writer_)); + auto batch = RecordBatch::Make(schema, dictionary->length(), {dictionary}); + RETURN_NOT_OK(WriteRecordBatch(*batch, writer_)); writer_->EndObject(); return Status::OK(); } @@ -1435,7 +1435,7 @@ Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr& RETURN_NOT_OK(ReadArray(pool, json_columns[i], type, &columns[i])); } - *batch = std::make_shared(schema, num_rows, columns); + *batch = RecordBatch::Make(schema, num_rows, columns); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc index 30a1bb81e1a1e..ea2947d5d4c36 100644 --- a/cpp/src/arrow/ipc/json.cc +++ b/cpp/src/arrow/ipc/json.cc @@ -24,8 +24,8 @@ #include "arrow/buffer.h" #include "arrow/ipc/json-internal.h" #include "arrow/memory_pool.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/type.h" #include "arrow/util/logging.h" diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 8e10d7d66f907..5960e81883d6b 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -37,8 +37,8 @@ #include "arrow/ipc/message.h" #include "arrow/ipc/metadata-internal.h" #include "arrow/ipc/util.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/tensor.h" #include "arrow/type.h" #include "arrow/util/bit-util.h" @@ -307,7 +307,7 @@ static Status LoadRecordBatchFromSource(const std::shared_ptr& schema, arrays[i] = std::move(arr); } - *out = std::make_shared(schema, num_rows, std::move(arrays)); + *out = RecordBatch::Make(schema, num_rows, std::move(arrays)); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h index 7581fbda5b140..627f67e2517e9 100644 --- a/cpp/src/arrow/ipc/reader.h +++ b/cpp/src/arrow/ipc/reader.h @@ -24,13 +24,12 @@ #include #include "arrow/ipc/message.h" -#include "arrow/table.h" +#include "arrow/record_batch.h" #include "arrow/util/visibility.h" namespace arrow { class Buffer; -class RecordBatch; class Schema; class Status; class Tensor; diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h index 7fc139381052c..6f8a0dcc61fbc 100644 --- a/cpp/src/arrow/ipc/test-common.h +++ b/cpp/src/arrow/ipc/test-common.h @@ -30,8 +30,8 @@ #include "arrow/builder.h" #include "arrow/memory_pool.h" #include "arrow/pretty_print.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/test-util.h" #include "arrow/type.h" #include "arrow/util/bit-util.h" @@ -184,7 +184,7 @@ Status MakeBooleanBatchSized(const int length, std::shared_ptr* out std::shared_ptr a0, a1; RETURN_NOT_OK(MakeRandomBooleanArray(length, true, &a0)); RETURN_NOT_OK(MakeRandomBooleanArray(length, false, &a1)); - out->reset(new RecordBatch(schema, length, {a0, a1})); + *out = RecordBatch::Make(schema, length, {a0, a1}); return Status::OK(); } @@ -203,7 +203,7 @@ Status MakeIntBatchSized(int length, std::shared_ptr* out) { MemoryPool* pool = default_memory_pool(); RETURN_NOT_OK(MakeRandomInt32Array(length, false, pool, &a0)); RETURN_NOT_OK(MakeRandomInt32Array(length, true, pool, &a1)); - out->reset(new RecordBatch(schema, length, {a0, a1})); + *out = RecordBatch::Make(schema, length, {a0, a1}); return Status::OK(); } @@ -252,7 +252,7 @@ Status MakeStringTypesRecordBatch(std::shared_ptr* out) { auto s = MakeRandomBinaryArray(length, true, pool, &a1); RETURN_NOT_OK(s); } - out->reset(new RecordBatch(schema, length, {a0, a1})); + *out = RecordBatch::Make(schema, length, {a0, a1}); return Status::OK(); } @@ -261,7 +261,7 @@ Status MakeNullRecordBatch(std::shared_ptr* out) { auto f0 = field("f0", null()); auto schema = ::arrow::schema({f0}); std::shared_ptr a0 = std::make_shared(length); - out->reset(new RecordBatch(schema, length, {a0})); + *out = RecordBatch::Make(schema, length, {a0}); return Status::OK(); } @@ -284,7 +284,7 @@ Status MakeListRecordBatch(std::shared_ptr* out) { RETURN_NOT_OK( MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array)); RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array)); - out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array})); + *out = RecordBatch::Make(schema, length, {list_array, list_list_array, flat_array}); return Status::OK(); } @@ -304,7 +304,7 @@ Status MakeZeroLengthRecordBatch(std::shared_ptr* out) { RETURN_NOT_OK( MakeRandomListArray(list_array, 0, include_nulls, pool, &list_list_array)); RETURN_NOT_OK(MakeRandomInt32Array(0, include_nulls, pool, &flat_array)); - out->reset(new RecordBatch(schema, 0, {list_array, list_list_array, flat_array})); + *out = RecordBatch::Make(schema, 0, {list_array, list_list_array, flat_array}); return Status::OK(); } @@ -327,7 +327,7 @@ Status MakeNonNullRecordBatch(std::shared_ptr* out) { RETURN_NOT_OK( MakeRandomListArray(list_array, length, include_nulls, pool, &list_list_array)); RETURN_NOT_OK(MakeRandomInt32Array(length, include_nulls, pool, &flat_array)); - out->reset(new RecordBatch(schema, length, {list_array, list_list_array, flat_array})); + *out = RecordBatch::Make(schema, length, {list_array, list_list_array, flat_array}); return Status::OK(); } @@ -347,7 +347,7 @@ Status MakeDeeplyNestedList(std::shared_ptr* out) { auto f0 = field("f0", type); auto schema = ::arrow::schema({f0}); std::vector> arrays = {array}; - out->reset(new RecordBatch(schema, batch_length, arrays)); + *out = RecordBatch::Make(schema, batch_length, arrays); return Status::OK(); } @@ -377,7 +377,7 @@ Status MakeStruct(std::shared_ptr* out) { // construct batch std::vector> arrays = {no_nulls, with_nulls}; - out->reset(new RecordBatch(schema, list_batch->num_rows(), arrays)); + *out = RecordBatch::Make(schema, list_batch->num_rows(), arrays); return Status::OK(); } @@ -445,7 +445,7 @@ Status MakeUnion(std::shared_ptr* out) { // construct batch std::vector> arrays = {sparse_no_nulls, sparse, dense}; - out->reset(new RecordBatch(schema, length, arrays)); + *out = RecordBatch::Make(schema, length, arrays); return Status::OK(); } @@ -526,7 +526,7 @@ Status MakeDictionary(std::shared_ptr* out) { std::vector> arrays = {a0, a1, a2, a3, a4}; - out->reset(new RecordBatch(schema, length, arrays)); + *out = RecordBatch::Make(schema, length, arrays); return Status::OK(); } @@ -564,7 +564,7 @@ Status MakeDictionaryFlat(std::shared_ptr* out) { {field("dict1", f0_type), field("sparse", f1_type), field("dense", f2_type)}); std::vector> arrays = {a0, a1, a2}; - out->reset(new RecordBatch(schema, length, arrays)); + *out = RecordBatch::Make(schema, length, arrays); return Status::OK(); } @@ -584,8 +584,7 @@ Status MakeDates(std::shared_ptr* out) { std::shared_ptr date64_array; ArrayFromVector(is_valid, date64_values, &date64_array); - std::vector> arrays = {date32_array, date64_array}; - *out = std::make_shared(schema, date32_array->length(), arrays); + *out = RecordBatch::Make(schema, date32_array->length(), {date32_array, date64_array}); return Status::OK(); } @@ -604,8 +603,7 @@ Status MakeTimestamps(std::shared_ptr* out) { ArrayFromVector(f1->type(), is_valid, ts_values, &a1); ArrayFromVector(f2->type(), is_valid, ts_values, &a2); - ArrayVector arrays = {a0, a1, a2}; - *out = std::make_shared(schema, a0->length(), arrays); + *out = RecordBatch::Make(schema, a0->length(), {a0, a1, a2}); return Status::OK(); } @@ -628,8 +626,7 @@ Status MakeTimes(std::shared_ptr* out) { ArrayFromVector(f2->type(), is_valid, t32_values, &a2); ArrayFromVector(f3->type(), is_valid, t64_values, &a3); - ArrayVector arrays = {a0, a1, a2, a3}; - *out = std::make_shared(schema, a0->length(), arrays); + *out = RecordBatch::Make(schema, a0->length(), {a0, a1, a2, a3}); return Status::OK(); } @@ -665,8 +662,7 @@ Status MakeFWBinary(std::shared_ptr* out) { RETURN_NOT_OK(b1.Finish(&a1)); RETURN_NOT_OK(b2.Finish(&a2)); - ArrayVector arrays = {a1, a2}; - *out = std::make_shared(schema, a1->length(), arrays); + *out = RecordBatch::Make(schema, a1->length(), {a1, a2}); return Status::OK(); } @@ -695,8 +691,7 @@ Status MakeDecimal(std::shared_ptr* out) { auto a2 = std::make_shared(f1->type(), length, data); - ArrayVector arrays = {a1, a2}; - *out = std::make_shared(schema, length, arrays); + *out = RecordBatch::Make(schema, length, {a1, a2}); return Status::OK(); } @@ -716,8 +711,7 @@ Status MakeNull(std::shared_ptr* out) { std::shared_ptr a2; ArrayFromVector(f1->type(), is_valid, int_values, &a2); - ArrayVector arrays = {a1, a2}; - *out = std::make_shared(schema, a1->length(), arrays); + *out = RecordBatch::Make(schema, a1->length(), {a1, a2}); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index 323116f589b73..3c1db06159e65 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -32,6 +32,7 @@ #include "arrow/ipc/metadata-internal.h" #include "arrow/ipc/util.h" #include "arrow/memory_pool.h" +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/tensor.h" @@ -508,12 +509,9 @@ class DictionaryWriter : public RecordBatchSerializer { dictionary_id_ = dictionary_id; // Make a dummy record batch. A bit tedious as we have to make a schema - std::vector> fields = { - arrow::field("dictionary", dictionary->type())}; - auto schema = std::make_shared(fields); - RecordBatch batch(schema, dictionary->length(), {dictionary}); - - return RecordBatchSerializer::Write(batch, dst, metadata_length, body_length); + auto schema = arrow::schema({arrow::field("dictionary", dictionary->type())}); + auto batch = RecordBatch::Make(schema, dictionary->length(), {dictionary}); + return RecordBatchSerializer::Write(*batch, dst, metadata_length, body_length); } private: diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc index cfbc30315fcd7..bd5f8ce10ea68 100644 --- a/cpp/src/arrow/pretty_print.cc +++ b/cpp/src/arrow/pretty_print.cc @@ -22,8 +22,8 @@ #include "arrow/array.h" #include "arrow/pretty_print.h" +#include "arrow/record_batch.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/logging.h" diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc index 86391a18598fd..3b7d7d884ef09 100644 --- a/cpp/src/arrow/python/python-test.cc +++ b/cpp/src/arrow/python/python-test.cc @@ -23,6 +23,7 @@ #include "arrow/array.h" #include "arrow/builder.h" +#include "arrow/table.h" #include "arrow/test-util.h" #include "arrow/python/arrow_to_pandas.h" @@ -81,8 +82,8 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) { std::vector> fields = {f1, f2, f3}; std::vector> cols = {arr, arr, arr}; - auto schema = std::make_shared(fields); - auto table = std::make_shared(schema, cols); + auto schema = ::arrow::schema(fields); + auto table = Table::Make(schema, cols); PyObject* out; Py_BEGIN_ALLOW_THREADS; diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc index b0c6287f088a1..72cc5b6e1db67 100644 --- a/cpp/src/arrow/python/python_to_arrow.cc +++ b/cpp/src/arrow/python/python_to_arrow.cc @@ -32,13 +32,15 @@ #include "arrow/builder.h" #include "arrow/io/interfaces.h" #include "arrow/ipc/writer.h" +#include "arrow/record_batch.h" +#include "arrow/tensor.h" +#include "arrow/util/logging.h" + #include "arrow/python/common.h" #include "arrow/python/helpers.h" #include "arrow/python/numpy_convert.h" #include "arrow/python/platform.h" #include "arrow/python/util/datetime.h" -#include "arrow/tensor.h" -#include "arrow/util/logging.h" constexpr int32_t kMaxRecursionDepth = 100; @@ -694,7 +696,7 @@ Status SerializeDict(PyObject* context, std::vector dicts, std::shared_ptr MakeBatch(std::shared_ptr data) { auto field = std::make_shared("list", data->type()); auto schema = ::arrow::schema({field}); - return std::shared_ptr(new RecordBatch(schema, data->length(), {data})); + return RecordBatch::Make(schema, data->length(), {data}); } Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out) { diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc new file mode 100644 index 0000000000000..60932bdf3e4bb --- /dev/null +++ b/cpp/src/arrow/record_batch.cc @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/record_batch.h" + +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/status.h" +#include "arrow/type.h" +#include "arrow/util/logging.h" + +namespace arrow { + +/// \class SimpleRecordBatch +/// \brief A basic, non-lazy in-memory record batch +class SimpleRecordBatch : public RecordBatch { + public: + SimpleRecordBatch(const std::shared_ptr& schema, int64_t num_rows, + const std::vector>& columns) + : RecordBatch(schema, num_rows) { + columns_.resize(columns.size()); + boxed_columns_.resize(schema->num_fields()); + for (size_t i = 0; i < columns.size(); ++i) { + columns_[i] = columns[i]->data(); + } + } + + SimpleRecordBatch(const std::shared_ptr& schema, int64_t num_rows, + std::vector>&& columns) + : RecordBatch(schema, num_rows) { + columns_.resize(columns.size()); + boxed_columns_.resize(schema->num_fields()); + for (size_t i = 0; i < columns.size(); ++i) { + columns_[i] = columns[i]->data(); + } + } + + SimpleRecordBatch(const std::shared_ptr& schema, int64_t num_rows, + std::vector>&& columns) + : RecordBatch(schema, num_rows) { + columns_ = std::move(columns); + boxed_columns_.resize(schema->num_fields()); + } + + SimpleRecordBatch(const std::shared_ptr& schema, int64_t num_rows, + const std::vector>& columns) + : RecordBatch(schema, num_rows) { + columns_ = columns; + boxed_columns_.resize(schema->num_fields()); + } + + std::shared_ptr column(int i) const override { + if (!boxed_columns_[i]) { + boxed_columns_[i] = MakeArray(columns_[i]); + } + DCHECK(boxed_columns_[i]); + return boxed_columns_[i]; + } + + std::shared_ptr column_data(int i) const override { return columns_[i]; } + + std::shared_ptr ReplaceSchemaMetadata( + const std::shared_ptr& metadata) const override { + auto new_schema = schema_->AddMetadata(metadata); + return RecordBatch::Make(new_schema, num_rows_, columns_); + } + + std::shared_ptr Slice(int64_t offset, int64_t length) const override { + std::vector> arrays; + arrays.reserve(num_columns()); + for (const auto& field : columns_) { + int64_t col_length = std::min(field->length - offset, length); + int64_t col_offset = field->offset + offset; + + auto new_data = std::make_shared(*field); + new_data->length = col_length; + new_data->offset = col_offset; + new_data->null_count = kUnknownNullCount; + arrays.emplace_back(new_data); + } + int64_t num_rows = std::min(num_rows_ - offset, length); + return std::make_shared(schema_, num_rows, std::move(arrays)); + } + + Status Validate() const override { + if (static_cast(columns_.size()) != schema_->num_fields()) { + return Status::Invalid("Number of columns did not match schema"); + } + return RecordBatch::Validate(); + } + + private: + std::vector> columns_; + + // Caching boxed array data + mutable std::vector> boxed_columns_; +}; + +RecordBatch::RecordBatch(const std::shared_ptr& schema, int64_t num_rows) + : schema_(schema), num_rows_(num_rows) {} + +std::shared_ptr RecordBatch::Make( + const std::shared_ptr& schema, int64_t num_rows, + const std::vector>& columns) { + return std::make_shared(schema, num_rows, columns); +} + +std::shared_ptr RecordBatch::Make( + const std::shared_ptr& schema, int64_t num_rows, + std::vector>&& columns) { + return std::make_shared(schema, num_rows, std::move(columns)); +} + +std::shared_ptr RecordBatch::Make( + const std::shared_ptr& schema, int64_t num_rows, + std::vector>&& columns) { + return std::make_shared(schema, num_rows, std::move(columns)); +} + +std::shared_ptr RecordBatch::Make( + const std::shared_ptr& schema, int64_t num_rows, + const std::vector>& columns) { + return std::make_shared(schema, num_rows, columns); +} + +const std::string& RecordBatch::column_name(int i) const { + return schema_->field(i)->name(); +} + +bool RecordBatch::Equals(const RecordBatch& other) const { + if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) { + return false; + } + + for (int i = 0; i < num_columns(); ++i) { + if (!column(i)->Equals(other.column(i))) { + return false; + } + } + + return true; +} + +bool RecordBatch::ApproxEquals(const RecordBatch& other) const { + if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) { + return false; + } + + for (int i = 0; i < num_columns(); ++i) { + if (!column(i)->ApproxEquals(other.column(i))) { + return false; + } + } + + return true; +} + +std::shared_ptr RecordBatch::Slice(int64_t offset) const { + return Slice(offset, this->num_rows() - offset); +} + +Status RecordBatch::Validate() const { + for (int i = 0; i < num_columns(); ++i) { + auto arr_shared = this->column_data(i); + const ArrayData& arr = *arr_shared; + if (arr.length != num_rows_) { + std::stringstream ss; + ss << "Number of rows in column " << i << " did not match batch: " << arr.length + << " vs " << num_rows_; + return Status::Invalid(ss.str()); + } + const auto& schema_type = *schema_->field(i)->type(); + if (!arr.type->Equals(schema_type)) { + std::stringstream ss; + ss << "Column " << i << " type not match schema: " << arr.type->ToString() << " vs " + << schema_type.ToString(); + return Status::Invalid(ss.str()); + } + } + return Status::OK(); +} + +// ---------------------------------------------------------------------- +// Base record batch reader + +RecordBatchReader::~RecordBatchReader() {} + +} // namespace arrow diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h new file mode 100644 index 0000000000000..b2c4c76b3f2d3 --- /dev/null +++ b/cpp/src/arrow/record_batch.h @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_RECORD_BATCH_H +#define ARROW_RECORD_BATCH_H + +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/type.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +class KeyValueMetadata; +class Status; + +/// \class RecordBatch +/// \brief Collection of equal-length arrays matching a particular Schema +/// +/// A record batch is table-like data structure that is semantically a sequence +/// of fields, each a contiguous Arrow array +class ARROW_EXPORT RecordBatch { + public: + virtual ~RecordBatch() = default; + + /// \param[in] schema The record batch schema + /// \param[in] num_rows length of fields in the record batch. Each array + /// should have the same length as num_rows + /// \param[in] columns the record batch fields as vector of arrays + static std::shared_ptr Make( + const std::shared_ptr& schema, int64_t num_rows, + const std::vector>& columns); + + /// \brief Move-based constructor for a vector of Array instances + static std::shared_ptr Make(const std::shared_ptr& schema, + int64_t num_rows, + std::vector>&& columns); + + /// \brief Construct record batch from vector of internal data structures + /// \since 0.5.0 + /// + /// This class is only provided with an rvalue-reference for the input data, + /// and is intended for internal use, or advanced users. + /// + /// \param schema the record batch schema + /// \param num_rows the number of semantic rows in the record batch. This + /// should be equal to the length of each field + /// \param columns the data for the batch's columns + static std::shared_ptr Make( + const std::shared_ptr& schema, int64_t num_rows, + std::vector>&& columns); + + /// \brief Construct record batch by copying vector of array data + /// \since 0.5.0 + static std::shared_ptr Make( + const std::shared_ptr& schema, int64_t num_rows, + const std::vector>& columns); + + /// \brief Determine if two record batches are exactly equal + /// \return true if batches are equal + bool Equals(const RecordBatch& other) const; + + /// \brief Determine if two record batches are approximately equal + bool ApproxEquals(const RecordBatch& other) const; + + // \return the table's schema + /// \return true if batches are equal + std::shared_ptr schema() const { return schema_; } + + /// \brief Retrieve an array from the record batch + /// \param[in] i field index, does not boundscheck + /// \return an Array object + virtual std::shared_ptr column(int i) const = 0; + + /// \brief Retrieve an array's internaldata from the record batch + /// \param[in] i field index, does not boundscheck + /// \return an internal ArrayData object + virtual std::shared_ptr column_data(int i) const = 0; + + virtual std::shared_ptr ReplaceSchemaMetadata( + const std::shared_ptr& metadata) const = 0; + + /// \brief Name in i-th column + const std::string& column_name(int i) const; + + /// \return the number of columns in the table + int num_columns() const { return schema_->num_fields(); } + + /// \return the number of rows (the corresponding length of each column) + int64_t num_rows() const { return num_rows_; } + + /// \brief Slice each of the arrays in the record batch + /// \param[in] offset the starting offset to slice, through end of batch + /// \return new record batch + virtual std::shared_ptr Slice(int64_t offset) const; + + /// \brief Slice each of the arrays in the record batch + /// \param[in] offset the starting offset to slice + /// \param[in] length the number of elements to slice from offset + /// \return new record batch + virtual std::shared_ptr Slice(int64_t offset, int64_t length) const = 0; + + /// \brief Check for schema or length inconsistencies + /// \return Status + virtual Status Validate() const; + + protected: + RecordBatch(const std::shared_ptr& schema, int64_t num_rows); + + std::shared_ptr schema_; + int64_t num_rows_; + + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(RecordBatch); +}; + +/// \brief Abstract interface for reading stream of record batches +class ARROW_EXPORT RecordBatchReader { + public: + virtual ~RecordBatchReader(); + + /// \return the shared schema of the record batches in the stream + virtual std::shared_ptr schema() const = 0; + + /// Read the next record batch in the stream. Return null for batch when + /// reaching end of stream + /// + /// \param[out] batch the next loaded batch, null at end of stream + /// \return Status + virtual Status ReadNext(std::shared_ptr* batch) = 0; +}; + +} // namespace arrow + +#endif // ARROW_RECORD_BATCH_H diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc index b490310c26ae6..e77d3aa8bbc08 100644 --- a/cpp/src/arrow/table-test.cc +++ b/cpp/src/arrow/table-test.cc @@ -22,6 +22,7 @@ #include "gtest/gtest.h" #include "arrow/array.h" +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/test-common.h" @@ -216,8 +217,8 @@ class TestTable : public TestBase { TEST_F(TestTable, EmptySchema) { auto empty_schema = ::arrow::schema({}); - table_.reset(new Table(empty_schema, columns_)); - ASSERT_OK(table_->ValidateColumns()); + table_ = Table::Make(empty_schema, columns_); + ASSERT_OK(table_->Validate()); ASSERT_EQ(0, table_->num_rows()); ASSERT_EQ(0, table_->num_columns()); } @@ -226,20 +227,20 @@ TEST_F(TestTable, Ctors) { const int length = 100; MakeExample1(length); - table_.reset(new Table(schema_, columns_)); - ASSERT_OK(table_->ValidateColumns()); + table_ = Table::Make(schema_, columns_); + ASSERT_OK(table_->Validate()); ASSERT_EQ(length, table_->num_rows()); ASSERT_EQ(3, table_->num_columns()); - auto array_ctor = std::make_shared
    (schema_, arrays_); + auto array_ctor = Table::Make(schema_, arrays_); ASSERT_TRUE(table_->Equals(*array_ctor)); - table_.reset(new Table(schema_, columns_, length)); - ASSERT_OK(table_->ValidateColumns()); + table_ = Table::Make(schema_, columns_, length); + ASSERT_OK(table_->Validate()); ASSERT_EQ(length, table_->num_rows()); - ASSERT_OK(MakeTable(schema_, arrays_, &table_)); - ASSERT_OK(table_->ValidateColumns()); + table_ = Table::Make(schema_, arrays_); + ASSERT_OK(table_->Validate()); ASSERT_EQ(length, table_->num_rows()); ASSERT_EQ(3, table_->num_columns()); } @@ -248,7 +249,7 @@ TEST_F(TestTable, Metadata) { const int length = 100; MakeExample1(length); - table_.reset(new Table(schema_, columns_)); + table_ = Table::Make(schema_, columns_); ASSERT_TRUE(table_->schema()->Equals(*schema_)); @@ -262,14 +263,14 @@ TEST_F(TestTable, InvalidColumns) { const int length = 100; MakeExample1(length); - table_.reset(new Table(schema_, columns_, length - 1)); - ASSERT_RAISES(Invalid, table_->ValidateColumns()); + table_ = Table::Make(schema_, columns_, length - 1); + ASSERT_RAISES(Invalid, table_->Validate()); columns_.clear(); // Wrong number of columns - table_.reset(new Table(schema_, columns_, length)); - ASSERT_RAISES(Invalid, table_->ValidateColumns()); + table_ = Table::Make(schema_, columns_, length); + ASSERT_RAISES(Invalid, table_->Validate()); columns_ = { std::make_shared(schema_->field(0), MakeRandomArray(length)), @@ -277,15 +278,15 @@ TEST_F(TestTable, InvalidColumns) { std::make_shared(schema_->field(2), MakeRandomArray(length - 1))}; - table_.reset(new Table(schema_, columns_, length)); - ASSERT_RAISES(Invalid, table_->ValidateColumns()); + table_ = Table::Make(schema_, columns_, length); + ASSERT_RAISES(Invalid, table_->Validate()); } TEST_F(TestTable, Equals) { const int length = 100; MakeExample1(length); - table_.reset(new Table(schema_, columns_)); + table_ = Table::Make(schema_, columns_); ASSERT_TRUE(table_->Equals(*table_)); // Differing schema @@ -294,7 +295,8 @@ TEST_F(TestTable, Equals) { auto f2 = field("f5", int16()); vector> fields = {f0, f1, f2}; auto other_schema = std::make_shared(fields); - ASSERT_FALSE(table_->Equals(Table(other_schema, columns_))); + auto other = Table::Make(other_schema, columns_); + ASSERT_FALSE(table_->Equals(*other)); // Differing columns std::vector> other_columns = { std::make_shared(schema_->field(0), @@ -303,19 +305,21 @@ TEST_F(TestTable, Equals) { MakeRandomArray(length, 10)), std::make_shared(schema_->field(2), MakeRandomArray(length, 10))}; - ASSERT_FALSE(table_->Equals(Table(schema_, other_columns))); + + other = Table::Make(schema_, other_columns); + ASSERT_FALSE(table_->Equals(*other)); } TEST_F(TestTable, FromRecordBatches) { const int64_t length = 10; MakeExample1(length); - auto batch1 = std::make_shared(schema_, length, arrays_); + auto batch1 = RecordBatch::Make(schema_, length, arrays_); std::shared_ptr
    result, expected; ASSERT_OK(Table::FromRecordBatches({batch1}, &result)); - expected = std::make_shared
    (schema_, columns_); + expected = Table::Make(schema_, columns_); ASSERT_TRUE(result->Equals(*expected)); std::vector> other_columns; @@ -325,18 +329,17 @@ TEST_F(TestTable, FromRecordBatches) { } ASSERT_OK(Table::FromRecordBatches({batch1, batch1}, &result)); - expected = std::make_shared
    (schema_, other_columns); + expected = Table::Make(schema_, other_columns); ASSERT_TRUE(result->Equals(*expected)); // Error states std::vector> empty_batches; ASSERT_RAISES(Invalid, Table::FromRecordBatches(empty_batches, &result)); - std::vector> fields = {schema_->field(0), schema_->field(1)}; - auto other_schema = std::make_shared(fields); + auto other_schema = ::arrow::schema({schema_->field(0), schema_->field(1)}); std::vector> other_arrays = {arrays_[0], arrays_[1]}; - auto batch2 = std::make_shared(other_schema, length, other_arrays); + auto batch2 = RecordBatch::Make(other_schema, length, other_arrays); ASSERT_RAISES(Invalid, Table::FromRecordBatches({batch1, batch2}, &result)); } @@ -344,11 +347,11 @@ TEST_F(TestTable, ConcatenateTables) { const int64_t length = 10; MakeExample1(length); - auto batch1 = std::make_shared(schema_, length, arrays_); + auto batch1 = RecordBatch::Make(schema_, length, arrays_); // generate different data MakeExample1(length); - auto batch2 = std::make_shared(schema_, length, arrays_); + auto batch2 = RecordBatch::Make(schema_, length, arrays_); std::shared_ptr
    t1, t2, t3, result, expected; ASSERT_OK(Table::FromRecordBatches({batch1}, &t1)); @@ -362,11 +365,10 @@ TEST_F(TestTable, ConcatenateTables) { std::vector> empty_tables; ASSERT_RAISES(Invalid, ConcatenateTables(empty_tables, &result)); - std::vector> fields = {schema_->field(0), schema_->field(1)}; - auto other_schema = std::make_shared(fields); + auto other_schema = ::arrow::schema({schema_->field(0), schema_->field(1)}); std::vector> other_arrays = {arrays_[0], arrays_[1]}; - auto batch3 = std::make_shared(other_schema, length, other_arrays); + auto batch3 = RecordBatch::Make(other_schema, length, other_arrays); ASSERT_OK(Table::FromRecordBatches({batch3}, &t3)); ASSERT_RAISES(Invalid, ConcatenateTables({t1, t3}, &result)); @@ -376,31 +378,38 @@ TEST_F(TestTable, RemoveColumn) { const int64_t length = 10; MakeExample1(length); - Table table(schema_, columns_); + auto table_sp = Table::Make(schema_, columns_); + const Table& table = *table_sp; std::shared_ptr
    result; ASSERT_OK(table.RemoveColumn(0, &result)); auto ex_schema = ::arrow::schema({schema_->field(1), schema_->field(2)}); std::vector> ex_columns = {table.column(1), table.column(2)}; - ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns))); + + auto expected = Table::Make(ex_schema, ex_columns); + ASSERT_TRUE(result->Equals(*expected)); ASSERT_OK(table.RemoveColumn(1, &result)); ex_schema = ::arrow::schema({schema_->field(0), schema_->field(2)}); ex_columns = {table.column(0), table.column(2)}; - ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns))); + + expected = Table::Make(ex_schema, ex_columns); + ASSERT_TRUE(result->Equals(*expected)); ASSERT_OK(table.RemoveColumn(2, &result)); ex_schema = ::arrow::schema({schema_->field(0), schema_->field(1)}); ex_columns = {table.column(0), table.column(1)}; - ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns))); + expected = Table::Make(ex_schema, ex_columns); + ASSERT_TRUE(result->Equals(*expected)); } TEST_F(TestTable, AddColumn) { const int64_t length = 10; MakeExample1(length); - Table table(schema_, columns_); + auto table_sp = Table::Make(schema_, columns_); + const Table& table = *table_sp; std::shared_ptr
    result; // Some negative tests with invalid index @@ -419,50 +428,32 @@ TEST_F(TestTable, AddColumn) { ASSERT_OK(table.AddColumn(0, columns_[0], &result)); auto ex_schema = ::arrow::schema( {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)}); - std::vector> ex_columns = {table.column(0), table.column(0), - table.column(1), table.column(2)}; - ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns))); + + auto expected = Table::Make( + ex_schema, {table.column(0), table.column(0), table.column(1), table.column(2)}); + ASSERT_TRUE(result->Equals(*expected)); ASSERT_OK(table.AddColumn(1, columns_[0], &result)); ex_schema = ::arrow::schema( {schema_->field(0), schema_->field(0), schema_->field(1), schema_->field(2)}); - ex_columns = {table.column(0), table.column(0), table.column(1), table.column(2)}; - ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns))); + + expected = Table::Make( + ex_schema, {table.column(0), table.column(0), table.column(1), table.column(2)}); + ASSERT_TRUE(result->Equals(*expected)); ASSERT_OK(table.AddColumn(2, columns_[0], &result)); ex_schema = ::arrow::schema( {schema_->field(0), schema_->field(1), schema_->field(0), schema_->field(2)}); - ex_columns = {table.column(0), table.column(1), table.column(0), table.column(2)}; - ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns))); + expected = Table::Make( + ex_schema, {table.column(0), table.column(1), table.column(0), table.column(2)}); + ASSERT_TRUE(result->Equals(*expected)); ASSERT_OK(table.AddColumn(3, columns_[0], &result)); ex_schema = ::arrow::schema( {schema_->field(0), schema_->field(1), schema_->field(2), schema_->field(0)}); - ex_columns = {table.column(0), table.column(1), table.column(2), table.column(0)}; - ASSERT_TRUE(result->Equals(Table(ex_schema, ex_columns))); -} - -TEST_F(TestTable, IsChunked) { - ArrayVector c1, c2; - - auto a1 = MakeRandomArray(10); - auto a2 = MakeRandomArray(20); - - auto sch1 = arrow::schema({field("f1", int32()), field("f2", int32())}); - - std::vector> columns; - - std::shared_ptr batch; - - columns = {column(sch1->field(0), {a1}), column(sch1->field(1), {a1})}; - auto t1 = std::make_shared
    (sch1, columns); - - ASSERT_FALSE(t1->IsChunked()); - - columns = {column(sch1->field(0), {a2}), column(sch1->field(1), {a1, a1})}; - auto t2 = std::make_shared
    (sch1, columns); - - ASSERT_TRUE(t2->IsChunked()); + expected = Table::Make( + ex_schema, {table.column(0), table.column(1), table.column(2), table.column(0)}); + ASSERT_TRUE(result->Equals(*expected)); } class TestRecordBatch : public TestBase {}; @@ -475,24 +466,22 @@ TEST_F(TestRecordBatch, Equals) { auto f2 = field("f2", int16()); vector> fields = {f0, f1, f2}; - auto schema = std::make_shared(fields); + auto schema = ::arrow::schema({f0, f1, f2}); + auto schema2 = ::arrow::schema({f0, f1}); auto a0 = MakeRandomArray(length); auto a1 = MakeRandomArray(length); auto a2 = MakeRandomArray(length); - RecordBatch b1(schema, length, {a0, a1, a2}); - RecordBatch b3(schema, length, {a0, a1}); - RecordBatch b4(schema, length, {a0, a1, a1}); + auto b1 = RecordBatch::Make(schema, length, {a0, a1, a2}); + auto b3 = RecordBatch::Make(schema2, length, {a0, a1}); + auto b4 = RecordBatch::Make(schema, length, {a0, a1, a1}); - ASSERT_TRUE(b1.Equals(b1)); - ASSERT_FALSE(b1.Equals(b3)); - ASSERT_FALSE(b1.Equals(b4)); + ASSERT_TRUE(b1->Equals(*b1)); + ASSERT_FALSE(b1->Equals(*b3)); + ASSERT_FALSE(b1->Equals(*b4)); } -#ifdef NDEBUG -// In debug builds, RecordBatch ctor aborts if you construct an invalid one - TEST_F(TestRecordBatch, Validate) { const int length = 10; @@ -507,21 +496,19 @@ TEST_F(TestRecordBatch, Validate) { auto a2 = MakeRandomArray(length); auto a3 = MakeRandomArray(5); - RecordBatch b1(schema, length, {a0, a1, a2}); + auto b1 = RecordBatch::Make(schema, length, {a0, a1, a2}); - ASSERT_OK(b1.Validate()); + ASSERT_OK(b1->Validate()); // Length mismatch - RecordBatch b2(schema, length, {a0, a1, a3}); - ASSERT_RAISES(Invalid, b2.Validate()); + auto b2 = RecordBatch::Make(schema, length, {a0, a1, a3}); + ASSERT_RAISES(Invalid, b2->Validate()); // Type mismatch - RecordBatch b3(schema, length, {a0, a1, a0}); - ASSERT_RAISES(Invalid, b3.Validate()); + auto b3 = RecordBatch::Make(schema, length, {a0, a1, a0}); + ASSERT_RAISES(Invalid, b3->Validate()); } -#endif - TEST_F(TestRecordBatch, Slice) { const int length = 10; @@ -529,19 +516,19 @@ TEST_F(TestRecordBatch, Slice) { auto f1 = field("f1", uint8()); vector> fields = {f0, f1}; - auto schema = std::make_shared(fields); + auto schema = ::arrow::schema(fields); auto a0 = MakeRandomArray(length); auto a1 = MakeRandomArray(length); - RecordBatch batch(schema, length, {a0, a1}); + auto batch = RecordBatch::Make(schema, length, {a0, a1}); - auto batch_slice = batch.Slice(2); - auto batch_slice2 = batch.Slice(1, 5); + auto batch_slice = batch->Slice(2); + auto batch_slice2 = batch->Slice(1, 5); - ASSERT_EQ(batch_slice->num_rows(), batch.num_rows() - 2); + ASSERT_EQ(batch_slice->num_rows(), batch->num_rows() - 2); - for (int i = 0; i < batch.num_columns(); ++i) { + for (int i = 0; i < batch->num_columns(); ++i) { ASSERT_EQ(2, batch_slice->column(i)->offset()); ASSERT_EQ(length - 2, batch_slice->column(i)->length()); @@ -567,9 +554,9 @@ TEST_F(TestTableBatchReader, ReadNext) { std::shared_ptr batch; columns = {column(sch1->field(0), {a1, a4, a2}), column(sch1->field(1), {a2, a2})}; - Table t1(sch1, columns); + auto t1 = Table::Make(sch1, columns); - TableBatchReader i1(t1); + TableBatchReader i1(*t1); ASSERT_OK(i1.ReadNext(&batch)); ASSERT_EQ(10, batch->num_rows()); @@ -584,9 +571,9 @@ TEST_F(TestTableBatchReader, ReadNext) { ASSERT_EQ(nullptr, batch); columns = {column(sch1->field(0), {a1}), column(sch1->field(1), {a4})}; - Table t2(sch1, columns); + auto t2 = Table::Make(sch1, columns); - TableBatchReader i2(t2); + TableBatchReader i2(*t2); ASSERT_OK(i2.ReadNext(&batch)); ASSERT_EQ(10, batch->num_rows()); diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc index fe19bf4ce0b3f..8f3f195765aa9 100644 --- a/cpp/src/arrow/table.cc +++ b/cpp/src/arrow/table.cc @@ -23,6 +23,7 @@ #include #include "arrow/array.h" +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/type.h" #include "arrow/util/logging.h" @@ -153,171 +154,126 @@ Status Column::ValidateData() { } // ---------------------------------------------------------------------- -// RecordBatch methods - -RecordBatch::RecordBatch(const std::shared_ptr& schema, int64_t num_rows) - : schema_(schema), num_rows_(num_rows) { - boxed_columns_.resize(schema->num_fields()); -} - -RecordBatch::RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - const std::vector>& columns) - : RecordBatch(schema, num_rows) { - columns_.resize(columns.size()); - for (size_t i = 0; i < columns.size(); ++i) { - columns_[i] = columns[i]->data(); - } -} - -RecordBatch::RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - std::vector>&& columns) - : RecordBatch(schema, num_rows) { - columns_.resize(columns.size()); - for (size_t i = 0; i < columns.size(); ++i) { - columns_[i] = columns[i]->data(); - } -} - -RecordBatch::RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - std::vector>&& columns) - : RecordBatch(schema, num_rows) { - columns_ = std::move(columns); -} - -RecordBatch::RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - const std::vector>& columns) - : RecordBatch(schema, num_rows) { - columns_ = columns; -} - -std::shared_ptr RecordBatch::column(int i) const { - if (!boxed_columns_[i]) { - boxed_columns_[i] = MakeArray(columns_[i]); - } - DCHECK(boxed_columns_[i]); - return boxed_columns_[i]; -} - -const std::string& RecordBatch::column_name(int i) const { - return schema_->field(i)->name(); -} - -bool RecordBatch::Equals(const RecordBatch& other) const { - if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) { - return false; - } +// Table methods - for (int i = 0; i < num_columns(); ++i) { - if (!column(i)->Equals(other.column(i))) { - return false; +/// \class SimpleTable +/// \brief A basic, non-lazy in-memory table, like SimpleRecordBatch +class SimpleTable : public Table { + public: + SimpleTable(const std::shared_ptr& schema, + const std::vector>& columns, int64_t num_rows = -1) + : columns_(columns) { + schema_ = schema; + if (num_rows < 0) { + if (columns.size() == 0) { + num_rows_ = 0; + } else { + num_rows_ = columns[0]->length(); + } + } else { + num_rows_ = num_rows; } } - return true; -} - -bool RecordBatch::ApproxEquals(const RecordBatch& other) const { - if (num_columns() != other.num_columns() || num_rows_ != other.num_rows()) { - return false; - } + SimpleTable(const std::shared_ptr& schema, + const std::vector>& columns, int64_t num_rows = -1) { + schema_ = schema; + if (num_rows < 0) { + if (columns.size() == 0) { + num_rows_ = 0; + } else { + num_rows_ = columns[0]->length(); + } + } else { + num_rows_ = num_rows; + } - for (int i = 0; i < num_columns(); ++i) { - if (!column(i)->ApproxEquals(other.column(i))) { - return false; + columns_.resize(columns.size()); + for (size_t i = 0; i < columns.size(); ++i) { + columns_[i] = + std::make_shared(schema->field(static_cast(i)), columns[i]); } } - return true; -} - -std::shared_ptr RecordBatch::ReplaceSchemaMetadata( - const std::shared_ptr& metadata) const { - auto new_schema = schema_->AddMetadata(metadata); - return std::make_shared(new_schema, num_rows_, columns_); -} + std::shared_ptr column(int i) const override { return columns_[i]; } -std::shared_ptr RecordBatch::Slice(int64_t offset) const { - return Slice(offset, this->num_rows() - offset); -} + Status RemoveColumn(int i, std::shared_ptr
    * out) const override { + std::shared_ptr new_schema; + RETURN_NOT_OK(schema_->RemoveField(i, &new_schema)); -std::shared_ptr RecordBatch::Slice(int64_t offset, int64_t length) const { - std::vector> arrays; - arrays.reserve(num_columns()); - for (const auto& field : columns_) { - int64_t col_length = std::min(field->length - offset, length); - int64_t col_offset = field->offset + offset; - - auto new_data = std::make_shared(*field); - new_data->length = col_length; - new_data->offset = col_offset; - new_data->null_count = kUnknownNullCount; - arrays.emplace_back(new_data); + *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i)); + return Status::OK(); } - int64_t num_rows = std::min(num_rows_ - offset, length); - return std::make_shared(schema_, num_rows, std::move(arrays)); -} -Status RecordBatch::Validate() const { - for (int i = 0; i < num_columns(); ++i) { - const ArrayData& arr = *columns_[i]; - if (arr.length != num_rows_) { + Status AddColumn(int i, const std::shared_ptr& col, + std::shared_ptr
    * out) const override { + if (i < 0 || i > num_columns() + 1) { + return Status::Invalid("Invalid column index."); + } + if (col == nullptr) { std::stringstream ss; - ss << "Number of rows in column " << i << " did not match batch: " << arr.length - << " vs " << num_rows_; + ss << "Column " << i << " was null"; return Status::Invalid(ss.str()); } - const auto& schema_type = *schema_->field(i)->type(); - if (!arr.type->Equals(schema_type)) { + if (col->length() != num_rows_) { std::stringstream ss; - ss << "Column " << i << " type not match schema: " << arr.type->ToString() << " vs " - << schema_type.ToString(); + ss << "Added column's length must match table's length. Expected length " + << num_rows_ << " but got length " << col->length(); return Status::Invalid(ss.str()); } + + std::shared_ptr new_schema; + RETURN_NOT_OK(schema_->AddField(i, col->field(), &new_schema)); + + *out = Table::Make(new_schema, internal::AddVectorElement(columns_, i, col)); + return Status::OK(); } - return Status::OK(); -} -// ---------------------------------------------------------------------- -// Table methods + std::shared_ptr
    ReplaceSchemaMetadata( + const std::shared_ptr& metadata) const override { + auto new_schema = schema_->AddMetadata(metadata); + return Table::Make(new_schema, columns_); + } -Table::Table(const std::shared_ptr& schema, - const std::vector>& columns, int64_t num_rows) - : schema_(schema), columns_(columns) { - if (num_rows < 0) { - if (columns.size() == 0) { - num_rows_ = 0; - } else { - num_rows_ = columns[0]->length(); + Status Validate() const override { + if (static_cast(columns_.size()) != schema_->num_fields()) { + return Status::Invalid("Number of columns did not match schema"); } - } else { - num_rows_ = num_rows; - } -} -Table::Table(const std::shared_ptr& schema, - const std::vector>& columns, int64_t num_rows) - : schema_(schema) { - if (num_rows < 0) { - if (columns.size() == 0) { - num_rows_ = 0; - } else { - num_rows_ = columns[0]->length(); + // Make sure columns are all the same length + for (int i = 0; i < num_columns(); ++i) { + const Column* col = columns_[i].get(); + if (col == nullptr) { + std::stringstream ss; + ss << "Column " << i << " was null"; + return Status::Invalid(ss.str()); + } + if (col->length() != num_rows_) { + std::stringstream ss; + ss << "Column " << i << " named " << col->name() << " expected length " + << num_rows_ << " but got length " << col->length(); + return Status::Invalid(ss.str()); + } } - } else { - num_rows_ = num_rows; + return Status::OK(); } - columns_.resize(columns.size()); - for (size_t i = 0; i < columns.size(); ++i) { - columns_[i] = - std::make_shared(schema->field(static_cast(i)), columns[i]); - } + private: + std::vector> columns_; +}; + +Table::Table() {} + +std::shared_ptr
    Table::Make(const std::shared_ptr& schema, + const std::vector>& columns, + int64_t num_rows) { + return std::make_shared(schema, columns, num_rows); } -std::shared_ptr
    Table::ReplaceSchemaMetadata( - const std::shared_ptr& metadata) const { - auto new_schema = schema_->AddMetadata(metadata); - return std::make_shared
    (new_schema, columns_); +std::shared_ptr
    Table::Make(const std::shared_ptr& schema, + const std::vector>& arrays, + int64_t num_rows) { + return std::make_shared(schema, arrays, num_rows); } Status Table::FromRecordBatches(const std::vector>& batches, @@ -351,7 +307,7 @@ Status Table::FromRecordBatches(const std::vector>& columns[i] = std::make_shared(schema->field(i), column_arrays); } - *table = std::make_shared
    (schema, columns); + *table = Table::Make(schema, columns); return Status::OK(); } @@ -388,7 +344,7 @@ Status ConcatenateTables(const std::vector>& tables, } columns[i] = std::make_shared(schema->field(i), column_arrays); } - *table = std::make_shared
    (schema, columns); + *table = Table::Make(schema, columns); return Status::OK(); } @@ -399,82 +355,19 @@ bool Table::Equals(const Table& other) const { if (!schema_->Equals(*other.schema())) { return false; } - if (static_cast(columns_.size()) != other.num_columns()) { + if (this->num_columns() != other.num_columns()) { return false; } - for (int i = 0; i < static_cast(columns_.size()); i++) { - if (!columns_[i]->Equals(other.column(i))) { + for (int i = 0; i < this->num_columns(); i++) { + if (!this->column(i)->Equals(other.column(i))) { return false; } } return true; } -Status Table::RemoveColumn(int i, std::shared_ptr
    * out) const { - std::shared_ptr new_schema; - RETURN_NOT_OK(schema_->RemoveField(i, &new_schema)); - - *out = std::make_shared
    (new_schema, internal::DeleteVectorElement(columns_, i)); - return Status::OK(); -} - -Status Table::AddColumn(int i, const std::shared_ptr& col, - std::shared_ptr
    * out) const { - if (i < 0 || i > num_columns() + 1) { - return Status::Invalid("Invalid column index."); - } - if (col == nullptr) { - std::stringstream ss; - ss << "Column " << i << " was null"; - return Status::Invalid(ss.str()); - } - if (col->length() != num_rows_) { - std::stringstream ss; - ss << "Added column's length must match table's length. Expected length " << num_rows_ - << " but got length " << col->length(); - return Status::Invalid(ss.str()); - } - - std::shared_ptr new_schema; - RETURN_NOT_OK(schema_->AddField(i, col->field(), &new_schema)); - - *out = - std::make_shared
    (new_schema, internal::AddVectorElement(columns_, i, col)); - return Status::OK(); -} - -Status Table::ValidateColumns() const { - if (num_columns() != schema_->num_fields()) { - return Status::Invalid("Number of columns did not match schema"); - } - - // Make sure columns are all the same length - for (size_t i = 0; i < columns_.size(); ++i) { - const Column* col = columns_[i].get(); - if (col == nullptr) { - std::stringstream ss; - ss << "Column " << i << " was null"; - return Status::Invalid(ss.str()); - } - if (col->length() != num_rows_) { - std::stringstream ss; - ss << "Column " << i << " named " << col->name() << " expected length " << num_rows_ - << " but got length " << col->length(); - return Status::Invalid(ss.str()); - } - } - return Status::OK(); -} - -bool Table::IsChunked() const { - for (size_t i = 0; i < columns_.size(); ++i) { - if (columns_[i]->data()->num_chunks() > 1) { - return true; - } - } - return false; -} +#ifndef ARROW_NO_DEPRECATED_API Status MakeTable(const std::shared_ptr& schema, const std::vector>& arrays, @@ -493,15 +386,12 @@ Status MakeTable(const std::shared_ptr& schema, columns.emplace_back(std::make_shared(schema->field(i), arrays[i])); } - *table = std::make_shared
    (schema, columns); + *table = Table::Make(schema, columns); return Status::OK(); } -// ---------------------------------------------------------------------- -// Base record batch reader - -RecordBatchReader::~RecordBatchReader() {} +#endif // ARROW_NO_DEPRECATED_API // ---------------------------------------------------------------------- // Convert a table to a sequence of record batches @@ -565,8 +455,7 @@ class TableBatchReader::TableBatchReaderImpl { } absolute_row_position_ += chunksize; - *out = - std::make_shared(table_.schema(), chunksize, std::move(batch_data)); + *out = RecordBatch::Make(table_.schema(), chunksize, std::move(batch_data)); return Status::OK(); } diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h index 2cff32f74ef14..d0312d93cb93c 100644 --- a/cpp/src/arrow/table.h +++ b/cpp/src/arrow/table.h @@ -24,6 +24,7 @@ #include #include "arrow/array.h" +#include "arrow/record_batch.h" #include "arrow/type.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h" @@ -33,8 +34,6 @@ namespace arrow { class KeyValueMetadata; class Status; -using ArrayVector = std::vector>; - /// \class ChunkedArray /// \brief A data structure managing a list of primitive Arrow arrays logically /// as one large array @@ -113,123 +112,28 @@ class ARROW_EXPORT Column { ARROW_DISALLOW_COPY_AND_ASSIGN(Column); }; -/// \class RecordBatch -/// \brief Collection of equal-length arrays matching a particular Schema -/// -/// A record batch is table-like data structure consisting of an internal -/// sequence of fields, each a contiguous Arrow array -class ARROW_EXPORT RecordBatch { - public: - /// \param[in] schema The record batch schema - /// \param[in] num_rows length of fields in the record batch. Each array - /// should have the same length as num_rows - /// \param[in] columns the record batch fields as vector of arrays - RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - const std::vector>& columns); - - /// \brief Move-based constructor for a vector of Array instances - RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - std::vector>&& columns); - - /// \brief Construct record batch from vector of internal data structures - /// \since 0.5.0 - /// - /// This class is only provided with an rvalue-reference for the input data, - /// and is intended for internal use, or advanced users. - /// - /// \param schema the record batch schema - /// \param num_rows the number of semantic rows in the record batch. This - /// should be equal to the length of each field - /// \param columns the data for the batch's columns - RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - std::vector>&& columns); - - /// \brief Construct record batch by copying vector of array data - /// \since 0.5.0 - RecordBatch(const std::shared_ptr& schema, int64_t num_rows, - const std::vector>& columns); - - /// \brief Determine if two record batches are exactly equal - /// \return true if batches are equal - bool Equals(const RecordBatch& other) const; - - /// \brief Determine if two record batches are approximately equal - bool ApproxEquals(const RecordBatch& other) const; - - // \return the table's schema - /// \return true if batches are equal - std::shared_ptr schema() const { return schema_; } - - /// \brief Retrieve an array from the record batch - /// \param[in] i field index, does not boundscheck - /// \return an Array object - std::shared_ptr column(int i) const; - - std::shared_ptr column_data(int i) const { return columns_[i]; } - - /// \brief Name in i-th column - const std::string& column_name(int i) const; - - /// \return the number of columns in the table - int num_columns() const { return static_cast(columns_.size()); } - - /// \return the number of rows (the corresponding length of each column) - int64_t num_rows() const { return num_rows_; } - - /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL) - /// \since 0.5.0 - /// - /// \param[in] metadata new KeyValueMetadata - /// \return new RecordBatch - std::shared_ptr ReplaceSchemaMetadata( - const std::shared_ptr& metadata) const; - - /// \brief Slice each of the arrays in the record batch - /// \param[in] offset the starting offset to slice, through end of batch - /// \return new record batch - std::shared_ptr Slice(int64_t offset) const; - - /// \brief Slice each of the arrays in the record batch - /// \param[in] offset the starting offset to slice - /// \param[in] length the number of elements to slice from offset - /// \return new record batch - std::shared_ptr Slice(int64_t offset, int64_t length) const; - - /// \brief Check for schema or length inconsistencies - /// \return Status - Status Validate() const; - - private: - ARROW_DISALLOW_COPY_AND_ASSIGN(RecordBatch); - - RecordBatch(const std::shared_ptr& schema, int64_t num_rows); - - std::shared_ptr schema_; - int64_t num_rows_; - std::vector> columns_; - - // Caching boxed array data - mutable std::vector> boxed_columns_; -}; - /// \class Table /// \brief Logical table as sequence of chunked arrays class ARROW_EXPORT Table { public: + virtual ~Table() = default; + /// \brief Construct Table from schema and columns /// If columns is zero-length, the table's number of rows is zero /// \param schema The table schema (column types) /// \param columns The table's columns /// \param num_rows number of rows in table, -1 (default) to infer from columns - Table(const std::shared_ptr& schema, - const std::vector>& columns, int64_t num_rows = -1); + static std::shared_ptr
    Make(const std::shared_ptr& schema, + const std::vector>& columns, + int64_t num_rows = -1); /// \brief Construct Table from schema and arrays /// \param schema The table schema (column types) /// \param arrays The table's columns as arrays /// \param num_rows number of rows in table, -1 (default) to infer from columns - Table(const std::shared_ptr& schema, - const std::vector>& arrays, int64_t num_rows = -1); + static std::shared_ptr
    Make(const std::shared_ptr& schema, + const std::vector>& arrays, + int64_t num_rows = -1); // Construct table from RecordBatch, but only if all of the batch schemas are // equal. Returns Status::Invalid if there is some problem @@ -242,25 +146,28 @@ class ARROW_EXPORT Table { /// \param[in] i column index, does not boundscheck /// \return the i-th column - std::shared_ptr column(int i) const { return columns_[i]; } + virtual std::shared_ptr column(int i) const = 0; /// \brief Remove column from the table, producing a new Table - Status RemoveColumn(int i, std::shared_ptr
    * out) const; + virtual Status RemoveColumn(int i, std::shared_ptr
    * out) const = 0; /// \brief Add column to the table, producing a new Table - Status AddColumn(int i, const std::shared_ptr& column, - std::shared_ptr
    * out) const; + virtual Status AddColumn(int i, const std::shared_ptr& column, + std::shared_ptr
    * out) const = 0; /// \brief Replace schema key-value metadata with new metadata (EXPERIMENTAL) /// \since 0.5.0 /// /// \param[in] metadata new KeyValueMetadata /// \return new Table - std::shared_ptr
    ReplaceSchemaMetadata( - const std::shared_ptr& metadata) const; + virtual std::shared_ptr
    ReplaceSchemaMetadata( + const std::shared_ptr& metadata) const = 0; + + /// \brief Perform any checks to validate the input arguments + virtual Status Validate() const = 0; /// \return the number of columns in the table - int num_columns() const { return static_cast(columns_.size()); } + int num_columns() const { return schema_->num_fields(); } /// \return the number of rows (the corresponding length of each column) int64_t num_rows() const { return num_rows_; } @@ -268,35 +175,14 @@ class ARROW_EXPORT Table { /// \brief Determine if semantic contents of tables are exactly equal bool Equals(const Table& other) const; - /// \brief Perform any checks to validate the input arguments - Status ValidateColumns() const; - - /// \brief Return true if any column has multiple chunks - bool IsChunked() const; - - private: - ARROW_DISALLOW_COPY_AND_ASSIGN(Table); + protected: + Table(); std::shared_ptr schema_; - std::vector> columns_; - int64_t num_rows_; -}; - -/// \brief Abstract interface for reading stream of record batches -class ARROW_EXPORT RecordBatchReader { - public: - virtual ~RecordBatchReader(); - /// \return the shared schema of the record batches in the stream - virtual std::shared_ptr schema() const = 0; - - /// Read the next record batch in the stream. Return null for batch when - /// reaching end of stream - /// - /// \param[out] batch the next loaded batch, null at end of stream - /// \return Status - virtual Status ReadNext(std::shared_ptr* batch) = 0; + private: + ARROW_DISALLOW_COPY_AND_ASSIGN(Table); }; /// \brief Compute a sequence of record batches from a (possibly chunked) Table @@ -322,13 +208,18 @@ ARROW_EXPORT Status ConcatenateTables(const std::vector>& tables, std::shared_ptr
    * table); +#ifndef ARROW_NO_DEPRECATED_API + /// \brief Construct table from multiple input tables. /// \return Status, fails if any schemas are different +/// \note Deprecated since 0.8.0 ARROW_EXPORT Status MakeTable(const std::shared_ptr& schema, const std::vector>& arrays, std::shared_ptr
    * table); +#endif + } // namespace arrow #endif // ARROW_TABLE_H diff --git a/cpp/src/arrow/table_builder-test.cc b/cpp/src/arrow/table_builder-test.cc index 07d9b6b2d6568..8167577e9064e 100644 --- a/cpp/src/arrow/table_builder-test.cc +++ b/cpp/src/arrow/table_builder-test.cc @@ -22,6 +22,7 @@ #include "gtest/gtest.h" #include "arrow/array.h" +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/table_builder.h" @@ -98,7 +99,7 @@ TEST_F(TestRecordBatchBuilder, Basics) { ASSERT_OK(ex_b1.Finish(&a1)); ASSERT_OK(ex_b2.Finish(&a2)); - RecordBatch expected(schema, 4, {a0, a1, a2}); + auto expected = RecordBatch::Make(schema, 4, {a0, a1, a2}); // Builder attributes ASSERT_EQ(3, builder->num_fields()); @@ -119,7 +120,7 @@ TEST_F(TestRecordBatchBuilder, Basics) { ASSERT_OK(builder->Flush(&batch)); } - ASSERT_BATCHES_EQUAL(expected, *batch); + ASSERT_BATCHES_EQUAL(*expected, *batch); } // Test setting initial capacity diff --git a/cpp/src/arrow/table_builder.cc b/cpp/src/arrow/table_builder.cc index a1bd95940a6db..379d886deacba 100644 --- a/cpp/src/arrow/table_builder.cc +++ b/cpp/src/arrow/table_builder.cc @@ -24,6 +24,7 @@ #include "arrow/array.h" #include "arrow/builder.h" +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/table.h" #include "arrow/type.h" @@ -64,7 +65,7 @@ Status RecordBatchBuilder::Flush(bool reset_builders, } length = fields[i]->length(); } - *batch = std::make_shared(schema_, length, std::move(fields)); + *batch = RecordBatch::Make(schema_, length, std::move(fields)); if (reset_builders) { return InitBuilders(); } else { diff --git a/cpp/src/arrow/test-common.h b/cpp/src/arrow/test-common.h index a4c4fddff7348..911adf7b6057a 100644 --- a/cpp/src/arrow/test-common.h +++ b/cpp/src/arrow/test-common.h @@ -30,7 +30,6 @@ #include "arrow/buffer.h" #include "arrow/builder.h" #include "arrow/memory_pool.h" -#include "arrow/table.h" #include "arrow/test-util.h" namespace arrow { diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h index 77f489ab17797..1a34808488a83 100644 --- a/cpp/src/arrow/test-util.h +++ b/cpp/src/arrow/test-util.h @@ -35,7 +35,6 @@ #include "arrow/memory_pool.h" #include "arrow/pretty_print.h" #include "arrow/status.h" -#include "arrow/table.h" #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit-util.h" @@ -375,7 +374,7 @@ void AssertArraysEqual(const Array& expected, const Array& actual) { #define ASSERT_BATCHES_EQUAL(LEFT, RIGHT) \ do { \ - if (!LEFT.ApproxEquals(RIGHT)) { \ + if (!(LEFT).ApproxEquals(RIGHT)) { \ std::stringstream ss; \ ss << "Left:\n"; \ ASSERT_OK(PrettyPrint(LEFT, 0, &ss)); \ diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 70f275c0fa495..8dcc1592da01f 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -498,9 +498,9 @@ class ARROW_EXPORT StructType : public NestedType { std::vector GetBufferLayout() const override; }; -class ARROW_EXPORT DecimalBaseType : public FixedSizeBinaryType { +class ARROW_EXPORT DecimalType : public FixedSizeBinaryType { public: - explicit DecimalBaseType(int32_t byte_width, int32_t precision, int32_t scale) + explicit DecimalType(int32_t byte_width, int32_t precision, int32_t scale) : FixedSizeBinaryType(byte_width, Type::DECIMAL), precision_(precision), scale_(scale) {} @@ -513,21 +513,18 @@ class ARROW_EXPORT DecimalBaseType : public FixedSizeBinaryType { int32_t scale_; }; -class ARROW_EXPORT Decimal128Type : public DecimalBaseType { +class ARROW_EXPORT Decimal128Type : public DecimalType { public: static constexpr Type::type type_id = Type::DECIMAL; explicit Decimal128Type(int32_t precision, int32_t scale) - : DecimalBaseType(16, precision, scale) {} + : DecimalType(16, precision, scale) {} Status Accept(TypeVisitor* visitor) const override; std::string ToString() const override; std::string name() const override { return "decimal"; } }; -// TODO(wesm): Remove this -using DecimalType = Decimal128Type; - struct UnionMode { enum type { SPARSE, DENSE }; }; diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index dbfd89cc378cb..73e34c7b21033 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -403,8 +403,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CChunkedArray] data() cdef cppclass CRecordBatch" arrow::RecordBatch": - CRecordBatch(const shared_ptr[CSchema]& schema, int64_t num_rows, - const vector[shared_ptr[CArray]]& columns) + @staticmethod + shared_ptr[CRecordBatch] Make( + const shared_ptr[CSchema]& schema, int64_t num_rows, + const vector[shared_ptr[CArray]]& columns) c_bool Equals(const CRecordBatch& other) @@ -427,6 +429,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: CTable(const shared_ptr[CSchema]& schema, const vector[shared_ptr[CColumn]]& columns) + @staticmethod + shared_ptr[CTable] Make( + const shared_ptr[CSchema]& schema, + const vector[shared_ptr[CColumn]]& columns) + @staticmethod CStatus FromRecordBatches( const vector[shared_ptr[CRecordBatch]]& batches, diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 591f3297587a3..8c5b8bbc343c0 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -724,7 +724,6 @@ cdef class RecordBatch: Array arr c_string c_name shared_ptr[CSchema] schema - shared_ptr[CRecordBatch] batch vector[shared_ptr[CArray]] c_arrays int64_t num_rows int64_t i @@ -740,8 +739,8 @@ cdef class RecordBatch: for arr in arrays: c_arrays.push_back(arr.sp_array) - batch.reset(new CRecordBatch(schema, num_rows, c_arrays)) - return pyarrow_wrap_batch(batch) + return pyarrow_wrap_batch( + CRecordBatch.Make(schema, num_rows, c_arrays)) def table_to_blocks(PandasOptions options, Table table, int nthreads, @@ -946,8 +945,7 @@ cdef class Table: else: raise ValueError(type(arrays[i])) - table.reset(new CTable(c_schema, columns)) - return pyarrow_wrap_table(table) + return pyarrow_wrap_table(CTable.Make(c_schema, columns)) @staticmethod def from_batches(batches): From 1516306d40e4b785416282fe84daa9db342e5b90 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Wed, 22 Nov 2017 13:46:56 -0500 Subject: [PATCH 072/177] ARROW-1047: [Java] Add Generic Reader Interface for Stream Format This change decouples the reading of messages from the ReadChannel so that it is possible to build a reader that is not tied to a specific stream. This adds a new interface `MessageReader` that will return a `Message` and the message body. The `MessageChannelReader` implements this interface to read from a `ReadChannel` to match the current functionality. A re-org of reading and writing packages is also done to better organize classes under an `ipc` package. There is a slight change in behavior for the `ArrowFileReader` that should not affect any usage. Previously, the schema was read during initialization, and all dictionaries read just before the first record batch was read. Now, all dictionaries are read directly after the schema, and not specifically tied to reading the first record batch. Author: Bryan Cutler Closes #1259 from BryanCutler/java-generic-stream-interfaces-ARROW-1047 and squashes the following commits: 43314ca1 [Bryan Cutler] ARROW-1047: [Java] Add Generic Reader Interface for Stream Format --- .../org/apache/arrow/tools/EchoServer.java | 4 +- .../org/apache/arrow/tools/FileRoundtrip.java | 4 +- .../org/apache/arrow/tools/FileToStream.java | 4 +- .../org/apache/arrow/tools/Integration.java | 10 +- .../org/apache/arrow/tools/StreamToFile.java | 4 +- .../arrow/tools/ArrowFileTestFixtures.java | 6 +- .../apache/arrow/tools/EchoServerTest.java | 4 +- .../main/codegen/templates/UnionVector.java | 2 +- .../arrow/vector/BaseDataValueVector.java | 2 +- .../vector/BaseNullableFixedWidthVector.java | 2 +- .../BaseNullableVariableWidthVector.java | 2 +- .../org/apache/arrow/vector/BitVector.java | 2 +- .../apache/arrow/vector/BitVectorHelper.java | 2 +- .../org/apache/arrow/vector/BufferBacked.java | 2 +- .../org/apache/arrow/vector/FieldVector.java | 2 +- .../org/apache/arrow/vector/VectorLoader.java | 6 +- .../apache/arrow/vector/VectorUnloader.java | 6 +- .../org/apache/arrow/vector/ZeroVector.java | 2 +- .../vector/complex/FixedSizeListVector.java | 2 +- .../arrow/vector/complex/ListVector.java | 2 +- .../vector/complex/NullableMapVector.java | 2 +- .../vector/{file => ipc}/ArrowFileReader.java | 59 +++++-- .../vector/{file => ipc}/ArrowFileWriter.java | 4 +- .../vector/{file => ipc}/ArrowMagic.java | 4 +- .../vector/{file => ipc}/ArrowReader.java | 150 ++++++++++-------- .../arrow/vector/ipc/ArrowStreamReader.java | 148 +++++++++++++++++ .../{stream => ipc}/ArrowStreamWriter.java | 11 +- .../vector/{file => ipc}/ArrowWriter.java | 9 +- .../InvalidArrowFileException.java | 2 +- .../{file/json => ipc}/JsonFileReader.java | 10 +- .../{file/json => ipc}/JsonFileWriter.java | 6 +- .../vector/{file => ipc}/ReadChannel.java | 2 +- .../{file => ipc}/SeekableReadChannel.java | 2 +- .../vector/{file => ipc}/WriteChannel.java | 4 +- .../{file => ipc/message}/ArrowBlock.java | 3 +- .../{schema => ipc/message}/ArrowBuffer.java | 2 +- .../message}/ArrowDictionaryBatch.java | 2 +- .../message}/ArrowFieldNode.java | 2 +- .../{file => ipc/message}/ArrowFooter.java | 5 +- .../{schema => ipc/message}/ArrowMessage.java | 2 +- .../message}/ArrowRecordBatch.java | 8 +- .../message}/ArrowVectorType.java | 2 +- .../message}/FBSerializable.java | 2 +- .../message}/FBSerializables.java | 2 +- .../ipc/message/MessageChannelReader.java | 115 ++++++++++++++ .../vector/ipc/message/MessageReader.java | 65 ++++++++ .../message}/MessageSerializer.java | 91 +++++++---- .../{schema => ipc/message}/TypeLayout.java | 46 +++--- .../{schema => ipc/message}/VectorLayout.java | 23 ++- .../vector/stream/ArrowStreamReader.java | 66 -------- .../apache/arrow/vector/types/pojo/Field.java | 8 +- .../apache/arrow/vector/TestValueVector.java | 6 +- .../arrow/vector/TestVectorUnloadLoad.java | 4 +- .../vector/{file => ipc}/BaseFileTest.java | 2 +- .../MessageSerializerTest.java | 13 +- .../vector/{file => ipc}/TestArrowFile.java | 61 +++---- .../vector/{file => ipc}/TestArrowFooter.java | 4 +- .../{file => ipc}/TestArrowReaderWriter.java | 21 ++- .../vector/{file => ipc}/TestArrowStream.java | 15 +- .../{file => ipc}/TestArrowStreamPipe.java | 50 +++--- .../{file/json => ipc}/TestJSONFile.java | 3 +- 61 files changed, 698 insertions(+), 408 deletions(-) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowFileReader.java (77%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowFileWriter.java (94%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowMagic.java (93%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowReader.java (65%) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java rename java/vector/src/main/java/org/apache/arrow/vector/{stream => ipc}/ArrowStreamWriter.java (84%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ArrowWriter.java (95%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/InvalidArrowFileException.java (96%) rename java/vector/src/main/java/org/apache/arrow/vector/{file/json => ipc}/JsonFileReader.java (98%) rename java/vector/src/main/java/org/apache/arrow/vector/{file/json => ipc}/JsonFileWriter.java (98%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/ReadChannel.java (98%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/SeekableReadChannel.java (97%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc}/WriteChannel.java (97%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc/message}/ArrowBlock.java (96%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowBuffer.java (97%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowDictionaryBatch.java (97%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowFieldNode.java (97%) rename java/vector/src/main/java/org/apache/arrow/vector/{file => ipc/message}/ArrowFooter.java (96%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowMessage.java (96%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowRecordBatch.java (94%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/ArrowVectorType.java (98%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/FBSerializable.java (95%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/FBSerializables.java (96%) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java rename java/vector/src/main/java/org/apache/arrow/vector/{stream => ipc/message}/MessageSerializer.java (86%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/TypeLayout.java (80%) rename java/vector/src/main/java/org/apache/arrow/vector/{schema => ipc/message}/VectorLayout.java (89%) delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/BaseFileTest.java (99%) rename java/vector/src/test/java/org/apache/arrow/vector/{stream => ipc}/MessageSerializerTest.java (94%) rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowFile.java (94%) rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowFooter.java (93%) rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowReaderWriter.java (85%) rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowStream.java (88%) rename java/vector/src/test/java/org/apache/arrow/vector/{file => ipc}/TestArrowStreamPipe.java (78%) rename java/vector/src/test/java/org/apache/arrow/vector/{file/json => ipc}/TestJSONFile.java (99%) diff --git a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java index 3091bc4dab123..ce6b5164a8cbb 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java @@ -23,8 +23,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.stream.ArrowStreamReader; -import org.apache.arrow.vector.stream.ArrowStreamWriter; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java index ab8fa6e45cecd..6e45305bf6cd4 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java @@ -22,8 +22,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.file.ArrowFileReader; -import org.apache.arrow.vector.file.ArrowFileWriter; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java index 6722b30fa7f50..3db01f40c590e 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/FileToStream.java @@ -21,8 +21,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.file.ArrowFileReader; -import org.apache.arrow.vector.stream.ArrowStreamWriter; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; import java.io.File; import java.io.FileInputStream; diff --git a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java index d2b35e65a8172..666f1ddeabcc9 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/Integration.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/Integration.java @@ -22,11 +22,11 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.file.ArrowBlock; -import org.apache.arrow.vector.file.ArrowFileReader; -import org.apache.arrow.vector.file.ArrowFileWriter; -import org.apache.arrow.vector.file.json.JsonFileReader; -import org.apache.arrow.vector.file.json.JsonFileWriter; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.ipc.JsonFileReader; +import org.apache.arrow.vector.ipc.JsonFileWriter; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; diff --git a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java index ef1a11f6bfac8..42d336af9b040 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/StreamToFile.java @@ -21,8 +21,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.file.ArrowFileWriter; -import org.apache.arrow.vector.stream.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.ipc.ArrowStreamReader; import java.io.File; import java.io.FileInputStream; diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java index c56a5a3303b75..eac517d96bd48 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java @@ -28,9 +28,9 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BigIntWriter; import org.apache.arrow.vector.complex.writer.IntWriter; -import org.apache.arrow.vector.file.ArrowBlock; -import org.apache.arrow.vector.file.ArrowFileReader; -import org.apache.arrow.vector.file.ArrowFileWriter; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Assert; diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index 89714e4479180..d8693c596155b 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -44,8 +44,8 @@ import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; -import org.apache.arrow.vector.stream.ArrowStreamReader; -import org.apache.arrow.vector.stream.ArrowStreamWriter; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.Int; diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index e44edbd47b67a..73165315ea556 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -32,7 +32,7 @@ import org.apache.arrow.vector.BaseDataValueVector; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.util.OversizedAllocationException; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java index 38524ff8adf82..6d9eb1db03af5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import io.netty.buffer.ArrowBuf; import org.apache.arrow.vector.util.CallBack; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index 209758e4ece9e..f82077f692f3d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -28,7 +28,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index edf4987de5756..b9e5442ecf666 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -25,7 +25,7 @@ import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.NullableMapVector; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index c6d404e15430c..26c817008836c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -24,7 +24,7 @@ import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.BitHolder; import org.apache.arrow.vector.holders.NullableBitHolder; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.OversizedAllocationException; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 23252ca697b8d..2d4db85c583bf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -20,7 +20,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; /** * Helper class for performing generic operations on a bit vector buffer. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java index a0dbf2bdcf101..332ca228a43fd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferBacked.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import io.netty.buffer.ArrowBuf; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java index c2ed17eb4dd31..509eeda75000b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java @@ -20,7 +20,7 @@ import java.util.List; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import io.netty.buffer.ArrowBuf; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java index 58fc80bbba17c..2cd4099c66975 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java @@ -24,9 +24,9 @@ import java.util.Iterator; import java.util.List; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowRecordBatch; -import org.apache.arrow.vector.schema.VectorLayout; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.VectorLayout; import org.apache.arrow.vector.types.pojo.Field; import com.google.common.collect.Iterators; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index fd9677312ed33..2b034894ab14d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -22,9 +22,9 @@ import java.util.List; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowRecordBatch; -import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.ArrowVectorType; public class VectorUnloader { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 3cc93a2a34f43..0ab3a7b6843c3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -28,7 +28,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.complex.impl.NullReader; import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType.Null; import org.apache.arrow.vector.types.pojo.Field; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 6713b1c7871b5..774a10dbfb98f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -33,7 +33,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index afe86a692c3e1..d50d4c447fefd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -42,7 +42,7 @@ import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.FieldWriter; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index f95302f55f887..e223d1ce67449 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -34,7 +34,7 @@ import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; import org.apache.arrow.vector.complex.impl.NullableMapWriter; import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileReader.java similarity index 77% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileReader.java index d711b9c6c1e26..4cd70262261e8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.IOException; import java.nio.ByteBuffer; @@ -26,32 +26,45 @@ import org.apache.arrow.flatbuf.Footer; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.schema.ArrowDictionaryBatch; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.schema.ArrowRecordBatch; -import org.apache.arrow.vector.stream.MessageSerializer; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.message.ArrowFooter; +import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.types.pojo.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ArrowFileReader extends ArrowReader { +public class ArrowFileReader extends ArrowReader { private static final Logger LOGGER = LoggerFactory.getLogger(ArrowFileReader.class); + private SeekableReadChannel in; private ArrowFooter footer; private int currentDictionaryBatch = 0; private int currentRecordBatch = 0; + public ArrowFileReader(SeekableReadChannel in, BufferAllocator allocator) { + super(allocator); + this.in = in; + } + public ArrowFileReader(SeekableByteChannel in, BufferAllocator allocator) { - super(new SeekableReadChannel(in), allocator); + this(new SeekableReadChannel(in), allocator); } - public ArrowFileReader(SeekableReadChannel in, BufferAllocator allocator) { - super(in, allocator); + @Override + public long bytesRead() { + return in.bytesRead(); + } + + @Override + protected void closeReadSource() throws IOException { + in.close(); } @Override - protected Schema readSchema(SeekableReadChannel in) throws IOException { + protected Schema readSchema() throws IOException { if (footer == null) { if (in.size() <= (ArrowMagic.MAGIC_LENGTH * 2 + 4)) { throw new InvalidArrowFileException("file too small: " + in.size()); @@ -82,18 +95,30 @@ protected Schema readSchema(SeekableReadChannel in) throws IOException { } @Override - protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException { - if (currentDictionaryBatch < footer.getDictionaries().size()) { - ArrowBlock block = footer.getDictionaries().get(currentDictionaryBatch++); - return readDictionaryBatch(in, block, allocator); - } else if (currentRecordBatch < footer.getRecordBatches().size()) { + public ArrowDictionaryBatch readDictionary() throws IOException { + if (currentDictionaryBatch >= footer.getDictionaries().size()) { + throw new IOException("Requested more dictionaries than defined in footer: " + currentDictionaryBatch); + } + ArrowBlock block = footer.getDictionaries().get(currentDictionaryBatch++); + return readDictionaryBatch(in, block, allocator); + } + + // Returns true if a batch was read, false if no more batches + @Override + public boolean loadNextBatch() throws IOException { + prepareLoadNextBatch(); + + if (currentRecordBatch < footer.getRecordBatches().size()) { ArrowBlock block = footer.getRecordBatches().get(currentRecordBatch++); - return readRecordBatch(in, block, allocator); + ArrowRecordBatch batch = readRecordBatch(in, block, allocator); + loadRecordBatch(batch); + return true; } else { - return null; + return false; } } + public List getDictionaryBlocks() throws IOException { ensureInitialized(); return footer.getDictionaries(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java similarity index 94% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java index 1d92d2bde1c6f..1b687c9f2697a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.IOException; import java.nio.channels.WritableByteChannel; @@ -24,6 +24,8 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.message.ArrowFooter; import org.apache.arrow.vector.types.pojo.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java similarity index 93% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java index 68313e7878b71..a9310a608ae03 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowMagic.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java @@ -16,7 +16,9 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; + +import org.apache.arrow.vector.ipc.WriteChannel; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java similarity index 65% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java index 21fb2207eb019..6d708a03cadb0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.IOException; import java.util.ArrayList; @@ -33,32 +33,25 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; -import org.apache.arrow.vector.schema.ArrowDictionaryBatch; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.schema.ArrowMessage.ArrowMessageVisitor; -import org.apache.arrow.vector.schema.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.DictionaryUtility; /** - * Abstract class to read ArrowRecordBatches from a ReadChannel. + * Abstract class to read Schema and ArrowRecordBatches. * - * @param Type of ReadChannel to use */ -public abstract class ArrowReader implements DictionaryProvider, AutoCloseable { - - private final T in; - private final BufferAllocator allocator; +public abstract class ArrowReader implements DictionaryProvider, AutoCloseable { + protected final BufferAllocator allocator; private VectorLoader loader; private VectorSchemaRoot root; private Map dictionaries; - private boolean initialized = false; - protected ArrowReader(T in, BufferAllocator allocator) { - this.in = in; + protected ArrowReader(BufferAllocator allocator) { this.allocator = allocator; } @@ -105,58 +98,18 @@ public Dictionary lookup(long id) { * @return true if a batch was read, false on EOS * @throws IOException */ - public boolean loadNextBatch() throws IOException { - ensureInitialized(); - // read in all dictionary batches, then stop after our first record batch - ArrowMessageVisitor visitor = new ArrowMessageVisitor() { - @Override - public Boolean visit(ArrowDictionaryBatch message) { - try { - load(message); - } finally { - message.close(); - } - return true; - } - - @Override - public Boolean visit(ArrowRecordBatch message) { - try { - loader.load(message); - } finally { - message.close(); - } - return false; - } - }; - root.setRowCount(0); - ArrowMessage message = readMessage(in, allocator); - - boolean readBatch = false; - while (message != null) { - if (!message.accepts(visitor)) { - readBatch = true; - break; - } - // else read a dictionary - message = readMessage(in, allocator); - } - - return readBatch; - } + public abstract boolean loadNextBatch() throws IOException; /** * Return the number of bytes read from the ReadChannel. * * @return number of bytes read */ - public long bytesRead() { - return in.bytesRead(); - } + public abstract long bytesRead(); /** * Close resources, including vector schema root and dictionary vectors, and the - * underlying ReadChannel. + * underlying read source. * * @throws IOException */ @@ -167,12 +120,12 @@ public void close() throws IOException { /** * Close resources, including vector schema root and dictionary vectors. If the flag - * closeReadChannel is true then close the underlying ReadChannel, otherwise leave it open. + * closeReadChannel is true then close the underlying read source, otherwise leave it open. * - * @param closeReadChannel Flag to control if closing the underlying ReadChannel + * @param closeReadSource Flag to control if closing the underlying read source * @throws IOException */ - public void close(boolean closeReadChannel) throws IOException { + public void close(boolean closeReadSource) throws IOException { if (initialized) { root.close(); for (Dictionary dictionary : dictionaries.values()) { @@ -180,15 +133,40 @@ public void close(boolean closeReadChannel) throws IOException { } } - if (closeReadChannel) { - in.close(); + if (closeReadSource) { + closeReadSource(); } } - protected abstract Schema readSchema(T in) throws IOException; + /** + * Close the underlying read source. + * + * @throws IOException + */ + protected abstract void closeReadSource() throws IOException; + + /** + * Read the Schema from the source, will be invoked at the beginning the initialization. + * + * @return the read Schema + * @throws IOException + */ + protected abstract Schema readSchema() throws IOException; - protected abstract ArrowMessage readMessage(T in, BufferAllocator allocator) throws IOException; + /** + * Read a dictionary batch from the source, will be invoked after the schema has been read and + * called N times, where N is the number of dictionaries indicated by the schema Fields. + * + * @return the read ArrowDictionaryBatch + * @throws IOException + */ + protected abstract ArrowDictionaryBatch readDictionary() throws IOException; + /** + * Initialize if not done previously. + * + * @throws IOException + */ protected void ensureInitialized() throws IOException { if (!initialized) { initialize(); @@ -200,7 +178,7 @@ protected void ensureInitialized() throws IOException { * Reads the schema and initializes the vectors */ private void initialize() throws IOException { - Schema originalSchema = readSchema(in); + Schema originalSchema = readSchema(); List fields = new ArrayList<>(); List vectors = new ArrayList<>(); Map dictionaries = new HashMap<>(); @@ -216,9 +194,43 @@ private void initialize() throws IOException { this.root = new VectorSchemaRoot(schema, vectors, 0); this.loader = new VectorLoader(root); this.dictionaries = Collections.unmodifiableMap(dictionaries); + + // Read and load all dictionaries from schema + for (int i = 0; i < dictionaries.size(); i++) { + ArrowDictionaryBatch dictionaryBatch = readDictionary(); + loadDictionary(dictionaryBatch); + } } - private void load(ArrowDictionaryBatch dictionaryBatch) { + /** + * Ensure the reader has been initialized and reset the VectorSchemaRoot row count to 0. + * + * @throws IOException + */ + protected void prepareLoadNextBatch() throws IOException { + ensureInitialized(); + root.setRowCount(0); + } + + /** + * Load an ArrowRecordBatch to the readers VectorSchemaRoot. + * + * @param batch the record batch to load + */ + protected void loadRecordBatch(ArrowRecordBatch batch) { + try { + loader.load(batch); + } finally { + batch.close(); + } + } + + /** + * Load an ArrowDictionaryBatch to the readers dictionary vectors. + * + * @param dictionaryBatch + */ + protected void loadDictionary(ArrowDictionaryBatch dictionaryBatch) { long id = dictionaryBatch.getDictionaryId(); Dictionary dictionary = dictionaries.get(id); if (dictionary == null) { @@ -227,6 +239,10 @@ private void load(ArrowDictionaryBatch dictionaryBatch) { FieldVector vector = dictionary.getVector(); VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), 0); VectorLoader loader = new VectorLoader(root); - loader.load(dictionaryBatch.getDictionary()); + try { + loader.load(dictionaryBatch.getDictionary()); + } finally { + dictionaryBatch.close(); + } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java new file mode 100644 index 0000000000000..d1e48021885d5 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.ipc; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; + +import org.apache.arrow.flatbuf.Message; +import org.apache.arrow.flatbuf.MessageHeader; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.MessageChannelReader; +import org.apache.arrow.vector.ipc.message.MessageReader; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.ipc.ReadChannel; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * This classes reads from an input stream and produces ArrowRecordBatches. + */ +public class ArrowStreamReader extends ArrowReader { + + private MessageReader messageReader; + + /** + * Constructs a streaming reader using the MessageReader interface. Non-blocking. + * + * @param messageReader interface to get read messages + * @param allocator to allocate new buffers + */ + public ArrowStreamReader(MessageReader messageReader, BufferAllocator allocator) { + super(allocator); + this.messageReader = messageReader; + } + + /** + * Constructs a streaming reader from a ReadableByteChannel input. Non-blocking. + * + * @param in ReadableByteChannel to read messages from + * @param allocator to allocate new buffers + */ + public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) { + this(new MessageChannelReader(new ReadChannel(in)), allocator); + } + + /** + * Constructs a streaming reader from an InputStream. Non-blocking. + * + * @param in InputStream to read messages from + * @param allocator to allocate new buffers + */ + public ArrowStreamReader(InputStream in, BufferAllocator allocator) { + this(Channels.newChannel(in), allocator); + } + + /** + * Get the number of bytes read from the stream since constructing the reader. + * + * @return number of bytes + */ + @Override + public long bytesRead() { + return messageReader.bytesRead(); + } + + /** + * Closes the underlying read source. + * + * @throws IOException + */ + @Override + protected void closeReadSource() throws IOException { + messageReader.close(); + } + + /** + * Load the next ArrowRecordBatch to the vector schema root if available. + * + * @return true if a batch was read, false on EOS + * @throws IOException + */ + public boolean loadNextBatch() throws IOException { + prepareLoadNextBatch(); + + Message message = messageReader.readNextMessage(); + + // Reached EOS + if (message == null) { + return false; + } + + if (message.headerType() != MessageHeader.RecordBatch) { + throw new IOException("Expected RecordBatch but header was " + message.headerType()); + } + + ArrowRecordBatch batch = MessageSerializer.deserializeRecordBatch(messageReader, message, allocator); + loadRecordBatch(batch); + return true; + } + + /** + * Reads the schema message from the beginning of the stream. + * + * @return the deserialized arrow schema + */ + @Override + protected Schema readSchema() throws IOException { + return MessageSerializer.deserializeSchema(messageReader); + } + + /** + * Read a dictionary batch message, will be invoked after the schema and before normal record + * batches are read. + * + * @return the deserialized dictionary batch + * @throws IOException + */ + @Override + protected ArrowDictionaryBatch readDictionary() throws IOException { + Message message = messageReader.readNextMessage(); + + if (message.headerType() != MessageHeader.DictionaryBatch) { + throw new IOException("Expected DictionaryBatch but header was " + message.headerType()); + } + + return MessageSerializer.deserializeDictionaryBatch(messageReader, message, allocator); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java similarity index 84% rename from java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java index b854cd2bb6e74..d731d05b81f16 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java @@ -16,16 +16,13 @@ * limitations under the License. */ -package org.apache.arrow.vector.stream; +package org.apache.arrow.vector.ipc; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.dictionary.DictionaryProvider; -import org.apache.arrow.vector.file.ArrowBlock; -import org.apache.arrow.vector.file.ArrowWriter; -import org.apache.arrow.vector.file.WriteChannel; -import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.ArrowWriter; +import org.apache.arrow.vector.ipc.WriteChannel; import org.apache.arrow.vector.types.pojo.Schema; import java.io.IOException; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java similarity index 95% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java index 7dc10b5e6299f..4b483d0105004 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.IOException; import java.nio.channels.WritableByteChannel; @@ -30,9 +30,10 @@ import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; -import org.apache.arrow.vector.schema.ArrowDictionaryBatch; -import org.apache.arrow.vector.schema.ArrowRecordBatch; -import org.apache.arrow.vector.stream.MessageSerializer; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.DictionaryUtility; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/InvalidArrowFileException.java similarity index 96% rename from java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/InvalidArrowFileException.java index 607207f41b06c..ad9d8776e33f4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/InvalidArrowFileException.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/InvalidArrowFileException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; public class InvalidArrowFileException extends RuntimeException { private static final long serialVersionUID = 1L; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java similarity index 98% rename from java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index 8017b385d148a..cb11a253058e5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -16,19 +16,18 @@ * limitations under the License. ******************************************************************************/ -package org.apache.arrow.vector.file.json; +package org.apache.arrow.vector.ipc; import static com.fasterxml.jackson.core.JsonToken.END_ARRAY; import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.arrow.vector.schema.ArrowVectorType.*; +import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*; import java.io.File; import java.io.IOException; import java.math.BigDecimal; -import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -42,9 +41,8 @@ import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; -import org.apache.arrow.vector.file.InvalidArrowFileException; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowVectorType; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java similarity index 98% rename from java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 0c8507b51f598..22423b844e5db 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -16,9 +16,9 @@ * limitations under the License. ******************************************************************************/ -package org.apache.arrow.vector.file.json; +package org.apache.arrow.vector.ipc; -import static org.apache.arrow.vector.schema.ArrowVectorType.*; +import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*; import java.io.File; import java.io.IOException; @@ -33,7 +33,7 @@ import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; -import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.ipc.message.ArrowVectorType; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ReadChannel.java similarity index 98% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/ReadChannel.java index b0eb8f3d84d9a..395fd7db5975f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ReadChannel.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ReadChannel.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/SeekableReadChannel.java similarity index 97% rename from java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/SeekableReadChannel.java index 46bea1314da63..62ba3b73e5377 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/SeekableReadChannel.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/SeekableReadChannel.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.IOException; import java.nio.channels.SeekableByteChannel; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java similarity index 97% rename from java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java index 89c9d1f9b7a44..da500aa97bee9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/WriteChannel.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.IOException; import java.nio.ByteBuffer; @@ -25,7 +25,7 @@ import com.google.flatbuffers.FlatBufferBuilder; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.schema.FBSerializable; +import org.apache.arrow.vector.ipc.message.FBSerializable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java similarity index 96% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java index e1b4d6a8b215e..8731f77ac2c4c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowBlock.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java @@ -16,10 +16,9 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc.message; import org.apache.arrow.flatbuf.Block; -import org.apache.arrow.vector.schema.FBSerializable; import com.google.flatbuffers.FlatBufferBuilder; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java similarity index 97% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java index 4e0187e791b5a..6b0eeaad4d177 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowBuffer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; import org.apache.arrow.flatbuf.Buffer; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java similarity index 97% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java index 635fa3fb42307..cd23cb96b6bd7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowDictionaryBatch.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; import com.google.flatbuffers.FlatBufferBuilder; import org.apache.arrow.flatbuf.DictionaryBatch; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFieldNode.java similarity index 97% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFieldNode.java index 3ed384ed7e280..ca0087f7089ce 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowFieldNode.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFieldNode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; import org.apache.arrow.flatbuf.FieldNode; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java similarity index 96% rename from java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java index 1e95321fdec5b..f7794f7364bd3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowFooter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java @@ -16,16 +16,15 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc.message; -import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector; +import static org.apache.arrow.vector.ipc.message.FBSerializables.writeAllStructsToVector; import java.util.ArrayList; import java.util.List; import org.apache.arrow.flatbuf.Block; import org.apache.arrow.flatbuf.Footer; -import org.apache.arrow.vector.schema.FBSerializable; import org.apache.arrow.vector.types.pojo.Schema; import com.google.flatbuffers.FlatBufferBuilder; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowMessage.java similarity index 96% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowMessage.java index f59b4b6c1721e..92fb58e16fe6b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowMessage.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowMessage.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; public interface ArrowMessage extends FBSerializable, AutoCloseable { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java similarity index 94% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java index bf0967a2797fe..6c6481e74dd93 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java @@ -16,9 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; - -import static org.apache.arrow.vector.schema.FBSerializables.writeAllStructsToVector; +package org.apache.arrow.vector.ipc.message; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -113,9 +111,9 @@ public List getBuffersLayout() { @Override public int writeTo(FlatBufferBuilder builder) { RecordBatch.startNodesVector(builder, nodes.size()); - int nodesOffset = writeAllStructsToVector(builder, nodes); + int nodesOffset = FBSerializables.writeAllStructsToVector(builder, nodes); RecordBatch.startBuffersVector(builder, buffers.size()); - int buffersOffset = writeAllStructsToVector(builder, buffersLayout); + int buffersOffset = FBSerializables.writeAllStructsToVector(builder, buffersLayout); RecordBatch.startRecordBatch(builder); RecordBatch.addLength(builder, length); RecordBatch.addNodes(builder, nodesOffset); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java similarity index 98% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java index 9d2fdfaafe4aa..3342652bedd52 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowVectorType.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; import java.util.Map; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializable.java similarity index 95% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializable.java index 91d60ea995b89..31f55bd522c68 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializable.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializable.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; import com.google.flatbuffers.FlatBufferBuilder; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializables.java similarity index 96% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializables.java index ae5aa555e745e..6717ed7ab313e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/FBSerializables.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/FBSerializables.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; import java.util.ArrayList; import java.util.Collections; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java new file mode 100644 index 0000000000000..5bc3e1fff6f96 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageChannelReader.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.ipc.message; + + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.flatbuf.Message; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ReadChannel; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Reads a sequence of messages using a ReadChannel. + */ +public class MessageChannelReader implements MessageReader { + + private ReadChannel in; + + /** + * Construct from an existing ReadChannel. + * + * @param in Channel to read messages from + */ + public MessageChannelReader(ReadChannel in) { + this.in = in; + } + + /** + * Read the next message from the ReadChannel. + * + * @return A Message or null if ReadChannel has no more messages, indicated by message length of 0 + * @throws IOException + */ + @Override + public Message readNextMessage() throws IOException { + // Read the message size. There is an i32 little endian prefix. + ByteBuffer buffer = ByteBuffer.allocate(4); + if (in.readFully(buffer) != 4) { + return null; + } + int messageLength = MessageSerializer.bytesToInt(buffer.array()); + if (messageLength == 0) { + return null; + } + + buffer = ByteBuffer.allocate(messageLength); + if (in.readFully(buffer) != messageLength) { + throw new IOException( + "Unexpected end of stream trying to read message."); + } + buffer.rewind(); + + return Message.getRootAsMessage(buffer); + } + + /** + * Read a message body from the ReadChannel. + * + * @param message Read message that is followed by a body of data + * @param allocator BufferAllocator to allocate memory for body data + * @return ArrowBuf containing the message body data + * @throws IOException + */ + @Override + public ArrowBuf readMessageBody(Message message, BufferAllocator allocator) throws IOException { + + int bodyLength = (int) message.bodyLength(); + + // Now read the record batch body + ArrowBuf buffer = allocator.buffer(bodyLength); + if (in.readFully(buffer, bodyLength) != bodyLength) { + throw new IOException("Unexpected end of input trying to read batch."); + } + + return buffer; + } + + /** + * Get the number of bytes read from the ReadChannel. + * + * @return number of bytes + */ + @Override + public long bytesRead() { + return in.bytesRead(); + } + + /** + * Close the ReadChannel. + * + * @throws IOException + */ + @Override + public void close() throws IOException { + in.close(); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java new file mode 100644 index 0000000000000..b277c58295059 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageReader.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.ipc.message; + + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.flatbuf.Message; +import org.apache.arrow.memory.BufferAllocator; + +import java.io.IOException; + +/** + * Interface for reading a sequence of messages. + */ +public interface MessageReader { + + /** + * Read the next message in the sequence. + * + * @return The read message or null if reached the end of the message sequence + * @throws IOException + */ + Message readNextMessage() throws IOException; + + /** + * When a message is followed by a body of data, read that data into an ArrowBuf. This should + * only be called when a Message has a body length > 0. + * + * @param message Read message that is followed by a body of data + * @param allocator BufferAllocator to allocate memory for body data + * @return An ArrowBuf containing the body of the message that was read + * @throws IOException + */ + ArrowBuf readMessageBody(Message message, BufferAllocator allocator) throws IOException; + + /** + * Return the current number of bytes that have been read. + * + * @return number of bytes read + */ + long bytesRead(); + + /** + * Close any resource opened by the message reader, not including message body allocations. + * + * @throws IOException + */ + void close() throws IOException; +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java similarity index 86% rename from java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java index c397cec72f0ed..e2f8f7d9a8d7e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/stream/MessageSerializer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.stream; +package org.apache.arrow.vector.ipc.message; import java.io.IOException; import java.nio.ByteBuffer; @@ -31,14 +31,8 @@ import org.apache.arrow.flatbuf.MetadataVersion; import org.apache.arrow.flatbuf.RecordBatch; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.file.ArrowBlock; -import org.apache.arrow.vector.file.ReadChannel; -import org.apache.arrow.vector.file.WriteChannel; -import org.apache.arrow.vector.schema.ArrowBuffer; -import org.apache.arrow.vector.schema.ArrowDictionaryBatch; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.schema.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.ReadChannel; +import org.apache.arrow.vector.ipc.WriteChannel; import org.apache.arrow.vector.types.pojo.Schema; import com.google.flatbuffers.FlatBufferBuilder; @@ -102,12 +96,12 @@ public static long serialize(WriteChannel out, Schema schema) throws IOException /** * Deserializes a schema object. Format is from serialize(). * - * @param in the channel to deserialize from + * @param reader the reader interface to deserialize from * @return the deserialized object * @throws IOException if something went wrong */ - public static Schema deserializeSchema(ReadChannel in) throws IOException { - Message message = deserializeMessage(in); + public static Schema deserializeSchema(MessageReader reader) throws IOException { + Message message = reader.readNextMessage(); if (message == null) { throw new IOException("Unexpected end of input. Missing schema."); } @@ -119,6 +113,16 @@ public static Schema deserializeSchema(ReadChannel in) throws IOException { message.header(new org.apache.arrow.flatbuf.Schema())); } + /** + * Deserializes a schema object. Format is from serialize(). + * + * @param in the channel to deserialize from + * @return the deserialized object + * @throws IOException if something went wrong + */ + public static Schema deserializeSchema(ReadChannel in) throws IOException { + return deserializeSchema(new MessageChannelReader(in)); + } /** * Serializes an ArrowRecordBatch. Returns the offset and length of the written batch. @@ -184,25 +188,20 @@ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) t } /** - * Deserializes a RecordBatch + * Deserializes a RecordBatch. * - * @param in the channel to deserialize from + * @param reader the reader interface to deserialize from * @param message the object to derialize to * @param alloc to allocate buffers * @return the deserialized object * @throws IOException if something went wrong */ - public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, Message message, BufferAllocator alloc) + public static ArrowRecordBatch deserializeRecordBatch(MessageReader reader, Message message, BufferAllocator alloc) throws IOException { RecordBatch recordBatchFB = (RecordBatch) message.header(new RecordBatch()); - int bodyLength = (int) message.bodyLength(); - // Now read the record batch body - ArrowBuf buffer = alloc.buffer(bodyLength); - if (in.readFully(buffer, bodyLength) != bodyLength) { - throw new IOException("Unexpected end of input trying to read batch."); - } + ArrowBuf buffer = reader.readMessageBody(message, alloc); return deserializeRecordBatch(recordBatchFB, buffer); } @@ -243,7 +242,14 @@ public static ArrowRecordBatch deserializeRecordBatch(ReadChannel in, ArrowBlock return deserializeRecordBatch(recordBatchFB, body); } - // Deserializes a record batch given the Flatbuffer metadata and in-memory body + /** + * Deserializes a record batch given the Flatbuffer metadata and in-memory body. + * + * @param recordBatchFB Deserialized FlatBuffer record batch + * @param body Read body of the record batch + * @return ArrowRecordBatch from metadata and in-memory body + * @throws IOException + */ public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB, ArrowBuf body) throws IOException { // Now read the body @@ -314,26 +320,21 @@ public static ArrowBlock serialize(WriteChannel out, ArrowDictionaryBatch batch) } /** - * Deserializes a DictionaryBatch + * Deserializes a DictionaryBatch. * - * @param in where to read from + * @param reader where to read from * @param message the message message metadata to deserialize * @param alloc the allocator for new buffers * @return the corresponding dictionary batch * @throws IOException if something went wrong */ - public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in, + public static ArrowDictionaryBatch deserializeDictionaryBatch(MessageReader reader, Message message, BufferAllocator alloc) throws IOException { DictionaryBatch dictionaryBatchFB = (DictionaryBatch) message.header(new DictionaryBatch()); - int bodyLength = (int) message.bodyLength(); - // Now read the record batch body - ArrowBuf body = alloc.buffer(bodyLength); - if (in.readFully(body, bodyLength) != bodyLength) { - throw new IOException("Unexpected end of input trying to read batch."); - } + ArrowBuf body = reader.readMessageBody(message, alloc); ArrowRecordBatch recordBatch = deserializeRecordBatch(dictionaryBatchFB.data(), body); return new ArrowDictionaryBatch(dictionaryBatchFB.id(), recordBatch); } @@ -377,8 +378,16 @@ public static ArrowDictionaryBatch deserializeDictionaryBatch(ReadChannel in, return new ArrowDictionaryBatch(dictionaryBatchFB.id(), recordBatch); } - public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocator alloc) throws IOException { - Message message = deserializeMessage(in); + /** + * Deserialize a message that is either an ArrowDictionaryBatch or ArrowRecordBatch. + * + * @param reader Interface to read messages from + * @param alloc Allocator for message data + * @return The deserialized record batch + * @throws IOException if the message is not an ArrowDictionaryBatch or ArrowRecordBatch + */ + public static ArrowMessage deserializeMessageBatch(MessageReader reader, BufferAllocator alloc) throws IOException { + Message message = reader.readNextMessage(); if (message == null) { return null; } else if (message.bodyLength() > Integer.MAX_VALUE) { @@ -391,14 +400,26 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat switch (message.headerType()) { case MessageHeader.RecordBatch: - return deserializeRecordBatch(in, message, alloc); + return deserializeRecordBatch(reader, message, alloc); case MessageHeader.DictionaryBatch: - return deserializeDictionaryBatch(in, message, alloc); + return deserializeDictionaryBatch(reader, message, alloc); default: throw new IOException("Unexpected message header type " + message.headerType()); } } + /** + * Deserialize a message that is either an ArrowDictionaryBatch or ArrowRecordBatch. + * + * @param in ReadChannel to read messages from + * @param alloc Allocator for message data + * @return The deserialized record batch + * @throws IOException if the message is not an ArrowDictionaryBatch or ArrowRecordBatch + */ + public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocator alloc) throws IOException { + return deserializeMessageBatch(new MessageChannelReader(in), alloc); + } + /** * Serializes a message header. * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java similarity index 80% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java index 29407bf1ab4e1..06fe9481686d7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/TypeLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java @@ -16,15 +16,9 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; +package org.apache.arrow.vector.ipc.message; import static java.util.Arrays.asList; -import static org.apache.arrow.vector.schema.VectorLayout.booleanVector; -import static org.apache.arrow.vector.schema.VectorLayout.byteVector; -import static org.apache.arrow.vector.schema.VectorLayout.dataVector; -import static org.apache.arrow.vector.schema.VectorLayout.offsetVector; -import static org.apache.arrow.vector.schema.VectorLayout.typeVector; -import static org.apache.arrow.vector.schema.VectorLayout.validityVector; import java.util.ArrayList; import java.util.Collections; @@ -64,7 +58,7 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) { @Override public TypeLayout visit(Int type) { - return newFixedWidthTypeLayout(dataVector(type.getBitWidth())); + return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth())); } @Override @@ -74,14 +68,14 @@ public TypeLayout visit(Union type) { case Dense: vectors = asList( // TODO: validate this - validityVector(), - typeVector(), - offsetVector() // offset to find the vector + VectorLayout.validityVector(), + VectorLayout.typeVector(), + VectorLayout.offsetVector() // offset to find the vector ); break; case Sparse: vectors = asList( - typeVector() // type of the value at the index or 0 if null + VectorLayout.typeVector() // type of the value at the index or 0 if null ); break; default: @@ -93,21 +87,21 @@ public TypeLayout visit(Union type) { @Override public TypeLayout visit(Struct type) { List vectors = asList( - validityVector() + VectorLayout.validityVector() ); return new TypeLayout(vectors); } @Override public TypeLayout visit(Timestamp type) { - return newFixedWidthTypeLayout(dataVector(64)); + return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); } @Override public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { List vectors = asList( - validityVector(), - offsetVector() + VectorLayout.validityVector(), + VectorLayout.offsetVector() ); return new TypeLayout(vectors); } @@ -115,7 +109,7 @@ public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) @Override public TypeLayout visit(FixedSizeList type) { List vectors = asList( - validityVector() + VectorLayout.validityVector() ); return new TypeLayout(vectors); } @@ -136,18 +130,18 @@ public TypeLayout visit(FloatingPoint type) { default: throw new UnsupportedOperationException("Unsupported Precision: " + type.getPrecision()); } - return newFixedWidthTypeLayout(dataVector(bitWidth)); + return newFixedWidthTypeLayout(VectorLayout.dataVector(bitWidth)); } @Override public TypeLayout visit(Decimal type) { // TODO: check size - return newFixedWidthTypeLayout(dataVector(64)); // actually depends on the type fields + return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); // actually depends on the type fields } @Override public TypeLayout visit(Bool type) { - return newFixedWidthTypeLayout(booleanVector()); + return newFixedWidthTypeLayout(VectorLayout.booleanVector()); } @Override @@ -161,7 +155,7 @@ public TypeLayout visit(Utf8 type) { } private TypeLayout newVariableWidthTypeLayout() { - return newPrimitiveTypeLayout(validityVector(), offsetVector(), byteVector()); + return newPrimitiveTypeLayout(VectorLayout.validityVector(), VectorLayout.offsetVector(), VectorLayout.byteVector()); } private TypeLayout newPrimitiveTypeLayout(VectorLayout... vectors) { @@ -169,7 +163,7 @@ private TypeLayout newPrimitiveTypeLayout(VectorLayout... vectors) { } public TypeLayout newFixedWidthTypeLayout(VectorLayout dataVector) { - return newPrimitiveTypeLayout(validityVector(), dataVector); + return newPrimitiveTypeLayout(VectorLayout.validityVector(), dataVector); } @Override @@ -179,21 +173,21 @@ public TypeLayout visit(Null type) { @Override public TypeLayout visit(Date type) { - return newFixedWidthTypeLayout(dataVector(64)); + return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); } @Override public TypeLayout visit(Time type) { - return newFixedWidthTypeLayout(dataVector(type.getBitWidth())); + return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth())); } @Override public TypeLayout visit(Interval type) { // TODO: check size switch (type.getUnit()) { case DAY_TIME: - return newFixedWidthTypeLayout(dataVector(64)); + return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); case YEAR_MONTH: - return newFixedWidthTypeLayout(dataVector(64)); + return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); default: throw new UnsupportedOperationException("Unknown unit " + type.getUnit()); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java similarity index 89% rename from java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java rename to java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java index 0871baf38edaa..e4f2f98fde37a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/VectorLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java @@ -16,12 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.schema; - -import static org.apache.arrow.vector.schema.ArrowVectorType.DATA; -import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET; -import static org.apache.arrow.vector.schema.ArrowVectorType.TYPE; -import static org.apache.arrow.vector.schema.ArrowVectorType.VALIDITY; +package org.apache.arrow.vector.ipc.message; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -30,14 +25,14 @@ public class VectorLayout implements FBSerializable { - private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(VALIDITY, 1); - private static final VectorLayout OFFSET_VECTOR = new VectorLayout(OFFSET, 32); - private static final VectorLayout TYPE_VECTOR = new VectorLayout(TYPE, 32); - private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(DATA, 1); - private static final VectorLayout VALUES_64 = new VectorLayout(DATA, 64); - private static final VectorLayout VALUES_32 = new VectorLayout(DATA, 32); - private static final VectorLayout VALUES_16 = new VectorLayout(DATA, 16); - private static final VectorLayout VALUES_8 = new VectorLayout(DATA, 8); + private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(ArrowVectorType.VALIDITY, 1); + private static final VectorLayout OFFSET_VECTOR = new VectorLayout(ArrowVectorType.OFFSET, 32); + private static final VectorLayout TYPE_VECTOR = new VectorLayout(ArrowVectorType.TYPE, 32); + private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(ArrowVectorType.DATA, 1); + private static final VectorLayout VALUES_64 = new VectorLayout(ArrowVectorType.DATA, 64); + private static final VectorLayout VALUES_32 = new VectorLayout(ArrowVectorType.DATA, 32); + private static final VectorLayout VALUES_16 = new VectorLayout(ArrowVectorType.DATA, 16); + private static final VectorLayout VALUES_8 = new VectorLayout(ArrowVectorType.DATA, 8); public static VectorLayout typeVector() { return TYPE_VECTOR; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java deleted file mode 100644 index 5b6300076b6c2..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/stream/ArrowStreamReader.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.stream; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.file.ArrowReader; -import org.apache.arrow.vector.file.ReadChannel; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.types.pojo.Schema; - -/** - * This classes reads from an input stream and produces ArrowRecordBatches. - */ -public class ArrowStreamReader extends ArrowReader { - - /** - * Constructs a streaming read, reading bytes from 'in'. Non-blocking. - * - * @param in the stream to read from - * @param allocator to allocate new buffers - */ - public ArrowStreamReader(ReadableByteChannel in, BufferAllocator allocator) { - super(new ReadChannel(in), allocator); - } - - public ArrowStreamReader(InputStream in, BufferAllocator allocator) { - this(Channels.newChannel(in), allocator); - } - - /** - * Reads the schema message from the beginning of the stream. - * - * @param in to allocate new buffers - * @return the deserialized arrow schema - */ - @Override - protected Schema readSchema(ReadChannel in) throws IOException { - return MessageSerializer.deserializeSchema(in); - } - - @Override - protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException { - return MessageSerializer.deserializeMessageBatch(in, allocator); - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java index eba149bf79f65..574612833cd19 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java @@ -43,8 +43,8 @@ import org.apache.arrow.flatbuf.Type; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.schema.TypeLayout; -import org.apache.arrow.vector.schema.VectorLayout; +import org.apache.arrow.vector.ipc.message.VectorLayout; +import org.apache.arrow.vector.ipc.message.TypeLayout; import org.apache.arrow.vector.types.pojo.ArrowType.Int; public class Field { @@ -117,9 +117,9 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) { } dictionary = new DictionaryEncoding(dictionaryFB.id(), dictionaryFB.isOrdered(), indexType); } - ImmutableList.Builder layout = ImmutableList.builder(); + ImmutableList.Builder layout = ImmutableList.builder(); for (int i = 0; i < field.layoutLength(); ++i) { - layout.add(new org.apache.arrow.vector.schema.VectorLayout(field.layout(i))); + layout.add(new VectorLayout(field.layout(i))); } ImmutableList.Builder childrenBuilder = ImmutableList.builder(); for (int i = 0; i < field.childrenLength(); i++) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index c7ee202f94661..f51a874367622 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -17,7 +17,6 @@ */ package org.apache.arrow.vector; -import org.apache.arrow.vector.holders.VarCharHolder; import org.apache.arrow.vector.util.OversizedAllocationException; import static org.apache.arrow.vector.TestUtils.newNullableVarBinaryVector; @@ -38,15 +37,14 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.schema.ArrowRecordBatch; -import org.apache.arrow.vector.schema.TypeLayout; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.TypeLayout; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.TransferPair; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 3853eecac0552..e61dbecf44101 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -39,8 +39,8 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BigIntWriter; import org.apache.arrow.vector.complex.writer.IntWriter; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java similarity index 99% rename from java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index 874ba99e20f29..233b682c99bc6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java similarity index 94% rename from java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java index f968768f5e67d..239d3034ad135 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.stream; +package org.apache.arrow.vector.ipc; import static java.util.Arrays.asList; import static org.junit.Assert.assertArrayEquals; @@ -33,12 +33,11 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.file.ArrowBlock; -import org.apache.arrow.vector.file.ReadChannel; -import org.apache.arrow.vector.file.WriteChannel; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.schema.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowMessage; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java similarity index 94% rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java index 8559969a20424..4387db03655d1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -40,16 +40,14 @@ import org.apache.arrow.vector.NullableIntVector; import org.apache.arrow.vector.NullableTinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; -import org.apache.arrow.vector.schema.ArrowBuffer; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.schema.ArrowRecordBatch; -import org.apache.arrow.vector.stream.ArrowStreamReader; -import org.apache.arrow.vector.stream.ArrowStreamWriter; -import org.apache.arrow.vector.stream.MessageSerializerTest; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.message.ArrowBuffer; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -108,52 +106,41 @@ public void testWriteRead() throws IOException { // read try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE); FileInputStream fileInputStream = new FileInputStream(file); - ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator) { - @Override - protected ArrowMessage readMessage(SeekableReadChannel in, BufferAllocator allocator) throws IOException { - ArrowMessage message = super.readMessage(in, allocator); - if (message != null) { - ArrowRecordBatch batch = (ArrowRecordBatch) message; - List buffersLayout = batch.getBuffersLayout(); - for (ArrowBuffer arrowBuffer : buffersLayout) { - Assert.assertEquals(0, arrowBuffer.getOffset() % 8); - } - } - return message; - } - }) { - Schema schema = arrowReader.getVectorSchemaRoot().getSchema(); - LOGGER.debug("reading schema: " + schema); + ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) { + VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); + VectorUnloader unloader = new VectorUnloader(root); + Schema schema = root.getSchema(); + LOGGER.debug("reading schema: " + schema); for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) { arrowReader.loadRecordBatch(rbBlock); Assert.assertEquals(count, root.getRowCount()); + ArrowRecordBatch batch = unloader.getRecordBatch(); + List buffersLayout = batch.getBuffersLayout(); + for (ArrowBuffer arrowBuffer : buffersLayout) { + Assert.assertEquals(0, arrowBuffer.getOffset() % 8); + } validateContent(count, root); + batch.close(); } } // Read from stream. try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE); ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray()); - ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator) { - @Override - protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException { - ArrowMessage message = super.readMessage(in, allocator); - if (message != null) { - ArrowRecordBatch batch = (ArrowRecordBatch) message; - List buffersLayout = batch.getBuffersLayout(); - for (ArrowBuffer arrowBuffer : buffersLayout) { - Assert.assertEquals(0, arrowBuffer.getOffset() % 8); - } - } - return message; - } - }) { + ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) { VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); + VectorUnloader unloader = new VectorUnloader(root); Schema schema = root.getSchema(); LOGGER.debug("reading schema: " + schema); Assert.assertTrue(arrowReader.loadNextBatch()); + ArrowRecordBatch batch = unloader.getRecordBatch(); + List buffersLayout = batch.getBuffersLayout(); + for (ArrowBuffer arrowBuffer : buffersLayout) { + Assert.assertEquals(0, arrowBuffer.getOffset() % 8); + } + batch.close(); Assert.assertEquals(count, root.getRowCount()); validateContent(count, root); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java similarity index 93% rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java index 4612465323130..235e8c1646712 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; @@ -27,6 +27,8 @@ import java.util.List; import org.apache.arrow.flatbuf.Footer; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.message.ArrowFooter; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java similarity index 85% rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java index 3ce01a26835d5..49e194b51b823 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import static java.nio.channels.Channels.newChannel; import static java.util.Arrays.asList; @@ -37,9 +37,15 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.TestUtils; +import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowRecordBatch; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.ipc.SeekableReadChannel; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; @@ -87,7 +93,10 @@ public void test() throws IOException { ArrowFileWriter writer = new ArrowFileWriter(root, null, newChannel(out))) { ArrowBuf validityb = buf(validity); ArrowBuf valuesb = buf(values); - writer.writeRecordBatch(new ArrowRecordBatch(16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb))); + ArrowRecordBatch batch = new ArrowRecordBatch(16, asList(new ArrowFieldNode(16, 8)), asList(validityb, valuesb)); + VectorLoader loader = new VectorLoader(root); + loader.load(batch); + writer.writeBatch(); } byte[] byteArray = out.toByteArray(); @@ -100,7 +109,9 @@ public void test() throws IOException { // TODO: dictionaries List recordBatches = reader.getRecordBlocks(); assertEquals(1, recordBatches.size()); - ArrowRecordBatch recordBatch = (ArrowRecordBatch) reader.readMessage(channel, allocator); + reader.loadNextBatch(); + VectorUnloader unloader = new VectorUnloader(reader.getVectorSchemaRoot()); + ArrowRecordBatch recordBatch = unloader.getRecordBatch(); List nodes = recordBatch.getNodes(); assertEquals(1, nodes.size()); ArrowFieldNode node = nodes.get(0); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java similarity index 88% rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java index c7e34191a3f1a..7a8586a9e708e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; -import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,16 +26,12 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.NullableTinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.schema.ArrowRecordBatch; -import org.apache.arrow.vector.stream.ArrowStreamReader; -import org.apache.arrow.vector.stream.ArrowStreamWriter; -import org.apache.arrow.vector.stream.MessageSerializerTest; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.ipc.BaseFileTest; +import org.apache.arrow.vector.ipc.MessageSerializerTest; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Assert; import org.junit.Test; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java similarity index 78% rename from java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java index f393733cc9021..65e6cea2ec2e6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file; +package org.apache.arrow.vector.ipc; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -30,10 +30,9 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.NullableTinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.schema.ArrowMessage; -import org.apache.arrow.vector.stream.ArrowStreamReader; -import org.apache.arrow.vector.stream.ArrowStreamWriter; -import org.apache.arrow.vector.stream.MessageSerializerTest; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.ipc.MessageSerializerTest; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Assert; import org.junit.Test; @@ -95,37 +94,28 @@ private final class ReaderThread extends Thread { public ReaderThread(ReadableByteChannel sourceChannel) throws IOException { reader = new ArrowStreamReader(sourceChannel, alloc) { - @Override - protected ArrowMessage readMessage(ReadChannel in, BufferAllocator allocator) throws IOException { - // Read all the batches. Each batch contains an incrementing id and then some - // constant data. Verify both. - ArrowMessage message = super.readMessage(in, allocator); - if (message == null) { - done = true; - } else { - batchesRead++; - } - return message; - } @Override public boolean loadNextBatch() throws IOException { - if (!super.loadNextBatch()) { + if (super.loadNextBatch()) { + batchesRead++; + } else { + done = true; return false; } - if (!done) { - VectorSchemaRoot root = getVectorSchemaRoot(); - Assert.assertEquals(16, root.getRowCount()); - NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); - Assert.assertEquals((byte) (batchesRead - 1), vector.get(0)); - for (int i = 1; i < 16; i++) { - if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.get(i)); - } else { - Assert.assertTrue(vector.isNull(i)); - } + + VectorSchemaRoot root = getVectorSchemaRoot(); + Assert.assertEquals(16, root.getRowCount()); + NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); + Assert.assertEquals((byte) (batchesRead - 1), vector.get(0)); + for (int i = 1; i < 16; i++) { + if (i < 8) { + Assert.assertEquals((byte) (i + 1), vector.get(i)); + } else { + Assert.assertTrue(vector.isNull(i)); } } + return true; } }; @@ -139,7 +129,7 @@ public void run() { reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(), reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0); while (!done) { - assertTrue(reader.loadNextBatch()); + assertTrue(reader.loadNextBatch() != done); } reader.close(); } catch (IOException e) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java similarity index 99% rename from java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java rename to java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java index 5c4c48cd26b60..c3e0b795174c0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.file.json; +package org.apache.arrow.vector.ipc; import java.io.File; import java.io.IOException; @@ -28,7 +28,6 @@ import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; -import org.apache.arrow.vector.file.BaseFileTest; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Validator; import org.junit.Assert; From 9b2dc77a4d95c7415edd5be087a5abafc5a7f64c Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 22 Nov 2017 17:22:19 -0500 Subject: [PATCH 073/177] =?UTF-8?q?ARROW-1845:=20[Python]=C2=A0Expose=20De?= =?UTF-8?q?cimal128Type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author: Uwe L. Korn Author: Wes McKinney Closes #1348 from xhochy/ARROW-1845 and squashes the following commits: 98514320 [Wes McKinney] Fix API page in Python docs for decimal->decimal128 5fade1f5 [Uwe L. Korn] ARROW-1845: [Python] Expose Decimal128Type --- python/doc/source/api.rst | 2 +- python/pyarrow/__init__.py | 2 +- python/pyarrow/includes/libarrow.pxd | 4 ++-- python/pyarrow/lib.pxd | 4 ++-- python/pyarrow/pandas_compat.py | 2 +- python/pyarrow/public-api.pxi | 2 +- python/pyarrow/tests/test_array.py | 2 +- python/pyarrow/tests/test_convert_builtin.py | 12 ++++++------ python/pyarrow/tests/test_convert_pandas.py | 6 +++--- python/pyarrow/tests/test_schema.py | 2 +- python/pyarrow/tests/test_types.py | 2 +- python/pyarrow/types.pxi | 18 +++++++++--------- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst index 8f2f23d9fd470..fb2a286770e96 100644 --- a/python/doc/source/api.rst +++ b/python/doc/source/api.rst @@ -50,7 +50,7 @@ Type and Schema Factory Functions date64 binary string - decimal + decimal128 list_ struct dictionary diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index c8ded2d3cb88d..c4db36e55a0d6 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -35,7 +35,7 @@ uint8, uint16, uint32, uint64, time32, time64, timestamp, date32, date64, float16, float32, float64, - binary, string, decimal, + binary, string, decimal128, list_, struct, union, dictionary, field, type_for_alias, DataType, NAType, diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 73e34c7b21033..f1f59384bd639 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -209,10 +209,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int byte_width() int bit_width() - cdef cppclass CDecimalType" arrow::DecimalType"(CFixedSizeBinaryType): + cdef cppclass CDecimal128Type" arrow::Decimal128Type"(CFixedSizeBinaryType): + CDecimal128Type(int precision, int scale) int precision() int scale() - CDecimalType(int precision, int scale) cdef cppclass CField" arrow::Field": const c_string& name() diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 6413b838f0595..5abb72ba48165 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -81,9 +81,9 @@ cdef class FixedSizeBinaryType(DataType): const CFixedSizeBinaryType* fixed_size_binary_type -cdef class DecimalType(FixedSizeBinaryType): +cdef class Decimal128Type(FixedSizeBinaryType): cdef: - const CDecimalType* decimal_type + const CDecimal128Type* decimal128_type cdef class Field: diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 0aab9a41b8025..a50ef96e79f59 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -80,7 +80,7 @@ def get_logical_type(arrow_type): return 'list[{}]'.format(get_logical_type(arrow_type.value_type)) elif isinstance(arrow_type, pa.lib.TimestampType): return 'datetimetz' if arrow_type.tz is not None else 'datetime' - elif isinstance(arrow_type, pa.lib.DecimalType): + elif isinstance(arrow_type, pa.lib.Decimal128Type): return 'decimal' raise NotImplementedError(str(arrow_type)) diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi index 90aff9e936d95..bf670c5c4e290 100644 --- a/python/pyarrow/public-api.pxi +++ b/python/pyarrow/public-api.pxi @@ -78,7 +78,7 @@ cdef public api object pyarrow_wrap_data_type( elif type.get().id() == _Type_FIXED_SIZE_BINARY: out = FixedSizeBinaryType() elif type.get().id() == _Type_DECIMAL: - out = DecimalType() + out = Decimal128Type() else: out = DataType() diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index b7b0b18333677..a4d781a3354b0 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -470,7 +470,7 @@ def test_simple_type_construction(): (pa.binary(length=4), 'bytes'), (pa.string(), 'unicode'), (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'), - (pa.decimal(18, 3), 'decimal'), + (pa.decimal128(18, 3), 'decimal'), (pa.timestamp('ms'), 'datetime'), (pa.timestamp('us', 'UTC'), 'datetimetz'), (pa.time32('s'), 'time'), diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py index c7a0d49b40db1..4c3d9e5631ce3 100644 --- a/python/pyarrow/tests/test_convert_builtin.py +++ b/python/pyarrow/tests/test_convert_builtin.py @@ -314,7 +314,7 @@ def test_mixed_types_fails(self): def test_decimal(self): data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')] - type = pa.decimal(precision=7, scale=3) + type = pa.decimal128(precision=7, scale=3) arr = pa.array(data, type=type) assert arr.to_pylist() == data @@ -322,32 +322,32 @@ def test_decimal_different_precisions(self): data = [ decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234') ] - type = pa.decimal(precision=13, scale=3) + type = pa.decimal128(precision=13, scale=3) arr = pa.array(data, type=type) assert arr.to_pylist() == data def test_decimal_no_scale(self): data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')] - type = pa.decimal(precision=10) + type = pa.decimal128(precision=10) arr = pa.array(data, type=type) assert arr.to_pylist() == data def test_decimal_negative(self): data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')] - type = pa.decimal(precision=10, scale=6) + type = pa.decimal128(precision=10, scale=6) arr = pa.array(data, type=type) assert arr.to_pylist() == data def test_decimal_no_whole_part(self): data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')] - type = pa.decimal(precision=7, scale=7) + type = pa.decimal128(precision=7, scale=7) arr = pa.array(data, type=type) assert arr.to_pylist() == data def test_decimal_large_integer(self): data = [decimal.Decimal('-394029506937548693.42983'), decimal.Decimal('32358695912932.01033')] - type = pa.decimal(precision=23, scale=5) + type = pa.decimal128(precision=23, scale=5) arr = pa.array(data, type=type) assert arr.to_pylist() == data diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index b9c3a12213243..a2a1d6b6910fa 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -828,7 +828,7 @@ def test_decimal_32_from_pandas(self): ] }) converted = pa.Table.from_pandas(expected, preserve_index=False) - field = pa.field('decimals', pa.decimal(7, 3)) + field = pa.field('decimals', pa.decimal128(7, 3)) schema = pa.schema([field]) assert converted.schema.equals(schema) @@ -851,7 +851,7 @@ def test_decimal_64_from_pandas(self): ] }) converted = pa.Table.from_pandas(expected, preserve_index=False) - field = pa.field('decimals', pa.decimal(12, 6)) + field = pa.field('decimals', pa.decimal128(12, 6)) schema = pa.schema([field]) assert converted.schema.equals(schema) @@ -874,7 +874,7 @@ def test_decimal_128_from_pandas(self): ] }) converted = pa.Table.from_pandas(expected, preserve_index=False) - field = pa.field('decimals', pa.decimal(26, 11)) + field = pa.field('decimals', pa.decimal128(26, 11)) schema = pa.schema([field]) assert converted.schema.equals(schema) diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py index 116f3978333a8..dbca139e20570 100644 --- a/python/pyarrow/tests/test_schema.py +++ b/python/pyarrow/tests/test_schema.py @@ -333,7 +333,7 @@ def test_type_schema_pickling(): pa.date64(), pa.timestamp('ms'), pa.timestamp('ns'), - pa.decimal(12, 2), + pa.decimal128(12, 2), pa.field('a', 'string', metadata={b'foo': b'bar'}) ] diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index 9eefa33b66187..e352e35a3a3a0 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -56,7 +56,7 @@ def test_is_null(): def test_is_decimal(): - assert types.is_decimal(pa.decimal(19, 4)) + assert types.is_decimal(pa.decimal128(19, 4)) assert not types.is_decimal(pa.int32()) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index abd678bac6e58..1563b57855cd9 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -290,28 +290,28 @@ cdef class FixedSizeBinaryType(DataType): return self.fixed_size_binary_type.byte_width() -cdef class DecimalType(FixedSizeBinaryType): +cdef class Decimal128Type(FixedSizeBinaryType): cdef void init(self, const shared_ptr[CDataType]& type): DataType.init(self, type) - self.decimal_type = type.get() + self.decimal128_type = type.get() def __getstate__(self): return (self.precision, self.scale) def __setstate__(self, state): - cdef DataType reconstituted = decimal(*state) + cdef DataType reconstituted = decimal128(*state) self.init(reconstituted.sp_type) property precision: def __get__(self): - return self.decimal_type.precision() + return self.decimal128_type.precision() property scale: def __get__(self): - return self.decimal_type.scale() + return self.decimal128_type.scale() cdef class Field: @@ -953,9 +953,9 @@ def float64(): return primitive_type(_Type_DOUBLE) -cpdef DataType decimal(int precision, int scale=0): +cpdef DataType decimal128(int precision, int scale=0): """ - Create decimal type with precision and scale + Create decimal type with precision and scale and 128bit width Parameters ---------- @@ -964,10 +964,10 @@ cpdef DataType decimal(int precision, int scale=0): Returns ------- - decimal_type : DecimalType + decimal_type : Decimal128Type """ cdef shared_ptr[CDataType] decimal_type - decimal_type.reset(new CDecimalType(precision, scale)) + decimal_type.reset(new CDecimal128Type(precision, scale)) return pyarrow_wrap_data_type(decimal_type) From dda2d34c712a6e64bcfeafe2b1e764ba8cd017e3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 23 Nov 2017 09:41:46 -0500 Subject: [PATCH 074/177] ARROW-1828: [C++] Hash kernel specialization for BooleanType This is a bit tedious because we want to preserve the order in which the unique values were observed. Author: Wes McKinney Closes #1350 from wesm/ARROW-1828 and squashes the following commits: 576ab330 [Wes McKinney] Fix typo 498a9092 [Wes McKinney] clang-format, fix Python flakes c6a2b8fe [Wes McKinney] Add tests without nulls b6cd4db6 [Wes McKinney] Finish boolean hash kernel implementation, tests passing e4d4db6f [Wes McKinney] Scaffolding --- cpp/src/arrow/compute/compute-test.cc | 34 ++++++++ cpp/src/arrow/compute/kernels/hash.cc | 77 ++++++++++++++++++- cpp/src/arrow/compute/kernels/util-internal.h | 4 + python/pyarrow/includes/libarrow.pxd | 3 +- 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index fa408ae40cd85..96edd8f01ed38 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -843,6 +843,40 @@ TEST_F(TestHashKernel, UniqueTimeTimestamp) { {}); } +TEST_F(TestHashKernel, UniqueBoolean) { + CheckUnique(&this->ctx_, boolean(), {true, true, false, true}, + {true, false, true, true}, {true, false}, {}); + + CheckUnique(&this->ctx_, boolean(), {false, true, false, true}, + {true, false, true, true}, {false, true}, {}); + + // No nulls + CheckUnique(&this->ctx_, boolean(), {true, true, false, true}, {}, + {true, false}, {}); + + CheckUnique(&this->ctx_, boolean(), {false, true, false, true}, {}, + {false, true}, {}); +} + +TEST_F(TestHashKernel, DictEncodeBoolean) { + CheckDictEncode( + &this->ctx_, boolean(), {true, true, false, true, false}, + {true, false, true, true, true}, {true, false}, {}, {0, 0, 1, 0, 1}); + + CheckDictEncode( + &this->ctx_, boolean(), {false, true, false, true, false}, + {true, false, true, true, true}, {false, true}, {}, {0, 0, 0, 1, 0}); + + // No nulls + CheckDictEncode(&this->ctx_, boolean(), + {true, true, false, true, false}, {}, {true, false}, + {}, {0, 0, 1, 0, 1}); + + CheckDictEncode(&this->ctx_, boolean(), + {false, true, false, true, false}, {}, {false, true}, + {}, {0, 1, 0, 1, 0}); +} + TEST_F(TestHashKernel, UniqueBinary) { CheckUnique(&this->ctx_, binary(), {"test", "", "test2", "test"}, diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc index 95f039932152c..e47759d4d4706 100644 --- a/cpp/src/arrow/compute/kernels/hash.cc +++ b/cpp/src/arrow/compute/kernels/hash.cc @@ -368,6 +368,79 @@ class HashTableKernel> : public HashTab HashDictionary dict_; }; +// ---------------------------------------------------------------------- +// Hash table for boolean types + +template +class HashTableKernel> : public HashTable { + public: + HashTableKernel(const std::shared_ptr& type, MemoryPool* pool) + : HashTable(type, pool) { + std::fill(table_, table_ + 2, kHashSlotEmpty); + } + + Status Append(const ArrayData& arr) override { + auto action = static_cast(this); + + RETURN_NOT_OK(action->Reserve(arr.length)); + + internal::BitmapReader value_reader(arr.buffers[1]->data(), arr.offset, arr.length); + +#define HASH_INNER_LOOP() \ + if (slot == kHashSlotEmpty) { \ + if (!Action::allow_expand) { \ + throw HashException("Encountered new dictionary value"); \ + } \ + table_[j] = slot = static_cast(dict_.size()); \ + dict_.push_back(value); \ + action->ObserveNotFound(slot); \ + } else { \ + action->ObserveFound(slot); \ + } + + if (arr.null_count != 0) { + internal::BitmapReader valid_reader(arr.buffers[0]->data(), arr.offset, arr.length); + for (int64_t i = 0; i < arr.length; ++i) { + const bool is_null = valid_reader.IsNotSet(); + const bool value = value_reader.IsSet(); + const int j = value ? 1 : 0; + hash_slot_t slot = table_[j]; + valid_reader.Next(); + value_reader.Next(); + if (is_null) { + action->ObserveNull(); + continue; + } + HASH_INNER_LOOP(); + } + } else { + for (int64_t i = 0; i < arr.length; ++i) { + const bool value = value_reader.IsSet(); + const int j = value ? 1 : 0; + hash_slot_t slot = table_[j]; + value_reader.Next(); + HASH_INNER_LOOP(); + } + } + +#undef HASH_INNER_LOOP + + return Status::OK(); + } + + Status GetDictionary(std::shared_ptr* out) override { + BooleanBuilder builder(pool_); + for (const bool value : dict_) { + RETURN_NOT_OK(builder.Append(value)); + } + return builder.FinishInternal(out); + } + + private: + hash_slot_t table_[2]; + std::vector dict_; +}; + // ---------------------------------------------------------------------- // Hash table pass for variable-length binary types @@ -698,7 +771,7 @@ Status GetUniqueKernel(FunctionContext* ctx, const std::shared_ptr& ty switch (type->id()) { UNIQUE_CASE(NullType); - // UNIQUE_CASE(BooleanType); + UNIQUE_CASE(BooleanType); UNIQUE_CASE(UInt8Type); UNIQUE_CASE(Int8Type); UNIQUE_CASE(UInt16Type); @@ -741,7 +814,7 @@ Status GetDictionaryEncodeKernel(FunctionContext* ctx, switch (type->id()) { DICTIONARY_ENCODE_CASE(NullType); - // DICTIONARY_ENCODE_CASE(BooleanType); + DICTIONARY_ENCODE_CASE(BooleanType); DICTIONARY_ENCODE_CASE(UInt8Type); DICTIONARY_ENCODE_CASE(Int8Type); DICTIONARY_ENCODE_CASE(UInt16Type); diff --git a/cpp/src/arrow/compute/kernels/util-internal.h b/cpp/src/arrow/compute/kernels/util-internal.h index 70c506286ede6..7633fed4a8fe7 100644 --- a/cpp/src/arrow/compute/kernels/util-internal.h +++ b/cpp/src/arrow/compute/kernels/util-internal.h @@ -59,6 +59,10 @@ template using enable_if_binary = typename std::enable_if::value>::type; +template +using enable_if_boolean = + typename std::enable_if::value>::type; + template using enable_if_fixed_size_binary = typename std::enable_if::value>::type; diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index f1f59384bd639..32464817845f0 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -209,7 +209,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: int byte_width() int bit_width() - cdef cppclass CDecimal128Type" arrow::Decimal128Type"(CFixedSizeBinaryType): + cdef cppclass CDecimal128Type \ + " arrow::Decimal128Type"(CFixedSizeBinaryType): CDecimal128Type(int precision, int scale) int precision() int scale() From 1524ed7fc6fe5aac4528bd05b0c5690fb0f625da Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 23 Nov 2017 09:48:01 -0500 Subject: [PATCH 075/177] ARROW-1782: [Python] Add pyarrow.compress, decompress APIs This enables bytes, Buffer, or buffer-like objects to be compressed either to PyBytes or `pyarrow.Buffer`. Wanted some feedback on the API (argument names, etc.). The compression API in Arrow in general requires knowing the size of the decompressed data, but some compressors (like Snappy) are able to tell you how big the result will be based only on the input buffer Author: Wes McKinney Closes #1345 from wesm/ARROW-1782 and squashes the following commits: f297103b [Wes McKinney] More assertions, nicer error message when decompressed size not passed 7cd0fdb7 [Wes McKinney] Tests for compress/decompress passing 38480411 [Wes McKinney] Draft implementation of pyarrow.decompress 29b82dca [Wes McKinney] Docstring for pyarrow.decompress 2eeaf9a8 [Wes McKinney] Expose ResizableBuffer as Python class, test 6968be3f [Wes McKinney] Work on compress/decompress methods --- cpp/src/arrow/util/compression.h | 2 +- python/doc/source/api.rst | 4 + python/pyarrow/__init__.py | 17 ++- python/pyarrow/includes/libarrow.pxd | 43 ++++-- python/pyarrow/io.pxi | 209 ++++++++++++++++++++++++++- python/pyarrow/lib.pxd | 7 + python/pyarrow/plasma.pyx | 2 +- python/pyarrow/public-api.pxi | 7 + python/pyarrow/tests/test_io.py | 36 +++++ 9 files changed, 301 insertions(+), 26 deletions(-) diff --git a/cpp/src/arrow/util/compression.h b/cpp/src/arrow/util/compression.h index ae187a7fcdf1c..de3837ec727b1 100644 --- a/cpp/src/arrow/util/compression.h +++ b/cpp/src/arrow/util/compression.h @@ -27,7 +27,7 @@ namespace arrow { struct Compression { - enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI, ZSTD, LZ4 }; + enum type { UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4, LZO }; }; class ARROW_EXPORT Codec { diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst index fb2a286770e96..bb2a0420b2a08 100644 --- a/python/doc/source/api.rst +++ b/python/doc/source/api.rst @@ -195,7 +195,11 @@ Input / Output and Shared Memory :toctree: generated/ allocate_buffer + compress + decompress + frombuffer Buffer + ResizableBuffer BufferReader BufferOutputStream NativeFile diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index c4db36e55a0d6..0456a658f1425 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -71,18 +71,21 @@ # ARROW-1683: Remove after 0.8.0? from pyarrow.lib import TimestampType -from pyarrow.lib import (HdfsFile, NativeFile, PythonFile, - FixedSizeBufferWriter, - Buffer, BufferReader, BufferOutputStream, - OSFile, MemoryMappedFile, memory_map, - allocate_buffer, frombuffer, - memory_map, create_memory_map, - have_libhdfs, have_libhdfs3, MockOutputStream) +# Buffers, allocation +from pyarrow.lib import (Buffer, ResizableBuffer, compress, decompress, + allocate_buffer, frombuffer) from pyarrow.lib import (MemoryPool, total_allocated_bytes, set_memory_pool, default_memory_pool, log_memory_allocations) +from pyarrow.lib import (HdfsFile, NativeFile, PythonFile, + FixedSizeBufferWriter, + BufferReader, BufferOutputStream, + OSFile, MemoryMappedFile, memory_map, + create_memory_map, have_libhdfs, have_libhdfs3, + MockOutputStream) + from pyarrow.lib import (ChunkedArray, Column, RecordBatch, Table, concat_tables) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 32464817845f0..5d68607efa386 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -169,26 +169,27 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef cppclass CBuffer" arrow::Buffer": CBuffer(const uint8_t* data, int64_t size) - uint8_t* data() + const uint8_t* data() + uint8_t* mutable_data() int64_t size() shared_ptr[CBuffer] parent() c_bool is_mutable() const + c_bool Equals(const CBuffer& other) cdef cppclass CMutableBuffer" arrow::MutableBuffer"(CBuffer): CMutableBuffer(const uint8_t* data, int64_t size) - uint8_t* mutable_data() + + cdef cppclass CResizableBuffer" arrow::ResizableBuffer"(CMutableBuffer): + CStatus Resize(const int64_t new_size, c_bool shrink_to_fit) + CStatus Reserve(const int64_t new_size) CStatus AllocateBuffer(CMemoryPool* pool, const int64_t size, shared_ptr[CBuffer]* out) CStatus AllocateResizableBuffer(CMemoryPool* pool, const int64_t size, - shared_ptr[ResizableBuffer]* out) + shared_ptr[CResizableBuffer]* out) - cdef cppclass ResizableBuffer(CBuffer): - CStatus Resize(int64_t nbytes) - CStatus Reserve(int64_t nbytes) - - cdef cppclass PoolBuffer(ResizableBuffer): + cdef cppclass PoolBuffer(CResizableBuffer): PoolBuffer() PoolBuffer(CMemoryPool*) @@ -635,7 +636,7 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: cdef cppclass CBufferOutputStream \ " arrow::io::BufferOutputStream"(OutputStream): - CBufferOutputStream(const shared_ptr[ResizableBuffer]& buffer) + CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer) cdef cppclass CMockOutputStream \ " arrow::io::MockOutputStream"(OutputStream): @@ -661,6 +662,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: MessageType_V1" arrow::ipc::MetadataVersion::V1" MessageType_V2" arrow::ipc::MetadataVersion::V2" MessageType_V3" arrow::ipc::MetadataVersion::V3" + MessageType_V4" arrow::ipc::MetadataVersion::V4" cdef cppclass CMessage" arrow::ipc::Message": CStatus Open(const shared_ptr[CBuffer]& metadata, @@ -926,3 +928,26 @@ cdef extern from 'arrow/python/init.h': cdef extern from 'arrow/python/config.h' namespace 'arrow::py': void set_numpy_nan(object o) + + +cdef extern from 'arrow/util/compression.h' namespace 'arrow' nogil: + enum CompressionType" arrow::Compression::type": + CompressionType_UNCOMPRESSED" arrow::Compression::UNCOMPRESSED" + CompressionType_SNAPPY" arrow::Compression::SNAPPY" + CompressionType_GZIP" arrow::Compression::GZIP" + CompressionType_BROTLI" arrow::Compression::BROTLI" + CompressionType_ZSTD" arrow::Compression::ZSTD" + CompressionType_LZ4" arrow::Compression::LZ4" + + cdef cppclass CCodec" arrow::Codec": + @staticmethod + CStatus Create(CompressionType codec, unique_ptr[CCodec]* out) + + CStatus Decompress(int64_t input_len, const uint8_t* input, + int64_t output_len, uint8_t* output_buffer) + + CStatus Compress(int64_t input_len, const uint8_t* input, + int64_t output_buffer_len, uint8_t* output_buffer, + int64_t* output_length) + + int64_t MaxCompressedLen(int64_t input_len, const uint8_t* input) diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index 495e31b5a2176..619ba365c2df7 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -600,6 +600,23 @@ cdef class Buffer: # TODO(wesm): buffer slicing raise NotImplementedError + def equals(self, Buffer other): + """ + Determine if two buffers contain exactly the same data + + Parameters + ---------- + other : Buffer + + Returns + ------- + are_equal : True if buffer contents and size are equal + """ + cdef c_bool result = False + with nogil: + result = self.buffer.get().Equals(deref(other.buffer.get())) + return result + def to_pybytes(self): return cp.PyBytes_FromStringAndSize( self.buffer.get().data(), @@ -644,13 +661,37 @@ cdef class Buffer: return self.size +cdef class ResizableBuffer(Buffer): + + cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer): + self.init( buffer) + + def resize(self, int64_t new_size, shrink_to_fit=False): + """ + Resize buffer to indicated size + + Parameters + ---------- + new_size : int64_t + New size of buffer (padding may be added internally) + shrink_to_fit : boolean, default False + If new_size is less than the current size, shrink internal + capacity, otherwise leave at current capacity + """ + cdef c_bool c_shrink_to_fit = shrink_to_fit + with nogil: + check_status(( self.buffer.get()) + .Resize(new_size, c_shrink_to_fit)) + + cdef shared_ptr[PoolBuffer] _allocate_buffer(CMemoryPool* pool): cdef shared_ptr[PoolBuffer] result result.reset(new PoolBuffer(pool)) return result -def allocate_buffer(int64_t size, MemoryPool pool=None): +def allocate_buffer(int64_t size, MemoryPool memory_pool=None, + resizable=False): """ Allocate mutable fixed-size buffer @@ -658,17 +699,27 @@ def allocate_buffer(int64_t size, MemoryPool pool=None): ---------- size : int Number of bytes to allocate (plus internal padding) - pool : MemoryPool, optional + memory_pool : MemoryPool, optional Uses default memory pool if not provided + resizable : boolean, default False + + Returns + ------- + buffer : Buffer or ResizableBuffer """ cdef: shared_ptr[CBuffer] buffer - CMemoryPool* cpool = maybe_unbox_memory_pool(pool) + shared_ptr[CResizableBuffer] rz_buffer + CMemoryPool* cpool = maybe_unbox_memory_pool(memory_pool) - with nogil: - check_status(AllocateBuffer(cpool, size, &buffer)) - - return pyarrow_wrap_buffer(buffer) + if resizable: + with nogil: + check_status(AllocateResizableBuffer(cpool, size, &rz_buffer)) + return pyarrow_wrap_resizable_buffer(rz_buffer) + else: + with nogil: + check_status(AllocateBuffer(cpool, size, &buffer)) + return pyarrow_wrap_buffer(buffer) cdef class BufferOutputStream(NativeFile): @@ -679,7 +730,7 @@ cdef class BufferOutputStream(NativeFile): def __cinit__(self, MemoryPool memory_pool=None): self.buffer = _allocate_buffer(maybe_unbox_memory_pool(memory_pool)) self.wr_file.reset(new CBufferOutputStream( - self.buffer)) + self.buffer)) self.is_readable = 0 self.is_writeable = 1 self.is_open = True @@ -783,3 +834,145 @@ cdef get_writer(object source, shared_ptr[OutputStream]* writer): else: raise TypeError('Unable to read from object of type: {0}' .format(type(source))) + + +# --------------------------------------------------------------------- + +cdef CompressionType _get_compression_type(object name): + if name is None or name == 'uncompressed': + return CompressionType_UNCOMPRESSED + elif name == 'snappy': + return CompressionType_SNAPPY + elif name == 'gzip': + return CompressionType_GZIP + elif name == 'brotli': + return CompressionType_BROTLI + elif name == 'zstd': + return CompressionType_ZSTD + elif name == 'lz4': + return CompressionType_LZ4 + else: + raise ValueError("Unrecognized compression type: {0}" + .format(str(name))) + + +def compress(object buf, codec='lz4', asbytes=False, memory_pool=None): + """ + Compress pyarrow.Buffer or Python object supporting the buffer (memoryview) + protocol + + Parameters + ---------- + buf : pyarrow.Buffer, bytes, or other object supporting buffer protocol + codec : string, default 'lz4' + Compression codec. + Supported types: {'brotli, 'gzip', 'lz4', 'snappy', 'zstd'} + asbytes : boolean, default False + Return result as Python bytes object, otherwise Buffer + memory_pool : MemoryPool, default None + Memory pool to use for buffer allocations, if any + + Returns + ------- + compressed : pyarrow.Buffer or bytes (if asbytes=True) + """ + cdef: + CompressionType c_codec = _get_compression_type(codec) + unique_ptr[CCodec] compressor + cdef CBuffer* c_buf + cdef PyObject* pyobj + cdef ResizableBuffer out_buf + + with nogil: + check_status(CCodec.Create(c_codec, &compressor)) + + if not isinstance(buf, Buffer): + buf = frombuffer(buf) + + c_buf = ( buf).buffer.get() + + cdef int64_t max_output_size = (compressor.get() + .MaxCompressedLen(c_buf.size(), + c_buf.data())) + cdef uint8_t* output_buffer = NULL + + if asbytes: + pyobj = PyBytes_FromStringAndSizeNative(NULL, max_output_size) + output_buffer = cp.PyBytes_AS_STRING( pyobj) + else: + out_buf = allocate_buffer(max_output_size, memory_pool=memory_pool, + resizable=True) + output_buffer = out_buf.buffer.get().mutable_data() + + cdef int64_t output_length = 0 + with nogil: + check_status(compressor.get() + .Compress(c_buf.size(), c_buf.data(), + max_output_size, output_buffer, + &output_length)) + + if asbytes: + cp._PyBytes_Resize(&pyobj, output_length) + return PyObject_to_object(pyobj) + else: + out_buf.resize(output_length) + return out_buf + + +def decompress(object buf, decompressed_size=None, codec='lz4', + asbytes=False, memory_pool=None): + """ + Decompress data from buffer-like object + + Parameters + ---------- + buf : pyarrow.Buffer, bytes, or memoryview-compatible object + decompressed_size : int64_t, default None + If not specified, will be computed if the codec is able to determine + the uncompressed buffer size + codec : string, default 'lz4' + Compression codec. + Supported types: {'brotli, 'gzip', 'lz4', 'snappy', 'zstd'} + asbytes : boolean, default False + Return result as Python bytes object, otherwise Buffer + memory_pool : MemoryPool, default None + Memory pool to use for buffer allocations, if any + + Returns + ------- + uncompressed : pyarrow.Buffer or bytes (if asbytes=True) + """ + cdef: + CompressionType c_codec = _get_compression_type(codec) + unique_ptr[CCodec] compressor + cdef CBuffer* c_buf + cdef Buffer out_buf + + with nogil: + check_status(CCodec.Create(c_codec, &compressor)) + + if not isinstance(buf, Buffer): + buf = frombuffer(buf) + + c_buf = ( buf).buffer.get() + + if decompressed_size is None: + raise ValueError("Must pass decompressed_size for {0} codec" + .format(codec)) + + cdef int64_t output_size = decompressed_size + cdef uint8_t* output_buffer = NULL + + if asbytes: + pybuf = cp.PyBytes_FromStringAndSize(NULL, output_size) + output_buffer = cp.PyBytes_AS_STRING(pybuf) + else: + out_buf = allocate_buffer(output_size, memory_pool=memory_pool) + output_buffer = out_buf.buffer.get().mutable_data() + + with nogil: + check_status(compressor.get() + .Decompress(c_buf.size(), c_buf.data(), + output_size, output_buffer)) + + return pybuf if asbytes else out_buf diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 5abb72ba48165..90f749d6db633 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -323,6 +323,11 @@ cdef class Buffer: cdef void init(self, const shared_ptr[CBuffer]& buffer) +cdef class ResizableBuffer(Buffer): + + cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer) + + cdef class NativeFile: cdef: shared_ptr[RandomAccessFile] rd_file @@ -343,6 +348,8 @@ cdef get_reader(object source, shared_ptr[RandomAccessFile]* reader) cdef get_writer(object source, shared_ptr[OutputStream]* writer) cdef public object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf) +cdef public object pyarrow_wrap_resizable_buffer( + const shared_ptr[CResizableBuffer]& buf) cdef public object pyarrow_wrap_data_type(const shared_ptr[CDataType]& type) cdef public object pyarrow_wrap_field(const shared_ptr[CField]& field) cdef public object pyarrow_wrap_schema(const shared_ptr[CSchema]& type) diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx index bc0e94e64906e..f2e8653d8a828 100644 --- a/python/pyarrow/plasma.pyx +++ b/python/pyarrow/plasma.pyx @@ -30,7 +30,7 @@ import collections import pyarrow from pyarrow.lib cimport Buffer, NativeFile, check_status -from pyarrow.includes.libarrow cimport (CMutableBuffer, CBuffer, +from pyarrow.includes.libarrow cimport (CBuffer, CMutableBuffer, CFixedSizeBufferWriter, CStatus) diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi index bf670c5c4e290..9776f2ad76e3c 100644 --- a/python/pyarrow/public-api.pxi +++ b/python/pyarrow/public-api.pxi @@ -43,6 +43,13 @@ cdef public api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf): return result +cdef public api object pyarrow_wrap_resizable_buffer( + const shared_ptr[CResizableBuffer]& buf): + cdef ResizableBuffer result = ResizableBuffer() + result.init_rz(buf) + return result + + cdef public api bint pyarrow_is_data_type(object type_): return isinstance(type_, DataType) diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py index 98c465adcdb3d..e60dd35de66fe 100644 --- a/python/pyarrow/tests/test_io.py +++ b/python/pyarrow/tests/test_io.py @@ -182,6 +182,42 @@ def test_allocate_buffer(): assert buf.to_pybytes()[:5] == bit +def test_allocate_buffer_resizable(): + buf = pa.allocate_buffer(100, resizable=True) + assert isinstance(buf, pa.ResizableBuffer) + + buf.resize(200) + assert buf.size == 200 + + +def test_compress_decompress(): + INPUT_SIZE = 10000 + test_data = (np.random.randint(0, 255, size=INPUT_SIZE) + .astype(np.uint8) + .tostring()) + test_buf = pa.frombuffer(test_data) + + codecs = ['lz4', 'snappy', 'gzip', 'zstd', 'brotli'] + for codec in codecs: + compressed_buf = pa.compress(test_buf, codec=codec) + compressed_bytes = pa.compress(test_data, codec=codec, asbytes=True) + + assert isinstance(compressed_bytes, bytes) + + decompressed_buf = pa.decompress(compressed_buf, INPUT_SIZE, + codec=codec) + decompressed_bytes = pa.decompress(compressed_bytes, INPUT_SIZE, + codec=codec, asbytes=True) + + assert isinstance(decompressed_bytes, bytes) + + assert decompressed_buf.equals(test_buf) + assert decompressed_bytes == test_data + + with pytest.raises(ValueError): + pa.decompress(compressed_bytes, codec=codec) + + def test_buffer_memoryview_is_immutable(): val = b'some data' From ea0fb370e2ee0d11b2fbdb149247af3695fd394a Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Thu, 23 Nov 2017 10:24:39 -0500 Subject: [PATCH 076/177] ARROW-1577: [JS] add ASF release scripts @wesm does [this](https://github.com/apache/arrow/commit/b270dbad4f4fd70e19613b93148ec2ae3596e1fd#diff-fc8acbd4f42fb5e6b0cad14928b68115R59) look good? Author: Paul Taylor Author: Wes McKinney Closes #1346 from trxcllnt/js-asf-release-scripts and squashes the following commits: ac246cd1 [Wes McKinney] Update npm-release.sh while integration testing from release tarball is hard 469beeac [Wes McKinney] Add to READMEs about how to release and verify JS release 160f5c30 [Paul Taylor] remove tag-version-prefix from npmrc 854020d2 [Wes McKinney] Run unit tests without integration tests for now 94859222 [Wes McKinney] Add JavaScript release verification script dd06c468 [Wes McKinney] Create tag in source release script 2c6ce86c [Paul Taylor] number of npm-release.sh script arguments is 0 e1c3a96c [Paul Taylor] npm install before release, use npm version --git-tag-version with tag-version-prefix 58700435 [Wes McKinney] Number of arguments is 2 1133fb8c [Wes McKinney] Remove upstream Arrow version tag. Bump JS version to 0.2.0 30b602f1 [Paul Taylor] add dev/release/js-source-release.sh script be82c087 [Paul Taylor] add npm-release.sh script to publish npm modules f72fc833 [Paul Taylor] update test_js commands in verify-release-candidate.sh 5194951d [Paul Taylor] npm pack the js sources into the release tarball d73c47a0 [Paul Taylor] prepare js lib for release 7d9c84a0 [Paul Taylor] remove tslib patches as it doesn't work on windows --- ci/travis_script_integration.sh | 2 +- dev/README.md | 8 ++ dev/release/README | 13 +++ dev/release/js-source-release.sh | 103 +++++++++++++++++++++ dev/release/js-verify-release-candidate.sh | 100 ++++++++++++++++++++ dev/release/verify-release-candidate.sh | 11 ++- js/.npmrc | 2 +- js/README.md | 22 +++-- js/gulp/package-task.js | 8 +- js/gulp/test-task.js | 3 +- js/gulp/util.js | 16 +++- js/gulpfile.js | 4 +- js/npm-release.sh | 12 ++- js/package.json | 23 +++-- 14 files changed, 292 insertions(+), 35 deletions(-) create mode 100755 dev/release/js-source-release.sh create mode 100755 dev/release/js-verify-release-candidate.sh mode change 100644 => 100755 js/npm-release.sh diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh index 105a0cc6086df..9514a30a88b08 100755 --- a/ci/travis_script_integration.sh +++ b/ci/travis_script_integration.sh @@ -51,7 +51,7 @@ pushd $ARROW_JS_DIR npm run lint npm run build # create initial test data -npm run test:createTestData +npm run create:testdata # run once to write the snapshots npm test -- -t ts -u --integration # run again to test all builds against the snapshots diff --git a/dev/README.md b/dev/README.md index f3e963cf491bb..62ffb0a8f1416 100644 --- a/dev/README.md +++ b/dev/README.md @@ -113,3 +113,11 @@ Studio 2015): ``` dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz ``` + +### Verifying the JavaScript release + +For JavaScript-specific releases, use a different verification script: + +```shell +bash dev/release/js-verify-release-candidate.sh 0.7.0 0 +``` \ No newline at end of file diff --git a/dev/release/README b/dev/release/README index dd2f512a0bb1b..8e0104ed83318 100644 --- a/dev/release/README +++ b/dev/release/README @@ -49,3 +49,16 @@ Logon to the apache repository: https://repository.apache.org/#stagingRepositori Select the arrow staging repository you just just created: orgapachearrow-100x Click the "close" button Once validation has passed, click the "release" button + +# Releasing JavaScript + +* Make release branch then tag the release + +git checkout -b release-js-X.Y.Z +git tag -a apache-arrow-js-X.Y.Z + +* Build the source release (requires NodeJS) + +dev/release/js-source-release.sh X.Y.Z $RC_NUM + +* After release vote push tag, and rebase master on release branch \ No newline at end of file diff --git a/dev/release/js-source-release.sh b/dev/release/js-source-release.sh new file mode 100755 index 0000000000000..bf32acd052403 --- /dev/null +++ b/dev/release/js-source-release.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +set -e + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit +fi + +js_version=$1 +rc=$2 + +tag=apache-arrow-js-${js_version} +tagrc=${tag}-rc${rc} + +echo "Preparing source for tag ${tag}" + +tarball=${tag}.tar.gz + +# cd to $ARROW_HOME/js +cd $SOURCE_DIR/../../js +JS_SRC_DIR="$PWD" +# npm pack the js source files +npm install + +npm version --no-git-tag-version $js_version +git add package.json +git commit -m "[Release] Apache Arrow JavaScript $js_version" +git tag -a ${tag} + +release_hash=`git rev-list $tag 2> /dev/null | head -n 1 ` + +if [ -z "$release_hash" ]; then + echo "Cannot continue: unknown git tag: $tag" + exit +fi + +echo "Using commit $release_hash" + +cd $SOURCE_DIR + +rm -rf js-tmp +# `npm pack` writes the .tgz file to the current dir, so cd into js-tmp +mkdir -p js-tmp +cd js-tmp +# run npm pack on `arrow/js` +npm pack ${JS_SRC_DIR} +# unzip and remove the npm pack tarball +tar -xzf *.tgz && rm *.tgz +# `npm pack` puts files in a dir called "package" +cp $JS_SRC_DIR/../NOTICE.txt package +cp $JS_SRC_DIR/../LICENSE.txt package +# rename "package" to $tag +mv package ${tag} +tar czf ${tarball} ${tag} +rm -rf ${tag} + +${SOURCE_DIR}/run-rat.sh ${tarball} + +# sign the archive +gpg --armor --output ${tarball}.asc --detach-sig ${tarball} +gpg --print-md MD5 ${tarball} > ${tarball}.md5 +sha1sum $tarball > ${tarball}.sha1 +sha256sum $tarball > ${tarball}.sha256 +sha512sum $tarball > ${tarball}.sha512 + +# check out the arrow RC folder +svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow js-rc-tmp + +# add the release candidate for the tag +mkdir -p js-rc-tmp/${tagrc} +cp ${tarball}* js-rc-tmp/${tagrc} +svn add js-rc-tmp/${tagrc} +svn ci -m 'Apache Arrow JavaScript ${version} RC${rc}' js-rc-tmp/${tagrc} + +cd - + +# clean up +rm -rf js-tmp + +echo "Success! The release candidate is available here:" +echo " https://dist.apache.org/repos/dist/dev/arrow/${tagrc}" +echo "" +echo "Commit SHA1: ${release_hash}" diff --git a/dev/release/js-verify-release-candidate.sh b/dev/release/js-verify-release-candidate.sh new file mode 100755 index 0000000000000..5a37e10f74afb --- /dev/null +++ b/dev/release/js-verify-release-candidate.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Requirements +# - nodejs >= 6.0.0 (best way is to use nvm) + +case $# in + 2) VERSION="$1" + RC_NUMBER="$2" + ;; + + *) echo "Usage: $0 X.Y.Z RC_NUMBER" + exit 1 + ;; +esac + +set -ex + +HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd) + +ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' + +download_dist_file() { + curl -f -O $ARROW_DIST_URL/$1 +} + +download_rc_file() { + download_dist_file apache-arrow-js-${VERSION}-rc${RC_NUMBER}/$1 +} + +import_gpg_keys() { + download_dist_file KEYS + gpg --import KEYS +} + +fetch_archive() { + local dist_name=$1 + download_rc_file ${dist_name}.tar.gz + download_rc_file ${dist_name}.tar.gz.asc + download_rc_file ${dist_name}.tar.gz.md5 + download_rc_file ${dist_name}.tar.gz.sha512 + gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz + gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5 + if [ "$(uname)" == "Darwin" ]; then + shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512 + else + sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512 + fi +} + +setup_tempdir() { + cleanup() { + rm -fr "$TMPDIR" + } + trap cleanup EXIT + TMPDIR=$(mktemp -d -t "$1.XXXXX") +} + +setup_tempdir "arrow-js-$VERSION" +echo "Working in sandbox $TMPDIR" +cd $TMPDIR + +VERSION=$1 +RC_NUMBER=$2 + +TARBALL=apache-arrow-js-$1.tar.gz + +import_gpg_keys + +DIST_NAME="apache-arrow-js-${VERSION}" +fetch_archive $DIST_NAME +tar xvzf ${DIST_NAME}.tar.gz +cd ${DIST_NAME} + +npm install +# npx run-s clean:all lint create:testdata build +# npm run test -- -t ts -u --integration +# npm run test -- --integration +npx run-s clean:all lint build +npm run test + +echo 'Release candidate looks good!' +exit 0 diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 8d1bed75b8b05..7c5e7fd0abaaf 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -173,7 +173,16 @@ test_glib() { test_js() { pushd js npm install - npm run validate + # clean, lint, and build JS source + npm run clean:all + npm run lint + npm run build + # create initial integration test data + npm run create:testdata + # run once to write the snapshots + npm test -- -t ts -u --integration + # run again to test all builds against the snapshots + npm test -- --integration popd } diff --git a/js/.npmrc b/js/.npmrc index 71ffabdd55d95..b6b25d1f1816d 100644 --- a/js/.npmrc +++ b/js/.npmrc @@ -1,2 +1,2 @@ save-prefix= -package-lock=false +package-lock=false \ No newline at end of file diff --git a/js/README.md b/js/README.md index e07be6bc0aaaf..9b1753f68335e 100644 --- a/js/README.md +++ b/js/README.md @@ -183,22 +183,24 @@ The base `apache-arrow` package includes all the compilation targets for conveni The targets are also published under the `@apache-arrow` namespace: ```sh -npm install @apache-arrow/es5-cjs # ES5 CommonJS target -npm install @apache-arrow/es5-esm # ES5 ESModules target -npm install @apache-arrow/es5-umd # ES5 UMD target -npm install @apache-arrow/es2015-cjs # ES2015 CommonJS target -npm install @apache-arrow/es2015-esm # ES2015 ESModules target -npm install @apache-arrow/es2015-umd # ES2015 UMD target -npm install @apache-arrow/esnext-esm # ESNext CommonJS target -npm install @apache-arrow/esnext-esm # ESNext ESModules target -npm install @apache-arrow/esnext-umd # ESNext UMD target +npm install apache-arrow # <-- combined es5/CommonJS + UMD, es2015/ESModules + UMD, and TypeScript package +npm install @apache-arrow/ts # standalone TypeScript package +npm install @apache-arrow/es5-cjs # standalone es5/CommonJS package +npm install @apache-arrow/es5-esm # standalone es5/ESModules package +npm install @apache-arrow/es5-umd # standalone es5/UMD package +npm install @apache-arrow/es2015-cjs # standalone es2015/CommonJS package +npm install @apache-arrow/es2015-esm # standalone es2015/ESModules package +npm install @apache-arrow/es2015-umd # standalone es2015/UMD package +npm install @apache-arrow/esnext-esm # standalone esNext/CommonJS package +npm install @apache-arrow/esnext-esm # standalone esNext/ESModules package +npm install @apache-arrow/esnext-umd # standalone esNext/UMD package ``` ### Why we package like this The JS community is a diverse group with a varied list of target environments and tool chains. Publishing multiple packages accommodates projects of all stripes. -If you think we missed a compilation target and it's a blocker for adoption, please open an issue. We're here for you ❤️. +If you think we missed a compilation target and it's a blocker for adoption, please open an issue. # License diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js index ad56d172eaec7..824284e6a158a 100644 --- a/js/gulp/package-task.js +++ b/js/gulp/package-task.js @@ -48,13 +48,7 @@ const createMainPackageJson = (target, format) => (orig) => ({ module: `${mainExport}.mjs`, browser: `${mainExport}.es5.min.js`, [`browser:es2015`]: `${mainExport}.es2015.min.js`, - [`@std/esm`]: { esm: `mjs` }, - // Temporary workaround until https://github.com/Microsoft/tslib/pull/44 is merged - scripts: { - postinstall: `npm i shx && npm run tslib_mjs && npm run tslib_pkg && npm r shx`, - tslib_mjs: `shx cp $(node -e \"console.log(require.resolve('tslib/tslib.es6.js'))\") $(node -e \"var r=require,p=r('path');console.log(p.join(p.dirname(r.resolve('tslib')),'tslib.mjs'))\")`, - tslib_pkg: `node -e \"var r=require,p=r('path'),f=r('fs'),k=p.join(p.dirname(r.resolve('tslib')),'package.json'),x=JSON.parse(f.readFileSync(k));x.main='tslib';f.writeFileSync(k,JSON.stringify(x))\"` - } + [`@std/esm`]: { esm: `mjs` } }); const createTypeScriptPackageJson = (target, format) => (orig) => ({ diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js index ae6b8071d2ab7..ac9755e11ded3 100644 --- a/js/gulp/test-task.js +++ b/js/gulp/test-task.js @@ -73,12 +73,13 @@ const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file'); const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream'); const testFilesDir = path.join(ARROW_HOME, 'js/test/data'); +const snapshotsDir = path.join(ARROW_HOME, 'js/test/__snapshots__'); const cppFilesDir = path.join(testFilesDir, 'cpp'); const javaFilesDir = path.join(testFilesDir, 'java'); const jsonFilesDir = path.join(testFilesDir, 'json'); async function cleanTestData() { - return await del(`${testFilesDir}/**`); + return await del([`${testFilesDir}/**`, `${snapshotsDir}/**`]); } async function createTestJSON() { diff --git a/js/gulp/util.js b/js/gulp/util.js index e23fc398562d4..23476da2df53b 100644 --- a/js/gulp/util.js +++ b/js/gulp/util.js @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +const fs = require('fs'); const path = require(`path`); const pump = require(`pump`); const { Observable, ReplaySubject } = require('rxjs'); @@ -27,13 +28,26 @@ const releasesRootDir = `targets`; const knownTargets = [`es5`, `es2015`, `esnext`]; const knownModules = [`cjs`, `esm`, `cls`, `umd`]; const moduleFormatsToSkipCombosOf = { cls: true }; -const metadataFiles = [`../LICENSE.txt`, `../NOTICE.txt`, `README.md`]; const packageJSONFields = [ `version`, `license`, `description`, `author`, `homepage`, `repository`, `bugs`, `keywords`, `dependencies` ]; +const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => { + let err = false, prefixes = [`./`, `../`]; + let p = prefixes.find((prefix) => { + try { + fs.statSync(path.resolve(path.join(prefix, filename))); + } catch (e) { return false; } + return true; + }); + if (!p) { + throw new Error(`Couldn't find ${filename} in ./ or ../`); + } + return path.join(p, filename); +}); + // see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988 const gCCLanguageNames = { es5: `ECMASCRIPT5`, diff --git a/js/gulpfile.js b/js/gulpfile.js index cd0ac70029aba..a60363214bb94 100644 --- a/js/gulpfile.js +++ b/js/gulpfile.js @@ -86,8 +86,8 @@ const buildConcurrent = (tasks) => () => .merge(...knownTargets.map((target) => del(`${targetDir(target, `cls`)}/**`))))); -gulp.task(`test:cleanTestData`, cleanTestData); -gulp.task(`test:createTestData`, createTestData); +gulp.task(`clean:testdata`, cleanTestData); +gulp.task(`create:testdata`, createTestData); gulp.task( `test`, gulp.series(getTasks(`test`))); gulp.task(`debug`, gulp.series(getTasks(`debug`))); gulp.task(`clean`, gulp.parallel(getTasks(`clean`))); diff --git a/js/npm-release.sh b/js/npm-release.sh old mode 100644 new mode 100755 index 0bf70d9798538..42cd73c0cfe59 --- a/js/npm-release.sh +++ b/js/npm-release.sh @@ -16,5 +16,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +set -e -lerna publish --yes --skip-git --force-publish=* \ No newline at end of file +# validate the targets pass all tests before publishing +npm install +# npx run-s clean:all lint create:testdata build +# npm run test -- -t ts -u --integration +# npm run test -- --integration +npx run-s clean:all lint build +npm run test + +# publish the JS target modules to npm +npx lerna publish --yes --skip-git --force-publish=* diff --git a/js/package.json b/js/package.json index 1a110b2d5de89..9933f75a2ce57 100644 --- a/js/package.json +++ b/js/package.json @@ -17,16 +17,16 @@ "debug": "gulp debug", "perf": "node ./perf/index.js", "release": "./npm-release.sh", - "validate": "run-s --silent lint build clean", + "clean:all": "run-p clean clean:testdata", + "clean:testdata": "gulp clean:testdata", + "create:testdata": "gulp create:testdata", "test:coverage": "gulp test -t ts --coverage", - "test:cleanTestData": "gulp test:cleanTestData", - "test:createTestData": "gulp test:createTestData", "doc": "shx rm -rf ./doc && esdoc", - "lint": "npm-run-all -p lint:*", + "lint": "run-p lint:*", "lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"", "lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"", "prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1", - "postinstall": "shx cp node_modules/tslib/tslib.es6.js node_modules/tslib/tslib.mjs" + "version": "npm install && npm run clean:all" }, "repository": { "type": "git", @@ -43,12 +43,15 @@ }, "homepage": "https://github.com/apache/arrow/blob/master/js/README.md", "files": [ + "bin", "src", - "dist", - "types", - "targets", - "LICENSE", - "README.md" + "gulp", + "test", + "*.json", + "tsconfig", + "README.md", + "gulpfile.js", + "npm-release.sh" ], "peerDependencies": { "command-line-usage": "4.0.1" From 6ec4f343ba5bff628d397a7441852b5603dc7488 Mon Sep 17 00:00:00 2001 From: Li Jin Date: Thu, 23 Nov 2017 13:26:30 -0500 Subject: [PATCH 077/177] ARROW-1047: [Java] [FollowUp] Change ArrowMagic to be non-public class Move ArrowMagic from vector.ipc to vector.ipc.message package. Author: Li Jin Closes #1349 from icexelloss/ARROW-1047-followup and squashes the following commits: 0b5df959 [Li Jin] Change ArrowMagic to be non public class --- .../src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java index a9310a608ae03..f71318ee678ad 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowMagic.java @@ -18,13 +18,11 @@ package org.apache.arrow.vector.ipc; -import org.apache.arrow.vector.ipc.WriteChannel; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; -public class ArrowMagic { +class ArrowMagic { private static final byte[] MAGIC = "ARROW1".getBytes(StandardCharsets.UTF_8); From ac4bb6990fef485a3178361881943e843da0f3c7 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 24 Nov 2017 09:12:51 -0500 Subject: [PATCH 078/177] ARROW-1852: [C++] Make retrieval of Plasma manager fd a const operation This small patch makes it possible to retrieve the manager fd in a `const` context. For example: ```cpp class handle { public: bool connected() const { return client_.get_manager_fd() != -1; } private: plasma::PlasmaClient client_; }; ``` Without this patch, it's impossible to implement the above function. An alternative would be to simply provide such a utility function as part of `PlasmaClient`, but this changes the API. Let me know what you prefer. Author: Matthias Vallentin Closes #1352 from mavam/topic/plasma-manager-fd and squashes the following commits: f4d4ad37 [Matthias Vallentin] Make retrieval of manager fd a const operation --- cpp/src/plasma/client.cc | 2 +- cpp/src/plasma/client.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc index dd32bdc8149a3..9bbafac38004a 100644 --- a/cpp/src/plasma/client.cc +++ b/cpp/src/plasma/client.cc @@ -617,7 +617,7 @@ Status PlasmaClient::Fetch(int num_object_ids, const ObjectID* object_ids) { return SendFetchRequest(manager_conn_, object_ids, num_object_ids); } -int PlasmaClient::get_manager_fd() { return manager_conn_; } +int PlasmaClient::get_manager_fd() const { return manager_conn_; } Status PlasmaClient::Info(const ObjectID& object_id, int* object_status) { ARROW_CHECK(manager_conn_ >= 0); diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h index 89df2b0b0039f..cfd11c16dbace 100644 --- a/cpp/src/plasma/client.h +++ b/cpp/src/plasma/client.h @@ -313,7 +313,7 @@ class ARROW_EXPORT PlasmaClient { /// /// \return The file descriptor for the manager connection. If there is no /// connection to the manager, this is -1. - int get_manager_fd(); + int get_manager_fd() const; private: /// This is a helper method for unmapping objects for which all references have From 05bfb2619b19154eead36e4aa067f440fdedf49a Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Fri, 24 Nov 2017 09:32:36 -0500 Subject: [PATCH 079/177] ARROW-1849: [GLib] Add input checks to GArrowRecordBatch Author: Kouhei Sutou Closes #1351 from kou/glib-add-input-check-to-record-batch and squashes the following commits: c85dde22 [Kouhei Sutou] [GLib] Follow API change in Go example 3bf1eeb4 [Kouhei Sutou] [GLib] Add index check to column readers for GArrowRecordBatch 94d9f238 [Kouhei Sutou] Always validate on creating new record batch --- c_glib/arrow-glib/record-batch.cpp | 54 ++++++++++++++++---- c_glib/arrow-glib/record-batch.h | 7 +-- c_glib/example/go/read-batch.go | 4 +- c_glib/example/go/read-stream.go | 4 +- c_glib/example/go/write-batch.go | 10 +++- c_glib/example/go/write-stream.go | 10 +++- c_glib/test/test-record-batch.rb | 81 ++++++++++++++++++++++-------- 7 files changed, 128 insertions(+), 42 deletions(-) diff --git a/c_glib/arrow-glib/record-batch.cpp b/c_glib/arrow-glib/record-batch.cpp index f23a0cf7582e2..64f2020ad3efc 100644 --- a/c_glib/arrow-glib/record-batch.cpp +++ b/c_glib/arrow-glib/record-batch.cpp @@ -28,6 +28,23 @@ #include +static inline bool +garrow_record_batch_adjust_index(const std::shared_ptr arrow_record_batch, + gint &i) +{ + auto n_columns = arrow_record_batch->num_columns(); + if (i < 0) { + i += n_columns; + if (i < 0) { + return false; + } + } + if (i >= n_columns) { + return false; + } + return true; +} + G_BEGIN_DECLS /** @@ -135,13 +152,15 @@ garrow_record_batch_class_init(GArrowRecordBatchClass *klass) * @schema: The schema of the record batch. * @n_rows: The number of the rows in the record batch. * @columns: (element-type GArrowArray): The columns in the record batch. + * @error: (nullable): Return location for a #GError or %NULL. * - * Returns: A newly created #GArrowRecordBatch. + * Returns: (nullable): A newly created #GArrowRecordBatch or %NULL on error. */ GArrowRecordBatch * garrow_record_batch_new(GArrowSchema *schema, guint32 n_rows, - GList *columns) + GList *columns, + GError **error) { std::vector> arrow_columns; for (GList *node = columns; node; node = node->next) { @@ -152,7 +171,12 @@ garrow_record_batch_new(GArrowSchema *schema, auto arrow_record_batch = arrow::RecordBatch::Make(garrow_schema_get_raw(schema), n_rows, arrow_columns); - return garrow_record_batch_new_raw(&arrow_record_batch); + auto status = arrow_record_batch->Validate(); + if (garrow_error_check(error, status, "[record-batch][new]")) { + return garrow_record_batch_new_raw(&arrow_record_batch); + } else { + return NULL; + } } /** @@ -192,15 +216,21 @@ garrow_record_batch_get_schema(GArrowRecordBatch *record_batch) /** * garrow_record_batch_get_column: * @record_batch: A #GArrowRecordBatch. - * @i: The index of the target column. + * @i: The index of the target column. If it's negative, index is + * counted backward from the end of the columns. `-1` means the last + * column. * - * Returns: (transfer full): The i-th column in the record batch. + * Returns: (transfer full) (nullable): The i-th column in the record batch + * on success, %NULL on out of index. */ GArrowArray * garrow_record_batch_get_column(GArrowRecordBatch *record_batch, - guint i) + gint i) { const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch); + if (!garrow_record_batch_adjust_index(arrow_record_batch, i)) { + return NULL; + } auto arrow_column = arrow_record_batch->column(i); return garrow_array_new_raw(&arrow_column); } @@ -230,15 +260,21 @@ garrow_record_batch_get_columns(GArrowRecordBatch *record_batch) /** * garrow_record_batch_get_column_name: * @record_batch: A #GArrowRecordBatch. - * @i: The index of the target column. + * @i: The index of the target column. If it's negative, index is + * counted backward from the end of the columns. `-1` means the last + * column. * - * Returns: The name of the i-th column in the record batch. + * Returns: (nullable): The name of the i-th column in the record batch + * on success, %NULL on out of index */ const gchar * garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch, - guint i) + gint i) { const auto arrow_record_batch = garrow_record_batch_get_raw(record_batch); + if (!garrow_record_batch_adjust_index(arrow_record_batch, i)) { + return NULL; + } return arrow_record_batch->column_name(i).c_str(); } diff --git a/c_glib/arrow-glib/record-batch.h b/c_glib/arrow-glib/record-batch.h index 021f894f3f5a3..d31edf43973d0 100644 --- a/c_glib/arrow-glib/record-batch.h +++ b/c_glib/arrow-glib/record-batch.h @@ -68,17 +68,18 @@ GType garrow_record_batch_get_type(void) G_GNUC_CONST; GArrowRecordBatch *garrow_record_batch_new(GArrowSchema *schema, guint32 n_rows, - GList *columns); + GList *columns, + GError **error); gboolean garrow_record_batch_equal(GArrowRecordBatch *record_batch, GArrowRecordBatch *other_record_batch); GArrowSchema *garrow_record_batch_get_schema (GArrowRecordBatch *record_batch); GArrowArray *garrow_record_batch_get_column (GArrowRecordBatch *record_batch, - guint i); + gint i); GList *garrow_record_batch_get_columns (GArrowRecordBatch *record_batch); const gchar *garrow_record_batch_get_column_name(GArrowRecordBatch *record_batch, - guint i); + gint i); guint garrow_record_batch_get_n_columns (GArrowRecordBatch *record_batch); gint64 garrow_record_batch_get_n_rows (GArrowRecordBatch *record_batch); GArrowRecordBatch *garrow_record_batch_slice (GArrowRecordBatch *record_batch, diff --git a/c_glib/example/go/read-batch.go b/c_glib/example/go/read-batch.go index ef1a7fb5a69f9..1472939cd9284 100644 --- a/c_glib/example/go/read-batch.go +++ b/c_glib/example/go/read-batch.go @@ -57,8 +57,8 @@ func PrintColumnValue(column *arrow.Array, i int64) { func PrintRecordBatch(recordBatch *arrow.RecordBatch) { nColumns := recordBatch.GetNColumns() for i := uint32(0); i < nColumns; i++ { - column := recordBatch.GetColumn(i) - columnName := recordBatch.GetColumnName(i) + column := recordBatch.GetColumn(int32(i)) + columnName := recordBatch.GetColumnName(int32(i)) fmt.Printf(" %s: [", columnName) nRows := recordBatch.GetNRows() for j := int64(0); j < nRows; j++ { diff --git a/c_glib/example/go/read-stream.go b/c_glib/example/go/read-stream.go index 7bd076473f667..ed75a96c95c39 100644 --- a/c_glib/example/go/read-stream.go +++ b/c_glib/example/go/read-stream.go @@ -57,8 +57,8 @@ func PrintColumnValue(column *arrow.Array, i int64) { func PrintRecordBatch(recordBatch *arrow.RecordBatch) { nColumns := recordBatch.GetNColumns() for i := uint32(0); i < nColumns; i++ { - column := recordBatch.GetColumn(i) - columnName := recordBatch.GetColumnName(i) + column := recordBatch.GetColumn(int32(i)) + columnName := recordBatch.GetColumnName(int32(i)) fmt.Printf(" %s: [", columnName) nRows := recordBatch.GetNRows() for j := int64(0); j < nRows; j++ { diff --git a/c_glib/example/go/write-batch.go b/c_glib/example/go/write-batch.go index 9dbc3c00acc50..f4d03ed922eb5 100644 --- a/c_glib/example/go/write-batch.go +++ b/c_glib/example/go/write-batch.go @@ -188,7 +188,10 @@ func main() { BuildDoubleArray(), } - recordBatch := arrow.NewRecordBatch(schema, 4, columns) + recordBatch, err := arrow.NewRecordBatch(schema, 4, columns) + if err != nil { + log.Fatalf("Failed to create record batch #1: %v", err) + } _, err = writer.WriteRecordBatch(recordBatch) if err != nil { log.Fatalf("Failed to write record batch #1: %v", err) @@ -198,7 +201,10 @@ func main() { for i, column := range columns { slicedColumns[i] = column.Slice(1, 3) } - recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns) + recordBatch, err = arrow.NewRecordBatch(schema, 3, slicedColumns) + if err != nil { + log.Fatalf("Failed to create record batch #2: %v", err) + } _, err = writer.WriteRecordBatch(recordBatch) if err != nil { log.Fatalf("Failed to write record batch #2: %v", err) diff --git a/c_glib/example/go/write-stream.go b/c_glib/example/go/write-stream.go index 244741e8cfeb0..7225156a7be84 100644 --- a/c_glib/example/go/write-stream.go +++ b/c_glib/example/go/write-stream.go @@ -188,7 +188,10 @@ func main() { BuildDoubleArray(), } - recordBatch := arrow.NewRecordBatch(schema, 4, columns) + recordBatch, err := arrow.NewRecordBatch(schema, 4, columns) + if err != nil { + log.Fatalf("Failed to create record batch #1: %v", err) + } _, err = writer.WriteRecordBatch(recordBatch) if err != nil { log.Fatalf("Failed to write record batch #1: %v", err) @@ -198,7 +201,10 @@ func main() { for i, column := range columns { slicedColumns[i] = column.Slice(1, 3) } - recordBatch = arrow.NewRecordBatch(schema, 3, slicedColumns) + recordBatch, err = arrow.NewRecordBatch(schema, 3, slicedColumns) + if err != nil { + log.Fatalf("Failed to create record batch #2: %v", err) + } writer.WriteRecordBatch(recordBatch) _, err = writer.WriteRecordBatch(recordBatch) if err != nil { diff --git a/c_glib/test/test-record-batch.rb b/c_glib/test/test-record-batch.rb index 9fd34b7d45cff..365922f496fe2 100644 --- a/c_glib/test/test-record-batch.rb +++ b/c_glib/test/test-record-batch.rb @@ -18,32 +18,53 @@ class TestTable < Test::Unit::TestCase include Helper::Buildable - def test_new - fields = [ - Arrow::Field.new("visible", Arrow::BooleanDataType.new), - Arrow::Field.new("valid", Arrow::BooleanDataType.new), - ] - schema = Arrow::Schema.new(fields) - columns = [ - build_boolean_array([true]), - build_boolean_array([false]), - ] - record_batch = Arrow::RecordBatch.new(schema, 1, columns) - assert_equal(1, record_batch.n_rows) + sub_test_case(".new") do + def test_valid + fields = [ + Arrow::Field.new("visible", Arrow::BooleanDataType.new), + Arrow::Field.new("valid", Arrow::BooleanDataType.new), + ] + schema = Arrow::Schema.new(fields) + columns = [ + build_boolean_array([true]), + build_boolean_array([false]), + ] + record_batch = Arrow::RecordBatch.new(schema, 1, columns) + assert_equal(1, record_batch.n_rows) + end + + def test_no_columns + fields = [ + Arrow::Field.new("visible", Arrow::BooleanDataType.new), + ] + schema = Arrow::Schema.new(fields) + message = "[record-batch][new]: " + + "Invalid: Number of columns did not match schema" + assert_raise(Arrow::Error::Invalid.new(message)) do + Arrow::RecordBatch.new(schema, 0, []) + end + end end sub_test_case("instance methods") do def setup + @visible_field = Arrow::Field.new("visible", Arrow::BooleanDataType.new) + @visible_values = [true, false, true, false, true] + @valid_field = Arrow::Field.new("valid", Arrow::BooleanDataType.new) + @valid_values = [false, true, false, true, false] + fields = [ - Arrow::Field.new("visible", Arrow::BooleanDataType.new), - Arrow::Field.new("valid", Arrow::BooleanDataType.new), + @visible_field, + @valid_field, ] schema = Arrow::Schema.new(fields) columns = [ - build_boolean_array([true, false, true, false, true, false]), - build_boolean_array([false, true, false, true, false]), + build_boolean_array(@visible_values), + build_boolean_array(@valid_values), ] - @record_batch = Arrow::RecordBatch.new(schema, 5, columns) + @record_batch = Arrow::RecordBatch.new(schema, + @visible_values.size, + columns) end def test_equal @@ -53,7 +74,7 @@ def test_equal ] schema = Arrow::Schema.new(fields) columns = [ - build_boolean_array([true, false, true, false, true, false]), + build_boolean_array([true, false, true, false, true]), build_boolean_array([false, true, false, true, false]), ] other_record_batch = Arrow::RecordBatch.new(schema, 5, columns) @@ -66,12 +87,28 @@ def test_schema @record_batch.schema.fields.collect(&:name)) end - def test_column - assert_equal(5, @record_batch.get_column(1).length) + sub_test_case("#column") do + def test_positive + assert_equal(build_boolean_array(@valid_values), + @record_batch.get_column(1)) + end + + def test_negative + assert_equal(build_boolean_array(@visible_values), + @record_batch.get_column(-2)) + end + + def test_positive_out_of_index + assert_nil(@record_batch.get_column(2)) + end + + def test_negative_out_of_index + assert_nil(@record_batch.get_column(-3)) + end end def test_columns - assert_equal([6, 5], + assert_equal([5, 5], @record_batch.columns.collect(&:length)) end @@ -94,7 +131,7 @@ def test_slice def test_to_s assert_equal(<<-PRETTY_PRINT, @record_batch.to_s) -visible: [true, false, true, false, true, false] +visible: [true, false, true, false, true] valid: [false, true, false, true, false] PRETTY_PRINT end From aaa04431acd998ebeef57f03456d95b534ad41f3 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Sat, 25 Nov 2017 12:09:52 -0500 Subject: [PATCH 080/177] ARROW-1855: [GLib] Add workaround for build failure on macOS Arrow-1.0.gir generation may be failed on macOS. Because Python's distutils' build option tries building universal binary implicitly. g-ir-scanner uses Python's distutils internally. We can disable it by defining ARCHFLAGS environment variable. See also ARROW-1766. If we put this workaround in c_glib/, users don't need to specify ARCHFLAGS environment variable explicitly. Author: Kouhei Sutou Closes #1355 from kou/glib-add-macos-workaround and squashes the following commits: ac4ecc6e [Kouhei Sutou] [GLib] Add workaround for macOS --- c_glib/arrow-glib/Makefile.am | 5 +++++ c_glib/arrow-gpu-glib/Makefile.am | 4 ++++ c_glib/configure.ac | 13 +++++++++++++ ci/travis_before_script_c_glib.sh | 8 +------- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am index 5ecb1a6614268..e8073115d2a15 100644 --- a/c_glib/arrow-glib/Makefile.am +++ b/c_glib/arrow-glib/Makefile.am @@ -207,6 +207,11 @@ if HAVE_INTROSPECTION -include $(INTROSPECTION_MAKEFILE) INTROSPECTION_GIRS = INTROSPECTION_SCANNER_ARGS = +INTROSPECTION_SCANNER_ENV = +if OS_MACOS +INTROSPECTION_SCANNER_ENV += \ + ARCHFLAGS= +endif INTROSPECTION_COMPILER_ARGS = Arrow-1.0.gir: libarrow-glib.la diff --git a/c_glib/arrow-gpu-glib/Makefile.am b/c_glib/arrow-gpu-glib/Makefile.am index ec9615987ee5e..11ffa5ecbe367 100644 --- a/c_glib/arrow-gpu-glib/Makefile.am +++ b/c_glib/arrow-gpu-glib/Makefile.am @@ -72,6 +72,10 @@ INTROSPECTION_GIRS = INTROSPECTION_SCANNER_ARGS = INTROSPECTION_SCANNER_ENV = \ PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH} +if OS_MACOS +INTROSPECTION_SCANNER_ENV += \ + ARCHFLAGS= +endif INTROSPECTION_COMPILER_ARGS = \ --includedir=$(abs_builddir)/../arrow-glib diff --git a/c_glib/configure.ac b/c_glib/configure.ac index c6fa0192c944d..f06a017a10ba1 100644 --- a/c_glib/configure.ac +++ b/c_glib/configure.ac @@ -34,6 +34,19 @@ AC_CONFIG_HEADERS([config.h]) AM_INIT_AUTOMAKE([1.13 foreign]) AM_SILENT_RULES([yes]) +AC_CANONICAL_HOST +AC_MSG_CHECKING([for macOS]) +case "$host_os" in +darwin*) + os_macos=yes + ;; +*) + os_macos=no + ;; +esac +AC_MSG_RESULT([$os_macos]) +AM_CONDITIONAL(OS_MACOS, test "$os_macos" = "yes") + AC_PROG_CC AC_PROG_CXX AX_CXX_COMPILE_STDCXX_11([ext], [mandatory]) diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh index 99d05397a9f23..9d07e02a84302 100755 --- a/ci/travis_before_script_c_glib.sh +++ b/ci/travis_before_script_c_glib.sh @@ -97,13 +97,7 @@ if [ $BUILD_SYSTEM = "autotools" ]; then CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS CXXFLAGS=-DARROW_NO_DEPRECATED_API" ./configure $CONFIGURE_OPTIONS - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then - ARCHFLAGS="-arch x86_64" make -j4 - else - make -j4 - fi - + make -j4 make install else MESON_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL" From 82e42c54b8dbd95e6b79a2c46aeb98e2a419e930 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sat, 25 Nov 2017 12:18:36 -0500 Subject: [PATCH 081/177] ARROW-1777: [C++] Add ArrayData::Make static ctor for more convenient construction Author: Wes McKinney Closes #1353 from wesm/ARROW-1777 and squashes the following commits: d086b1e4 [Wes McKinney] Add ArrayData::Make static ctor for more convenient construction --- cpp/src/arrow/array.cc | 53 ++++++++++++++------------ cpp/src/arrow/array.h | 11 ++++++ cpp/src/arrow/builder.cc | 33 ++++++---------- cpp/src/arrow/compute/compute-test.cc | 2 +- cpp/src/arrow/compute/kernels/cast.cc | 2 +- cpp/src/arrow/compute/kernels/hash.cc | 7 ++-- cpp/src/arrow/ipc/feather-test.cc | 7 ++-- cpp/src/arrow/ipc/feather.cc | 2 +- cpp/src/arrow/python/numpy_to_arrow.cc | 7 +--- python/pyarrow/public-api.pxi | 2 +- 10 files changed, 62 insertions(+), 64 deletions(-) diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index 4ceb071ac3338..4b1fabfdcd441 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -36,6 +36,20 @@ namespace arrow { +std::shared_ptr ArrayData::Make(const std::shared_ptr& type, + int64_t length, + std::vector>&& buffers, + int64_t null_count, int64_t offset) { + return std::make_shared(type, length, std::move(buffers), null_count, + offset); +} + +std::shared_ptr ArrayData::Make(const std::shared_ptr& type, + int64_t length, int64_t null_count, + int64_t offset) { + return std::make_shared(type, length, null_count, offset); +} + // ---------------------------------------------------------------------- // Base array class @@ -112,8 +126,7 @@ std::string Array::ToString() const { } NullArray::NullArray(int64_t length) { - BufferVector buffers = {nullptr}; - SetData(std::make_shared(null(), length, std::move(buffers), length)); + SetData(ArrayData::Make(null(), length, {nullptr}, length)); } // ---------------------------------------------------------------------- @@ -123,9 +136,7 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr& type, int64_t le const std::shared_ptr& data, const std::shared_ptr& null_bitmap, int64_t null_count, int64_t offset) { - BufferVector buffers = {null_bitmap, data}; - SetData( - std::make_shared(type, length, std::move(buffers), null_count, offset)); + SetData(ArrayData::Make(type, length, {null_bitmap, data}, null_count, offset)); } const uint8_t* PrimitiveArray::raw_values() const { @@ -165,9 +176,8 @@ ListArray::ListArray(const std::shared_ptr& type, int64_t length, const std::shared_ptr& values, const std::shared_ptr& null_bitmap, int64_t null_count, int64_t offset) { - BufferVector buffers = {null_bitmap, value_offsets}; auto internal_data = - std::make_shared(type, length, std::move(buffers), null_count, offset); + ArrayData::Make(type, length, {null_bitmap, value_offsets}, null_count, offset); internal_data->child_data.emplace_back(values->data()); SetData(internal_data); } @@ -219,9 +229,8 @@ Status ListArray::FromArrays(const Array& offsets, const Array& values, MemoryPo } auto list_type = list(values.type()); - auto internal_data = - std::make_shared(list_type, num_offsets - 1, std::move(buffers), - offsets.null_count(), offsets.offset()); + auto internal_data = ArrayData::Make(list_type, num_offsets - 1, std::move(buffers), + offsets.null_count(), offsets.offset()); internal_data->child_data.push_back(values.data()); *out = std::make_shared(internal_data); @@ -276,9 +285,8 @@ BinaryArray::BinaryArray(const std::shared_ptr& type, int64_t length, const std::shared_ptr& data, const std::shared_ptr& null_bitmap, int64_t null_count, int64_t offset) { - BufferVector buffers = {null_bitmap, value_offsets, data}; - SetData( - std::make_shared(type, length, std::move(buffers), null_count, offset)); + SetData(ArrayData::Make(type, length, {null_bitmap, value_offsets, data}, null_count, + offset)); } StringArray::StringArray(const std::shared_ptr& data) { @@ -338,9 +346,7 @@ StructArray::StructArray(const std::shared_ptr& type, int64_t length, const std::vector>& children, std::shared_ptr null_bitmap, int64_t null_count, int64_t offset) { - BufferVector buffers = {null_bitmap}; - SetData( - std::make_shared(type, length, std::move(buffers), null_count, offset)); + SetData(ArrayData::Make(type, length, {null_bitmap}, null_count, offset)); for (const auto& child : children) { data_->child_data.push_back(child->data()); } @@ -384,9 +390,8 @@ UnionArray::UnionArray(const std::shared_ptr& type, int64_t length, const std::shared_ptr& value_offsets, const std::shared_ptr& null_bitmap, int64_t null_count, int64_t offset) { - BufferVector buffers = {null_bitmap, type_ids, value_offsets}; - auto internal_data = - std::make_shared(type, length, std::move(buffers), null_count, offset); + auto internal_data = ArrayData::Make( + type, length, {null_bitmap, type_ids, value_offsets}, null_count, offset); for (const auto& child : children) { internal_data->child_data.push_back(child->data()); } @@ -416,9 +421,8 @@ Status UnionArray::MakeDense(const Array& type_ids, const Array& value_offsets, static_cast(type_ids).values(), static_cast(value_offsets).values()}; auto union_type = union_(children, UnionMode::DENSE); - auto internal_data = - std::make_shared(union_type, type_ids.length(), std::move(buffers), - type_ids.null_count(), type_ids.offset()); + auto internal_data = ArrayData::Make(union_type, type_ids.length(), std::move(buffers), + type_ids.null_count(), type_ids.offset()); for (const auto& child : children) { internal_data->child_data.push_back(child->data()); } @@ -435,9 +439,8 @@ Status UnionArray::MakeSparse(const Array& type_ids, BufferVector buffers = {type_ids.null_bitmap(), static_cast(type_ids).values(), nullptr}; auto union_type = union_(children, UnionMode::SPARSE); - auto internal_data = - std::make_shared(union_type, type_ids.length(), std::move(buffers), - type_ids.null_count(), type_ids.offset()); + auto internal_data = ArrayData::Make(union_type, type_ids.length(), std::move(buffers), + type_ids.null_count(), type_ids.offset()); for (const auto& child : children) { internal_data->child_data.push_back(child->data()); if (child->length() != type_ids.length()) { diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index dda9dd38be4a7..ec5381d6e28a3 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -104,6 +104,17 @@ struct ARROW_EXPORT ArrayData { this->buffers = std::move(buffers); } + static std::shared_ptr Make(const std::shared_ptr& type, + int64_t length, + std::vector>&& buffers, + int64_t null_count = kUnknownNullCount, + int64_t offset = 0); + + static std::shared_ptr Make(const std::shared_ptr& type, + int64_t length, + int64_t null_count = kUnknownNullCount, + int64_t offset = 0); + // Move constructor ArrayData(ArrayData&& other) noexcept : type(std::move(other.type)), diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index a42f9024545ec..4d7fd5ff840ea 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -219,8 +219,7 @@ void ArrayBuilder::UnsafeSetNotNull(int64_t length) { // Null builder Status NullBuilder::FinishInternal(std::shared_ptr* out) { - BufferVector buffers = {nullptr}; - *out = std::make_shared(null(), length_, std::move(buffers), length_); + *out = ArrayData::Make(null(), length_, {nullptr}, length_); length_ = null_count_ = 0; return Status::OK(); } @@ -314,8 +313,7 @@ Status PrimitiveBuilder::FinishInternal(std::shared_ptr* out) { // Trim buffers RETURN_NOT_OK(data_->Resize(bytes_required)); } - BufferVector buffers = {null_bitmap_, data_}; - *out = std::make_shared(type_, length_, std::move(buffers), null_count_); + *out = ArrayData::Make(type_, length_, {null_bitmap_, data_}, null_count_); data_ = null_bitmap_ = nullptr; capacity_ = length_ = null_count_ = 0; @@ -404,9 +402,7 @@ Status AdaptiveIntBuilder::FinishInternal(std::shared_ptr* out) { return Status::NotImplemented("Only ints of size 1,2,4,8 are supported"); } - BufferVector buffers = {null_bitmap_, data_}; - *out = - std::make_shared(output_type, length_, std::move(buffers), null_count_); + *out = ArrayData::Make(output_type, length_, {null_bitmap_, data_}, null_count_); data_ = null_bitmap_ = nullptr; capacity_ = length_ = null_count_ = 0; @@ -562,9 +558,7 @@ Status AdaptiveUIntBuilder::FinishInternal(std::shared_ptr* out) { return Status::NotImplemented("Only ints of size 1,2,4,8 are supported"); } - BufferVector buffers = {null_bitmap_, data_}; - *out = - std::make_shared(output_type, length_, std::move(buffers), null_count_); + *out = ArrayData::Make(output_type, length_, {null_bitmap_, data_}, null_count_); data_ = null_bitmap_ = nullptr; capacity_ = length_ = null_count_ = 0; @@ -741,8 +735,7 @@ Status BooleanBuilder::FinishInternal(std::shared_ptr* out) { // Trim buffers RETURN_NOT_OK(data_->Resize(bytes_required)); } - BufferVector buffers = {null_bitmap_, data_}; - *out = std::make_shared(boolean(), length_, std::move(buffers), null_count_); + *out = ArrayData::Make(boolean(), length_, {null_bitmap_, data_}, null_count_); data_ = null_bitmap_ = nullptr; capacity_ = length_ = null_count_ = 0; @@ -828,8 +821,7 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr* out) { std::shared_ptr data; RETURN_NOT_OK(byte_builder_.Finish(&data)); - BufferVector buffers = {null_bitmap_, data}; - *out = std::make_shared(type_, length_, std::move(buffers), null_count_); + *out = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_); return Status::OK(); } @@ -896,8 +888,7 @@ Status ListBuilder::FinishInternal(std::shared_ptr* out) { RETURN_NOT_OK(value_builder_->FinishInternal(&items)); } - BufferVector buffers = {null_bitmap_, offsets}; - *out = std::make_shared(type_, length_, std::move(buffers), null_count_); + *out = ArrayData::Make(type_, length_, {null_bitmap_, offsets}, null_count_); (*out)->child_data.emplace_back(std::move(items)); Reset(); return Status::OK(); @@ -969,8 +960,8 @@ Status BinaryBuilder::FinishInternal(std::shared_ptr* out) { RETURN_NOT_OK(offsets_builder_.Finish(&offsets)); RETURN_NOT_OK(value_data_builder_.Finish(&value_data)); - BufferVector buffers = {null_bitmap_, offsets, value_data}; - *out = std::make_shared(type_, length_, std::move(buffers), null_count_, 0); + *out = ArrayData::Make(type_, length_, {null_bitmap_, offsets, value_data}, null_count_, + 0); Reset(); return Status::OK(); } @@ -1040,8 +1031,7 @@ Status FixedSizeBinaryBuilder::FinishInternal(std::shared_ptr* out) { std::shared_ptr data; RETURN_NOT_OK(byte_builder_.Finish(&data)); - BufferVector buffers = {null_bitmap_, data}; - *out = std::make_shared(type_, length_, std::move(buffers), null_count_); + *out = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count_); return Status::OK(); } @@ -1060,8 +1050,7 @@ StructBuilder::StructBuilder(const std::shared_ptr& type, MemoryPool* } Status StructBuilder::FinishInternal(std::shared_ptr* out) { - BufferVector buffers = {null_bitmap_}; - *out = std::make_shared(type_, length_, std::move(buffers), null_count_); + *out = ArrayData::Make(type_, length_, {null_bitmap_}, null_count_); (*out)->child_data.resize(field_builders_.size()); for (size_t i = 0; i < field_builders_.size(); ++i) { diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index 96edd8f01ed38..d5158978c8de8 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -688,7 +688,7 @@ TEST_F(TestCast, PreallocatedMemory) { std::unique_ptr kernel; ASSERT_OK(GetCastFunction(*int32(), out_type, options, &kernel)); - auto out_data = std::make_shared(out_type, length); + auto out_data = ArrayData::Make(out_type, length); shared_ptr out_values; ASSERT_OK(this->ctx_.Allocate(length * sizeof(int64_t), &out_values)); diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc index d595d2ea507c1..d48d669922aa6 100644 --- a/cpp/src/arrow/compute/kernels/cast.cc +++ b/cpp/src/arrow/compute/kernels/cast.cc @@ -747,7 +747,7 @@ class CastKernel : public UnaryKernel { ArrayData* result; if (out->kind() == Datum::NONE) { - out->value = std::make_shared(out_type_, in_data.length); + out->value = ArrayData::Make(out_type_, in_data.length); } result = out->array().get(); diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc index e47759d4d4706..66c907369e3ee 100644 --- a/cpp/src/arrow/compute/kernels/hash.cc +++ b/cpp/src/arrow/compute/kernels/hash.cc @@ -342,8 +342,7 @@ class HashTableKernel> : public HashTab auto dict_data = dict_.buffer; RETURN_NOT_OK(dict_data->Resize(dict_.size * sizeof(T), false)); - BufferVector buffers = {nullptr, dict_data}; - *out = std::make_shared(type_, dict_.size, std::move(buffers), 0); + *out = ArrayData::Make(type_, dict_.size, {nullptr, dict_data}, 0); return Status::OK(); } @@ -528,7 +527,7 @@ class HashTableKernel> : public HashTable { RETURN_NOT_OK(dict_offsets_.Finish(&buffers[1])); RETURN_NOT_OK(dict_data_.Finish(&buffers[2])); - *out = std::make_shared(type_, dict_size_, std::move(buffers), 0); + *out = ArrayData::Make(type_, dict_size_, std::move(buffers), 0); return Status::OK(); } @@ -634,7 +633,7 @@ class HashTableKernel> BufferVector buffers = {nullptr, nullptr}; RETURN_NOT_OK(dict_data_.Finish(&buffers[1])); - *out = std::make_shared(type_, dict_size_, std::move(buffers), 0); + *out = ArrayData::Make(type_, dict_size_, std::move(buffers), 0); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc index e3de17f1f75cd..8ec3b0e4ae8b2 100644 --- a/cpp/src/arrow/ipc/feather-test.cc +++ b/cpp/src/arrow/ipc/feather-test.cc @@ -366,15 +366,14 @@ TEST_F(TestTableWriter, TimeTypes) { ArrayFromVector(is_valid, date_values_vec, &date_array); const auto& prim_values = static_cast(*values); - std::vector> buffers = {prim_values.null_bitmap(), - prim_values.values()}; + BufferVector buffers = {prim_values.null_bitmap(), prim_values.values()}; std::vector> arrays; arrays.push_back(date_array->data()); for (int i = 1; i < schema->num_fields(); ++i) { - arrays.emplace_back(std::make_shared( - schema->field(i)->type(), values->length(), buffers, values->null_count(), 0)); + arrays.emplace_back(ArrayData::Make(schema->field(i)->type(), values->length(), + BufferVector(buffers), values->null_count(), 0)); } auto batch = RecordBatch::Make(schema, values->length(), std::move(arrays)); diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc index 077dc39305b09..d339449c084d2 100644 --- a/cpp/src/arrow/ipc/feather.cc +++ b/cpp/src/arrow/ipc/feather.cc @@ -371,7 +371,7 @@ class TableReader::TableReaderImpl { buffers.push_back(SliceBuffer(buffer, offset, buffer->size() - offset)); auto arr_data = - std::make_shared(type, meta->length(), buffers, meta->null_count()); + ArrayData::Make(type, meta->length(), std::move(buffers), meta->null_count()); *out = MakeArray(arr_data); return Status::OK(); } diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index 0d2df93151913..0c0d1a9b3fdf2 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -392,9 +392,7 @@ class NumPyConverter { null_count = ValuesToBitmap(arr_, null_bitmap_data_); } - BufferVector buffers = {null_bitmap_, data}; - auto arr_data = - std::make_shared(type_, length_, std::move(buffers), null_count, 0); + auto arr_data = ArrayData::Make(type_, length_, {null_bitmap_, data}, null_count, 0); return PushArray(arr_data); } @@ -472,8 +470,7 @@ Status CastBuffer(const std::shared_ptr& in_type, const std::shared_ptr& out_type, MemoryPool* pool, std::shared_ptr* out) { // Must cast - std::vector> buffers = {valid_bitmap, input}; - auto tmp_data = std::make_shared(in_type, length, buffers, null_count); + auto tmp_data = ArrayData::Make(in_type, length, {valid_bitmap, input}, null_count); std::shared_ptr tmp_array = MakeArray(tmp_data); std::shared_ptr casted_array; diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi index 9776f2ad76e3c..2fdb606a7d1c7 100644 --- a/python/pyarrow/public-api.pxi +++ b/python/pyarrow/public-api.pxi @@ -44,7 +44,7 @@ cdef public api object pyarrow_wrap_buffer(const shared_ptr[CBuffer]& buf): cdef public api object pyarrow_wrap_resizable_buffer( - const shared_ptr[CResizableBuffer]& buf): + const shared_ptr[CResizableBuffer]& buf): cdef ResizableBuffer result = ResizableBuffer() result.init_rz(buf) return result From b20beff1f801e888fa0deb5d48331634d0213cff Mon Sep 17 00:00:00 2001 From: Max Risuhin Date: Sat, 25 Nov 2017 17:33:33 -0500 Subject: [PATCH 082/177] ARROW-1836: [C++] Remove deprecated static_visitor struct to avoid msvc C4996 warning Author: Max Risuhin Closes #1359 from MaxRis/ARROW-1836 and squashes the following commits: bbd5850b [Max Risuhin] ARROW-1836: [C++] Remove deprecated static_visitor struct to avoid msvc C4996 warning --- cpp/cmake_modules/SetupCxxFlags.cmake | 9 --------- cpp/src/arrow/util/variant.h | 10 ---------- 2 files changed, 19 deletions(-) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 6b0974b3d4c19..4b1950f7a17bd 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -68,9 +68,6 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3") # Treat all compiler warnings as errors set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX") - - # MSVC version of -Wno-deprecated - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996") elseif ("${COMPILER_FAMILY}" STREQUAL "clang") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \ -Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \ @@ -118,9 +115,6 @@ elseif ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "EVERYTHING") # /wdnnnn disables a warning where "nnnn" is a warning number # Treat all compiler warnings as errors set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX") - - # MSVC version of -Wno-deprecated - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996") elseif ("${COMPILER_FAMILY}" STREQUAL "clang") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic") # Treat all compiler warnings as errors @@ -140,9 +134,6 @@ else() # /wdnnnn disables a warning where "nnnn" is a warning number string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3") - - # MSVC version of -Wno-deprecated - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996") elseif ("${COMPILER_FAMILY}" STREQUAL "clang") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall") elseif ("${COMPILER_FAMILY}" STREQUAL "gcc") diff --git a/cpp/src/arrow/util/variant.h b/cpp/src/arrow/util/variant.h index 923a8685f0cdb..9bfc52cb1c9a2 100644 --- a/cpp/src/arrow/util/variant.h +++ b/cpp/src/arrow/util/variant.h @@ -101,16 +101,6 @@ class bad_variant_access : public std::runtime_error }; // class bad_variant_access -template -struct ARROW_VARIANT_DEPRECATED static_visitor -{ - using result_type = R; - -protected: - static_visitor() {} - ~static_visitor() {} -}; - #if !defined(ARROW_VARIANT_MINIMIZE_SIZE) using type_index_t = unsigned int; #else From ad82c9abaf1e564cd725591e543f828e99a3eeb2 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sat, 25 Nov 2017 15:43:01 -0800 Subject: [PATCH 083/177] ARROW-1853: [Plasma] Fix off-by-one error in retry processing When a user construct a plasma client that should not perform a single retry, by passing `num_retries = 0`, nothing happens due to an off-by-one error in the retry processing. This commit fixes the issue and polishes the error output. Author: Matthias Vallentin Closes #1358 from wesm/topic/plasma-num-retries-fix and squashes the following commits: 082ee4c [Matthias Vallentin] Fix off-by-one error in retry processing --- cpp/src/plasma/io.cc | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/cpp/src/plasma/io.cc b/cpp/src/plasma/io.cc index 2228ad94c6346..2cba8970e26ae 100644 --- a/cpp/src/plasma/io.cc +++ b/cpp/src/plasma/io.cc @@ -156,19 +156,14 @@ Status ConnectIpcSocketRetry(const std::string& pathname, int num_retries, if (timeout < 0) { timeout = CONNECT_TIMEOUT_MS; } - - *fd = -1; - for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) { - *fd = connect_ipc_sock(pathname); - if (*fd >= 0) { - break; - } - if (num_attempts == 0) { - ARROW_LOG(ERROR) << "Connection to IPC socket failed for pathname " << pathname - << ", retrying " << num_retries << " times"; - } + *fd = connect_ipc_sock(pathname); + while (*fd < 0 && num_retries > 0) { + ARROW_LOG(ERROR) << "Connection to IPC socket failed for pathname " << pathname + << ", retrying " << num_retries << " more times"; /* Sleep for timeout milliseconds. */ usleep(static_cast(timeout * 1000)); + *fd = connect_ipc_sock(pathname); + --num_retries; } /* If we could not connect to the socket, exit. */ if (*fd == -1) { From bf1cf3befc5a5d1e7eee92b6ab4b0935c9821ce5 Mon Sep 17 00:00:00 2001 From: Andrew Andrade Date: Sun, 26 Nov 2017 19:02:52 +0000 Subject: [PATCH 084/177] [Python] Add more detail to development docs (#1356) * updated file path to brew install for repos dir * added information about bundled wheel build --- python/doc/source/development.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst index 885613725a10a..1c1a308fc4a56 100644 --- a/python/doc/source/development.rst +++ b/python/doc/source/development.rst @@ -108,7 +108,7 @@ building Arrow C++: .. code-block:: shell - brew update && brew bundle --file=python/Brewfile + brew update && brew bundle --file=arrow/python/Brewfile On Debian/Ubuntu, you need the following minimal set of dependencies. All other dependencies will be automatically built by Arrow's third-party toolchain. @@ -230,7 +230,7 @@ You should be able to run the unit tests with: ====================== 181 passed, 17 skipped in 0.98 seconds =========== -You can build a wheel by running: +To build a self-contained wheel (include Arrow C++ and Parquet C++), one can set `--bundle-arrow-cpp`: .. code-block:: shell From ebb6c764687c197fbd5269b6500568d11dafaf31 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Sun, 26 Nov 2017 14:17:28 -0500 Subject: [PATCH 085/177] ARROW-1859: [GLib] Add GArrowDictionaryDataType Author: Kouhei Sutou Closes #1361 from kou/glib-dictionary-data-type and squashes the following commits: 6ccce1f8 [Kouhei Sutou] [GLib] Add GArrowDictionaryDataType --- c_glib/arrow-glib/Makefile.am | 6 +- c_glib/arrow-glib/array.h | 1225 +---------------- c_glib/arrow-glib/array.hpp | 8 +- .../arrow-glib/{array.cpp => basic-array.cpp} | 206 +-- c_glib/arrow-glib/basic-array.h | 1115 +++++++++++++++ c_glib/arrow-glib/basic-array.hpp | 27 + c_glib/arrow-glib/basic-data-type.cpp | 38 +- c_glib/arrow-glib/basic-data-type.h | 104 +- c_glib/arrow-glib/composite-array.cpp | 237 ++++ c_glib/arrow-glib/composite-array.h | 132 ++ c_glib/arrow-glib/composite-data-type.cpp | 86 ++ c_glib/arrow-glib/composite-data-type.h | 25 + c_glib/arrow-glib/meson.build | 6 +- c_glib/doc/reference/arrow-glib-docs.xml | 3 +- c_glib/test/test-dictionary-data-type.rb | 56 + 15 files changed, 1765 insertions(+), 1509 deletions(-) rename c_glib/arrow-glib/{array.cpp => basic-array.cpp} (90%) create mode 100644 c_glib/arrow-glib/basic-array.h create mode 100644 c_glib/arrow-glib/basic-array.hpp create mode 100644 c_glib/arrow-glib/composite-array.cpp create mode 100644 c_glib/arrow-glib/composite-array.h create mode 100644 c_glib/test/test-dictionary-data-type.rb diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am index e8073115d2a15..39222a5480162 100644 --- a/c_glib/arrow-glib/Makefile.am +++ b/c_glib/arrow-glib/Makefile.am @@ -45,10 +45,12 @@ libarrow_glib_la_headers = \ array.h \ array-builder.h \ arrow-glib.h \ + basic-array.h \ basic-data-type.h \ buffer.h \ chunked-array.h \ column.h \ + composite-array.h \ composite-data-type.h \ data-type.h \ error.h \ @@ -86,12 +88,13 @@ libarrow_glib_la_generated_sources = \ $(libarrow_glib_la_generated_headers) libarrow_glib_la_sources = \ - array.cpp \ array-builder.cpp \ + basic-array.cpp \ basic-data-type.cpp \ buffer.cpp \ chunked-array.cpp \ column.cpp \ + composite-array.cpp \ composite-data-type.cpp \ error.cpp \ field.cpp \ @@ -125,6 +128,7 @@ libarrow_glib_la_cpp_headers = \ array.hpp \ array-builder.hpp \ arrow-glib.hpp \ + basic-array.hpp \ basic-data-type.hpp \ buffer.hpp \ chunked-array.hpp \ diff --git a/c_glib/arrow-glib/array.h b/c_glib/arrow-glib/array.h index e988a8aae931d..9a845597d4f0f 100644 --- a/c_glib/arrow-glib/array.h +++ b/c_glib/arrow-glib/array.h @@ -19,1226 +19,5 @@ #pragma once -#include -#include -#include - -G_BEGIN_DECLS - -#define GARROW_TYPE_ARRAY \ - (garrow_array_get_type()) -#define GARROW_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_ARRAY, GArrowArray)) -#define GARROW_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_ARRAY, GArrowArrayClass)) -#define GARROW_IS_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_ARRAY)) -#define GARROW_IS_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_ARRAY)) -#define GARROW_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_ARRAY, GArrowArrayClass)) - -typedef struct _GArrowArray GArrowArray; -typedef struct _GArrowArrayClass GArrowArrayClass; - -/** - * GArrowArray: - * - * It wraps `arrow::Array`. - */ -struct _GArrowArray -{ - /*< private >*/ - GObject parent_instance; -}; - -struct _GArrowArrayClass -{ - GObjectClass parent_class; -}; - -GType garrow_array_get_type (void) G_GNUC_CONST; - -gboolean garrow_array_equal (GArrowArray *array, - GArrowArray *other_array); -gboolean garrow_array_equal_approx(GArrowArray *array, - GArrowArray *other_array); -gboolean garrow_array_equal_range (GArrowArray *array, - gint64 start_index, - GArrowArray *other_array, - gint64 other_start_index, - gint64 end_index); - -gboolean garrow_array_is_null (GArrowArray *array, - gint64 i); -gboolean garrow_array_is_valid (GArrowArray *array, - gint64 i); -gint64 garrow_array_get_length (GArrowArray *array); -gint64 garrow_array_get_offset (GArrowArray *array); -gint64 garrow_array_get_n_nulls (GArrowArray *array); -GArrowBuffer *garrow_array_get_null_bitmap(GArrowArray *array); -GArrowDataType *garrow_array_get_value_data_type(GArrowArray *array); -GArrowType garrow_array_get_value_type(GArrowArray *array); -GArrowArray *garrow_array_slice (GArrowArray *array, - gint64 offset, - gint64 length); -gchar *garrow_array_to_string (GArrowArray *array, - GError **error); - -GArrowArray *garrow_array_cast (GArrowArray *array, - GArrowDataType *target_data_type, - GArrowCastOptions *options, - GError **error); - -#define GARROW_TYPE_NULL_ARRAY \ - (garrow_null_array_get_type()) -#define GARROW_NULL_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_NULL_ARRAY, \ - GArrowNullArray)) -#define GARROW_NULL_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_NULL_ARRAY, \ - GArrowNullArrayClass)) -#define GARROW_IS_NULL_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_NULL_ARRAY)) -#define GARROW_IS_NULL_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_NULL_ARRAY)) -#define GARROW_NULL_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_NULL_ARRAY, \ - GArrowNullArrayClass)) - -typedef struct _GArrowNullArray GArrowNullArray; -typedef struct _GArrowNullArrayClass GArrowNullArrayClass; - -/** - * GArrowNullArray: - * - * It wraps `arrow::NullArray`. - */ -struct _GArrowNullArray -{ - /*< private >*/ - GArrowArray parent_instance; -}; - -struct _GArrowNullArrayClass -{ - GArrowArrayClass parent_class; -}; - -GType garrow_null_array_get_type(void) G_GNUC_CONST; - -GArrowNullArray *garrow_null_array_new(gint64 length); - - -#define GARROW_TYPE_PRIMITIVE_ARRAY \ - (garrow_primitive_array_get_type()) -#define GARROW_PRIMITIVE_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_PRIMITIVE_ARRAY, \ - GArrowPrimitiveArray)) -#define GARROW_PRIMITIVE_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_PRIMITIVE_ARRAY, \ - GArrowPrimitiveArrayClass)) -#define GARROW_IS_PRIMITIVE_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_PRIMITIVE_ARRAY)) -#define GARROW_IS_PRIMITIVE_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_PRIMITIVE_ARRAY)) -#define GARROW_PRIMITIVE_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_PRIMITIVE_ARRAY, \ - GArrowPrimitiveArrayClass)) - -typedef struct _GArrowPrimitiveArray GArrowPrimitiveArray; -typedef struct _GArrowPrimitiveArrayClass GArrowPrimitiveArrayClass; - -/** - * GArrowPrimitiveArray: - * - * It wraps `arrow::PrimitiveArray`. - */ -struct _GArrowPrimitiveArray -{ - /*< private >*/ - GArrowArray parent_instance; -}; - -struct _GArrowPrimitiveArrayClass -{ - GArrowArrayClass parent_class; -}; - -GType garrow_primitive_array_get_type(void) G_GNUC_CONST; - -GArrowBuffer *garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array); - - -#define GARROW_TYPE_BOOLEAN_ARRAY \ - (garrow_boolean_array_get_type()) -#define GARROW_BOOLEAN_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_BOOLEAN_ARRAY, \ - GArrowBooleanArray)) -#define GARROW_BOOLEAN_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_BOOLEAN_ARRAY, \ - GArrowBooleanArrayClass)) -#define GARROW_IS_BOOLEAN_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_BOOLEAN_ARRAY)) -#define GARROW_IS_BOOLEAN_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_BOOLEAN_ARRAY)) -#define GARROW_BOOLEAN_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_BOOLEAN_ARRAY, \ - GArrowBooleanArrayClass)) - -typedef struct _GArrowBooleanArray GArrowBooleanArray; -typedef struct _GArrowBooleanArrayClass GArrowBooleanArrayClass; - -/** - * GArrowBooleanArray: - * - * It wraps `arrow::BooleanArray`. - */ -struct _GArrowBooleanArray -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowBooleanArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_boolean_array_get_type (void) G_GNUC_CONST; - -GArrowBooleanArray *garrow_boolean_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gboolean garrow_boolean_array_get_value (GArrowBooleanArray *array, - gint64 i); -gboolean *garrow_boolean_array_get_values(GArrowBooleanArray *array, - gint64 *length); - - -#define GARROW_TYPE_INT8_ARRAY \ - (garrow_int8_array_get_type()) -#define GARROW_INT8_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_INT8_ARRAY, \ - GArrowInt8Array)) -#define GARROW_INT8_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_INT8_ARRAY, \ - GArrowInt8ArrayClass)) -#define GARROW_IS_INT8_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_INT8_ARRAY)) -#define GARROW_IS_INT8_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_INT8_ARRAY)) -#define GARROW_INT8_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_INT8_ARRAY, \ - GArrowInt8ArrayClass)) - -typedef struct _GArrowInt8Array GArrowInt8Array; -typedef struct _GArrowInt8ArrayClass GArrowInt8ArrayClass; - -/** - * GArrowInt8Array: - * - * It wraps `arrow::Int8Array`. - */ -struct _GArrowInt8Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowInt8ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_int8_array_get_type(void) G_GNUC_CONST; - -GArrowInt8Array *garrow_int8_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint8 garrow_int8_array_get_value(GArrowInt8Array *array, - gint64 i); -const gint8 *garrow_int8_array_get_values(GArrowInt8Array *array, - gint64 *length); - - -#define GARROW_TYPE_UINT8_ARRAY \ - (garrow_uint8_array_get_type()) -#define GARROW_UINT8_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_UINT8_ARRAY, \ - GArrowUInt8Array)) -#define GARROW_UINT8_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_UINT8_ARRAY, \ - GArrowUInt8ArrayClass)) -#define GARROW_IS_UINT8_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_UINT8_ARRAY)) -#define GARROW_IS_UINT8_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_UINT8_ARRAY)) -#define GARROW_UINT8_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_UINT8_ARRAY, \ - GArrowUInt8ArrayClass)) - -typedef struct _GArrowUInt8Array GArrowUInt8Array; -typedef struct _GArrowUInt8ArrayClass GArrowUInt8ArrayClass; - -/** - * GArrowUInt8Array: - * - * It wraps `arrow::UInt8Array`. - */ -struct _GArrowUInt8Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowUInt8ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_uint8_array_get_type(void) G_GNUC_CONST; - -GArrowUInt8Array *garrow_uint8_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array, - gint64 i); -const guint8 *garrow_uint8_array_get_values(GArrowUInt8Array *array, - gint64 *length); - - -#define GARROW_TYPE_INT16_ARRAY \ - (garrow_int16_array_get_type()) -#define GARROW_INT16_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_INT16_ARRAY, \ - GArrowInt16Array)) -#define GARROW_INT16_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_INT16_ARRAY, \ - GArrowInt16ArrayClass)) -#define GARROW_IS_INT16_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_INT16_ARRAY)) -#define GARROW_IS_INT16_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_INT16_ARRAY)) -#define GARROW_INT16_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_INT16_ARRAY, \ - GArrowInt16ArrayClass)) - -typedef struct _GArrowInt16Array GArrowInt16Array; -typedef struct _GArrowInt16ArrayClass GArrowInt16ArrayClass; - -/** - * GArrowInt16Array: - * - * It wraps `arrow::Int16Array`. - */ -struct _GArrowInt16Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowInt16ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_int16_array_get_type(void) G_GNUC_CONST; - -GArrowInt16Array *garrow_int16_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint16 garrow_int16_array_get_value(GArrowInt16Array *array, - gint64 i); -const gint16 *garrow_int16_array_get_values(GArrowInt16Array *array, - gint64 *length); - - -#define GARROW_TYPE_UINT16_ARRAY \ - (garrow_uint16_array_get_type()) -#define GARROW_UINT16_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_UINT16_ARRAY, \ - GArrowUInt16Array)) -#define GARROW_UINT16_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_UINT16_ARRAY, \ - GArrowUInt16ArrayClass)) -#define GARROW_IS_UINT16_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_UINT16_ARRAY)) -#define GARROW_IS_UINT16_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_UINT16_ARRAY)) -#define GARROW_UINT16_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_UINT16_ARRAY, \ - GArrowUInt16ArrayClass)) - -typedef struct _GArrowUInt16Array GArrowUInt16Array; -typedef struct _GArrowUInt16ArrayClass GArrowUInt16ArrayClass; - -/** - * GArrowUInt16Array: - * - * It wraps `arrow::UInt16Array`. - */ -struct _GArrowUInt16Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowUInt16ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_uint16_array_get_type(void) G_GNUC_CONST; - -GArrowUInt16Array *garrow_uint16_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array, - gint64 i); -const guint16 *garrow_uint16_array_get_values(GArrowUInt16Array *array, - gint64 *length); - - -#define GARROW_TYPE_INT32_ARRAY \ - (garrow_int32_array_get_type()) -#define GARROW_INT32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_INT32_ARRAY, \ - GArrowInt32Array)) -#define GARROW_INT32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_INT32_ARRAY, \ - GArrowInt32ArrayClass)) -#define GARROW_IS_INT32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_INT32_ARRAY)) -#define GARROW_IS_INT32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_INT32_ARRAY)) -#define GARROW_INT32_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_INT32_ARRAY, \ - GArrowInt32ArrayClass)) - -typedef struct _GArrowInt32Array GArrowInt32Array; -typedef struct _GArrowInt32ArrayClass GArrowInt32ArrayClass; - -/** - * GArrowInt32Array: - * - * It wraps `arrow::Int32Array`. - */ -struct _GArrowInt32Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowInt32ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_int32_array_get_type(void) G_GNUC_CONST; - -GArrowInt32Array *garrow_int32_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint32 garrow_int32_array_get_value(GArrowInt32Array *array, - gint64 i); -const gint32 *garrow_int32_array_get_values(GArrowInt32Array *array, - gint64 *length); - - -#define GARROW_TYPE_UINT32_ARRAY \ - (garrow_uint32_array_get_type()) -#define GARROW_UINT32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_UINT32_ARRAY, \ - GArrowUInt32Array)) -#define GARROW_UINT32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_UINT32_ARRAY, \ - GArrowUInt32ArrayClass)) -#define GARROW_IS_UINT32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_UINT32_ARRAY)) -#define GARROW_IS_UINT32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_UINT32_ARRAY)) -#define GARROW_UINT32_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_UINT32_ARRAY, \ - GArrowUInt32ArrayClass)) - -typedef struct _GArrowUInt32Array GArrowUInt32Array; -typedef struct _GArrowUInt32ArrayClass GArrowUInt32ArrayClass; - -/** - * GArrowUInt32Array: - * - * It wraps `arrow::UInt32Array`. - */ -struct _GArrowUInt32Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowUInt32ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_uint32_array_get_type(void) G_GNUC_CONST; - -GArrowUInt32Array *garrow_uint32_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array, - gint64 i); -const guint32 *garrow_uint32_array_get_values(GArrowUInt32Array *array, - gint64 *length); - - -#define GARROW_TYPE_INT64_ARRAY \ - (garrow_int64_array_get_type()) -#define GARROW_INT64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_INT64_ARRAY, \ - GArrowInt64Array)) -#define GARROW_INT64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_INT64_ARRAY, \ - GArrowInt64ArrayClass)) -#define GARROW_IS_INT64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_INT64_ARRAY)) -#define GARROW_IS_INT64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_INT64_ARRAY)) -#define GARROW_INT64_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_INT64_ARRAY, \ - GArrowInt64ArrayClass)) - -typedef struct _GArrowInt64Array GArrowInt64Array; -typedef struct _GArrowInt64ArrayClass GArrowInt64ArrayClass; - -/** - * GArrowInt64Array: - * - * It wraps `arrow::Int64Array`. - */ -struct _GArrowInt64Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowInt64ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_int64_array_get_type(void) G_GNUC_CONST; - -GArrowInt64Array *garrow_int64_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint64 garrow_int64_array_get_value(GArrowInt64Array *array, - gint64 i); -const gint64 *garrow_int64_array_get_values(GArrowInt64Array *array, - gint64 *length); - - -#define GARROW_TYPE_UINT64_ARRAY \ - (garrow_uint64_array_get_type()) -#define GARROW_UINT64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_UINT64_ARRAY, \ - GArrowUInt64Array)) -#define GARROW_UINT64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_UINT64_ARRAY, \ - GArrowUInt64ArrayClass)) -#define GARROW_IS_UINT64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_UINT64_ARRAY)) -#define GARROW_IS_UINT64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_UINT64_ARRAY)) -#define GARROW_UINT64_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_UINT64_ARRAY, \ - GArrowUInt64ArrayClass)) - -typedef struct _GArrowUInt64Array GArrowUInt64Array; -typedef struct _GArrowUInt64ArrayClass GArrowUInt64ArrayClass; - -/** - * GArrowUInt64Array: - * - * It wraps `arrow::UInt64Array`. - */ -struct _GArrowUInt64Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowUInt64ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_uint64_array_get_type(void) G_GNUC_CONST; - -GArrowUInt64Array *garrow_uint64_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array, - gint64 i); -const guint64 *garrow_uint64_array_get_values(GArrowUInt64Array *array, - gint64 *length); - - -#define GARROW_TYPE_FLOAT_ARRAY \ - (garrow_float_array_get_type()) -#define GARROW_FLOAT_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_FLOAT_ARRAY, \ - GArrowFloatArray)) -#define GARROW_FLOAT_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_FLOAT_ARRAY, \ - GArrowFloatArrayClass)) -#define GARROW_IS_FLOAT_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_FLOAT_ARRAY)) -#define GARROW_IS_FLOAT_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_FLOAT_ARRAY)) -#define GARROW_FLOAT_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_FLOAT_ARRAY, \ - GArrowFloatArrayClass)) - -typedef struct _GArrowFloatArray GArrowFloatArray; -typedef struct _GArrowFloatArrayClass GArrowFloatArrayClass; - -/** - * GArrowFloatArray: - * - * It wraps `arrow::FloatArray`. - */ -struct _GArrowFloatArray -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowFloatArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_float_array_get_type(void) G_GNUC_CONST; - -GArrowFloatArray *garrow_float_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gfloat garrow_float_array_get_value(GArrowFloatArray *array, - gint64 i); -const gfloat *garrow_float_array_get_values(GArrowFloatArray *array, - gint64 *length); - - -#define GARROW_TYPE_DOUBLE_ARRAY \ - (garrow_double_array_get_type()) -#define GARROW_DOUBLE_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_DOUBLE_ARRAY, \ - GArrowDoubleArray)) -#define GARROW_DOUBLE_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_DOUBLE_ARRAY, \ - GArrowDoubleArrayClass)) -#define GARROW_IS_DOUBLE_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_DOUBLE_ARRAY)) -#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_DOUBLE_ARRAY)) -#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_DOUBLE_ARRAY, \ - GArrowDoubleArrayClass)) - -typedef struct _GArrowDoubleArray GArrowDoubleArray; -typedef struct _GArrowDoubleArrayClass GArrowDoubleArrayClass; - -/** - * GArrowDoubleArray: - * - * It wraps `arrow::DoubleArray`. - */ -struct _GArrowDoubleArray -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowDoubleArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_double_array_get_type(void) G_GNUC_CONST; - -GArrowDoubleArray *garrow_double_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gdouble garrow_double_array_get_value(GArrowDoubleArray *array, - gint64 i); -const gdouble *garrow_double_array_get_values(GArrowDoubleArray *array, - gint64 *length); - - -#define GARROW_TYPE_BINARY_ARRAY \ - (garrow_binary_array_get_type()) -#define GARROW_BINARY_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_BINARY_ARRAY, \ - GArrowBinaryArray)) -#define GARROW_BINARY_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_BINARY_ARRAY, \ - GArrowBinaryArrayClass)) -#define GARROW_IS_BINARY_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_BINARY_ARRAY)) -#define GARROW_IS_BINARY_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_BINARY_ARRAY)) -#define GARROW_BINARY_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_BINARY_ARRAY, \ - GArrowBinaryArrayClass)) - -typedef struct _GArrowBinaryArray GArrowBinaryArray; -typedef struct _GArrowBinaryArrayClass GArrowBinaryArrayClass; - -/** - * GArrowBinaryArray: - * - * It wraps `arrow::BinaryArray`. - */ -struct _GArrowBinaryArray -{ - /*< private >*/ - GArrowArray parent_instance; -}; - -struct _GArrowBinaryArrayClass -{ - GArrowArrayClass parent_class; -}; - -GType garrow_binary_array_get_type(void) G_GNUC_CONST; - -GArrowBinaryArray *garrow_binary_array_new(gint64 length, - GArrowBuffer *value_offsets, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array, - gint64 i); -GArrowBuffer *garrow_binary_array_get_buffer(GArrowBinaryArray *array); -GArrowBuffer *garrow_binary_array_get_offsets_buffer(GArrowBinaryArray *array); - -#define GARROW_TYPE_STRING_ARRAY \ - (garrow_string_array_get_type()) -#define GARROW_STRING_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_STRING_ARRAY, \ - GArrowStringArray)) -#define GARROW_STRING_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_STRING_ARRAY, \ - GArrowStringArrayClass)) -#define GARROW_IS_STRING_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_STRING_ARRAY)) -#define GARROW_IS_STRING_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_STRING_ARRAY)) -#define GARROW_STRING_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_STRING_ARRAY, \ - GArrowStringArrayClass)) - -typedef struct _GArrowStringArray GArrowStringArray; -typedef struct _GArrowStringArrayClass GArrowStringArrayClass; - -/** - * GArrowStringArray: - * - * It wraps `arrow::StringArray`. - */ -struct _GArrowStringArray -{ - /*< private >*/ - GArrowBinaryArray parent_instance; -}; - -struct _GArrowStringArrayClass -{ - GArrowBinaryArrayClass parent_class; -}; - -GType garrow_string_array_get_type(void) G_GNUC_CONST; - -GArrowStringArray *garrow_string_array_new(gint64 length, - GArrowBuffer *value_offsets, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gchar *garrow_string_array_get_string(GArrowStringArray *array, - gint64 i); - - -#define GARROW_TYPE_DATE32_ARRAY \ - (garrow_date32_array_get_type()) -#define GARROW_DATE32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_DATE32_ARRAY, \ - GArrowDate32Array)) -#define GARROW_DATE32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_DATE32_ARRAY, \ - GArrowDate32ArrayClass)) -#define GARROW_IS_DATE32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_DATE32_ARRAY)) -#define GARROW_IS_DATE32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_DATE32_ARRAY)) -#define GARROW_DATE32_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_DATE32_ARRAY, \ - GArrowDate32ArrayClass)) - -typedef struct _GArrowDate32Array GArrowDate32Array; -typedef struct _GArrowDate32ArrayClass GArrowDate32ArrayClass; - -/** - * GArrowDate32Array: - * - * It wraps `arrow::Date32Array`. - */ -struct _GArrowDate32Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowDate32ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_date32_array_get_type(void) G_GNUC_CONST; - -GArrowDate32Array *garrow_date32_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint32 garrow_date32_array_get_value(GArrowDate32Array *array, - gint64 i); -const gint32 *garrow_date32_array_get_values(GArrowDate32Array *array, - gint64 *length); - - -#define GARROW_TYPE_DATE64_ARRAY \ - (garrow_date64_array_get_type()) -#define GARROW_DATE64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_DATE64_ARRAY, \ - GArrowDate64Array)) -#define GARROW_DATE64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_DATE64_ARRAY, \ - GArrowDate64ArrayClass)) -#define GARROW_IS_DATE64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_DATE64_ARRAY)) -#define GARROW_IS_DATE64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_DATE64_ARRAY)) -#define GARROW_DATE64_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_DATE64_ARRAY, \ - GArrowDate64ArrayClass)) - -typedef struct _GArrowDate64Array GArrowDate64Array; -typedef struct _GArrowDate64ArrayClass GArrowDate64ArrayClass; - -/** - * GArrowDate64Array: - * - * It wraps `arrow::Date64Array`. - */ -struct _GArrowDate64Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowDate64ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_date64_array_get_type(void) G_GNUC_CONST; - -GArrowDate64Array *garrow_date64_array_new(gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint64 garrow_date64_array_get_value(GArrowDate64Array *array, - gint64 i); -const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array, - gint64 *length); - - -#define GARROW_TYPE_TIMESTAMP_ARRAY \ - (garrow_timestamp_array_get_type()) -#define GARROW_TIMESTAMP_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_TIMESTAMP_ARRAY, \ - GArrowTimestampArray)) -#define GARROW_TIMESTAMP_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_TIMESTAMP_ARRAY, \ - GArrowTimestampArrayClass)) -#define GARROW_IS_TIMESTAMP_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_TIMESTAMP_ARRAY)) -#define GARROW_IS_TIMESTAMP_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_TIMESTAMP_ARRAY)) -#define GARROW_TIMESTAMP_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_TIMESTAMP_ARRAY, \ - GArrowTimestampArrayClass)) - -typedef struct _GArrowTimestampArray GArrowTimestampArray; -typedef struct _GArrowTimestampArrayClass GArrowTimestampArrayClass; - -/** - * GArrowTimestampArray: - * - * It wraps `arrow::TimestampArray`. - */ -struct _GArrowTimestampArray -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowTimestampArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_timestamp_array_get_type(void) G_GNUC_CONST; - -GArrowTimestampArray *garrow_timestamp_array_new(GArrowTimestampDataType *data_type, - gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint64 garrow_timestamp_array_get_value(GArrowTimestampArray *array, - gint64 i); -const gint64 *garrow_timestamp_array_get_values(GArrowTimestampArray *array, - gint64 *length); - - -#define GARROW_TYPE_TIME32_ARRAY \ - (garrow_time32_array_get_type()) -#define GARROW_TIME32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_TIME32_ARRAY, \ - GArrowTime32Array)) -#define GARROW_TIME32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_TIME32_ARRAY, \ - GArrowTime32ArrayClass)) -#define GARROW_IS_TIME32_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_TIME32_ARRAY)) -#define GARROW_IS_TIME32_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_TIME32_ARRAY)) -#define GARROW_TIME32_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_TIME32_ARRAY, \ - GArrowTime32ArrayClass)) - -typedef struct _GArrowTime32Array GArrowTime32Array; -typedef struct _GArrowTime32ArrayClass GArrowTime32ArrayClass; - -/** - * GArrowTime32Array: - * - * It wraps `arrow::Time32Array`. - */ -struct _GArrowTime32Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowTime32ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_time32_array_get_type(void) G_GNUC_CONST; - -GArrowTime32Array *garrow_time32_array_new(GArrowTime32DataType *data_type, - gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint32 garrow_time32_array_get_value(GArrowTime32Array *array, - gint64 i); -const gint32 *garrow_time32_array_get_values(GArrowTime32Array *array, - gint64 *length); - - -#define GARROW_TYPE_TIME64_ARRAY \ - (garrow_time64_array_get_type()) -#define GARROW_TIME64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_TIME64_ARRAY, \ - GArrowTime64Array)) -#define GARROW_TIME64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_TIME64_ARRAY, \ - GArrowTime64ArrayClass)) -#define GARROW_IS_TIME64_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_TIME64_ARRAY)) -#define GARROW_IS_TIME64_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_TIME64_ARRAY)) -#define GARROW_TIME64_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_TIME64_ARRAY, \ - GArrowTime64ArrayClass)) - -typedef struct _GArrowTime64Array GArrowTime64Array; -typedef struct _GArrowTime64ArrayClass GArrowTime64ArrayClass; - -/** - * GArrowTime64Array: - * - * It wraps `arrow::Time64Array`. - */ -struct _GArrowTime64Array -{ - /*< private >*/ - GArrowPrimitiveArray parent_instance; -}; - -struct _GArrowTime64ArrayClass -{ - GArrowPrimitiveArrayClass parent_class; -}; - -GType garrow_time64_array_get_type(void) G_GNUC_CONST; - -GArrowTime64Array *garrow_time64_array_new(GArrowTime64DataType *data_type, - gint64 length, - GArrowBuffer *data, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -gint64 garrow_time64_array_get_value(GArrowTime64Array *array, - gint64 i); -const gint64 *garrow_time64_array_get_values(GArrowTime64Array *array, - gint64 *length); - - -#define GARROW_TYPE_LIST_ARRAY \ - (garrow_list_array_get_type()) -#define GARROW_LIST_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_LIST_ARRAY, \ - GArrowListArray)) -#define GARROW_LIST_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_LIST_ARRAY, \ - GArrowListArrayClass)) -#define GARROW_IS_LIST_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_LIST_ARRAY)) -#define GARROW_IS_LIST_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_LIST_ARRAY)) -#define GARROW_LIST_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_LIST_ARRAY, \ - GArrowListArrayClass)) - -typedef struct _GArrowListArray GArrowListArray; -typedef struct _GArrowListArrayClass GArrowListArrayClass; - -/** - * GArrowListArray: - * - * It wraps `arrow::ListArray`. - */ -struct _GArrowListArray -{ - /*< private >*/ - GArrowArray parent_instance; -}; - -struct _GArrowListArrayClass -{ - GArrowArrayClass parent_class; -}; - -GType garrow_list_array_get_type(void) G_GNUC_CONST; - -GArrowListArray *garrow_list_array_new(gint64 length, - GArrowBuffer *value_offsets, - GArrowArray *values, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array); -GArrowArray *garrow_list_array_get_value(GArrowListArray *array, - gint64 i); - - -#define GARROW_TYPE_STRUCT_ARRAY \ - (garrow_struct_array_get_type()) -#define GARROW_STRUCT_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_STRUCT_ARRAY, \ - GArrowStructArray)) -#define GARROW_STRUCT_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_STRUCT_ARRAY, \ - GArrowStructArrayClass)) -#define GARROW_IS_STRUCT_ARRAY(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_STRUCT_ARRAY)) -#define GARROW_IS_STRUCT_ARRAY_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_STRUCT_ARRAY)) -#define GARROW_STRUCT_ARRAY_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_STRUCT_ARRAY, \ - GArrowStructArrayClass)) - -typedef struct _GArrowStructArray GArrowStructArray; -typedef struct _GArrowStructArrayClass GArrowStructArrayClass; - -/** - * GArrowStructArray: - * - * It wraps `arrow::StructArray`. - */ -struct _GArrowStructArray -{ - /*< private >*/ - GArrowArray parent_instance; -}; - -struct _GArrowStructArrayClass -{ - GArrowArrayClass parent_class; -}; - -GType garrow_struct_array_get_type(void) G_GNUC_CONST; - -GArrowStructArray *garrow_struct_array_new(GArrowDataType *data_type, - gint64 length, - GList *children, - GArrowBuffer *null_bitmap, - gint64 n_nulls); - -GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array, - gint i); -GList *garrow_struct_array_get_fields(GArrowStructArray *array); - -G_END_DECLS +#include +#include diff --git a/c_glib/arrow-glib/array.hpp b/c_glib/arrow-glib/array.hpp index d2dff22c48cf9..e575c425380f2 100644 --- a/c_glib/arrow-glib/array.hpp +++ b/c_glib/arrow-glib/array.hpp @@ -19,9 +19,5 @@ #pragma once -#include - -#include - -GArrowArray *garrow_array_new_raw(std::shared_ptr *arrow_array); -std::shared_ptr garrow_array_get_raw(GArrowArray *array); +#include +#include diff --git a/c_glib/arrow-glib/array.cpp b/c_glib/arrow-glib/basic-array.cpp similarity index 90% rename from c_glib/arrow-glib/array.cpp rename to c_glib/arrow-glib/basic-array.cpp index 4945ca3cfe903..0698a04098050 100644 --- a/c_glib/arrow-glib/array.cpp +++ b/c_glib/arrow-glib/basic-array.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include @@ -86,9 +86,9 @@ garrow_primitive_array_new(GArrowDataType *data_type, G_BEGIN_DECLS /** - * SECTION: array - * @section_id: array-classes - * @title: Array classes + * SECTION: basic-array + * @section_id: basic-array-classes + * @title: Basic array classes * @include: arrow-glib/arrow-glib.h * * #GArrowArray is a base class for all array classes such as @@ -194,15 +194,6 @@ G_BEGIN_DECLS * nanoseconds since midnight in 64-bit signed integer array. It can * store zero or more time data. If you don't have Arrow format data, * you need to use #GArrowTime64ArrayBuilder to create a new array. - * - * #GArrowListArray is a class for list array. It can store zero or - * more list data. If you don't have Arrow format data, you need to - * use #GArrowListArrayBuilder to create a new array. - * - * #GArrowStructArray is a class for struct array. It can store zero - * or more structs. One struct has zero or more fields. If you don't - * have Arrow format data, you need to use #GArrowStructArrayBuilder - * to create a new array. */ typedef struct GArrowArrayPrivate_ { @@ -2028,195 +2019,6 @@ garrow_time64_array_get_values(GArrowTime64Array *array, return reinterpret_cast(values); } - -G_DEFINE_TYPE(GArrowListArray, \ - garrow_list_array, \ - GARROW_TYPE_ARRAY) - -static void -garrow_list_array_init(GArrowListArray *object) -{ -} - -static void -garrow_list_array_class_init(GArrowListArrayClass *klass) -{ -} - -/** - * garrow_list_array_new: - * @length: The number of elements. - * @value_offsets: The offsets of @values in Arrow format. - * @values: The values as #GArrowArray. - * @null_bitmap: (nullable): The bitmap that shows null elements. The - * N-th element is null when the N-th bit is 0, not null otherwise. - * If the array has no null elements, the bitmap must be %NULL and - * @n_nulls is 0. - * @n_nulls: The number of null elements. If -1 is specified, the - * number of nulls are computed from @null_bitmap. - * - * Returns: A newly created #GArrowListArray. - * - * Since: 0.4.0 - */ -GArrowListArray * -garrow_list_array_new(gint64 length, - GArrowBuffer *value_offsets, - GArrowArray *values, - GArrowBuffer *null_bitmap, - gint64 n_nulls) -{ - const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets); - const auto arrow_values = garrow_array_get_raw(values); - const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap); - auto arrow_data_type = arrow::list(arrow_values->type()); - auto arrow_list_array = - std::make_shared(arrow_data_type, - length, - arrow_value_offsets, - arrow_values, - arrow_bitmap, - n_nulls); - auto arrow_array = - std::static_pointer_cast(arrow_list_array); - return GARROW_LIST_ARRAY(garrow_array_new_raw(&arrow_array)); -} - -/** - * garrow_list_array_get_value_type: - * @array: A #GArrowListArray. - * - * Returns: (transfer full): The data type of value in each list. - */ -GArrowDataType * -garrow_list_array_get_value_type(GArrowListArray *array) -{ - auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); - auto arrow_list_array = - static_cast(arrow_array.get()); - auto arrow_value_type = arrow_list_array->value_type(); - return garrow_data_type_new_raw(&arrow_value_type); -} - -/** - * garrow_list_array_get_value: - * @array: A #GArrowListArray. - * @i: The index of the target value. - * - * Returns: (transfer full): The i-th list. - */ -GArrowArray * -garrow_list_array_get_value(GArrowListArray *array, - gint64 i) -{ - auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); - auto arrow_list_array = - static_cast(arrow_array.get()); - auto arrow_list = - arrow_list_array->values()->Slice(arrow_list_array->value_offset(i), - arrow_list_array->value_length(i)); - return garrow_array_new_raw(&arrow_list); -} - - -G_DEFINE_TYPE(GArrowStructArray, \ - garrow_struct_array, \ - GARROW_TYPE_ARRAY) - -static void -garrow_struct_array_init(GArrowStructArray *object) -{ -} - -static void -garrow_struct_array_class_init(GArrowStructArrayClass *klass) -{ -} - -/** - * garrow_struct_array_new: - * @data_type: The data type of the struct. - * @length: The number of elements. - * @children: (element-type GArrowArray): The arrays for each field - * as #GList of #GArrowArray. - * @null_bitmap: (nullable): The bitmap that shows null elements. The - * N-th element is null when the N-th bit is 0, not null otherwise. - * If the array has no null elements, the bitmap must be %NULL and - * @n_nulls is 0. - * @n_nulls: The number of null elements. If -1 is specified, the - * number of nulls are computed from @null_bitmap. - * - * Returns: A newly created #GArrowStructArray. - * - * Since: 0.4.0 - */ -GArrowStructArray * -garrow_struct_array_new(GArrowDataType *data_type, - gint64 length, - GList *children, - GArrowBuffer *null_bitmap, - gint64 n_nulls) -{ - const auto arrow_data_type = garrow_data_type_get_raw(data_type); - std::vector> arrow_children; - for (GList *node = children; node; node = node->next) { - GArrowArray *child = GARROW_ARRAY(node->data); - arrow_children.push_back(garrow_array_get_raw(child)); - } - const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap); - auto arrow_struct_array = - std::make_shared(arrow_data_type, - length, - arrow_children, - arrow_bitmap, - n_nulls); - auto arrow_array = - std::static_pointer_cast(arrow_struct_array); - return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_array)); -} - -/** - * garrow_struct_array_get_field - * @array: A #GArrowStructArray. - * @i: The index of the field in the struct. - * - * Returns: (transfer full): The i-th field. - */ -GArrowArray * -garrow_struct_array_get_field(GArrowStructArray *array, - gint i) -{ - auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); - auto arrow_struct_array = - static_cast(arrow_array.get()); - auto arrow_field = arrow_struct_array->field(i); - return garrow_array_new_raw(&arrow_field); -} - -/** - * garrow_struct_array_get_fields - * @array: A #GArrowStructArray. - * - * Returns: (element-type GArrowArray) (transfer full): - * The fields in the struct. - */ -GList * -garrow_struct_array_get_fields(GArrowStructArray *array) -{ - const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); - const auto arrow_struct_array = - static_cast(arrow_array.get()); - - GList *fields = NULL; - for (int i = 0; i < arrow_struct_array->num_fields(); ++i) { - auto arrow_field = arrow_struct_array->field(i); - GArrowArray *field = garrow_array_new_raw(&arrow_field); - fields = g_list_prepend(fields, field); - } - - return g_list_reverse(fields); -} - G_END_DECLS GArrowArray * diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h new file mode 100644 index 0000000000000..c01a04f07d8b9 --- /dev/null +++ b/c_glib/arrow-glib/basic-array.h @@ -0,0 +1,1115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +G_BEGIN_DECLS + +#define GARROW_TYPE_ARRAY (garrow_array_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowArray, + garrow_array, + GARROW, + ARRAY, + GObject) +struct _GArrowArrayClass +{ + GObjectClass parent_class; +}; + +gboolean garrow_array_equal (GArrowArray *array, + GArrowArray *other_array); +gboolean garrow_array_equal_approx(GArrowArray *array, + GArrowArray *other_array); +gboolean garrow_array_equal_range (GArrowArray *array, + gint64 start_index, + GArrowArray *other_array, + gint64 other_start_index, + gint64 end_index); + +gboolean garrow_array_is_null (GArrowArray *array, + gint64 i); +gboolean garrow_array_is_valid (GArrowArray *array, + gint64 i); +gint64 garrow_array_get_length (GArrowArray *array); +gint64 garrow_array_get_offset (GArrowArray *array); +gint64 garrow_array_get_n_nulls (GArrowArray *array); +GArrowBuffer *garrow_array_get_null_bitmap(GArrowArray *array); +GArrowDataType *garrow_array_get_value_data_type(GArrowArray *array); +GArrowType garrow_array_get_value_type(GArrowArray *array); +GArrowArray *garrow_array_slice (GArrowArray *array, + gint64 offset, + gint64 length); +gchar *garrow_array_to_string (GArrowArray *array, + GError **error); + +GArrowArray *garrow_array_cast (GArrowArray *array, + GArrowDataType *target_data_type, + GArrowCastOptions *options, + GError **error); + +#define GARROW_TYPE_NULL_ARRAY \ + (garrow_null_array_get_type()) +#define GARROW_NULL_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_NULL_ARRAY, \ + GArrowNullArray)) +#define GARROW_NULL_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_NULL_ARRAY, \ + GArrowNullArrayClass)) +#define GARROW_IS_NULL_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_NULL_ARRAY)) +#define GARROW_IS_NULL_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_NULL_ARRAY)) +#define GARROW_NULL_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_NULL_ARRAY, \ + GArrowNullArrayClass)) + +typedef struct _GArrowNullArray GArrowNullArray; +typedef struct _GArrowNullArrayClass GArrowNullArrayClass; + +/** + * GArrowNullArray: + * + * It wraps `arrow::NullArray`. + */ +struct _GArrowNullArray +{ + /*< private >*/ + GArrowArray parent_instance; +}; + +struct _GArrowNullArrayClass +{ + GArrowArrayClass parent_class; +}; + +GType garrow_null_array_get_type(void) G_GNUC_CONST; + +GArrowNullArray *garrow_null_array_new(gint64 length); + + +#define GARROW_TYPE_PRIMITIVE_ARRAY \ + (garrow_primitive_array_get_type()) +#define GARROW_PRIMITIVE_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_PRIMITIVE_ARRAY, \ + GArrowPrimitiveArray)) +#define GARROW_PRIMITIVE_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_PRIMITIVE_ARRAY, \ + GArrowPrimitiveArrayClass)) +#define GARROW_IS_PRIMITIVE_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_PRIMITIVE_ARRAY)) +#define GARROW_IS_PRIMITIVE_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_PRIMITIVE_ARRAY)) +#define GARROW_PRIMITIVE_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_PRIMITIVE_ARRAY, \ + GArrowPrimitiveArrayClass)) + +typedef struct _GArrowPrimitiveArray GArrowPrimitiveArray; +typedef struct _GArrowPrimitiveArrayClass GArrowPrimitiveArrayClass; + +/** + * GArrowPrimitiveArray: + * + * It wraps `arrow::PrimitiveArray`. + */ +struct _GArrowPrimitiveArray +{ + /*< private >*/ + GArrowArray parent_instance; +}; + +struct _GArrowPrimitiveArrayClass +{ + GArrowArrayClass parent_class; +}; + +GType garrow_primitive_array_get_type(void) G_GNUC_CONST; + +GArrowBuffer *garrow_primitive_array_get_buffer(GArrowPrimitiveArray *array); + + +#define GARROW_TYPE_BOOLEAN_ARRAY \ + (garrow_boolean_array_get_type()) +#define GARROW_BOOLEAN_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_BOOLEAN_ARRAY, \ + GArrowBooleanArray)) +#define GARROW_BOOLEAN_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_BOOLEAN_ARRAY, \ + GArrowBooleanArrayClass)) +#define GARROW_IS_BOOLEAN_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_BOOLEAN_ARRAY)) +#define GARROW_IS_BOOLEAN_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_BOOLEAN_ARRAY)) +#define GARROW_BOOLEAN_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_BOOLEAN_ARRAY, \ + GArrowBooleanArrayClass)) + +typedef struct _GArrowBooleanArray GArrowBooleanArray; +typedef struct _GArrowBooleanArrayClass GArrowBooleanArrayClass; + +/** + * GArrowBooleanArray: + * + * It wraps `arrow::BooleanArray`. + */ +struct _GArrowBooleanArray +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowBooleanArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_boolean_array_get_type (void) G_GNUC_CONST; + +GArrowBooleanArray *garrow_boolean_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gboolean garrow_boolean_array_get_value (GArrowBooleanArray *array, + gint64 i); +gboolean *garrow_boolean_array_get_values(GArrowBooleanArray *array, + gint64 *length); + + +#define GARROW_TYPE_INT8_ARRAY \ + (garrow_int8_array_get_type()) +#define GARROW_INT8_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_INT8_ARRAY, \ + GArrowInt8Array)) +#define GARROW_INT8_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_INT8_ARRAY, \ + GArrowInt8ArrayClass)) +#define GARROW_IS_INT8_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_INT8_ARRAY)) +#define GARROW_IS_INT8_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_INT8_ARRAY)) +#define GARROW_INT8_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_INT8_ARRAY, \ + GArrowInt8ArrayClass)) + +typedef struct _GArrowInt8Array GArrowInt8Array; +typedef struct _GArrowInt8ArrayClass GArrowInt8ArrayClass; + +/** + * GArrowInt8Array: + * + * It wraps `arrow::Int8Array`. + */ +struct _GArrowInt8Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowInt8ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_int8_array_get_type(void) G_GNUC_CONST; + +GArrowInt8Array *garrow_int8_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint8 garrow_int8_array_get_value(GArrowInt8Array *array, + gint64 i); +const gint8 *garrow_int8_array_get_values(GArrowInt8Array *array, + gint64 *length); + + +#define GARROW_TYPE_UINT8_ARRAY \ + (garrow_uint8_array_get_type()) +#define GARROW_UINT8_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_UINT8_ARRAY, \ + GArrowUInt8Array)) +#define GARROW_UINT8_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_UINT8_ARRAY, \ + GArrowUInt8ArrayClass)) +#define GARROW_IS_UINT8_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_UINT8_ARRAY)) +#define GARROW_IS_UINT8_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_UINT8_ARRAY)) +#define GARROW_UINT8_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_UINT8_ARRAY, \ + GArrowUInt8ArrayClass)) + +typedef struct _GArrowUInt8Array GArrowUInt8Array; +typedef struct _GArrowUInt8ArrayClass GArrowUInt8ArrayClass; + +/** + * GArrowUInt8Array: + * + * It wraps `arrow::UInt8Array`. + */ +struct _GArrowUInt8Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowUInt8ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_uint8_array_get_type(void) G_GNUC_CONST; + +GArrowUInt8Array *garrow_uint8_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +guint8 garrow_uint8_array_get_value(GArrowUInt8Array *array, + gint64 i); +const guint8 *garrow_uint8_array_get_values(GArrowUInt8Array *array, + gint64 *length); + + +#define GARROW_TYPE_INT16_ARRAY \ + (garrow_int16_array_get_type()) +#define GARROW_INT16_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_INT16_ARRAY, \ + GArrowInt16Array)) +#define GARROW_INT16_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_INT16_ARRAY, \ + GArrowInt16ArrayClass)) +#define GARROW_IS_INT16_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_INT16_ARRAY)) +#define GARROW_IS_INT16_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_INT16_ARRAY)) +#define GARROW_INT16_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_INT16_ARRAY, \ + GArrowInt16ArrayClass)) + +typedef struct _GArrowInt16Array GArrowInt16Array; +typedef struct _GArrowInt16ArrayClass GArrowInt16ArrayClass; + +/** + * GArrowInt16Array: + * + * It wraps `arrow::Int16Array`. + */ +struct _GArrowInt16Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowInt16ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_int16_array_get_type(void) G_GNUC_CONST; + +GArrowInt16Array *garrow_int16_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint16 garrow_int16_array_get_value(GArrowInt16Array *array, + gint64 i); +const gint16 *garrow_int16_array_get_values(GArrowInt16Array *array, + gint64 *length); + + +#define GARROW_TYPE_UINT16_ARRAY \ + (garrow_uint16_array_get_type()) +#define GARROW_UINT16_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_UINT16_ARRAY, \ + GArrowUInt16Array)) +#define GARROW_UINT16_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_UINT16_ARRAY, \ + GArrowUInt16ArrayClass)) +#define GARROW_IS_UINT16_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_UINT16_ARRAY)) +#define GARROW_IS_UINT16_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_UINT16_ARRAY)) +#define GARROW_UINT16_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_UINT16_ARRAY, \ + GArrowUInt16ArrayClass)) + +typedef struct _GArrowUInt16Array GArrowUInt16Array; +typedef struct _GArrowUInt16ArrayClass GArrowUInt16ArrayClass; + +/** + * GArrowUInt16Array: + * + * It wraps `arrow::UInt16Array`. + */ +struct _GArrowUInt16Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowUInt16ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_uint16_array_get_type(void) G_GNUC_CONST; + +GArrowUInt16Array *garrow_uint16_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +guint16 garrow_uint16_array_get_value(GArrowUInt16Array *array, + gint64 i); +const guint16 *garrow_uint16_array_get_values(GArrowUInt16Array *array, + gint64 *length); + + +#define GARROW_TYPE_INT32_ARRAY \ + (garrow_int32_array_get_type()) +#define GARROW_INT32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_INT32_ARRAY, \ + GArrowInt32Array)) +#define GARROW_INT32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_INT32_ARRAY, \ + GArrowInt32ArrayClass)) +#define GARROW_IS_INT32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_INT32_ARRAY)) +#define GARROW_IS_INT32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_INT32_ARRAY)) +#define GARROW_INT32_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_INT32_ARRAY, \ + GArrowInt32ArrayClass)) + +typedef struct _GArrowInt32Array GArrowInt32Array; +typedef struct _GArrowInt32ArrayClass GArrowInt32ArrayClass; + +/** + * GArrowInt32Array: + * + * It wraps `arrow::Int32Array`. + */ +struct _GArrowInt32Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowInt32ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_int32_array_get_type(void) G_GNUC_CONST; + +GArrowInt32Array *garrow_int32_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint32 garrow_int32_array_get_value(GArrowInt32Array *array, + gint64 i); +const gint32 *garrow_int32_array_get_values(GArrowInt32Array *array, + gint64 *length); + + +#define GARROW_TYPE_UINT32_ARRAY \ + (garrow_uint32_array_get_type()) +#define GARROW_UINT32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_UINT32_ARRAY, \ + GArrowUInt32Array)) +#define GARROW_UINT32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_UINT32_ARRAY, \ + GArrowUInt32ArrayClass)) +#define GARROW_IS_UINT32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_UINT32_ARRAY)) +#define GARROW_IS_UINT32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_UINT32_ARRAY)) +#define GARROW_UINT32_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_UINT32_ARRAY, \ + GArrowUInt32ArrayClass)) + +typedef struct _GArrowUInt32Array GArrowUInt32Array; +typedef struct _GArrowUInt32ArrayClass GArrowUInt32ArrayClass; + +/** + * GArrowUInt32Array: + * + * It wraps `arrow::UInt32Array`. + */ +struct _GArrowUInt32Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowUInt32ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_uint32_array_get_type(void) G_GNUC_CONST; + +GArrowUInt32Array *garrow_uint32_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +guint32 garrow_uint32_array_get_value(GArrowUInt32Array *array, + gint64 i); +const guint32 *garrow_uint32_array_get_values(GArrowUInt32Array *array, + gint64 *length); + + +#define GARROW_TYPE_INT64_ARRAY \ + (garrow_int64_array_get_type()) +#define GARROW_INT64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_INT64_ARRAY, \ + GArrowInt64Array)) +#define GARROW_INT64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_INT64_ARRAY, \ + GArrowInt64ArrayClass)) +#define GARROW_IS_INT64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_INT64_ARRAY)) +#define GARROW_IS_INT64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_INT64_ARRAY)) +#define GARROW_INT64_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_INT64_ARRAY, \ + GArrowInt64ArrayClass)) + +typedef struct _GArrowInt64Array GArrowInt64Array; +typedef struct _GArrowInt64ArrayClass GArrowInt64ArrayClass; + +/** + * GArrowInt64Array: + * + * It wraps `arrow::Int64Array`. + */ +struct _GArrowInt64Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowInt64ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_int64_array_get_type(void) G_GNUC_CONST; + +GArrowInt64Array *garrow_int64_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint64 garrow_int64_array_get_value(GArrowInt64Array *array, + gint64 i); +const gint64 *garrow_int64_array_get_values(GArrowInt64Array *array, + gint64 *length); + + +#define GARROW_TYPE_UINT64_ARRAY \ + (garrow_uint64_array_get_type()) +#define GARROW_UINT64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_UINT64_ARRAY, \ + GArrowUInt64Array)) +#define GARROW_UINT64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_UINT64_ARRAY, \ + GArrowUInt64ArrayClass)) +#define GARROW_IS_UINT64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_UINT64_ARRAY)) +#define GARROW_IS_UINT64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_UINT64_ARRAY)) +#define GARROW_UINT64_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_UINT64_ARRAY, \ + GArrowUInt64ArrayClass)) + +typedef struct _GArrowUInt64Array GArrowUInt64Array; +typedef struct _GArrowUInt64ArrayClass GArrowUInt64ArrayClass; + +/** + * GArrowUInt64Array: + * + * It wraps `arrow::UInt64Array`. + */ +struct _GArrowUInt64Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowUInt64ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_uint64_array_get_type(void) G_GNUC_CONST; + +GArrowUInt64Array *garrow_uint64_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +guint64 garrow_uint64_array_get_value(GArrowUInt64Array *array, + gint64 i); +const guint64 *garrow_uint64_array_get_values(GArrowUInt64Array *array, + gint64 *length); + + +#define GARROW_TYPE_FLOAT_ARRAY \ + (garrow_float_array_get_type()) +#define GARROW_FLOAT_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_FLOAT_ARRAY, \ + GArrowFloatArray)) +#define GARROW_FLOAT_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_FLOAT_ARRAY, \ + GArrowFloatArrayClass)) +#define GARROW_IS_FLOAT_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_FLOAT_ARRAY)) +#define GARROW_IS_FLOAT_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_FLOAT_ARRAY)) +#define GARROW_FLOAT_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_FLOAT_ARRAY, \ + GArrowFloatArrayClass)) + +typedef struct _GArrowFloatArray GArrowFloatArray; +typedef struct _GArrowFloatArrayClass GArrowFloatArrayClass; + +/** + * GArrowFloatArray: + * + * It wraps `arrow::FloatArray`. + */ +struct _GArrowFloatArray +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowFloatArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_float_array_get_type(void) G_GNUC_CONST; + +GArrowFloatArray *garrow_float_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gfloat garrow_float_array_get_value(GArrowFloatArray *array, + gint64 i); +const gfloat *garrow_float_array_get_values(GArrowFloatArray *array, + gint64 *length); + + +#define GARROW_TYPE_DOUBLE_ARRAY \ + (garrow_double_array_get_type()) +#define GARROW_DOUBLE_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_DOUBLE_ARRAY, \ + GArrowDoubleArray)) +#define GARROW_DOUBLE_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_DOUBLE_ARRAY, \ + GArrowDoubleArrayClass)) +#define GARROW_IS_DOUBLE_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_DOUBLE_ARRAY)) +#define GARROW_IS_DOUBLE_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_DOUBLE_ARRAY)) +#define GARROW_DOUBLE_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_DOUBLE_ARRAY, \ + GArrowDoubleArrayClass)) + +typedef struct _GArrowDoubleArray GArrowDoubleArray; +typedef struct _GArrowDoubleArrayClass GArrowDoubleArrayClass; + +/** + * GArrowDoubleArray: + * + * It wraps `arrow::DoubleArray`. + */ +struct _GArrowDoubleArray +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowDoubleArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_double_array_get_type(void) G_GNUC_CONST; + +GArrowDoubleArray *garrow_double_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gdouble garrow_double_array_get_value(GArrowDoubleArray *array, + gint64 i); +const gdouble *garrow_double_array_get_values(GArrowDoubleArray *array, + gint64 *length); + + +#define GARROW_TYPE_BINARY_ARRAY \ + (garrow_binary_array_get_type()) +#define GARROW_BINARY_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_BINARY_ARRAY, \ + GArrowBinaryArray)) +#define GARROW_BINARY_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_BINARY_ARRAY, \ + GArrowBinaryArrayClass)) +#define GARROW_IS_BINARY_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_BINARY_ARRAY)) +#define GARROW_IS_BINARY_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_BINARY_ARRAY)) +#define GARROW_BINARY_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_BINARY_ARRAY, \ + GArrowBinaryArrayClass)) + +typedef struct _GArrowBinaryArray GArrowBinaryArray; +typedef struct _GArrowBinaryArrayClass GArrowBinaryArrayClass; + +/** + * GArrowBinaryArray: + * + * It wraps `arrow::BinaryArray`. + */ +struct _GArrowBinaryArray +{ + /*< private >*/ + GArrowArray parent_instance; +}; + +struct _GArrowBinaryArrayClass +{ + GArrowArrayClass parent_class; +}; + +GType garrow_binary_array_get_type(void) G_GNUC_CONST; + +GArrowBinaryArray *garrow_binary_array_new(gint64 length, + GArrowBuffer *value_offsets, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +GBytes *garrow_binary_array_get_value(GArrowBinaryArray *array, + gint64 i); +GArrowBuffer *garrow_binary_array_get_buffer(GArrowBinaryArray *array); +GArrowBuffer *garrow_binary_array_get_offsets_buffer(GArrowBinaryArray *array); + +#define GARROW_TYPE_STRING_ARRAY \ + (garrow_string_array_get_type()) +#define GARROW_STRING_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_STRING_ARRAY, \ + GArrowStringArray)) +#define GARROW_STRING_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_STRING_ARRAY, \ + GArrowStringArrayClass)) +#define GARROW_IS_STRING_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_STRING_ARRAY)) +#define GARROW_IS_STRING_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_STRING_ARRAY)) +#define GARROW_STRING_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_STRING_ARRAY, \ + GArrowStringArrayClass)) + +typedef struct _GArrowStringArray GArrowStringArray; +typedef struct _GArrowStringArrayClass GArrowStringArrayClass; + +/** + * GArrowStringArray: + * + * It wraps `arrow::StringArray`. + */ +struct _GArrowStringArray +{ + /*< private >*/ + GArrowBinaryArray parent_instance; +}; + +struct _GArrowStringArrayClass +{ + GArrowBinaryArrayClass parent_class; +}; + +GType garrow_string_array_get_type(void) G_GNUC_CONST; + +GArrowStringArray *garrow_string_array_new(gint64 length, + GArrowBuffer *value_offsets, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gchar *garrow_string_array_get_string(GArrowStringArray *array, + gint64 i); + + +#define GARROW_TYPE_DATE32_ARRAY \ + (garrow_date32_array_get_type()) +#define GARROW_DATE32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_DATE32_ARRAY, \ + GArrowDate32Array)) +#define GARROW_DATE32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_DATE32_ARRAY, \ + GArrowDate32ArrayClass)) +#define GARROW_IS_DATE32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_DATE32_ARRAY)) +#define GARROW_IS_DATE32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_DATE32_ARRAY)) +#define GARROW_DATE32_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_DATE32_ARRAY, \ + GArrowDate32ArrayClass)) + +typedef struct _GArrowDate32Array GArrowDate32Array; +typedef struct _GArrowDate32ArrayClass GArrowDate32ArrayClass; + +/** + * GArrowDate32Array: + * + * It wraps `arrow::Date32Array`. + */ +struct _GArrowDate32Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowDate32ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_date32_array_get_type(void) G_GNUC_CONST; + +GArrowDate32Array *garrow_date32_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint32 garrow_date32_array_get_value(GArrowDate32Array *array, + gint64 i); +const gint32 *garrow_date32_array_get_values(GArrowDate32Array *array, + gint64 *length); + + +#define GARROW_TYPE_DATE64_ARRAY \ + (garrow_date64_array_get_type()) +#define GARROW_DATE64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_DATE64_ARRAY, \ + GArrowDate64Array)) +#define GARROW_DATE64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_DATE64_ARRAY, \ + GArrowDate64ArrayClass)) +#define GARROW_IS_DATE64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_DATE64_ARRAY)) +#define GARROW_IS_DATE64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_DATE64_ARRAY)) +#define GARROW_DATE64_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_DATE64_ARRAY, \ + GArrowDate64ArrayClass)) + +typedef struct _GArrowDate64Array GArrowDate64Array; +typedef struct _GArrowDate64ArrayClass GArrowDate64ArrayClass; + +/** + * GArrowDate64Array: + * + * It wraps `arrow::Date64Array`. + */ +struct _GArrowDate64Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowDate64ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_date64_array_get_type(void) G_GNUC_CONST; + +GArrowDate64Array *garrow_date64_array_new(gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint64 garrow_date64_array_get_value(GArrowDate64Array *array, + gint64 i); +const gint64 *garrow_date64_array_get_values(GArrowDate64Array *array, + gint64 *length); + + +#define GARROW_TYPE_TIMESTAMP_ARRAY \ + (garrow_timestamp_array_get_type()) +#define GARROW_TIMESTAMP_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_TIMESTAMP_ARRAY, \ + GArrowTimestampArray)) +#define GARROW_TIMESTAMP_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_TIMESTAMP_ARRAY, \ + GArrowTimestampArrayClass)) +#define GARROW_IS_TIMESTAMP_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_TIMESTAMP_ARRAY)) +#define GARROW_IS_TIMESTAMP_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_TIMESTAMP_ARRAY)) +#define GARROW_TIMESTAMP_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_TIMESTAMP_ARRAY, \ + GArrowTimestampArrayClass)) + +typedef struct _GArrowTimestampArray GArrowTimestampArray; +typedef struct _GArrowTimestampArrayClass GArrowTimestampArrayClass; + +/** + * GArrowTimestampArray: + * + * It wraps `arrow::TimestampArray`. + */ +struct _GArrowTimestampArray +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowTimestampArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_timestamp_array_get_type(void) G_GNUC_CONST; + +GArrowTimestampArray *garrow_timestamp_array_new(GArrowTimestampDataType *data_type, + gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint64 garrow_timestamp_array_get_value(GArrowTimestampArray *array, + gint64 i); +const gint64 *garrow_timestamp_array_get_values(GArrowTimestampArray *array, + gint64 *length); + + +#define GARROW_TYPE_TIME32_ARRAY \ + (garrow_time32_array_get_type()) +#define GARROW_TIME32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_TIME32_ARRAY, \ + GArrowTime32Array)) +#define GARROW_TIME32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_TIME32_ARRAY, \ + GArrowTime32ArrayClass)) +#define GARROW_IS_TIME32_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_TIME32_ARRAY)) +#define GARROW_IS_TIME32_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_TIME32_ARRAY)) +#define GARROW_TIME32_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_TIME32_ARRAY, \ + GArrowTime32ArrayClass)) + +typedef struct _GArrowTime32Array GArrowTime32Array; +typedef struct _GArrowTime32ArrayClass GArrowTime32ArrayClass; + +/** + * GArrowTime32Array: + * + * It wraps `arrow::Time32Array`. + */ +struct _GArrowTime32Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowTime32ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_time32_array_get_type(void) G_GNUC_CONST; + +GArrowTime32Array *garrow_time32_array_new(GArrowTime32DataType *data_type, + gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint32 garrow_time32_array_get_value(GArrowTime32Array *array, + gint64 i); +const gint32 *garrow_time32_array_get_values(GArrowTime32Array *array, + gint64 *length); + + +#define GARROW_TYPE_TIME64_ARRAY \ + (garrow_time64_array_get_type()) +#define GARROW_TIME64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_TIME64_ARRAY, \ + GArrowTime64Array)) +#define GARROW_TIME64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_TIME64_ARRAY, \ + GArrowTime64ArrayClass)) +#define GARROW_IS_TIME64_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_TIME64_ARRAY)) +#define GARROW_IS_TIME64_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_TIME64_ARRAY)) +#define GARROW_TIME64_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_TIME64_ARRAY, \ + GArrowTime64ArrayClass)) + +typedef struct _GArrowTime64Array GArrowTime64Array; +typedef struct _GArrowTime64ArrayClass GArrowTime64ArrayClass; + +/** + * GArrowTime64Array: + * + * It wraps `arrow::Time64Array`. + */ +struct _GArrowTime64Array +{ + /*< private >*/ + GArrowPrimitiveArray parent_instance; +}; + +struct _GArrowTime64ArrayClass +{ + GArrowPrimitiveArrayClass parent_class; +}; + +GType garrow_time64_array_get_type(void) G_GNUC_CONST; + +GArrowTime64Array *garrow_time64_array_new(GArrowTime64DataType *data_type, + gint64 length, + GArrowBuffer *data, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +gint64 garrow_time64_array_get_value(GArrowTime64Array *array, + gint64 i); +const gint64 *garrow_time64_array_get_values(GArrowTime64Array *array, + gint64 *length); + +G_END_DECLS diff --git a/c_glib/arrow-glib/basic-array.hpp b/c_glib/arrow-glib/basic-array.hpp new file mode 100644 index 0000000000000..52b94a51f3dbf --- /dev/null +++ b/c_glib/arrow-glib/basic-array.hpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include + +GArrowArray *garrow_array_new_raw(std::shared_ptr *arrow_array); +std::shared_ptr garrow_array_get_raw(GArrowArray *array); diff --git a/c_glib/arrow-glib/basic-data-type.cpp b/c_glib/arrow-glib/basic-data-type.cpp index 62f8a7e4d2fbe..dcafe06d67581 100644 --- a/c_glib/arrow-glib/basic-data-type.cpp +++ b/c_glib/arrow-glib/basic-data-type.cpp @@ -220,6 +220,37 @@ garrow_data_type_get_id(GArrowDataType *data_type) } +G_DEFINE_ABSTRACT_TYPE(GArrowFixedWidthDataType, \ + garrow_fixed_width_data_type, \ + GARROW_TYPE_DATA_TYPE) + +static void +garrow_fixed_width_data_type_init(GArrowFixedWidthDataType *object) +{ +} + +static void +garrow_fixed_width_data_type_class_init(GArrowFixedWidthDataTypeClass *klass) +{ +} + +/** + * garrow_fixed_width_data_type_get_id: + * @data_type: A #GArrowFixedWidthDataType. + * + * Returns: The number of bits for one data. + */ +gint +garrow_fixed_width_data_type_get_bit_width(GArrowFixedWidthDataType *data_type) +{ + const auto arrow_data_type = + garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + const auto arrow_fixed_width_type = + std::static_pointer_cast(arrow_data_type); + return arrow_fixed_width_type->bit_width(); +} + + G_DEFINE_TYPE(GArrowNullDataType, \ garrow_null_data_type, \ GARROW_TYPE_DATA_TYPE) @@ -254,7 +285,7 @@ garrow_null_data_type_new(void) G_DEFINE_TYPE(GArrowBooleanDataType, \ garrow_boolean_data_type, \ - GARROW_TYPE_DATA_TYPE) + GARROW_TYPE_FIXED_WIDTH_DATA_TYPE) static void garrow_boolean_data_type_init(GArrowBooleanDataType *object) @@ -1015,12 +1046,15 @@ garrow_data_type_new_raw(std::shared_ptr *arrow_data_type) case arrow::Type::type::STRUCT: type = GARROW_TYPE_STRUCT_DATA_TYPE; break; + case arrow::Type::type::DICTIONARY: + type = GARROW_TYPE_DICTIONARY_DATA_TYPE; + break; default: type = GARROW_TYPE_DATA_TYPE; break; } data_type = GARROW_DATA_TYPE(g_object_new(type, - "data_type", arrow_data_type, + "data-type", arrow_data_type, NULL)); return data_type; } diff --git a/c_glib/arrow-glib/basic-data-type.h b/c_glib/arrow-glib/basic-data-type.h index d3d52c6120de7..bc0192f1b0062 100644 --- a/c_glib/arrow-glib/basic-data-type.h +++ b/c_glib/arrow-glib/basic-data-type.h @@ -19,57 +19,45 @@ #pragma once +#include #include G_BEGIN_DECLS -#define GARROW_TYPE_DATA_TYPE \ - (garrow_data_type_get_type()) -#define GARROW_DATA_TYPE(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_DATA_TYPE, \ - GArrowDataType)) -#define GARROW_DATA_TYPE_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_DATA_TYPE, \ - GArrowDataTypeClass)) -#define GARROW_IS_DATA_TYPE(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_DATA_TYPE)) -#define GARROW_IS_DATA_TYPE_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_DATA_TYPE)) -#define GARROW_DATA_TYPE_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_DATA_TYPE, \ - GArrowDataTypeClass)) - -typedef struct _GArrowDataType GArrowDataType; -typedef struct _GArrowDataTypeClass GArrowDataTypeClass; - -/** - * GArrowDataType: - * - * It wraps `arrow::DataType`. - */ -struct _GArrowDataType -{ - /*< private >*/ - GObject parent_instance; -}; - +#define GARROW_TYPE_DATA_TYPE (garrow_data_type_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowDataType, + garrow_data_type, + GARROW, + DATA_TYPE, + GObject) struct _GArrowDataTypeClass { GObjectClass parent_class; }; -GType garrow_data_type_get_type (void) G_GNUC_CONST; gboolean garrow_data_type_equal (GArrowDataType *data_type, GArrowDataType *other_data_type); gchar *garrow_data_type_to_string (GArrowDataType *data_type); GArrowType garrow_data_type_get_id (GArrowDataType *data_type); +#define GARROW_TYPE_FIXED_WIDTH_DATA_TYPE (garrow_fixed_width_data_type_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowFixedWidthDataType, + garrow_fixed_width_data_type, + GARROW, + FIXED_WIDTH_DATA_TYPE, + GArrowDataType) +struct _GArrowFixedWidthDataTypeClass +{ + GArrowDataTypeClass parent_class; +}; + +gint garrow_fixed_width_data_type_get_bit_width(GArrowFixedWidthDataType *data_type); +/* TODO: +GList *garrow_fixed_width_data_type_get_buffer_layout(GArrowFixedWidthDataType *data_type); +*/ + + #define GARROW_TYPE_NULL_DATA_TYPE \ (garrow_null_data_type_get_type()) #define GARROW_NULL_DATA_TYPE(obj) \ @@ -114,47 +102,17 @@ GType garrow_null_data_type_get_type (void) G_GNUC_CONST; GArrowNullDataType *garrow_null_data_type_new (void); -#define GARROW_TYPE_BOOLEAN_DATA_TYPE \ - (garrow_boolean_data_type_get_type()) -#define GARROW_BOOLEAN_DATA_TYPE(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_BOOLEAN_DATA_TYPE, \ - GArrowBooleanDataType)) -#define GARROW_BOOLEAN_DATA_TYPE_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_BOOLEAN_DATA_TYPE, \ - GArrowBooleanDataTypeClass)) -#define GARROW_IS_BOOLEAN_DATA_TYPE(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_BOOLEAN_DATA_TYPE)) -#define GARROW_IS_BOOLEAN_DATA_TYPE_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_BOOLEAN_DATA_TYPE)) -#define GARROW_BOOLEAN_DATA_TYPE_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_BOOLEAN_DATA_TYPE, \ - GArrowBooleanDataTypeClass)) - -typedef struct _GArrowBooleanDataType GArrowBooleanDataType; -typedef struct _GArrowBooleanDataTypeClass GArrowBooleanDataTypeClass; - -/** - * GArrowBooleanDataType: - * - * It wraps `arrow::BooleanType`. - */ -struct _GArrowBooleanDataType -{ - /*< private >*/ - GArrowDataType parent_instance; -}; - +#define GARROW_TYPE_BOOLEAN_DATA_TYPE (garrow_boolean_data_type_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowBooleanDataType, + garrow_boolean_data_type, + GARROW, + BOOLEAN_DATA_TYPE, + GArrowFixedWidthDataType) struct _GArrowBooleanDataTypeClass { - GArrowDataTypeClass parent_class; + GArrowFixedWidthDataTypeClass parent_class; }; -GType garrow_boolean_data_type_get_type (void) G_GNUC_CONST; GArrowBooleanDataType *garrow_boolean_data_type_new (void); diff --git a/c_glib/arrow-glib/composite-array.cpp b/c_glib/arrow-glib/composite-array.cpp new file mode 100644 index 0000000000000..445103d374037 --- /dev/null +++ b/c_glib/arrow-glib/composite-array.cpp @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include +#include +#include +#include +#include + +G_BEGIN_DECLS + +/** + * SECTION: composite-array + * @section_id: composite-array-classes + * @title: Composite array classes + * @include: arrow-glib/arrow-glib.h + * + * #GArrowListArray is a class for list array. It can store zero or + * more list data. If you don't have Arrow format data, you need to + * use #GArrowListArrayBuilder to create a new array. + * + * #GArrowStructArray is a class for struct array. It can store zero + * or more structs. One struct has zero or more fields. If you don't + * have Arrow format data, you need to use #GArrowStructArrayBuilder + * to create a new array. + */ + +G_DEFINE_TYPE(GArrowListArray, \ + garrow_list_array, \ + GARROW_TYPE_ARRAY) + +static void +garrow_list_array_init(GArrowListArray *object) +{ +} + +static void +garrow_list_array_class_init(GArrowListArrayClass *klass) +{ +} + +/** + * garrow_list_array_new: + * @length: The number of elements. + * @value_offsets: The offsets of @values in Arrow format. + * @values: The values as #GArrowArray. + * @null_bitmap: (nullable): The bitmap that shows null elements. The + * N-th element is null when the N-th bit is 0, not null otherwise. + * If the array has no null elements, the bitmap must be %NULL and + * @n_nulls is 0. + * @n_nulls: The number of null elements. If -1 is specified, the + * number of nulls are computed from @null_bitmap. + * + * Returns: A newly created #GArrowListArray. + * + * Since: 0.4.0 + */ +GArrowListArray * +garrow_list_array_new(gint64 length, + GArrowBuffer *value_offsets, + GArrowArray *values, + GArrowBuffer *null_bitmap, + gint64 n_nulls) +{ + const auto arrow_value_offsets = garrow_buffer_get_raw(value_offsets); + const auto arrow_values = garrow_array_get_raw(values); + const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap); + auto arrow_data_type = arrow::list(arrow_values->type()); + auto arrow_list_array = + std::make_shared(arrow_data_type, + length, + arrow_value_offsets, + arrow_values, + arrow_bitmap, + n_nulls); + auto arrow_array = + std::static_pointer_cast(arrow_list_array); + return GARROW_LIST_ARRAY(garrow_array_new_raw(&arrow_array)); +} + +/** + * garrow_list_array_get_value_type: + * @array: A #GArrowListArray. + * + * Returns: (transfer full): The data type of value in each list. + */ +GArrowDataType * +garrow_list_array_get_value_type(GArrowListArray *array) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_list_array = + static_cast(arrow_array.get()); + auto arrow_value_type = arrow_list_array->value_type(); + return garrow_data_type_new_raw(&arrow_value_type); +} + +/** + * garrow_list_array_get_value: + * @array: A #GArrowListArray. + * @i: The index of the target value. + * + * Returns: (transfer full): The i-th list. + */ +GArrowArray * +garrow_list_array_get_value(GArrowListArray *array, + gint64 i) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_list_array = + static_cast(arrow_array.get()); + auto arrow_list = + arrow_list_array->values()->Slice(arrow_list_array->value_offset(i), + arrow_list_array->value_length(i)); + return garrow_array_new_raw(&arrow_list); +} + + +G_DEFINE_TYPE(GArrowStructArray, \ + garrow_struct_array, \ + GARROW_TYPE_ARRAY) + +static void +garrow_struct_array_init(GArrowStructArray *object) +{ +} + +static void +garrow_struct_array_class_init(GArrowStructArrayClass *klass) +{ +} + +/** + * garrow_struct_array_new: + * @data_type: The data type of the struct. + * @length: The number of elements. + * @children: (element-type GArrowArray): The arrays for each field + * as #GList of #GArrowArray. + * @null_bitmap: (nullable): The bitmap that shows null elements. The + * N-th element is null when the N-th bit is 0, not null otherwise. + * If the array has no null elements, the bitmap must be %NULL and + * @n_nulls is 0. + * @n_nulls: The number of null elements. If -1 is specified, the + * number of nulls are computed from @null_bitmap. + * + * Returns: A newly created #GArrowStructArray. + * + * Since: 0.4.0 + */ +GArrowStructArray * +garrow_struct_array_new(GArrowDataType *data_type, + gint64 length, + GList *children, + GArrowBuffer *null_bitmap, + gint64 n_nulls) +{ + const auto arrow_data_type = garrow_data_type_get_raw(data_type); + std::vector> arrow_children; + for (GList *node = children; node; node = node->next) { + GArrowArray *child = GARROW_ARRAY(node->data); + arrow_children.push_back(garrow_array_get_raw(child)); + } + const auto arrow_bitmap = garrow_buffer_get_raw(null_bitmap); + auto arrow_struct_array = + std::make_shared(arrow_data_type, + length, + arrow_children, + arrow_bitmap, + n_nulls); + auto arrow_array = + std::static_pointer_cast(arrow_struct_array); + return GARROW_STRUCT_ARRAY(garrow_array_new_raw(&arrow_array)); +} + +/** + * garrow_struct_array_get_field + * @array: A #GArrowStructArray. + * @i: The index of the field in the struct. + * + * Returns: (transfer full): The i-th field. + */ +GArrowArray * +garrow_struct_array_get_field(GArrowStructArray *array, + gint i) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_struct_array = + static_cast(arrow_array.get()); + auto arrow_field = arrow_struct_array->field(i); + return garrow_array_new_raw(&arrow_field); +} + +/** + * garrow_struct_array_get_fields + * @array: A #GArrowStructArray. + * + * Returns: (element-type GArrowArray) (transfer full): + * The fields in the struct. + */ +GList * +garrow_struct_array_get_fields(GArrowStructArray *array) +{ + const auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + const auto arrow_struct_array = + static_cast(arrow_array.get()); + + GList *fields = NULL; + for (int i = 0; i < arrow_struct_array->num_fields(); ++i) { + auto arrow_field = arrow_struct_array->field(i); + GArrowArray *field = garrow_array_new_raw(&arrow_field); + fields = g_list_prepend(fields, field); + } + + return g_list_reverse(fields); +} + +G_END_DECLS diff --git a/c_glib/arrow-glib/composite-array.h b/c_glib/arrow-glib/composite-array.h new file mode 100644 index 0000000000000..ebf955498724b --- /dev/null +++ b/c_glib/arrow-glib/composite-array.h @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +G_BEGIN_DECLS + +#define GARROW_TYPE_LIST_ARRAY \ + (garrow_list_array_get_type()) +#define GARROW_LIST_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_LIST_ARRAY, \ + GArrowListArray)) +#define GARROW_LIST_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_LIST_ARRAY, \ + GArrowListArrayClass)) +#define GARROW_IS_LIST_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_LIST_ARRAY)) +#define GARROW_IS_LIST_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_LIST_ARRAY)) +#define GARROW_LIST_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_LIST_ARRAY, \ + GArrowListArrayClass)) + +typedef struct _GArrowListArray GArrowListArray; +typedef struct _GArrowListArrayClass GArrowListArrayClass; + +/** + * GArrowListArray: + * + * It wraps `arrow::ListArray`. + */ +struct _GArrowListArray +{ + /*< private >*/ + GArrowArray parent_instance; +}; + +struct _GArrowListArrayClass +{ + GArrowArrayClass parent_class; +}; + +GType garrow_list_array_get_type(void) G_GNUC_CONST; + +GArrowListArray *garrow_list_array_new(gint64 length, + GArrowBuffer *value_offsets, + GArrowArray *values, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +GArrowDataType *garrow_list_array_get_value_type(GArrowListArray *array); +GArrowArray *garrow_list_array_get_value(GArrowListArray *array, + gint64 i); + + +#define GARROW_TYPE_STRUCT_ARRAY \ + (garrow_struct_array_get_type()) +#define GARROW_STRUCT_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), \ + GARROW_TYPE_STRUCT_ARRAY, \ + GArrowStructArray)) +#define GARROW_STRUCT_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), \ + GARROW_TYPE_STRUCT_ARRAY, \ + GArrowStructArrayClass)) +#define GARROW_IS_STRUCT_ARRAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ + GARROW_TYPE_STRUCT_ARRAY)) +#define GARROW_IS_STRUCT_ARRAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), \ + GARROW_TYPE_STRUCT_ARRAY)) +#define GARROW_STRUCT_ARRAY_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), \ + GARROW_TYPE_STRUCT_ARRAY, \ + GArrowStructArrayClass)) + +typedef struct _GArrowStructArray GArrowStructArray; +typedef struct _GArrowStructArrayClass GArrowStructArrayClass; + +/** + * GArrowStructArray: + * + * It wraps `arrow::StructArray`. + */ +struct _GArrowStructArray +{ + /*< private >*/ + GArrowArray parent_instance; +}; + +struct _GArrowStructArrayClass +{ + GArrowArrayClass parent_class; +}; + +GType garrow_struct_array_get_type(void) G_GNUC_CONST; + +GArrowStructArray *garrow_struct_array_new(GArrowDataType *data_type, + gint64 length, + GList *children, + GArrowBuffer *null_bitmap, + gint64 n_nulls); + +GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array, + gint i); +GList *garrow_struct_array_get_fields(GArrowStructArray *array); + +G_END_DECLS diff --git a/c_glib/arrow-glib/composite-data-type.cpp b/c_glib/arrow-glib/composite-data-type.cpp index ce3d78ca08332..7ce8a978e2036 100644 --- a/c_glib/arrow-glib/composite-data-type.cpp +++ b/c_glib/arrow-glib/composite-data-type.cpp @@ -21,6 +21,7 @@ # include #endif +#include #include #include #include @@ -38,6 +39,8 @@ G_BEGIN_DECLS * #GArrowListDataType is a class for list data type. * * #GArrowStructDataType is a class for struct data type. + * + * #GArrowDictionaryDataType is a class for dictionary data type. */ G_DEFINE_TYPE(GArrowListDataType, \ @@ -133,4 +136,87 @@ garrow_struct_data_type_new(GList *fields) return data_type; } + +G_DEFINE_TYPE(GArrowDictionaryDataType, \ + garrow_dictionary_data_type, \ + GARROW_TYPE_FIXED_WIDTH_DATA_TYPE) + +static void +garrow_dictionary_data_type_init(GArrowDictionaryDataType *object) +{ +} + +static void +garrow_dictionary_data_type_class_init(GArrowDictionaryDataTypeClass *klass) +{ +} + +/** + * garrow_dictionary_data_type_new: + * @index_data_type: The data type of index. + * @dictionary: The dictionary. + * @ordered: Whether dictionary contents are ordered or not. + * + * Returns: The newly created dictionary data type. + */ +GArrowDictionaryDataType * +garrow_dictionary_data_type_new(GArrowDataType *index_data_type, + GArrowArray *dictionary, + gboolean ordered) +{ + auto arrow_index_data_type = garrow_data_type_get_raw(index_data_type); + auto arrow_dictionary = garrow_array_get_raw(dictionary); + auto arrow_data_type = arrow::dictionary(arrow_index_data_type, + arrow_dictionary, + ordered); + return GARROW_DICTIONARY_DATA_TYPE(garrow_data_type_new_raw(&arrow_data_type)); +} + +/** + * garrow_dictionary_data_type_get_index_data_type: + * @data_type: The #GArrowDictionaryDataType. + * + * Returns: (transfer full): The #GArrowDataType of index. + */ +GArrowDataType * +garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type) +{ + auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + auto arrow_dictionary_data_type = + std::static_pointer_cast(arrow_data_type); + auto arrow_index_data_type = arrow_dictionary_data_type->index_type(); + return garrow_data_type_new_raw(&arrow_index_data_type); +} + +/** + * garrow_dictionary_data_type_get_dictionary: + * @data_type: The #GArrowDictionaryDataType. + * + * Returns: (transfer full): The dictionary as #GArrowArray. + */ +GArrowArray * +garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type) +{ + auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + auto arrow_dictionary_data_type = + std::static_pointer_cast(arrow_data_type); + auto arrow_dictionary = arrow_dictionary_data_type->dictionary(); + return garrow_array_new_raw(&arrow_dictionary); +} + +/** + * garrow_dictionary_data_type_is_ordered: + * @data_type: The #GArrowDictionaryDataType. + * + * Returns: Whether dictionary contents are ordered or not. + */ +gboolean +garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type) +{ + auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + auto arrow_dictionary_data_type = + std::static_pointer_cast(arrow_data_type); + return arrow_dictionary_data_type->ordered(); +} + G_END_DECLS diff --git a/c_glib/arrow-glib/composite-data-type.h b/c_glib/arrow-glib/composite-data-type.h index 9dac5bd84bed2..11b83014ff830 100644 --- a/c_glib/arrow-glib/composite-data-type.h +++ b/c_glib/arrow-glib/composite-data-type.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include @@ -112,4 +113,28 @@ struct _GArrowStructDataTypeClass GType garrow_struct_data_type_get_type (void) G_GNUC_CONST; GArrowStructDataType *garrow_struct_data_type_new (GList *fields); + +#define GARROW_TYPE_DICTIONARY_DATA_TYPE (garrow_dictionary_data_type_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowDictionaryDataType, + garrow_dictionary_data_type, + GARROW, + DICTIONARY_DATA_TYPE, + GArrowFixedWidthDataType) +struct _GArrowDictionaryDataTypeClass +{ + GArrowFixedWidthDataTypeClass parent_class; +}; + +GArrowDictionaryDataType * +garrow_dictionary_data_type_new(GArrowDataType *index_data_type, + GArrowArray *dictionary, + gboolean ordered); +GArrowDataType * +garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type); +GArrowArray * +garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type); +gboolean +garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type); + + G_END_DECLS diff --git a/c_glib/arrow-glib/meson.build b/c_glib/arrow-glib/meson.build index aeec4172d6284..25968e69c1047 100644 --- a/c_glib/arrow-glib/meson.build +++ b/c_glib/arrow-glib/meson.build @@ -18,12 +18,13 @@ # under the License. sources = files( - 'array.cpp', 'array-builder.cpp', + 'basic-array.cpp', 'basic-data-type.cpp', 'buffer.cpp', 'chunked-array.cpp', 'column.cpp', + 'composite-array.cpp', 'composite-data-type.cpp', 'error.cpp', 'field.cpp', @@ -59,10 +60,12 @@ c_headers = files( 'array.h', 'array-builder.h', 'arrow-glib.h', + 'basic-array.h', 'basic-data-type.h', 'buffer.h', 'chunked-array.h', 'column.h', + 'composite-array.h', 'composite-data-type.h', 'data-type.h', 'error.h', @@ -102,6 +105,7 @@ cpp_headers = files( 'array.hpp', 'array-builder.hpp', 'arrow-glib.hpp', + 'basic-array.hpp', 'basic-data-type.hpp', 'buffer.hpp', 'chunked-array.hpp', diff --git a/c_glib/doc/reference/arrow-glib-docs.xml b/c_glib/doc/reference/arrow-glib-docs.xml index e267ea2f9d356..51e7b2a6a6cf5 100644 --- a/c_glib/doc/reference/arrow-glib-docs.xml +++ b/c_glib/doc/reference/arrow-glib-docs.xml @@ -40,7 +40,8 @@ Data Array - + + Array builder diff --git a/c_glib/test/test-dictionary-data-type.rb b/c_glib/test/test-dictionary-data-type.rb new file mode 100644 index 0000000000000..5530a0415cb28 --- /dev/null +++ b/c_glib/test/test-dictionary-data-type.rb @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDictionaryDataType < Test::Unit::TestCase + include Helper::Buildable + + def setup + @index_data_type = Arrow::Int32DataType.new + @dictionary = build_string_array(["C", "C++", "Ruby"]) + @ordered = true + @data_type = Arrow::DictionaryDataType.new(@index_data_type, + @dictionary, + @ordered) + end + + def test_type + assert_equal(Arrow::Type::DICTIONARY, @data_type.id) + end + + def test_to_s + assert_equal("dictionary", + @data_type.to_s) + end + + def test_bit_width + assert_equal(32, @data_type.bit_width) + end + + def test_index_data_type + assert_equal(@index_data_type, @data_type.index_data_type) + end + + def test_dictionary + assert_equal(@dictionary, @data_type.dictionary) + end + + def test_ordered? + assert do + @data_type.ordered? + end + end +end From 85e2d8960d5aeeb04d8b59e6e7d8a8266a7d095f Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sun, 26 Nov 2017 14:12:22 -0800 Subject: [PATCH 086/177] ARROW-1758: [Python] Remove pickle=True option for object serialization This closes [ARROW-1758](https://issues.apache.org/jira/browse/ARROW-1758). Author: Licht-T Author: Wes McKinney Closes #1347 from Licht-T/clean-pickle-option-for-object-serialization and squashes the following commits: 927f154 [Wes McKinney] Use cloudpickle for lambda serialization if available ba998dd [Licht-T] CLN: Remove pickle=True option for object serialization --- python/pyarrow/serialization.pxi | 14 ++------------ python/pyarrow/serialization.py | 13 +++++++++++-- python/pyarrow/tests/test_serialization.py | 9 ++++++--- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index 6b7227797a836..3ee5c7d4e2770 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -47,7 +47,6 @@ cdef class SerializationContext: cdef: object type_to_type_id object whitelisted_types - object types_to_pickle object custom_serializers object custom_deserializers @@ -55,11 +54,10 @@ cdef class SerializationContext: # Types with special serialization handlers self.type_to_type_id = dict() self.whitelisted_types = dict() - self.types_to_pickle = set() self.custom_serializers = dict() self.custom_deserializers = dict() - def register_type(self, type_, type_id, pickle=False, + def register_type(self, type_, type_id, custom_serializer=None, custom_deserializer=None): """EXPERIMENTAL: Add type to the list of types we can serialize. @@ -69,9 +67,6 @@ cdef class SerializationContext: The type that we can serialize. type_id : bytes A string of bytes used to identify the type. - pickle : bool - True if the serialization should be done with pickle. - False if it should be done efficiently with Arrow. custom_serializer : callable This argument is optional, but can be provided to serialize objects of the class in a particular way. @@ -81,8 +76,6 @@ cdef class SerializationContext: """ self.type_to_type_id[type_] = type_id self.whitelisted_types[type_id] = type_ - if pickle: - self.types_to_pickle.add(type_id) if custom_serializer is not None: self.custom_serializers[type_id] = custom_serializer self.custom_deserializers[type_id] = custom_deserializer @@ -102,9 +95,7 @@ cdef class SerializationContext: # use the closest match to type(obj) type_id = self.type_to_type_id[type_] - if type_id in self.types_to_pickle: - serialized_obj = {"data": pickle.dumps(obj), "pickle": True} - elif type_id in self.custom_serializers: + if type_id in self.custom_serializers: serialized_obj = {"data": self.custom_serializers[type_id](obj)} else: if is_named_tuple(type_): @@ -125,7 +116,6 @@ cdef class SerializationContext: # The object was pickled, so unpickle it. obj = pickle.loads(serialized_obj["data"]) else: - assert type_id not in self.types_to_pickle if type_id not in self.whitelisted_types: msg = "Type ID " + str(type_id) + " not registered in " \ "deserialization callback" diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py index 2b47513fd1c85..ab25b63d5710b 100644 --- a/python/pyarrow/serialization.py +++ b/python/pyarrow/serialization.py @@ -17,12 +17,18 @@ from collections import OrderedDict, defaultdict import sys +import pickle import numpy as np from pyarrow import serialize_pandas, deserialize_pandas from pyarrow.lib import _default_serialization_context +try: + import cloudpickle +except ImportError: + cloudpickle = pickle + def register_default_serialization_handlers(serialization_context): @@ -67,9 +73,12 @@ def _deserialize_default_dict(data): serialization_context.register_type( type(lambda: 0), "function", - pickle=True) + custom_serializer=cloudpickle.dumps, + custom_deserializer=cloudpickle.loads) - serialization_context.register_type(type, "type", pickle=True) + serialization_context.register_type(type, "type", + custom_serializer=cloudpickle.dumps, + custom_deserializer=cloudpickle.loads) # ---------------------------------------------------------------------- # Set up serialization for numpy with dtype object (primitive types are diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index b0c5bc49e6a58..ed4fd9ae59aee 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -23,6 +23,7 @@ import datetime import string import sys +import pickle import pyarrow as pa import numpy as np @@ -197,7 +198,9 @@ def make_serialization_context(): context.register_type(Baz, "Baz") context.register_type(Qux, "Quz") context.register_type(SubQux, "SubQux") - context.register_type(SubQuxPickle, "SubQuxPickle", pickle=True) + context.register_type(SubQuxPickle, "SubQuxPickle", + custom_serializer=pickle.dumps, + custom_deserializer=pickle.loads) context.register_type(Exception, "Exception") context.register_type(CustomError, "CustomError") context.register_type(Point, "Point") @@ -338,7 +341,7 @@ def deserialize_dummy_class(serialized_obj): return serialized_obj pa._default_serialization_context.register_type( - DummyClass, "DummyClass", pickle=False, + DummyClass, "DummyClass", custom_serializer=serialize_dummy_class, custom_deserializer=deserialize_dummy_class) @@ -357,7 +360,7 @@ def deserialize_buffer_class(serialized_obj): return serialized_obj pa._default_serialization_context.register_type( - BufferClass, "BufferClass", pickle=False, + BufferClass, "BufferClass", custom_serializer=serialize_buffer_class, custom_deserializer=deserialize_buffer_class) From 42fc57be9c768add32b278adbee2f6f30b10005d Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 27 Nov 2017 09:42:50 +0100 Subject: [PATCH 087/177] ARROW-1178: [C++/Python] Add option to set chunksize in TableBatchReader, Table.to_batches method This also fixes ARROW-504 by adding a chunksize option when writing tables to a RecordBatch stream in Python Author: Wes McKinney Closes #1364 from wesm/ARROW-1178 and squashes the following commits: a31e258 [Wes McKinney] Add chunksize argument to RecordBatchWriter.write_table dc6023a [Wes McKinney] Implement Table.to_batches, add tests --- cpp/src/arrow/ipc/writer.cc | 8 +++++- cpp/src/arrow/ipc/writer.h | 6 +++++ cpp/src/arrow/table-test.cc | 33 ++++++++++++++++++++++++ cpp/src/arrow/table.cc | 21 ++++++++++----- cpp/src/arrow/table.h | 2 ++ python/pyarrow/includes/libarrow.pxd | 9 ++++++- python/pyarrow/ipc.pxi | 12 +++++++-- python/pyarrow/table.pxi | 38 ++++++++++++++++++++++++++++ python/pyarrow/tests/test_ipc.py | 23 +++++++++++++++++ python/pyarrow/tests/test_table.py | 25 ++++++++++++++++++ 10 files changed, 167 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index 3c1db06159e65..fb766a9a7599c 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -651,9 +651,13 @@ Status GetTensorSize(const Tensor& tensor, int64_t* size) { RecordBatchWriter::~RecordBatchWriter() {} -Status RecordBatchWriter::WriteTable(const Table& table) { +Status RecordBatchWriter::WriteTable(const Table& table, int64_t max_chunksize) { TableBatchReader reader(table); + if (max_chunksize > 0) { + reader.set_chunksize(max_chunksize); + } + std::shared_ptr batch; while (true) { RETURN_NOT_OK(reader.ReadNext(&batch)); @@ -666,6 +670,8 @@ Status RecordBatchWriter::WriteTable(const Table& table) { return Status::OK(); } +Status RecordBatchWriter::WriteTable(const Table& table) { return WriteTable(table, -1); } + // ---------------------------------------------------------------------- // Stream writer implementation diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h index cedac45e712d6..457dcb4ec6a13 100644 --- a/cpp/src/arrow/ipc/writer.h +++ b/cpp/src/arrow/ipc/writer.h @@ -65,6 +65,12 @@ class ARROW_EXPORT RecordBatchWriter { /// \return Status Status WriteTable(const Table& table); + /// \brief Write Table with a particular chunksize + /// \param[in] table table to write + /// \param[in] max_chunksize maximum chunk size for table chunks + /// \return Status + Status WriteTable(const Table& table, int64_t max_chunksize); + /// \brief Perform any logic necessary to finish the stream /// /// \return Status diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc index e77d3aa8bbc08..8a2288710c896 100644 --- a/cpp/src/arrow/table-test.cc +++ b/cpp/src/arrow/table-test.cc @@ -586,4 +586,37 @@ TEST_F(TestTableBatchReader, ReadNext) { ASSERT_EQ(nullptr, batch); } +TEST_F(TestTableBatchReader, Chunksize) { + auto a1 = MakeRandomArray(10); + auto a2 = MakeRandomArray(20); + auto a3 = MakeRandomArray(10); + + auto sch1 = arrow::schema({field("f1", int32())}); + auto t1 = Table::Make(sch1, {column(sch1->field(0), {a1, a2, a3})}); + + TableBatchReader i1(*t1); + + i1.set_chunksize(15); + + std::shared_ptr batch; + ASSERT_OK(i1.ReadNext(&batch)); + ASSERT_OK(batch->Validate()); + ASSERT_EQ(10, batch->num_rows()); + + ASSERT_OK(i1.ReadNext(&batch)); + ASSERT_OK(batch->Validate()); + ASSERT_EQ(15, batch->num_rows()); + + ASSERT_OK(i1.ReadNext(&batch)); + ASSERT_OK(batch->Validate()); + ASSERT_EQ(5, batch->num_rows()); + + ASSERT_OK(i1.ReadNext(&batch)); + ASSERT_OK(batch->Validate()); + ASSERT_EQ(10, batch->num_rows()); + + ASSERT_OK(i1.ReadNext(&batch)); + ASSERT_EQ(nullptr, batch); +} + } // namespace arrow diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc index 8f3f195765aa9..129524b7e4320 100644 --- a/cpp/src/arrow/table.cc +++ b/cpp/src/arrow/table.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -403,7 +404,8 @@ class TableBatchReader::TableBatchReaderImpl { column_data_(table.num_columns()), chunk_numbers_(table.num_columns(), 0), chunk_offsets_(table.num_columns(), 0), - absolute_row_position_(0) { + absolute_row_position_(0), + max_chunksize_(std::numeric_limits::max()) { for (int i = 0; i < table.num_columns(); ++i) { column_data_[i] = table.column(i)->data().get(); } @@ -416,7 +418,7 @@ class TableBatchReader::TableBatchReaderImpl { } // Determine the minimum contiguous slice across all columns - int64_t chunksize = table_.num_rows(); + int64_t chunksize = std::min(table_.num_rows(), max_chunksize_); std::vector chunks(table_.num_columns()); for (int i = 0; i < table_.num_columns(); ++i) { auto chunk = column_data_[i]->chunk(chunk_numbers_[i]).get(); @@ -430,8 +432,7 @@ class TableBatchReader::TableBatchReaderImpl { } // Slice chunks and advance chunk index as appropriate - std::vector> batch_data; - batch_data.reserve(table_.num_columns()); + std::vector> batch_data(table_.num_columns()); for (int i = 0; i < table_.num_columns(); ++i) { // Exhausted chunk @@ -441,7 +442,7 @@ class TableBatchReader::TableBatchReaderImpl { if ((chunk->length() - offset) == chunksize) { ++chunk_numbers_[i]; chunk_offsets_[i] = 0; - if (chunk_offsets_[i] > 0) { + if (offset > 0) { // Need to slice slice_data = chunk->Slice(offset, chunksize)->data(); } else { @@ -449,9 +450,10 @@ class TableBatchReader::TableBatchReaderImpl { slice_data = chunk->data(); } } else { + chunk_offsets_[i] += chunksize; slice_data = chunk->Slice(offset, chunksize)->data(); } - batch_data.emplace_back(std::move(slice_data)); + batch_data[i] = std::move(slice_data); } absolute_row_position_ += chunksize; @@ -462,12 +464,15 @@ class TableBatchReader::TableBatchReaderImpl { std::shared_ptr schema() const { return table_.schema(); } + void set_chunksize(int64_t chunksize) { max_chunksize_ = chunksize; } + private: const Table& table_; std::vector column_data_; std::vector chunk_numbers_; std::vector chunk_offsets_; int64_t absolute_row_position_; + int64_t max_chunksize_; }; TableBatchReader::TableBatchReader(const Table& table) { @@ -478,6 +483,10 @@ TableBatchReader::~TableBatchReader() {} std::shared_ptr TableBatchReader::schema() const { return impl_->schema(); } +void TableBatchReader::set_chunksize(int64_t chunksize) { + impl_->set_chunksize(chunksize); +} + Status TableBatchReader::ReadNext(std::shared_ptr* out) { return impl_->ReadNext(out); } diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h index d0312d93cb93c..c813b32ad36dc 100644 --- a/cpp/src/arrow/table.h +++ b/cpp/src/arrow/table.h @@ -197,6 +197,8 @@ class ARROW_EXPORT TableBatchReader : public RecordBatchReader { Status ReadNext(std::shared_ptr* out) override; + void set_chunksize(int64_t chunksize); + private: class TableBatchReaderImpl; std::unique_ptr impl_; diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 5d68607efa386..14211787c89f2 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -456,6 +456,13 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CTable] ReplaceSchemaMetadata( const shared_ptr[CKeyValueMetadata]& metadata) + cdef cppclass RecordBatchReader: + CStatus ReadNext(shared_ptr[CRecordBatch]* out) + + cdef cppclass TableBatchReader(RecordBatchReader): + TableBatchReader(const CTable& table) + void set_chunksize(int64_t chunksize) + cdef cppclass CTensor" arrow::Tensor": shared_ptr[CDataType] type() shared_ptr[CBuffer] data() @@ -692,7 +699,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: CStatus Close() CStatus WriteRecordBatch(const CRecordBatch& batch, c_bool allow_64bit) - CStatus WriteTable(const CTable& table) + CStatus WriteTable(const CTable& table, int64_t max_chunksize) cdef cppclass CRecordBatchReader" arrow::ipc::RecordBatchReader": shared_ptr[CSchema] schema() diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi index 27e9167750991..e5639137dd3a1 100644 --- a/python/pyarrow/ipc.pxi +++ b/python/pyarrow/ipc.pxi @@ -202,7 +202,7 @@ cdef class _RecordBatchWriter: check_status(self.writer.get() .WriteRecordBatch(deref(batch.batch), 1)) - def write_table(self, Table table): + def write_table(self, Table table, chunksize=None): """ Write RecordBatch to stream @@ -210,8 +210,16 @@ cdef class _RecordBatchWriter: ---------- batch : RecordBatch """ + cdef: + # Chunksize must be > 0 to have any impact + int64_t c_chunksize = -1 + + if chunksize is not None: + c_chunksize = chunksize + with nogil: - check_status(self.writer.get().WriteTable(table.table[0])) + check_status(self.writer.get().WriteTable(table.table[0], + c_chunksize)) def close(self): """ diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 8c5b8bbc343c0..b03ee26702245 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -971,6 +971,44 @@ cdef class Table: return pyarrow_wrap_table(c_table) + def to_batches(self, chunksize=None): + """ + Convert Table to list of (contiguous) RecordBatch objects, with optimal + maximum chunk size + + Parameters + ---------- + chunksize : int, default None + Maximum size for RecordBatch chunks. Individual chunks may be + smaller depending on the chunk layout of individual columns + + Returns + ------- + batches : list of RecordBatch + """ + cdef: + unique_ptr[TableBatchReader] reader + int64_t c_chunksize + list result = [] + shared_ptr[CRecordBatch] batch + + reader.reset(new TableBatchReader(deref(self.table))) + + if chunksize is not None: + c_chunksize = chunksize + reader.get().set_chunksize(c_chunksize) + + while True: + with nogil: + check_status(reader.get().ReadNext(&batch)) + + if batch.get() == NULL: + break + + result.append(pyarrow_wrap_batch(batch)) + + return result + def to_pandas(self, nthreads=None, strings_to_categorical=False, memory_pool=None, zero_copy_only=False): """ diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py index 5033ea95783ab..9cd5f807662d6 100644 --- a/python/pyarrow/tests/test_ipc.py +++ b/python/pyarrow/tests/test_ipc.py @@ -168,6 +168,29 @@ def test_stream_write_dispatch(self): assert_frame_equal(table.to_pandas(), pd.concat([df, df], ignore_index=True)) + def test_stream_write_table_batches(self): + # ARROW-504 + df = pd.DataFrame({ + 'one': np.random.randn(20), + }) + + b1 = pa.RecordBatch.from_pandas(df[:10], preserve_index=False) + b2 = pa.RecordBatch.from_pandas(df, preserve_index=False) + + table = pa.Table.from_batches([b1, b2, b1]) + + writer = self._get_writer(self.sink, table.schema) + writer.write_table(table, chunksize=15) + writer.close() + + batches = list(pa.open_stream(pa.BufferReader(self._get_source()))) + + assert list(map(len, batches)) == [10, 15, 5, 10] + result_table = pa.Table.from_batches(batches) + assert_frame_equal(result_table.to_pandas(), + pd.concat([df[:10], df, df[:10]], + ignore_index=True)) + def test_simple_roundtrip(self): _, batches = self.write_batches() file_contents = pa.BufferReader(self._get_source()) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index cd05fb8e1fce5..ab012340c0ab9 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -213,6 +213,31 @@ def test_recordbatchlist_schema_equals(): pa.Table.from_batches([batch1, batch2]) +def test_table_to_batches(): + df1 = pd.DataFrame({'a': list(range(10))}) + df2 = pd.DataFrame({'a': list(range(10, 30))}) + + batch1 = pa.RecordBatch.from_pandas(df1, preserve_index=False) + batch2 = pa.RecordBatch.from_pandas(df2, preserve_index=False) + + table = pa.Table.from_batches([batch1, batch2, batch1]) + + expected_df = pd.concat([df1, df2, df1], ignore_index=True) + + batches = table.to_batches() + assert len(batches) == 3 + + assert_frame_equal(pa.Table.from_batches(batches).to_pandas(), + expected_df) + + batches = table.to_batches(chunksize=15) + assert list(map(len, batches)) == [10, 15, 5, 10] + + assert_frame_equal(table.to_pandas(), expected_df) + assert_frame_equal(pa.Table.from_batches(batches).to_pandas(), + expected_df) + + def test_table_basics(): data = [ pa.array(range(5)), From 61763504e53fb5531f1d642c399b8ec7b094b337 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 27 Nov 2017 10:36:33 -0500 Subject: [PATCH 088/177] [Release] Apache Arrow JavaScript 0.2.0 Change-Id: I1db065001e7fc196128e8f8c36b3406a89ccbdd5 --- js/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/package.json b/js/package.json index 9933f75a2ce57..8e5c42ef7d1e3 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "apache-arrow", - "version": "0.1.2", + "version": "0.2.0", "types": "./types/Arrow.d.ts", "typings": "./types/Arrow.d.ts", "main": "./targets/es5/cjs/Arrow.js", From 682e248a7fd78d9e6ad8ec41dd4f518951d77152 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 27 Nov 2017 11:45:54 -0500 Subject: [PATCH 089/177] ARROW-1850: [C++] Use void* / const void* for buffers in file APIs This makes for a more convenient / less rigid API without as need for as many usages of `reinterpret_cast`. This does not impact downstream projects (e.g. parquet-cpp is unaffected) unless they provide implementations of these virtual interfaces. Author: Wes McKinney Closes #1363 from wesm/ARROW-1850 and squashes the following commits: af5a3488 [Wes McKinney] Update glib, arrow-gpu for API changes 5d5cf2d0 [Wes McKinney] Use void* / const void* for buffers in file APIs --- c_glib/arrow-glib/input-stream.cpp | 4 ++-- c_glib/arrow-glib/output-stream.cpp | 4 ++-- c_glib/arrow-glib/writeable.cpp | 2 +- c_glib/arrow-glib/writeable.hpp | 6 ++--- cpp/src/arrow/gpu/CMakeLists.txt | 2 +- cpp/src/arrow/gpu/cuda_context.cc | 23 +++++++++---------- cpp/src/arrow/gpu/cuda_context.h | 10 ++++----- cpp/src/arrow/gpu/cuda_memory.cc | 14 ++++++------ cpp/src/arrow/gpu/cuda_memory.h | 10 ++++----- cpp/src/arrow/io/file.cc | 26 +++++++++++----------- cpp/src/arrow/io/file.h | 16 ++++++------- cpp/src/arrow/io/hdfs.cc | 17 +++++++------- cpp/src/arrow/io/hdfs.h | 8 +++---- cpp/src/arrow/io/interfaces.cc | 9 ++++---- cpp/src/arrow/io/interfaces.h | 14 ++++++------ cpp/src/arrow/io/io-file-test.cc | 11 +++++---- cpp/src/arrow/io/io-memory-test.cc | 2 +- cpp/src/arrow/io/memory.cc | 19 ++++++++-------- cpp/src/arrow/io/memory.h | 12 +++++----- cpp/src/arrow/ipc/feather.cc | 6 ++--- cpp/src/arrow/ipc/json-integration-test.cc | 6 ++--- cpp/src/arrow/ipc/metadata-internal.cc | 3 +-- cpp/src/arrow/ipc/writer.cc | 13 +++++------ cpp/src/arrow/python/io.cc | 8 +++---- cpp/src/arrow/python/io.h | 6 ++--- cpp/src/arrow/util/io-util.h | 4 ++-- 26 files changed, 122 insertions(+), 133 deletions(-) diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp index a7a894b9d4f3e..94422241b9cfa 100644 --- a/c_glib/arrow-glib/input-stream.cpp +++ b/c_glib/arrow-glib/input-stream.cpp @@ -420,7 +420,7 @@ namespace garrow { arrow::Status Read(int64_t n_bytes, int64_t *n_read_bytes, - uint8_t *out) override { + void *out) override { GError *error = NULL; *n_read_bytes = g_input_stream_read(input_stream_, out, @@ -437,7 +437,7 @@ namespace garrow { } arrow::Status ReadAt(int64_t position, int64_t n_bytes, - int64_t *n_read_bytes, uint8_t* out) override { + int64_t *n_read_bytes, void* out) override { return arrow::io::RandomAccessFile::ReadAt( position, n_bytes, n_read_bytes, out); } diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp index 739992fb62b0e..9939f4f086f8e 100644 --- a/c_glib/arrow-glib/output-stream.cpp +++ b/c_glib/arrow-glib/output-stream.cpp @@ -76,7 +76,7 @@ garrow_output_stream_file_interface_init(GArrowFileInterface *iface) iface->get_raw = garrow_output_stream_get_raw_file_interface; } -static std::shared_ptr +static std::shared_ptr garrow_output_stream_get_raw_writeable_interface(GArrowWriteable *writeable) { auto output_stream = GARROW_OUTPUT_STREAM(writeable); @@ -325,7 +325,7 @@ namespace garrow { return arrow::Status::OK(); } - arrow::Status Write(const uint8_t *data, + arrow::Status Write(const void *data, int64_t n_bytes) override { GError *error = NULL; gsize n_written_bytes; diff --git a/c_glib/arrow-glib/writeable.cpp b/c_glib/arrow-glib/writeable.cpp index eb6adfee8c985..a16e43ab17ae9 100644 --- a/c_glib/arrow-glib/writeable.cpp +++ b/c_glib/arrow-glib/writeable.cpp @@ -88,7 +88,7 @@ garrow_writeable_flush(GArrowWriteable *writeable, G_END_DECLS -std::shared_ptr +std::shared_ptr garrow_writeable_get_raw(GArrowWriteable *writeable) { auto *iface = GARROW_WRITEABLE_GET_IFACE(writeable); diff --git a/c_glib/arrow-glib/writeable.hpp b/c_glib/arrow-glib/writeable.hpp index 2b398f8b507c1..806d36fc07957 100644 --- a/c_glib/arrow-glib/writeable.hpp +++ b/c_glib/arrow-glib/writeable.hpp @@ -26,13 +26,13 @@ /** * GArrowWriteableInterface: * - * It wraps `arrow::io::Writeable`. + * It wraps `arrow::io::Writable`. */ struct _GArrowWriteableInterface { GTypeInterface parent_iface; - std::shared_ptr (*get_raw)(GArrowWriteable *file); + std::shared_ptr (*get_raw)(GArrowWriteable *file); }; -std::shared_ptr garrow_writeable_get_raw(GArrowWriteable *writeable); +std::shared_ptr garrow_writeable_get_raw(GArrowWriteable *writeable); diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt index 3f3069b91974b..3ddf2c7974f0e 100644 --- a/cpp/src/arrow/gpu/CMakeLists.txt +++ b/cpp/src/arrow/gpu/CMakeLists.txt @@ -54,7 +54,7 @@ configure_file(cuda_version.h.in @ONLY) install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/cuda_version.h" + "${CMAKE_CURRENT_BINARY_DIR}/cuda_version.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/gpu") install(FILES diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc index fff8ece6c1cff..2f5ccb0a95063 100644 --- a/cpp/src/arrow/gpu/cuda_context.cc +++ b/cpp/src/arrow/gpu/cuda_context.cc @@ -69,28 +69,27 @@ class CudaContext::CudaContextImpl { return Status::OK(); } - Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) { + Status CopyHostToDevice(void* dst, const void* src, int64_t nbytes) { CU_RETURN_NOT_OK(cuCtxSetCurrent(context_)); - CU_RETURN_NOT_OK(cuMemcpyHtoD(reinterpret_cast(dst), - reinterpret_cast(src), + CU_RETURN_NOT_OK(cuMemcpyHtoD(reinterpret_cast(dst), src, static_cast(nbytes))); return Status::OK(); } - Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) { + Status CopyDeviceToHost(void* dst, const void* src, int64_t nbytes) { CU_RETURN_NOT_OK(cuCtxSetCurrent(context_)); CU_RETURN_NOT_OK(cuMemcpyDtoH(dst, reinterpret_cast(src), static_cast(nbytes))); return Status::OK(); } - Status Free(uint8_t* device_ptr, int64_t nbytes) { + Status Free(void* device_ptr, int64_t nbytes) { CU_RETURN_NOT_OK(cuMemFree(reinterpret_cast(device_ptr))); bytes_allocated_ -= nbytes; return Status::OK(); } - Status ExportIpcBuffer(uint8_t* data, std::unique_ptr* handle) { + Status ExportIpcBuffer(void* data, std::unique_ptr* handle) { CU_RETURN_NOT_OK(cuCtxSetCurrent(context_)); CUipcMemHandle cu_handle; CU_RETURN_NOT_OK(cuIpcGetMemHandle(&cu_handle, reinterpret_cast(data))); @@ -145,7 +144,7 @@ class CudaDeviceManager::CudaDeviceManagerImpl { return Status::OK(); } - Status FreeHost(uint8_t* data, int64_t nbytes) { + Status FreeHost(void* data, int64_t nbytes) { CU_RETURN_NOT_OK(cuMemFreeHost(data)); host_bytes_allocated_ -= nbytes; return Status::OK(); @@ -221,7 +220,7 @@ Status CudaDeviceManager::AllocateHost(int64_t nbytes, return Status::OK(); } -Status CudaDeviceManager::FreeHost(uint8_t* data, int64_t nbytes) { +Status CudaDeviceManager::FreeHost(void* data, int64_t nbytes) { return impl_->FreeHost(data, nbytes); } @@ -241,22 +240,22 @@ Status CudaContext::Allocate(int64_t nbytes, std::shared_ptr* out) { return Status::OK(); } -Status CudaContext::ExportIpcBuffer(uint8_t* data, +Status CudaContext::ExportIpcBuffer(void* data, std::unique_ptr* handle) { return impl_->ExportIpcBuffer(data, handle); } -Status CudaContext::CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes) { +Status CudaContext::CopyHostToDevice(void* dst, const void* src, int64_t nbytes) { return impl_->CopyHostToDevice(dst, src, nbytes); } -Status CudaContext::CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes) { +Status CudaContext::CopyDeviceToHost(void* dst, const void* src, int64_t nbytes) { return impl_->CopyDeviceToHost(dst, src, nbytes); } Status CudaContext::Close() { return impl_->Close(); } -Status CudaContext::Free(uint8_t* device_ptr, int64_t nbytes) { +Status CudaContext::Free(void* device_ptr, int64_t nbytes) { return impl_->Free(device_ptr, nbytes); } diff --git a/cpp/src/arrow/gpu/cuda_context.h b/cpp/src/arrow/gpu/cuda_context.h index 6471059612349..6fc2e0d08abc9 100644 --- a/cpp/src/arrow/gpu/cuda_context.h +++ b/cpp/src/arrow/gpu/cuda_context.h @@ -46,7 +46,7 @@ class ARROW_EXPORT CudaDeviceManager { Status AllocateHost(int64_t nbytes, std::shared_ptr* buffer); - Status FreeHost(uint8_t* data, int64_t nbytes); + Status FreeHost(void* data, int64_t nbytes); int num_devices() const; @@ -88,10 +88,10 @@ class ARROW_EXPORT CudaContext : public std::enable_shared_from_this* handle); - Status CopyHostToDevice(uint8_t* dst, const uint8_t* src, int64_t nbytes); - Status CopyDeviceToHost(uint8_t* dst, const uint8_t* src, int64_t nbytes); - Status Free(uint8_t* device_ptr, int64_t nbytes); + Status ExportIpcBuffer(void* data, std::unique_ptr* handle); + Status CopyHostToDevice(void* dst, const void* src, int64_t nbytes); + Status CopyDeviceToHost(void* dst, const void* src, int64_t nbytes); + Status Free(void* device_ptr, int64_t nbytes); class CudaContextImpl; std::unique_ptr impl_; diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc index 949c1d7a4dd41..cbf044121a644 100644 --- a/cpp/src/arrow/gpu/cuda_memory.cc +++ b/cpp/src/arrow/gpu/cuda_memory.cc @@ -101,11 +101,11 @@ CudaBuffer::CudaBuffer(const std::shared_ptr& parent, const int64_t is_ipc_(false) {} Status CudaBuffer::CopyToHost(const int64_t position, const int64_t nbytes, - uint8_t* out) const { + void* out) const { return context_->CopyDeviceToHost(out, data_ + position, nbytes); } -Status CudaBuffer::CopyFromHost(const int64_t position, const uint8_t* data, +Status CudaBuffer::CopyFromHost(const int64_t position, const void* data, int64_t nbytes) { DCHECK_LE(nbytes, size_ - position) << "Copy would overflow buffer"; return context_->CopyHostToDevice(mutable_data_ + position, data, nbytes); @@ -134,7 +134,7 @@ CudaBufferReader::CudaBufferReader(const std::shared_ptr& buffer) CudaBufferReader::~CudaBufferReader() {} -Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) { +Status CudaBufferReader::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) { nbytes = std::min(nbytes, size_ - position_); *bytes_read = nbytes; RETURN_NOT_OK(context_->CopyDeviceToHost(buffer, data_ + position_, nbytes)); @@ -190,7 +190,7 @@ class CudaBufferWriter::CudaBufferWriterImpl { return Status::OK(); } - Status Write(const uint8_t* data, int64_t nbytes) { + Status Write(const void* data, int64_t nbytes) { if (nbytes == 0) { return Status::OK(); } @@ -214,7 +214,7 @@ class CudaBufferWriter::CudaBufferWriterImpl { return Status::OK(); } - Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) { + Status WriteAt(int64_t position, const void* data, int64_t nbytes) { std::lock_guard guard(lock_); RETURN_NOT_OK(Seek(position)); return Write(data, nbytes); @@ -269,11 +269,11 @@ Status CudaBufferWriter::Seek(int64_t position) { Status CudaBufferWriter::Tell(int64_t* position) const { return impl_->Tell(position); } -Status CudaBufferWriter::Write(const uint8_t* data, int64_t nbytes) { +Status CudaBufferWriter::Write(const void* data, int64_t nbytes) { return impl_->Write(data, nbytes); } -Status CudaBufferWriter::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) { +Status CudaBufferWriter::WriteAt(int64_t position, const void* data, int64_t nbytes) { return impl_->WriteAt(position, data, nbytes); } diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h index 9ebd2ccf0c77e..9376b4b3ffcb9 100644 --- a/cpp/src/arrow/gpu/cuda_memory.h +++ b/cpp/src/arrow/gpu/cuda_memory.h @@ -49,14 +49,14 @@ class ARROW_EXPORT CudaBuffer : public Buffer { /// \brief Copy memory from GPU device to CPU host /// \param[out] out a pre-allocated output buffer /// \return Status - Status CopyToHost(const int64_t position, const int64_t nbytes, uint8_t* out) const; + Status CopyToHost(const int64_t position, const int64_t nbytes, void* out) const; /// \brief Copy memory to device at position /// \param[in] position start position to copy bytes /// \param[in] data the host data to copy /// \param[in] nbytes number of bytes to copy /// \return Status - Status CopyFromHost(const int64_t position, const uint8_t* data, int64_t nbytes); + Status CopyFromHost(const int64_t position, const void* data, int64_t nbytes); /// \brief Expose this device buffer as IPC memory which can be used in other processes /// \param[out] handle the exported IPC handle @@ -130,7 +130,7 @@ class ARROW_EXPORT CudaBufferReader : public io::BufferReader { /// \param[in] nbytes number of bytes to read /// \param[out] bytes_read actual number of bytes read /// \param[out] buffer pre-allocated memory to write into - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override; + Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override; /// \brief Zero-copy read from device memory /// \param[in] nbytes number of bytes to read @@ -158,9 +158,9 @@ class ARROW_EXPORT CudaBufferWriter : public io::WriteableFile { Status Seek(int64_t position) override; - Status Write(const uint8_t* data, int64_t nbytes) override; + Status Write(const void* data, int64_t nbytes) override; - Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override; + Status WriteAt(int64_t position, const void* data, int64_t nbytes) override; Status Tell(int64_t* position) const override; diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc index 1ec5e23e587d5..65a302c8beac2 100644 --- a/cpp/src/arrow/io/file.cc +++ b/cpp/src/arrow/io/file.cc @@ -394,11 +394,11 @@ class OSFile { return Status::OK(); } - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) { - return FileRead(fd_, out, nbytes, bytes_read); + Status Read(int64_t nbytes, int64_t* bytes_read, void* out) { + return FileRead(fd_, reinterpret_cast(out), nbytes, bytes_read); } - Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* out) { + Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) { std::lock_guard guard(lock_); RETURN_NOT_OK(Seek(position)); return Read(nbytes, bytes_read, out); @@ -413,12 +413,12 @@ class OSFile { Status Tell(int64_t* pos) const { return FileTell(fd_, pos); } - Status Write(const uint8_t* data, int64_t length) { + Status Write(const void* data, int64_t length) { std::lock_guard guard(lock_); if (length < 0) { return Status::IOError("Length must be non-negative"); } - return FileWrite(fd_, data, length); + return FileWrite(fd_, reinterpret_cast(data), length); } int fd() const { return fd_; } @@ -504,13 +504,13 @@ Status ReadableFile::Close() { return impl_->Close(); } Status ReadableFile::Tell(int64_t* pos) const { return impl_->Tell(pos); } -Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) { +Status ReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) { std::lock_guard guard(impl_->lock()); return impl_->Read(nbytes, bytes_read, out); } Status ReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) { + void* out) { return impl_->ReadAt(position, nbytes, bytes_read, out); } @@ -570,7 +570,7 @@ Status FileOutputStream::Close() { return impl_->Close(); } Status FileOutputStream::Tell(int64_t* pos) const { return impl_->Tell(pos); } -Status FileOutputStream::Write(const uint8_t* data, int64_t length) { +Status FileOutputStream::Write(const void* data, int64_t length) { return impl_->Write(data, length); } @@ -710,7 +710,7 @@ Status MemoryMappedFile::Close() { return Status::OK(); } -Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) { +Status MemoryMappedFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) { nbytes = std::max( 0, std::min(nbytes, memory_map_->size() - memory_map_->position())); if (nbytes > 0) { @@ -735,7 +735,7 @@ Status MemoryMappedFile::Read(int64_t nbytes, std::shared_ptr* out) { } Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) { + void* out) { std::lock_guard guard(memory_map_->lock()); RETURN_NOT_OK(Seek(position)); return Read(nbytes, bytes_read, out); @@ -750,7 +750,7 @@ Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, bool MemoryMappedFile::supports_zero_copy() const { return true; } -Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) { +Status MemoryMappedFile::WriteAt(int64_t position, const void* data, int64_t nbytes) { std::lock_guard guard(memory_map_->lock()); if (!memory_map_->opened() || !memory_map_->writable()) { @@ -761,7 +761,7 @@ Status MemoryMappedFile::WriteAt(int64_t position, const uint8_t* data, int64_t return WriteInternal(data, nbytes); } -Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) { +Status MemoryMappedFile::Write(const void* data, int64_t nbytes) { std::lock_guard guard(memory_map_->lock()); if (!memory_map_->opened() || !memory_map_->writable()) { @@ -773,7 +773,7 @@ Status MemoryMappedFile::Write(const uint8_t* data, int64_t nbytes) { return WriteInternal(data, nbytes); } -Status MemoryMappedFile::WriteInternal(const uint8_t* data, int64_t nbytes) { +Status MemoryMappedFile::WriteInternal(const void* data, int64_t nbytes) { memcpy(memory_map_->head(), data, static_cast(nbytes)); memory_map_->advance(nbytes); return Status::OK(); diff --git a/cpp/src/arrow/io/file.h b/cpp/src/arrow/io/file.h index 7937fea74997d..265df4d65225d 100644 --- a/cpp/src/arrow/io/file.h +++ b/cpp/src/arrow/io/file.h @@ -59,7 +59,7 @@ class ARROW_EXPORT FileOutputStream : public OutputStream { Status Tell(int64_t* position) const override; // Write bytes to the stream. Thread-safe - Status Write(const uint8_t* data, int64_t nbytes) override; + Status Write(const void* data, int64_t nbytes) override; int file_descriptor() const; @@ -93,12 +93,12 @@ class ARROW_EXPORT ReadableFile : public RandomAccessFile { Status Tell(int64_t* position) const override; // Read bytes from the file. Thread-safe - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override; + Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override; Status Read(int64_t nbytes, std::shared_ptr* out) override; /// \brief Thread-safe implementation of ReadAt Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) override; + void* out) override; /// \brief Thread-safe implementation of ReadAt Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr* out) override; @@ -141,13 +141,13 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface { Status Seek(int64_t position) override; // Required by RandomAccessFile, copies memory into out. Not thread-safe - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override; + Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override; // Zero copy read. Not thread-safe Status Read(int64_t nbytes, std::shared_ptr* out) override; Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) override; + void* out) override; /// Default implementation is thread-safe Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr* out) override; @@ -155,10 +155,10 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface { bool supports_zero_copy() const override; /// Write data at the current position in the file. Thread-safe - Status Write(const uint8_t* data, int64_t nbytes) override; + Status Write(const void* data, int64_t nbytes) override; /// Write data at a particular position in the file. Thread-safe - Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override; + Status WriteAt(int64_t position, const void* data, int64_t nbytes) override; // @return: the size in bytes of the memory source Status GetSize(int64_t* size) override; @@ -168,7 +168,7 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface { private: MemoryMappedFile(); - Status WriteInternal(const uint8_t* data, int64_t nbytes); + Status WriteInternal(const void* data, int64_t nbytes); class ARROW_NO_EXPORT MemoryMap; std::shared_ptr memory_map_; diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc index 77d1f524aae85..6e3e4a7a1c7e7 100644 --- a/cpp/src/arrow/io/hdfs.cc +++ b/cpp/src/arrow/io/hdfs.cc @@ -119,7 +119,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl { return Status::OK(); } - Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) { + Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, void* buffer) { tSize ret; if (driver_->HasPread()) { ret = driver_->Pread(fs_, file_, static_cast(position), @@ -149,11 +149,11 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl { return Status::OK(); } - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) { + Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) { int64_t total_bytes = 0; while (total_bytes < nbytes) { tSize ret = driver_->Read( - fs_, file_, reinterpret_cast(buffer + total_bytes), + fs_, file_, reinterpret_cast(buffer) + total_bytes, static_cast(std::min(buffer_size_, nbytes - total_bytes))); RETURN_NOT_OK(CheckReadResult(ret)); total_bytes += ret; @@ -212,7 +212,7 @@ HdfsReadableFile::~HdfsReadableFile() { DCHECK(impl_->Close().ok()); } Status HdfsReadableFile::Close() { return impl_->Close(); } Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* buffer) { + void* buffer) { return impl_->ReadAt(position, nbytes, bytes_read, buffer); } @@ -223,7 +223,7 @@ Status HdfsReadableFile::ReadAt(int64_t position, int64_t nbytes, bool HdfsReadableFile::supports_zero_copy() const { return false; } -Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) { +Status HdfsReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) { return impl_->Read(nbytes, bytes_read, buffer); } @@ -261,7 +261,7 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl { return Status::OK(); } - Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) { + Status Write(const void* buffer, int64_t nbytes, int64_t* bytes_written) { std::lock_guard guard(lock_); tSize ret = driver_->Write(fs_, file_, reinterpret_cast(buffer), static_cast(nbytes)); @@ -277,12 +277,11 @@ HdfsOutputStream::~HdfsOutputStream() { DCHECK(impl_->Close().ok()); } Status HdfsOutputStream::Close() { return impl_->Close(); } -Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes, - int64_t* bytes_read) { +Status HdfsOutputStream::Write(const void* buffer, int64_t nbytes, int64_t* bytes_read) { return impl_->Write(buffer, nbytes, bytes_read); } -Status HdfsOutputStream::Write(const uint8_t* buffer, int64_t nbytes) { +Status HdfsOutputStream::Write(const void* buffer, int64_t nbytes) { int64_t bytes_written_dummy = 0; return Write(buffer, nbytes, &bytes_written_dummy); } diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h index 0708b11cca550..062473b20104d 100644 --- a/cpp/src/arrow/io/hdfs.h +++ b/cpp/src/arrow/io/hdfs.h @@ -182,12 +182,12 @@ class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile { // NOTE: If you wish to read a particular range of a file in a multithreaded // context, you may prefer to use ReadAt to avoid locking issues - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override; + Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override; Status Read(int64_t nbytes, std::shared_ptr* out) override; Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* buffer) override; + void* buffer) override; Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr* out) override; @@ -217,9 +217,9 @@ class ARROW_EXPORT HdfsOutputStream : public OutputStream { Status Close() override; - Status Write(const uint8_t* buffer, int64_t nbytes) override; + Status Write(const void* buffer, int64_t nbytes) override; - Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written); + Status Write(const void* buffer, int64_t nbytes, int64_t* bytes_written); Status Flush() override; diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc index 582cc2026d86d..04560209a62c0 100644 --- a/cpp/src/arrow/io/interfaces.cc +++ b/cpp/src/arrow/io/interfaces.cc @@ -38,7 +38,7 @@ RandomAccessFile::RandomAccessFile() : impl_(new RandomAccessFile::RandomAccessFileImpl()) {} Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) { + void* out) { std::lock_guard lock(impl_->lock_); RETURN_NOT_OK(Seek(position)); return Read(nbytes, bytes_read, out); @@ -51,12 +51,11 @@ Status RandomAccessFile::ReadAt(int64_t position, int64_t nbytes, return Read(nbytes, out); } -Status Writeable::Write(const std::string& data) { - return Write(reinterpret_cast(data.c_str()), - static_cast(data.size())); +Status Writable::Write(const std::string& data) { + return Write(data.c_str(), static_cast(data.size())); } -Status Writeable::Flush() { return Status::OK(); } +Status Writable::Flush() { return Status::OK(); } } // namespace io } // namespace arrow diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h index 82af875e7c07e..09536a44ef003 100644 --- a/cpp/src/arrow/io/interfaces.h +++ b/cpp/src/arrow/io/interfaces.h @@ -86,11 +86,11 @@ class ARROW_EXPORT Seekable { virtual Status Seek(int64_t position) = 0; }; -class ARROW_EXPORT Writeable { +class ARROW_EXPORT Writable { public: - virtual ~Writeable() = default; + virtual ~Writable() = default; - virtual Status Write(const uint8_t* data, int64_t nbytes) = 0; + virtual Status Write(const void* data, int64_t nbytes) = 0; /// \brief Flush buffered bytes, if any virtual Status Flush(); @@ -102,13 +102,13 @@ class ARROW_EXPORT Readable { public: virtual ~Readable() = default; - virtual Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) = 0; + virtual Status Read(int64_t nbytes, int64_t* bytes_read, void* out) = 0; // Does not copy if not necessary virtual Status Read(int64_t nbytes, std::shared_ptr* out) = 0; }; -class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writeable { +class ARROW_EXPORT OutputStream : virtual public FileInterface, public Writable { protected: OutputStream() = default; }; @@ -138,7 +138,7 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable { /// \param[out] out The buffer to read bytes into /// \return Status virtual Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) = 0; + void* out) = 0; /// \brief Read nbytes at position, provide default implementations using Read(...), but /// can be overridden. Default implementation is thread-safe. @@ -162,7 +162,7 @@ class ARROW_EXPORT RandomAccessFile : public InputStream, public Seekable { class ARROW_EXPORT WriteableFile : public OutputStream, public Seekable { public: - virtual Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) = 0; + virtual Status WriteAt(int64_t position, const void* data, int64_t nbytes) = 0; protected: WriteableFile() = default; diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc index ee3beabd9a80e..e70431e69580a 100644 --- a/cpp/src/arrow/io/io-file-test.cc +++ b/cpp/src/arrow/io/io-file-test.cc @@ -135,7 +135,7 @@ TEST_F(TestFileOutputStream, Close) { OpenFile(); const char* data = "testdata"; - ASSERT_OK(file_->Write(reinterpret_cast(data), strlen(data))); + ASSERT_OK(file_->Write(data, strlen(data))); int fd = file_->file_descriptor(); ASSERT_OK(file_->Close()); @@ -158,7 +158,7 @@ TEST_F(TestFileOutputStream, InvalidWrites) { const char* data = ""; - ASSERT_RAISES(IOError, file_->Write(reinterpret_cast(data), -1)); + ASSERT_RAISES(IOError, file_->Write(data, -1)); } TEST_F(TestFileOutputStream, Tell) { @@ -170,7 +170,7 @@ TEST_F(TestFileOutputStream, Tell) { ASSERT_EQ(0, position); const char* data = "testdata"; - ASSERT_OK(file_->Write(reinterpret_cast(data), 8)); + ASSERT_OK(file_->Write(data, 8)); ASSERT_OK(file_->Tell(&position)); ASSERT_EQ(8, position); } @@ -179,7 +179,7 @@ TEST_F(TestFileOutputStream, TruncatesNewFile) { ASSERT_OK(FileOutputStream::Open(path_, &file_)); const char* data = "testdata"; - ASSERT_OK(file_->Write(reinterpret_cast(data), strlen(data))); + ASSERT_OK(file_->Write(data, strlen(data))); ASSERT_OK(file_->Close()); ASSERT_OK(FileOutputStream::Open(path_, &file_)); @@ -583,8 +583,7 @@ TEST_F(TestMemoryMappedFile, ThreadSafety) { std::shared_ptr file; ASSERT_OK(MemoryMappedFile::Open(path, FileMode::READWRITE, &file)); - ASSERT_OK(file->Write(reinterpret_cast(data.c_str()), - static_cast(data.size()))); + ASSERT_OK(file->Write(data.c_str(), static_cast(data.size()))); std::atomic correct_count(0); constexpr int niter = 10000; diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc index 117972f1cf06e..8c2e8c3b0beba 100644 --- a/cpp/src/arrow/io/io-memory-test.cc +++ b/cpp/src/arrow/io/io-memory-test.cc @@ -93,7 +93,7 @@ TEST(TestFixedSizeBufferWriter, Basics) { std::string data = "data123456"; auto nbytes = static_cast(data.size()); - ASSERT_OK(writer.Write(reinterpret_cast(data.c_str()), nbytes)); + ASSERT_OK(writer.Write(data.c_str(), nbytes)); ASSERT_OK(writer.Tell(&position)); ASSERT_EQ(nbytes, position); diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc index d9c84b495d21a..74ec85c0e58ca 100644 --- a/cpp/src/arrow/io/memory.cc +++ b/cpp/src/arrow/io/memory.cc @@ -79,7 +79,7 @@ Status BufferOutputStream::Tell(int64_t* position) const { return Status::OK(); } -Status BufferOutputStream::Write(const uint8_t* data, int64_t nbytes) { +Status BufferOutputStream::Write(const void* data, int64_t nbytes) { if (ARROW_PREDICT_FALSE(!is_open_)) { return Status::IOError("OutputStream is closed"); } @@ -116,7 +116,7 @@ Status MockOutputStream::Tell(int64_t* position) const { return Status::OK(); } -Status MockOutputStream::Write(const uint8_t* data, int64_t nbytes) { +Status MockOutputStream::Write(const void* data, int64_t nbytes) { extent_bytes_written_ += nbytes; return Status::OK(); } @@ -162,9 +162,10 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl { return Status::OK(); } - Status Write(const uint8_t* data, int64_t nbytes) { + Status Write(const void* data, int64_t nbytes) { if (nbytes > memcopy_threshold_ && memcopy_num_threads_ > 1) { - internal::parallel_memcopy(mutable_data_ + position_, data, nbytes, + internal::parallel_memcopy(mutable_data_ + position_, + reinterpret_cast(data), nbytes, memcopy_blocksize_, memcopy_num_threads_); } else { memcpy(mutable_data_ + position_, data, nbytes); @@ -173,7 +174,7 @@ class FixedSizeBufferWriter::FixedSizeBufferWriterImpl { return Status::OK(); } - Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) { + Status WriteAt(int64_t position, const void* data, int64_t nbytes) { std::lock_guard guard(lock_); RETURN_NOT_OK(Seek(position)); return Write(data, nbytes); @@ -210,11 +211,11 @@ Status FixedSizeBufferWriter::Tell(int64_t* position) const { return impl_->Tell(position); } -Status FixedSizeBufferWriter::Write(const uint8_t* data, int64_t nbytes) { +Status FixedSizeBufferWriter::Write(const void* data, int64_t nbytes) { return impl_->Write(data, nbytes); } -Status FixedSizeBufferWriter::WriteAt(int64_t position, const uint8_t* data, +Status FixedSizeBufferWriter::WriteAt(int64_t position, const void* data, int64_t nbytes) { return impl_->WriteAt(position, data, nbytes); } @@ -252,7 +253,7 @@ Status BufferReader::Tell(int64_t* position) const { bool BufferReader::supports_zero_copy() const { return true; } -Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) { +Status BufferReader::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) { memcpy(buffer, data_ + position_, nbytes); *bytes_read = std::min(nbytes, size_ - position_); position_ += *bytes_read; @@ -273,7 +274,7 @@ Status BufferReader::Read(int64_t nbytes, std::shared_ptr* out) { } Status BufferReader::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) { + void* out) { return RandomAccessFile::ReadAt(position, nbytes, bytes_read, out); } diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h index 3aec91f7237e1..8dfd582f9ee05 100644 --- a/cpp/src/arrow/io/memory.h +++ b/cpp/src/arrow/io/memory.h @@ -48,7 +48,7 @@ class ARROW_EXPORT BufferOutputStream : public OutputStream { // Implement the OutputStream interface Status Close() override; Status Tell(int64_t* position) const override; - Status Write(const uint8_t* data, int64_t nbytes) override; + Status Write(const void* data, int64_t nbytes) override; /// Close the stream and return the buffer Status Finish(std::shared_ptr* result); @@ -72,7 +72,7 @@ class ARROW_EXPORT MockOutputStream : public OutputStream { // Implement the OutputStream interface Status Close() override; Status Tell(int64_t* position) const override; - Status Write(const uint8_t* data, int64_t nbytes) override; + Status Write(const void* data, int64_t nbytes) override; int64_t GetExtentBytesWritten() const { return extent_bytes_written_; } @@ -90,8 +90,8 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile { Status Close() override; Status Seek(int64_t position) override; Status Tell(int64_t* position) const override; - Status Write(const uint8_t* data, int64_t nbytes) override; - Status WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) override; + Status Write(const void* data, int64_t nbytes) override; + Status WriteAt(int64_t position, const void* data, int64_t nbytes) override; void set_memcopy_threads(int num_threads); void set_memcopy_blocksize(int64_t blocksize); @@ -111,12 +111,12 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile { Status Close() override; Status Tell(int64_t* position) const override; - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override; + Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override; // Zero copy read Status Read(int64_t nbytes, std::shared_ptr* out) override; Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) override; + void* out) override; /// Default implementation is thread-safe Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr* out) override; diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc index d339449c084d2..d3872503edf19 100644 --- a/cpp/src/arrow/ipc/feather.cc +++ b/cpp/src/arrow/ipc/feather.cc @@ -523,10 +523,8 @@ class TableWriter::TableWriterImpl : public ArrayVisitor { uint32_t buffer_size = static_cast(bytes_written); // Footer: metadata length, magic bytes - RETURN_NOT_OK( - stream_->Write(reinterpret_cast(&buffer_size), sizeof(uint32_t))); - return stream_->Write(reinterpret_cast(kFeatherMagicBytes), - strlen(kFeatherMagicBytes)); + RETURN_NOT_OK(stream_->Write(&buffer_size, sizeof(uint32_t))); + return stream_->Write(kFeatherMagicBytes, strlen(kFeatherMagicBytes)); } Status LoadArrayMetadata(const Array& values, ArrayMetadata* meta) { diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc index f487487dfda9f..f362d97015978 100644 --- a/cpp/src/arrow/ipc/json-integration-test.cc +++ b/cpp/src/arrow/ipc/json-integration-test.cc @@ -119,8 +119,7 @@ static Status ConvertArrowToJson(const std::string& arrow_path, std::string result; RETURN_NOT_OK(writer->Finish(&result)); - return out_file->Write(reinterpret_cast(result.c_str()), - static_cast(result.size())); + return out_file->Write(result.c_str(), static_cast(result.size())); } static Status ValidateArrowVsJson(const std::string& arrow_path, @@ -250,8 +249,7 @@ class TestJSONIntegration : public ::testing::Test { do { std::shared_ptr out; RETURN_NOT_OK(io::FileOutputStream::Open(path, &out)); - RETURN_NOT_OK(out->Write(reinterpret_cast(data), - static_cast(strlen(data)))); + RETURN_NOT_OK(out->Write(data, static_cast(strlen(data)))); } while (0); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index 87b4708bf6346..05202ea937217 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -945,8 +945,7 @@ Status WriteMessage(const Buffer& message, io::OutputStream* file, // Write the flatbuffer size prefix including padding int32_t flatbuffer_size = padded_message_length - 4; - RETURN_NOT_OK( - file->Write(reinterpret_cast(&flatbuffer_size), sizeof(int32_t))); + RETURN_NOT_OK(file->Write(&flatbuffer_size, sizeof(int32_t))); // Write the flatbuffer RETURN_NOT_OK(file->Write(message.data(), message.size())); diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index fb766a9a7599c..a36ed4159d108 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -693,7 +693,7 @@ class StreamBookKeeper { } // Write data and update position - Status Write(const uint8_t* data, int64_t nbytes) { + Status Write(const void* data, int64_t nbytes) { RETURN_NOT_OK(sink_->Write(data, nbytes)); position_ += nbytes; return Status::OK(); @@ -782,7 +782,7 @@ class RecordBatchStreamWriter::RecordBatchStreamWriterImpl : public StreamBookKe // Write 0 EOS message const int32_t kEos = 0; - return Write(reinterpret_cast(&kEos), sizeof(int32_t)); + return Write(&kEos, sizeof(int32_t)); } Status CheckStarted() { @@ -870,8 +870,7 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl Status Start() override { // It is only necessary to align to 8-byte boundary at the start of the file - RETURN_NOT_OK(Write(reinterpret_cast(kArrowMagicBytes), - strlen(kArrowMagicBytes))); + RETURN_NOT_OK(Write(kArrowMagicBytes, strlen(kArrowMagicBytes))); RETURN_NOT_OK(Align()); // We write the schema at the start of the file (and the end). This also @@ -895,12 +894,10 @@ class RecordBatchFileWriter::RecordBatchFileWriterImpl return Status::Invalid("Invalid file footer"); } - RETURN_NOT_OK( - Write(reinterpret_cast(&footer_length), sizeof(int32_t))); + RETURN_NOT_OK(Write(&footer_length, sizeof(int32_t))); // Write magic bytes to end file - return Write(reinterpret_cast(kArrowMagicBytes), - strlen(kArrowMagicBytes)); + return Write(kArrowMagicBytes, strlen(kArrowMagicBytes)); } }; diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc index b01358ab00b4b..cc3892928c455 100644 --- a/cpp/src/arrow/python/io.cc +++ b/cpp/src/arrow/python/io.cc @@ -76,7 +76,7 @@ class PythonFile { return Status::OK(); } - Status Write(const uint8_t* data, int64_t nbytes) { + Status Write(const void* data, int64_t nbytes) { PyObject* py_data = PyBytes_FromStringAndSize(reinterpret_cast(data), nbytes); PY_RETURN_IF_ERROR(StatusCode::IOError); @@ -130,7 +130,7 @@ Status PyReadableFile::Tell(int64_t* position) const { return file_->Tell(position); } -Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) { +Status PyReadableFile::Read(int64_t nbytes, int64_t* bytes_read, void* out) { PyAcquireGIL lock; PyObject* bytes_obj; ARROW_RETURN_NOT_OK(file_->Read(nbytes, &bytes_obj)); @@ -155,7 +155,7 @@ Status PyReadableFile::Read(int64_t nbytes, std::shared_ptr* out) { } Status PyReadableFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) { + void* out) { std::lock_guard guard(file_->lock()); RETURN_NOT_OK(Seek(position)); return Read(nbytes, bytes_read, out); @@ -208,7 +208,7 @@ Status PyOutputStream::Tell(int64_t* position) const { return Status::OK(); } -Status PyOutputStream::Write(const uint8_t* data, int64_t nbytes) { +Status PyOutputStream::Write(const void* data, int64_t nbytes) { PyAcquireGIL lock; position_ += nbytes; return file_->Write(data, nbytes); diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h index bf5db5313a9db..f550de7b2848c 100644 --- a/cpp/src/arrow/python/io.h +++ b/cpp/src/arrow/python/io.h @@ -41,12 +41,12 @@ class ARROW_EXPORT PyReadableFile : public io::RandomAccessFile { Status Close() override; - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override; + Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override; Status Read(int64_t nbytes, std::shared_ptr* out) override; // Thread-safe version Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, - uint8_t* out) override; + void* out) override; // Thread-safe version Status ReadAt(int64_t position, int64_t nbytes, std::shared_ptr* out) override; @@ -70,7 +70,7 @@ class ARROW_EXPORT PyOutputStream : public io::OutputStream { Status Close() override; Status Tell(int64_t* position) const override; - Status Write(const uint8_t* data, int64_t nbytes) override; + Status Write(const void* data, int64_t nbytes) override; private: std::unique_ptr file_; diff --git a/cpp/src/arrow/util/io-util.h b/cpp/src/arrow/util/io-util.h index dbca0d8be394e..7e2a94ca82320 100644 --- a/cpp/src/arrow/util/io-util.h +++ b/cpp/src/arrow/util/io-util.h @@ -40,7 +40,7 @@ class StdoutStream : public OutputStream { return Status::OK(); } - Status Write(const uint8_t* data, int64_t nbytes) override { + Status Write(const void* data, int64_t nbytes) override { pos_ += nbytes; std::cout.write(reinterpret_cast(data), nbytes); return Status::OK(); @@ -63,7 +63,7 @@ class StdinStream : public InputStream { return Status::OK(); } - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override { + Status Read(int64_t nbytes, int64_t* bytes_read, void* out) override { std::cin.read(reinterpret_cast(out), nbytes); if (std::cin) { *bytes_read = nbytes; From b19e183c152000193df227f1e931444759093af6 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 27 Nov 2017 16:15:16 -0500 Subject: [PATCH 090/177] ARROW-1783: [Python] Provide a "component" dict representation of a serialized Python object with minimal allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For systems (like Dask) that prefer to handle their own framed buffer transport, this provides a list of memoryview-compatible objects with minimal copying / allocation from the input data structure, which can similarly be zero-copy reconstructed to the original object. To motivate the use case, consider a dict of ndarrays: ``` data = {i: np.random.randn(1000, 1000) for i in range(50)} ``` Here, we have: ``` >>> %timeit serialized = pa.serialize(data) 52.7 µs ± 1.01 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each) ``` This is about 400MB of data. Some systems may not want to double memory by assembling this into a single large buffer, like with the `to_buffer` method: ``` >>> written = serialized.to_buffer() >>> written.size 400015456 ``` We provide a `to_components` method which contains a dict with a `'data'` field containing a list of `pyarrow.Buffer` objects. This can be converted back to the original Python object using `pyarrow.deserialize_components`: ``` >>> %timeit components = serialized.to_components() 73.8 µs ± 812 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each) >>> list(components.keys()) ['num_buffers', 'data', 'num_tensors'] >>> len(components['data']) 101 >>> type(components['data'][0]) pyarrow.lib.Buffer ``` and ``` >>> %timeit recons = pa.deserialize_components(components) 93.6 µs ± 260 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each) ``` The reason there are 101 data components (1 + 2 * 50) is that: * 1 buffer for the serialized Union stream representing the object * 2 buffers for each of the tensors: 1 for the metadata and 1 for the tensor body. The body is separate so that this is zero-copy from the input Next step after this is ARROW-1784 which is to transport a pandas.DataFrame using this mechanism cc @pitrou @jcrist @mrocklin Author: Wes McKinney Closes #1362 from wesm/ARROW-1783 and squashes the following commits: 4ec5a89f [Wes McKinney] Add missing decref on error e8c76d42 [Wes McKinney] Acquire GIL in GetSerializedFromComponents 1d2e0e27 [Wes McKinney] Fix function documentation fffc7bb6 [Wes McKinney] Typos, add deserialize_components to API 50d2fee5 [Wes McKinney] Finish componentwise serialization roundtrip 58174dde [Wes McKinney] More progress, stubs for reconstruction b1e31a34 [Wes McKinney] Draft GetTensorMessage 337e1d29 [Wes McKinney] Draft SerializedPyObject::GetComponents 598ef335 [Wes McKinney] Tweak --- cpp/src/arrow/io/memory.cc | 3 + cpp/src/arrow/io/memory.h | 1 + cpp/src/arrow/ipc/message.cc | 30 +++++++- cpp/src/arrow/ipc/message.h | 27 ++----- cpp/src/arrow/ipc/reader.cc | 13 ++-- cpp/src/arrow/ipc/reader.h | 10 ++- cpp/src/arrow/ipc/writer.cc | 64 ++++++++++++++--- cpp/src/arrow/ipc/writer.h | 11 +++ cpp/src/arrow/python/arrow_to_pandas.cc | 2 +- cpp/src/arrow/python/arrow_to_python.cc | 60 +++++++++++++++- cpp/src/arrow/python/arrow_to_python.h | 13 ++++ cpp/src/arrow/python/python_to_arrow.cc | 82 +++++++++++++++++++--- cpp/src/arrow/python/python_to_arrow.h | 28 ++++++-- python/doc/source/api.rst | 1 + python/pyarrow/__init__.py | 1 + python/pyarrow/includes/libarrow.pxd | 17 +++-- python/pyarrow/ipc.pxi | 4 +- python/pyarrow/serialization.pxi | 62 +++++++++++++++- python/pyarrow/tests/test_serialization.py | 33 +++++++++ 19 files changed, 394 insertions(+), 68 deletions(-) diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc index 74ec85c0e58ca..ecdf26f0a991b 100644 --- a/cpp/src/arrow/io/memory.cc +++ b/cpp/src/arrow/io/memory.cc @@ -241,6 +241,9 @@ BufferReader::BufferReader(const std::shared_ptr& buffer) BufferReader::BufferReader(const uint8_t* data, int64_t size) : buffer_(nullptr), data_(data), size_(size), position_(0) {} +BufferReader::BufferReader(const Buffer& buffer) + : BufferReader(buffer.data(), buffer.size()) {} + Status BufferReader::Close() { // no-op return Status::OK(); diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h index 8dfd582f9ee05..cf370b3b6388b 100644 --- a/cpp/src/arrow/io/memory.h +++ b/cpp/src/arrow/io/memory.h @@ -107,6 +107,7 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WriteableFile { class ARROW_EXPORT BufferReader : public RandomAccessFile { public: explicit BufferReader(const std::shared_ptr& buffer); + explicit BufferReader(const Buffer& buffer); BufferReader(const uint8_t* data, int64_t size); Status Close() override; diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 21d6a69a28603..1835cefde09ee 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -236,11 +236,35 @@ Status ReadMessage(io::InputStream* file, std::unique_ptr* message) { // ---------------------------------------------------------------------- // Implement InputStream message reader -Status InputStreamMessageReader::ReadNextMessage(std::unique_ptr* message) { - return ReadMessage(stream_, message); +/// \brief Implementation of MessageReader that reads from InputStream +class InputStreamMessageReader : public MessageReader { + public: + explicit InputStreamMessageReader(io::InputStream* stream) : stream_(stream) {} + + explicit InputStreamMessageReader(const std::shared_ptr& owned_stream) + : InputStreamMessageReader(owned_stream.get()) { + owned_stream_ = owned_stream; + } + + ~InputStreamMessageReader() {} + + Status ReadNextMessage(std::unique_ptr* message) { + return ReadMessage(stream_, message); + } + + private: + io::InputStream* stream_; + std::shared_ptr owned_stream_; +}; + +std::unique_ptr MessageReader::Open(io::InputStream* stream) { + return std::unique_ptr(new InputStreamMessageReader(stream)); } -InputStreamMessageReader::~InputStreamMessageReader() {} +std::unique_ptr MessageReader::Open( + const std::shared_ptr& owned_stream) { + return std::unique_ptr(new InputStreamMessageReader(owned_stream)); +} } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h index 495474e505157..159b39a81f95d 100644 --- a/cpp/src/arrow/ipc/message.h +++ b/cpp/src/arrow/ipc/message.h @@ -144,6 +144,13 @@ class ARROW_EXPORT MessageReader { public: virtual ~MessageReader() = default; + /// \brief Create MessageReader that reads from InputStream + static std::unique_ptr Open(io::InputStream* stream); + + /// \brief Create MessageReader that reads from owned InputStream + static std::unique_ptr Open( + const std::shared_ptr& owned_stream); + /// \brief Read next Message from the interface /// /// \param[out] message an arrow::ipc::Message instance @@ -151,26 +158,6 @@ class ARROW_EXPORT MessageReader { virtual Status ReadNextMessage(std::unique_ptr* message) = 0; }; -/// \brief Implementation of MessageReader that reads from InputStream -/// \since 0.5.0 -class ARROW_EXPORT InputStreamMessageReader : public MessageReader { - public: - explicit InputStreamMessageReader(io::InputStream* stream) : stream_(stream) {} - - explicit InputStreamMessageReader(const std::shared_ptr& owned_stream) - : InputStreamMessageReader(owned_stream.get()) { - owned_stream_ = owned_stream; - } - - ~InputStreamMessageReader(); - - Status ReadNextMessage(std::unique_ptr* message) override; - - private: - io::InputStream* stream_; - std::shared_ptr owned_stream_; -}; - /// \brief Read encapulated RPC message from position in file /// /// Read a length-prefixed message flatbuffer starting at the indicated file diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 5960e81883d6b..ae0f8f39806b7 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -480,14 +480,12 @@ Status RecordBatchStreamReader::Open(std::unique_ptr message_read Status RecordBatchStreamReader::Open(io::InputStream* stream, std::shared_ptr* out) { - std::unique_ptr message_reader(new InputStreamMessageReader(stream)); - return Open(std::move(message_reader), out); + return Open(MessageReader::Open(stream), out); } Status RecordBatchStreamReader::Open(const std::shared_ptr& stream, std::shared_ptr* out) { - std::unique_ptr message_reader(new InputStreamMessageReader(stream)); - return Open(std::move(message_reader), out); + return Open(MessageReader::Open(stream), out); } std::shared_ptr RecordBatchStreamReader::schema() const { @@ -717,14 +715,17 @@ Status ReadTensor(int64_t offset, io::RandomAccessFile* file, std::unique_ptr message; RETURN_NOT_OK(ReadContiguousPayload(file, &message)); + return ReadTensor(*message, out); +} +Status ReadTensor(const Message& message, std::shared_ptr* out) { std::shared_ptr type; std::vector shape; std::vector strides; std::vector dim_names; - RETURN_NOT_OK(internal::GetTensorMetadata(*message->metadata(), &type, &shape, &strides, + RETURN_NOT_OK(internal::GetTensorMetadata(*message.metadata(), &type, &shape, &strides, &dim_names)); - *out = std::make_shared(type, message->body(), shape, strides, dim_names); + *out = std::make_shared(type, message.body(), shape, strides, dim_names); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h index 627f67e2517e9..019c9bc1f32d8 100644 --- a/cpp/src/arrow/ipc/reader.h +++ b/cpp/src/arrow/ipc/reader.h @@ -219,7 +219,7 @@ Status ReadRecordBatch(const Buffer& metadata, const std::shared_ptr& sc int max_recursion_depth, io::RandomAccessFile* file, std::shared_ptr* out); -/// EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file +/// \brief EXPERIMENTAL: Read arrow::Tensor as encapsulated IPC message in file /// /// \param[in] offset the file location of the start of the message /// \param[in] file the file where the batch is located @@ -229,6 +229,14 @@ ARROW_EXPORT Status ReadTensor(int64_t offset, io::RandomAccessFile* file, std::shared_ptr* out); +/// \brief EXPERIMENTAL: Read arrow::Tensor from IPC message +/// +/// \param[in] message a Message containing the tensor metadata and body +/// \param[out] out the read tensor +/// \return Status +ARROW_EXPORT +Status ReadTensor(const Message& message, std::shared_ptr* out); + } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index a36ed4159d108..c6aa770127c88 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -560,9 +560,18 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs pool, kMaxNestingDepth, true); } -static Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_size, - const Tensor& tensor, uint8_t* scratch_space, - io::OutputStream* dst) { +namespace { + +Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst, + int32_t* metadata_length, int64_t* body_length) { + std::shared_ptr metadata; + RETURN_NOT_OK(internal::WriteTensorMessage(tensor, 0, &metadata)); + return internal::WriteMessage(*metadata, dst, metadata_length); +} + +Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_size, + const Tensor& tensor, uint8_t* scratch_space, + io::OutputStream* dst) { if (dim_index == tensor.ndim() - 1) { const uint8_t* data_ptr = tensor.raw_data() + offset; const int64_t stride = tensor.strides()[dim_index]; @@ -580,16 +589,37 @@ static Status WriteStridedTensorData(int dim_index, int64_t offset, int elem_siz return Status::OK(); } -Status WriteTensorHeader(const Tensor& tensor, io::OutputStream* dst, - int32_t* metadata_length, int64_t* body_length) { - RETURN_NOT_OK(AlignStreamPosition(dst)); - std::shared_ptr metadata; - RETURN_NOT_OK(internal::WriteTensorMessage(tensor, 0, &metadata)); - return internal::WriteMessage(*metadata, dst, metadata_length); +Status GetContiguousTensor(const Tensor& tensor, MemoryPool* pool, + std::unique_ptr* out) { + const auto& type = static_cast(*tensor.type()); + const int elem_size = type.bit_width() / 8; + + // TODO(wesm): Do we care enough about this temporary allocation to pass in + // a MemoryPool to this function? + std::shared_ptr scratch_space; + RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), + tensor.shape()[tensor.ndim() - 1] * elem_size, + &scratch_space)); + + std::shared_ptr contiguous_data; + RETURN_NOT_OK( + AllocateResizableBuffer(pool, tensor.size() * elem_size, &contiguous_data)); + + io::BufferOutputStream stream(contiguous_data); + RETURN_NOT_OK(WriteStridedTensorData(0, 0, elem_size, tensor, + scratch_space->mutable_data(), &stream)); + + out->reset(new Tensor(tensor.type(), contiguous_data, tensor.shape())); + + return Status::OK(); } +} // namespace + Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) { + RETURN_NOT_OK(AlignStreamPosition(dst)); + if (tensor.is_contiguous()) { RETURN_NOT_OK(WriteTensorHeader(tensor, dst, metadata_length, body_length)); auto data = tensor.data(); @@ -619,6 +649,22 @@ Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadat } } +Status GetTensorMessage(const Tensor& tensor, MemoryPool* pool, + std::unique_ptr* out) { + const Tensor* tensor_to_write = &tensor; + std::unique_ptr temp_tensor; + + if (!tensor.is_contiguous()) { + RETURN_NOT_OK(GetContiguousTensor(tensor, pool, &temp_tensor)); + tensor_to_write = temp_tensor.get(); + } + + std::shared_ptr metadata; + RETURN_NOT_OK(internal::WriteTensorMessage(*tensor_to_write, 0, &metadata)); + out->reset(new Message(metadata, tensor_to_write->data())); + return Status::OK(); +} + Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr& dictionary, int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length, MemoryPool* pool) { diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h index 457dcb4ec6a13..013783ee0a224 100644 --- a/cpp/src/arrow/ipc/writer.h +++ b/cpp/src/arrow/ipc/writer.h @@ -245,6 +245,17 @@ Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size); ARROW_EXPORT Status GetTensorSize(const Tensor& tensor, int64_t* size); +/// \brief EXPERIMENTAL: Convert arrow::Tensor to a Message with minimal memory +/// allocation +/// +/// \param[in] tensor the Tensor to write +/// \param[in] pool MemoryPool to allocate space for metadata +/// \param[out] out the resulting Message +/// \return Status +ARROW_EXPORT +Status GetTensorMessage(const Tensor& tensor, MemoryPool* pool, + std::unique_ptr* out); + /// \brief EXPERIMENTAL: Write arrow::Tensor as a contiguous message /// /// \param[in] tensor the Tensor to write diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 8814fc190ab3d..096bbd55c6d41 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -480,7 +480,7 @@ inline Status ConvertStruct(PandasOptions options, const ChunkedArray& data, Py_INCREF(Py_None); field_value.reset(Py_None); } - // PyDict_SetItemString does not steal the value reference + // PyDict_SetItemString increments reference count auto setitem_result = PyDict_SetItemString(dict_item.obj(), name.c_str(), field_value.obj()); RETURN_IF_PYERROR(); diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc index 9686050b9676f..ce539a597be37 100644 --- a/cpp/src/arrow/python/arrow_to_python.cc +++ b/cpp/src/arrow/python/arrow_to_python.cc @@ -29,15 +29,17 @@ #include "arrow/array.h" #include "arrow/io/interfaces.h" +#include "arrow/io/memory.h" #include "arrow/ipc/reader.h" +#include "arrow/table.h" +#include "arrow/util/logging.h" + #include "arrow/python/common.h" #include "arrow/python/helpers.h" #include "arrow/python/numpy_convert.h" #include "arrow/python/pyarrow.h" #include "arrow/python/python_to_arrow.h" #include "arrow/python/util/datetime.h" -#include "arrow/table.h" -#include "arrow/util/logging.h" namespace arrow { namespace py { @@ -286,5 +288,59 @@ Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObj obj, out); } +Status GetSerializedFromComponents(int num_tensors, int num_buffers, PyObject* data, + SerializedPyObject* out) { + PyAcquireGIL gil; + const Py_ssize_t data_length = PyList_Size(data); + RETURN_IF_PYERROR(); + + const Py_ssize_t expected_data_length = 1 + num_tensors * 2 + num_buffers; + if (data_length != expected_data_length) { + return Status::Invalid("Invalid number of buffers in data"); + } + + auto GetBuffer = [&data](Py_ssize_t index, std::shared_ptr* out) { + PyObject* py_buf = PyList_GET_ITEM(data, index); + return unwrap_buffer(py_buf, out); + }; + + Py_ssize_t buffer_index = 0; + + // Read the union batch describing object structure + { + std::shared_ptr data_buffer; + RETURN_NOT_OK(GetBuffer(buffer_index++, &data_buffer)); + gil.release(); + io::BufferReader buf_reader(data_buffer); + std::shared_ptr reader; + RETURN_NOT_OK(ipc::RecordBatchStreamReader::Open(&buf_reader, &reader)); + RETURN_NOT_OK(reader->ReadNext(&out->batch)); + gil.acquire(); + } + + // Zero-copy reconstruct tensors + for (int i = 0; i < num_tensors; ++i) { + std::shared_ptr metadata; + std::shared_ptr body; + std::shared_ptr tensor; + RETURN_NOT_OK(GetBuffer(buffer_index++, &metadata)); + RETURN_NOT_OK(GetBuffer(buffer_index++, &body)); + + ipc::Message message(metadata, body); + + RETURN_NOT_OK(ReadTensor(message, &tensor)); + out->tensors.emplace_back(std::move(tensor)); + } + + // Unwrap and append buffers + for (int i = 0; i < num_buffers; ++i) { + std::shared_ptr buffer; + RETURN_NOT_OK(GetBuffer(buffer_index++, &buffer)); + out->buffers.emplace_back(std::move(buffer)); + } + + return Status::OK(); +} + } // namespace py } // namespace arrow diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h index 7509f30eb4e90..9440ffb32abeb 100644 --- a/cpp/src/arrow/python/arrow_to_python.h +++ b/cpp/src/arrow/python/arrow_to_python.h @@ -48,6 +48,19 @@ namespace py { ARROW_EXPORT Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out); +/// \brief Reconstruct SerializedPyObject from representation produced by +/// SerializedPyObject::GetComponents. +/// +/// \param[in] num_tensors +/// \param[in] num_buffers +/// \param[in] data a list containing pyarrow.Buffer instances. Must be 1 + +/// num_tensors * 2 + num_buffers in length +/// \param[out] out the reconstructed object +/// \return Status +ARROW_EXPORT +Status GetSerializedFromComponents(int num_tensors, int num_buffers, PyObject* data, + SerializedPyObject* out); + /// \brief Reconstruct Python object from Arrow-serialized representation /// \param[in] context Serialization context which contains custom serialization /// and deserialization callbacks. Can be any Python object with a diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc index 72cc5b6e1db67..253e9d9a7da30 100644 --- a/cpp/src/arrow/python/python_to_arrow.cc +++ b/cpp/src/arrow/python/python_to_arrow.cc @@ -31,7 +31,9 @@ #include "arrow/array.h" #include "arrow/builder.h" #include "arrow/io/interfaces.h" +#include "arrow/io/memory.h" #include "arrow/ipc/writer.h" +#include "arrow/memory_pool.h" #include "arrow/record_batch.h" #include "arrow/tensor.h" #include "arrow/util/logging.h" @@ -710,27 +712,89 @@ Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject return Status::OK(); } -Status WriteSerializedObject(const SerializedPyObject& obj, io::OutputStream* dst) { - int32_t num_tensors = static_cast(obj.tensors.size()); - int32_t num_buffers = static_cast(obj.buffers.size()); - RETURN_NOT_OK(dst->Write(reinterpret_cast(&num_tensors), sizeof(int32_t))); - RETURN_NOT_OK(dst->Write(reinterpret_cast(&num_buffers), sizeof(int32_t))); - RETURN_NOT_OK(ipc::WriteRecordBatchStream({obj.batch}, dst)); +Status SerializedPyObject::WriteTo(io::OutputStream* dst) { + int32_t num_tensors = static_cast(this->tensors.size()); + int32_t num_buffers = static_cast(this->buffers.size()); + RETURN_NOT_OK( + dst->Write(reinterpret_cast(&num_tensors), sizeof(int32_t))); + RETURN_NOT_OK( + dst->Write(reinterpret_cast(&num_buffers), sizeof(int32_t))); + RETURN_NOT_OK(ipc::WriteRecordBatchStream({this->batch}, dst)); int32_t metadata_length; int64_t body_length; - for (const auto& tensor : obj.tensors) { + for (const auto& tensor : this->tensors) { RETURN_NOT_OK(ipc::WriteTensor(*tensor, dst, &metadata_length, &body_length)); } - for (const auto& buffer : obj.buffers) { + for (const auto& buffer : this->buffers) { int64_t size = buffer->size(); - RETURN_NOT_OK(dst->Write(reinterpret_cast(&size), sizeof(int64_t))); + RETURN_NOT_OK(dst->Write(reinterpret_cast(&size), sizeof(int64_t))); RETURN_NOT_OK(dst->Write(buffer->data(), size)); } return Status::OK(); } +Status SerializedPyObject::GetComponents(MemoryPool* memory_pool, PyObject** out) { + PyAcquireGIL py_gil; + + ScopedRef result(PyDict_New()); + PyObject* buffers = PyList_New(0); + + // TODO(wesm): Not sure how pedantic we need to be about checking the return + // values of these functions. There are other places where we do not check + // PyDict_SetItem/SetItemString return value, but these failures would be + // quite esoteric + PyDict_SetItemString(result.get(), "num_tensors", + PyLong_FromSize_t(this->tensors.size())); + PyDict_SetItemString(result.get(), "num_buffers", + PyLong_FromSize_t(this->buffers.size())); + PyDict_SetItemString(result.get(), "data", buffers); + RETURN_IF_PYERROR(); + + Py_DECREF(buffers); + + auto PushBuffer = [&buffers](const std::shared_ptr& buffer) { + PyObject* wrapped_buffer = wrap_buffer(buffer); + RETURN_IF_PYERROR(); + if (PyList_Append(buffers, wrapped_buffer) < 0) { + Py_DECREF(wrapped_buffer); + RETURN_IF_PYERROR(); + } + Py_DECREF(wrapped_buffer); + return Status::OK(); + }; + + constexpr int64_t kInitialCapacity = 1024; + + // Write the record batch describing the object structure + std::shared_ptr stream; + std::shared_ptr buffer; + + py_gil.release(); + RETURN_NOT_OK(io::BufferOutputStream::Create(kInitialCapacity, memory_pool, &stream)); + RETURN_NOT_OK(ipc::WriteRecordBatchStream({this->batch}, stream.get())); + RETURN_NOT_OK(stream->Finish(&buffer)); + py_gil.acquire(); + + RETURN_NOT_OK(PushBuffer(buffer)); + + // For each tensor, get a metadata buffer and a buffer for the body + for (const auto& tensor : this->tensors) { + std::unique_ptr message; + RETURN_NOT_OK(ipc::GetTensorMessage(*tensor, memory_pool, &message)); + RETURN_NOT_OK(PushBuffer(message->metadata())); + RETURN_NOT_OK(PushBuffer(message->body())); + } + + for (const auto& buf : this->buffers) { + RETURN_NOT_OK(PushBuffer(buf)); + } + + *out = result.release(); + return Status::OK(); +} + } // namespace py } // namespace arrow diff --git a/cpp/src/arrow/python/python_to_arrow.h b/cpp/src/arrow/python/python_to_arrow.h index c5b6396145b7f..ce7aefa0e2409 100644 --- a/cpp/src/arrow/python/python_to_arrow.h +++ b/cpp/src/arrow/python/python_to_arrow.h @@ -30,6 +30,7 @@ namespace arrow { +class MemoryPool; class RecordBatch; class Tensor; @@ -45,6 +46,26 @@ struct ARROW_EXPORT SerializedPyObject { std::shared_ptr batch; std::vector> tensors; std::vector> buffers; + + /// \brief Write serialized Python object to OutputStream + /// \param[in,out] dst an OutputStream + /// \return Status + Status WriteTo(io::OutputStream* dst); + + /// \brief Convert SerializedPyObject to a dict containing the message + /// components as Buffer instances with minimal memory allocation + /// + /// { + /// 'num_tensors': N, + /// 'num_buffers': K, + /// 'data': [Buffer] + /// } + /// + /// Each tensor is written as two buffers, one for the metadata and one for + /// the body. Therefore, the number of buffers in 'data' is 2 * N + K + 1, + /// with the first buffer containing the serialized record batch containing + /// the UnionArray that describes the whole object + Status GetComponents(MemoryPool* pool, PyObject** out); }; /// \brief Serialize Python sequence as a RecordBatch plus @@ -62,13 +83,6 @@ struct ARROW_EXPORT SerializedPyObject { ARROW_EXPORT Status SerializeObject(PyObject* context, PyObject* sequence, SerializedPyObject* out); -/// \brief Write serialized Python object to OutputStream -/// \param[in] object a serialized Python object to write out -/// \param[out] dst an OutputStream -/// \return Status -ARROW_EXPORT -Status WriteSerializedObject(const SerializedPyObject& object, io::OutputStream* dst); - } // namespace py } // namespace arrow diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst index bb2a0420b2a08..636f41d67bc6a 100644 --- a/python/doc/source/api.rst +++ b/python/doc/source/api.rst @@ -245,6 +245,7 @@ Serialization and IPC serialize serialize_to deserialize + deserialize_components deserialize_from read_serialized SerializedPyObject diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 0456a658f1425..bd31b21c196de 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -101,6 +101,7 @@ # Serialization from pyarrow.lib import (deserialize_from, deserialize, + deserialize_components, serialize, serialize_to, read_serialized, SerializedPyObject, SerializationContext, SerializationCallbackError, diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 14211787c89f2..91bc96dc63f89 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -689,11 +689,10 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: c_string FormatMessageType(MessageType type) cdef cppclass CMessageReader" arrow::ipc::MessageReader": - CStatus ReadNextMessage(unique_ptr[CMessage]* out) + @staticmethod + unique_ptr[CMessageReader] Open(const shared_ptr[InputStream]& stream) - cdef cppclass CInputStreamMessageReader \ - " arrow::ipc::InputStreamMessageReader": - CInputStreamMessageReader(const shared_ptr[InputStream]& stream) + CStatus ReadNextMessage(unique_ptr[CMessage]* out) cdef cppclass CRecordBatchWriter" arrow::ipc::RecordBatchWriter": CStatus Close() @@ -915,12 +914,12 @@ cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil: shared_ptr[CRecordBatch] batch vector[shared_ptr[CTensor]] tensors + CStatus WriteTo(OutputStream* dst) + CStatus GetComponents(CMemoryPool* pool, PyObject** dst) + CStatus SerializeObject(object context, object sequence, CSerializedPyObject* out) - CStatus WriteSerializedObject(const CSerializedPyObject& obj, - OutputStream* dst) - CStatus DeserializeObject(object context, const CSerializedPyObject& obj, PyObject* base, PyObject** out) @@ -928,6 +927,10 @@ cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil: CStatus ReadSerializedObject(RandomAccessFile* src, CSerializedPyObject* out) + CStatus GetSerializedFromComponents(int num_tensors, int num_buffers, + object buffers, + CSerializedPyObject* out) + cdef extern from 'arrow/python/init.h': int arrow_init_numpy() except -1 diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi index e5639137dd3a1..7534b0d0e87ec 100644 --- a/python/pyarrow/ipc.pxi +++ b/python/pyarrow/ipc.pxi @@ -125,9 +125,11 @@ cdef class MessageReader: def open_stream(source): cdef MessageReader result = MessageReader() cdef shared_ptr[InputStream] in_stream + cdef unique_ptr[CMessageReader] reader get_input_stream(source, &in_stream) with nogil: - result.reader.reset(new CInputStreamMessageReader(in_stream)) + reader = CMessageReader.Open(in_stream) + result.reader.reset(reader.release()) return result diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index 3ee5c7d4e2770..bb266b2f928ff 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -155,7 +155,7 @@ cdef class SerializedPyObject: def __get__(self): cdef CMockOutputStream mock_stream with nogil: - check_status(WriteSerializedObject(self.data, &mock_stream)) + check_status(self.data.WriteTo(&mock_stream)) return mock_stream.GetExtentBytesWritten() @@ -169,7 +169,7 @@ cdef class SerializedPyObject: cdef _write_to(self, OutputStream* stream): with nogil: - check_status(WriteSerializedObject(self.data, stream)) + check_status(self.data.WriteTo(stream)) def deserialize(self, SerializationContext context=None): """ @@ -199,6 +199,46 @@ cdef class SerializedPyObject: self.write_to(sink) return output + @staticmethod + def from_components(components): + """ + Reconstruct SerializedPyObject from output of + SerializedPyObject.to_components + """ + cdef: + int num_tensors = components['num_tensors'] + int num_buffers = components['num_buffers'] + list buffers = components['data'] + SerializedPyObject result = SerializedPyObject() + + with nogil: + check_status(GetSerializedFromComponents(num_tensors, num_buffers, + buffers, &result.data)) + + return result + + def to_components(self, memory_pool=None): + """ + Return the decomposed dict representation of the serialized object + containing a collection of Buffer objects which maximize opportunities + for zero-copy + + Parameters + ---------- + memory_pool : MemoryPool default None + Pool to use for necessary allocations + + Returns + + """ + cdef PyObject* result + cdef CMemoryPool* c_pool = maybe_unbox_memory_pool(memory_pool) + + with nogil: + check_status(self.data.GetComponents(c_pool, &result)) + + return PyObject_to_object(result) + def serialize(object value, SerializationContext context=None): """EXPERIMENTAL: Serialize a Python sequence @@ -291,6 +331,24 @@ def deserialize_from(source, object base, SerializationContext context=None): return serialized.deserialize(context) +def deserialize_components(components, SerializationContext context=None): + """ + Reconstruct Python object from output of SerializedPyObject.to_components + + Parameters + ---------- + components : dict + Output of SerializedPyObject.to_components + context : SerializationContext, default None + + Returns + ------- + object : the Python object that was originally serialized + """ + serialized = SerializedPyObject.from_components(components) + return serialized.deserialize(context) + + def deserialize(obj, SerializationContext context=None): """ EXPERIMENTAL: Deserialize Python object from Buffer or other Python object diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index ed4fd9ae59aee..d06beeac99280 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -219,6 +219,17 @@ def serialization_roundtrip(value, f): result = pa.deserialize_from(f, None, serialization_context) assert_equal(value, result) + _check_component_roundtrip(value) + + +def _check_component_roundtrip(value): + # Test to/from components + serialized = pa.serialize(value) + components = serialized.to_components() + from_comp = pa.SerializedPyObject.from_components(components) + recons = from_comp.deserialize() + assert_equal(value, recons) + @pytest.yield_fixture(scope='session') def large_memory_map(tmpdir_factory, size=100*1024*1024): @@ -485,3 +496,25 @@ def test_serialize_subclasses(): deserialized = serialized.deserialize() assert type(deserialized).__name__ == SerializableClass.__name__ assert deserialized.value == 3 + + +def test_serialize_to_components_invalid_cases(): + buf = pa.frombuffer(b'hello') + + components = { + 'num_tensors': 0, + 'num_buffers': 1, + 'data': [buf] + } + + with pytest.raises(pa.ArrowException): + pa.deserialize_components(components) + + components = { + 'num_tensors': 1, + 'num_buffers': 0, + 'data': [buf, buf] + } + + with pytest.raises(pa.ArrowException): + pa.deserialize_components(components) From a75325a8e273f845a50ee5cdaf050ea74cb82797 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Tue, 28 Nov 2017 12:32:42 -0500 Subject: [PATCH 091/177] ARROW-1710: [Java] Remove Non-Nullable Vectors This removes non-nullable vectors that are no longer part of the vector class hierarchy and renames Nullable*Vector classes to remove the Nullable prefix. Author: Bryan Cutler Closes #1341 from BryanCutler/java-nullable-vector-rename-ARROW-1710 and squashes the following commits: 7d930dc2 [Bryan Cutler] fixed realloc test ff2120df [Bryan Cutler] clean up test 374dfcc5 [Bryan Cutler] properly rename BitVector file 6b7a85e7 [Bryan Cutler] remove old BitVector.java before rebase 089f7fc9 [Bryan Cutler] some minor cleanup 4e580d93 [Bryan Cutler] removed legacy BitVector 74f771f5 [Bryan Cutler] fixed remaining tests 8c5dfef6 [Bryan Cutler] fix naming in support classes 6e498e55 [Bryan Cutler] removed nullable prefix dfed444c [Bryan Cutler] removed non-nullable vectors --- .../apache/arrow/tools/EchoServerTest.java | 30 +- .../templates/CaseSensitiveMapWriters.java | 2 +- .../codegen/templates/ComplexReaders.java | 12 +- .../codegen/templates/ComplexWriters.java | 10 +- .../codegen/templates/FixedValueVectors.java | 765 --------------- .../main/codegen/templates/MapWriters.java | 9 +- .../main/codegen/templates/UnionVector.java | 12 +- .../templates/VariableLengthVectors.java | 677 -------------- ...hVector.java => BaseFixedWidthVector.java} | 22 +- ...ctor.java => BaseVariableWidthVector.java} | 16 +- ...bleBigIntVector.java => BigIntVector.java} | 32 +- .../org/apache/arrow/vector/BitVector.java | 878 ++++++++---------- ...eDateDayVector.java => DateDayVector.java} | 32 +- ...eMilliVector.java => DateMilliVector.java} | 32 +- ...eDecimalVector.java => DecimalVector.java} | 34 +- ...bleFloat4Vector.java => Float4Vector.java} | 32 +- ...bleFloat8Vector.java => Float8Vector.java} | 32 +- .../arrow/vector/GenerateSampleData.java | 142 +-- ...{NullableIntVector.java => IntVector.java} | 32 +- ...lDayVector.java => IntervalDayVector.java} | 32 +- ...earVector.java => IntervalYearVector.java} | 32 +- .../arrow/vector/NullableBitVector.java | 513 ---------- .../apache/arrow/vector/NullableVector.java | 26 - ...mallIntVector.java => SmallIntVector.java} | 32 +- ...eMicroVector.java => TimeMicroVector.java} | 32 +- ...eMilliVector.java => TimeMilliVector.java} | 32 +- ...imeNanoVector.java => TimeNanoVector.java} | 32 +- ...eTimeSecVector.java => TimeSecVector.java} | 32 +- ...ector.java => TimeStampMicroTZVector.java} | 18 +- ...oVector.java => TimeStampMicroVector.java} | 18 +- ...ector.java => TimeStampMilliTZVector.java} | 18 +- ...iVector.java => TimeStampMilliVector.java} | 18 +- ...Vector.java => TimeStampNanoTZVector.java} | 18 +- ...noVector.java => TimeStampNanoVector.java} | 18 +- ...ZVector.java => TimeStampSecTZVector.java} | 18 +- ...SecVector.java => TimeStampSecVector.java} | 18 +- ...eStampVector.java => TimeStampVector.java} | 22 +- ...eTinyIntVector.java => TinyIntVector.java} | 32 +- ...lableUInt1Vector.java => UInt1Vector.java} | 26 +- ...lableUInt2Vector.java => UInt2Vector.java} | 26 +- ...lableUInt4Vector.java => UInt4Vector.java} | 26 +- ...lableUInt8Vector.java => UInt8Vector.java} | 26 +- ...BinaryVector.java => VarBinaryVector.java} | 32 +- ...eVarCharVector.java => VarCharVector.java} | 32 +- ...etter.java => VectorDefinitionSetter.java} | 2 +- .../complex/AbstractContainerVector.java | 4 +- .../vector/complex/EmptyValuePopulator.java | 10 +- .../arrow/vector/complex/MapVector.java | 562 +++++++---- .../vector/complex/NonNullableMapVector.java | 352 +++++++ .../vector/complex/NullableMapVector.java | 512 ---------- .../complex/impl/ComplexWriterImpl.java | 6 +- .../complex/impl/NullableMapReaderImpl.java | 8 +- .../impl/NullableMapWriterFactory.java | 4 +- .../vector/complex/impl/PromotableWriter.java | 4 +- .../complex/impl/SingleMapReaderImpl.java | 5 +- .../vector/complex/impl/UnionListReader.java | 1 - .../arrow/vector/ipc/JsonFileReader.java | 14 +- .../arrow/vector/ipc/JsonFileWriter.java | 52 +- .../org/apache/arrow/vector/types/Types.java | 186 ++-- .../arrow/vector/util/DecimalUtility.java | 1 - .../apache/arrow/vector/TestBitVector.java | 189 ++-- .../vector/TestBufferOwnershipTransfer.java | 8 +- .../org/apache/arrow/vector/TestCopyFrom.java | 86 +- .../arrow/vector/TestDecimalVector.java | 6 +- .../arrow/vector/TestDictionaryVector.java | 22 +- .../arrow/vector/TestFixedSizeListVector.java | 8 +- .../apache/arrow/vector/TestListVector.java | 8 +- .../apache/arrow/vector/TestMapVector.java | 4 +- ...TestOversizedAllocationForValueVector.java | 4 +- .../arrow/vector/TestSplitAndTransfer.java | 8 +- .../org/apache/arrow/vector/TestUtils.java | 8 +- .../apache/arrow/vector/TestValueVector.java | 308 +++--- .../arrow/vector/TestVectorReAlloc.java | 17 +- .../apache/arrow/vector/TestVectorReset.java | 5 +- .../arrow/vector/TestVectorUnloadLoad.java | 10 +- .../complex/impl/TestPromotableWriter.java | 4 +- .../complex/writer/TestComplexWriter.java | 22 +- .../apache/arrow/vector/ipc/BaseFileTest.java | 60 +- .../arrow/vector/ipc/TestArrowFile.java | 26 +- .../arrow/vector/ipc/TestArrowStream.java | 6 +- .../arrow/vector/ipc/TestArrowStreamPipe.java | 7 +- .../apache/arrow/vector/ipc/TestJSONFile.java | 10 +- 82 files changed, 2159 insertions(+), 4320 deletions(-) delete mode 100644 java/vector/src/main/codegen/templates/FixedValueVectors.java delete mode 100644 java/vector/src/main/codegen/templates/VariableLengthVectors.java rename java/vector/src/main/java/org/apache/arrow/vector/{BaseNullableFixedWidthVector.java => BaseFixedWidthVector.java} (97%) rename java/vector/src/main/java/org/apache/arrow/vector/{BaseNullableVariableWidthVector.java => BaseVariableWidthVector.java} (98%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableBigIntVector.java => BigIntVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableDateDayVector.java => DateDayVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableDateMilliVector.java => DateMilliVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableDecimalVector.java => DecimalVector.java} (91%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableFloat4Vector.java => Float4Vector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableFloat8Vector.java => Float8Vector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableIntVector.java => IntVector.java} (91%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableIntervalDayVector.java => IntervalDayVector.java} (91%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableIntervalYearVector.java => IntervalYearVector.java} (90%) delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java rename java/vector/src/main/java/org/apache/arrow/vector/{NullableSmallIntVector.java => SmallIntVector.java} (91%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeMicroVector.java => TimeMicroVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeMilliVector.java => TimeMilliVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeNanoVector.java => TimeNanoVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeSecVector.java => TimeSecVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMicroTZVector.java => TimeStampMicroTZVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMicroVector.java => TimeStampMicroVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMilliTZVector.java => TimeStampMilliTZVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampMilliVector.java => TimeStampMilliVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampNanoTZVector.java => TimeStampNanoTZVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampNanoVector.java => TimeStampNanoVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampSecTZVector.java => TimeStampSecTZVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampSecVector.java => TimeStampSecVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTimeStampVector.java => TimeStampVector.java} (88%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableTinyIntVector.java => TinyIntVector.java} (91%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt1Vector.java => UInt1Vector.java} (91%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt2Vector.java => UInt2Vector.java} (91%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt4Vector.java => UInt4Vector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableUInt8Vector.java => UInt8Vector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableVarBinaryVector.java => VarBinaryVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableVarCharVector.java => VarCharVector.java} (90%) rename java/vector/src/main/java/org/apache/arrow/vector/{NullableVectorDefinitionSetter.java => VectorDefinitionSetter.java} (94%) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index d8693c596155b..47b5541d17e4e 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -35,9 +35,9 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.NullableIntVector; -import org.apache.arrow.vector.NullableTinyIntVector; -import org.apache.arrow.vector.NullableVarCharVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.impl.UnionListWriter; @@ -92,7 +92,7 @@ public static void stopEchoServer() throws IOException, InterruptedException { private void testEchoServer(int serverPort, Field field, - NullableTinyIntVector vector, + TinyIntVector vector, int batches) throws UnknownHostException, IOException { VectorSchemaRoot root = new VectorSchemaRoot(asList(field), asList((FieldVector) vector), 0); @@ -115,7 +115,7 @@ private void testEchoServer(int serverPort, assertEquals(new Schema(asList(field)), reader.getVectorSchemaRoot().getSchema()); - NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot() + TinyIntVector readVector = (TinyIntVector) reader.getVectorSchemaRoot() .getFieldVectors().get(0); for (int i = 0; i < batches; i++) { Assert.assertTrue(reader.loadNextBatch()); @@ -140,8 +140,8 @@ public void basicTest() throws InterruptedException, IOException { "testField", new FieldType(true, new ArrowType.Int(8, true), null, null), Collections.emptyList()); - NullableTinyIntVector vector = - new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc); + TinyIntVector vector = + new TinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc); Schema schema = new Schema(asList(field)); // Try an empty stream, just the header. @@ -158,13 +158,13 @@ public void basicTest() throws InterruptedException, IOException { public void testFlatDictionary() throws IOException { DictionaryEncoding writeEncoding = new DictionaryEncoding(1L, false, null); try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); - NullableIntVector writeVector = - new NullableIntVector( + IntVector writeVector = + new IntVector( "varchar", new FieldType(true, MinorType.INT.getType(), writeEncoding, null), allocator); - NullableVarCharVector writeDictionaryVector = - new NullableVarCharVector( + VarCharVector writeDictionaryVector = + new VarCharVector( "dict", FieldType.nullable(VARCHAR.getType()), allocator)) { @@ -218,7 +218,7 @@ public void testFlatDictionary() throws IOException { Dictionary dictionary = reader.lookup(1L); Assert.assertNotNull(dictionary); - NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + VarCharVector dictionaryVector = ((VarCharVector) dictionary.getVector()); Assert.assertEquals(3, dictionaryVector.getValueCount()); Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); @@ -231,8 +231,8 @@ public void testFlatDictionary() throws IOException { public void testNestedDictionary() throws IOException { DictionaryEncoding writeEncoding = new DictionaryEncoding(2L, false, null); try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); - NullableVarCharVector writeDictionaryVector = - new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator); + VarCharVector writeDictionaryVector = + new VarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator); ListVector writeVector = ListVector.empty("list", allocator)) { // data being written: @@ -300,7 +300,7 @@ public void testNestedDictionary() throws IOException { Dictionary readDictionary = reader.lookup(2L); Assert.assertNotNull(readDictionary); - NullableVarCharVector dictionaryVector = ((NullableVarCharVector) readDictionary.getVector()); + VarCharVector dictionaryVector = ((VarCharVector) readDictionary.getVector()); Assert.assertEquals(2, dictionaryVector.getValueCount()); Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java index 5357f9b8a9d3a..7c652902eb8f5 100644 --- a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java +++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java @@ -23,7 +23,7 @@ <#if mode == "Single"> <#assign containerClass = "MapVector" /> <#else> -<#assign containerClass = "NullableMapVector" /> +<#assign containerClass = "MapVector" /> <#include "/@includes/license.ftl" /> diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index 60347a2fcb8e2..4863ecdb63284 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -53,9 +53,9 @@ @SuppressWarnings("unused") public class ${name}ReaderImpl extends AbstractFieldReader { - private final ${nullMode}${name}Vector vector; + private final ${name}Vector vector; - public ${name}ReaderImpl(${nullMode}${name}Vector vector){ + public ${name}ReaderImpl(${name}Vector vector){ super(); this.vector = vector; } @@ -69,11 +69,7 @@ public Field getField(){ } public boolean isSet(){ - <#if nullMode == "Nullable"> - return !vector.isNull(idx()); - <#else> - return true; - + return !vector.isNull(idx()); } public void copyAsValue(${minor.class?cap_first}Writer writer){ @@ -88,7 +84,7 @@ public void copyAsField(String name, MapWriter writer){ <#if nullMode != "Nullable"> public void read(${minor.class?cap_first}Holder h){ - vector.getAccessor().get(idx(), h); + vector.get(idx(), h); } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 406bbb39c7f4a..98672d54cc38e 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -39,9 +39,9 @@ @SuppressWarnings("unused") public class ${eName}WriterImpl extends AbstractFieldWriter { - final Nullable${name}Vector vector; + final ${name}Vector vector; - public ${eName}WriterImpl(Nullable${name}Vector vector) { + public ${eName}WriterImpl(${name}Vector vector) { this.vector = vector; } @@ -82,7 +82,7 @@ public void write(${minor.class?cap_first}Holder h) { vector.getMutator().setValueCount(idx()+1); } - public void write(Nullable${minor.class?cap_first}Holder h) { + public void write(${minor.class?cap_first}Holder h) { mutator.addSafe(idx(), h); vector.getMutator().setValueCount(idx()+1); } @@ -111,7 +111,7 @@ public void write(Nullable${minor.class}Holder h) { } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.setSafe(idx(), 1<#list fields as field><#if field.include!true >, ${field.name}); vector.setValueCount(idx()+1); } @@ -122,13 +122,11 @@ public void write(Nullable${minor.class}Holder h) { } - <#if mode == "Nullable"> public void writeNull() { vector.setNull(idx()); vector.setValueCount(idx()+1); } - } <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/writer/${eName}Writer.java" /> diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java deleted file mode 100644 index e07416ba984d4..0000000000000 --- a/java/vector/src/main/codegen/templates/FixedValueVectors.java +++ /dev/null @@ -1,765 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.arrow.vector.util.DecimalUtility; - -import java.lang.Override; -import java.util.concurrent.TimeUnit; - -<@pp.dropOutputFile /> -<#list vv.types as type> -<#list type.minor as minor> -<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> -<#assign className = "${minor.class}Vector" /> - -<#if type.major == "Fixed"> -<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> -<#include "/@includes/license.ftl" /> - -package org.apache.arrow.vector; - -<#include "/@includes/vv_imports.ftl" /> - -/** - * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed - * by position, starting from the logical start of the vector. Values should be pushed onto the - * vector sequentially, but may be randomly accessed. - * The width of each element is ${type.width} byte(s) - * The equivalent Java primitive is '${minor.javaType!type.javaType}' - * - * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. - */ -public final class ${className} extends BaseDataValueVector implements FixedWidthVector{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); - - public static final int TYPE_WIDTH = ${type.width}; - - private final Accessor accessor = new Accessor(); - private final Mutator mutator = new Mutator(); - - private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width}; - private int allocationMonitor = 0; - <#if minor.typeParams??> - - <#assign typeParams = minor.typeParams?reverse /> - <#list typeParams as typeParam> - private final ${typeParam.type} ${typeParam.name}; - - - public ${className}(String name, BufferAllocator allocator<#list typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { - super(name, allocator); - <#list typeParams as typeParam> - this.${typeParam.name} = ${typeParam.name}; - - } - <#else> - public ${className}(String name, BufferAllocator allocator) { - super(name, allocator); - } - - - @Override - public MinorType getMinorType() { - return MinorType.${minor.class?upper_case}; - } - - @Override - public Field getField() { - throw new UnsupportedOperationException("internal vector"); - } - - @Override - public FieldReader getReader(){ - throw new UnsupportedOperationException("non-nullable vectors cannot be used in readers"); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - return valueCount * ${type.width}; - } - - @Override - public ArrowBuf getValidityBuffer() { - /* this operation is not supported for non-nullable vectors */ - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getDataBuffer() { - /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */ - return getBuffer(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - /* this operation is not supported for fixed-width vectors */ - throw new UnsupportedOperationException(); - } - - @Override - public int getValueCapacity(){ - return (int) (data.capacity() *1.0 / ${type.width}); - } - - @Override - public Accessor getAccessor(){ - return accessor; - } - - @Override - public Mutator getMutator(){ - return mutator; - } - - int getAllocationSize() { - return allocationSizeInBytes; - } - - @Override - public void setInitialCapacity(final int valueCount) { - final long size = 1L * valueCount * ${type.width}; - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size"); - } - allocationSizeInBytes = (int)size; - } - - @Override - public void allocateNew() { - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating buffer."); - } - } - - @Override - public boolean allocateNewSafe() { - long curAllocationSize = allocationSizeInBytes; - if (allocationMonitor > 10) { - curAllocationSize = Math.max(8, curAllocationSize / 2); - allocationMonitor = 0; - } else if (allocationMonitor < -2) { - curAllocationSize = allocationSizeInBytes * 2L; - allocationMonitor = 0; - } - - try{ - allocateBytes(curAllocationSize); - } catch (RuntimeException ex) { - return false; - } - return true; - } - - /** - * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger - * depending on underlying buffer rounding size. Must be called prior to using the ValueVector. - * - * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width. - * - * @param valueCount the number of values to allocate for - * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer - */ - @Override - public void allocateNew(final int valueCount) { - allocateBytes(valueCount * ${type.width}); - } - - @Override - public void reset() { - allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width}; - allocationMonitor = 0; - zeroVector(); - super.reset(); - } - - private void allocateBytes(final long size) { - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size"); - } - - final int curSize = (int)size; - clear(); - data = allocator.buffer(curSize); - data.readerIndex(0); - allocationSizeInBytes = curSize; - } - - /** - * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one - * - * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer - */ - public void reAlloc() { - long baseSize = allocationSizeInBytes; - final int currentBufferCapacity = data.capacity(); - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached."); - } - - logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", name, allocationSizeInBytes, newAllocationSize); - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, data, 0, currentBufferCapacity); - final int halfNewCapacity = newBuf.capacity() / 2; - newBuf.setZero(halfNewCapacity, halfNewCapacity); - newBuf.writerIndex(data.writerIndex()); - data.release(1); - data = newBuf; - allocationSizeInBytes = (int)newAllocationSize; - } - - /** - * {@inheritDoc} - */ - @Override - public void zeroVector() { - data.setZero(0, data.capacity()); - } - - public TransferPair getTransferPair(BufferAllocator allocator){ - return new TransferImpl(name, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${className}) to); - } - - public void transferTo(${className} target){ - target.clear(); - target.data = data.transferOwnership(target.allocator).buffer; - target.data.writerIndex(data.writerIndex()); - clear(); - } - - public void splitAndTransferTo(int startIndex, int length, ${className} target) { - final int startPoint = startIndex * ${type.width}; - final int sliceLength = length * ${type.width}; - target.clear(); - target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; - target.data.writerIndex(sliceLength); - } - - private class TransferImpl implements TransferPair{ - private ${className} to; - - public TransferImpl(String name, BufferAllocator allocator){ - to = new ${className}(name, allocator<#if minor.typeParams??><#list typeParams as typeParam>, ${className}.this.${typeParam.name}); - } - - public TransferImpl(${className} to) { - this.to = to; - } - - @Override - public ${className} getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${className}.this); - } - } - - public void copyFrom(int fromIndex, int thisIndex, ${className} from){ - <#if (type.width > 8 || minor.class == "IntervalDay")> - from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width}); - <#else> <#-- type.width <= 8 --> - data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}, - from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width}) - ); - <#-- type.width --> - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ - while(thisIndex >= getValueCapacity()) { - reAlloc(); - } - copyFrom(fromIndex, thisIndex, from); - } - - public void decrementAllocationMonitor() { - if (allocationMonitor > 0) { - allocationMonitor = 0; - } - --allocationMonitor; - } - - private void incrementAllocationMonitor() { - ++allocationMonitor; - } - - public final class Accessor extends BaseDataValueVector.BaseAccessor { - @Override - public int getValueCount() { - return data.writerIndex() / ${type.width}; - } - - @Override - public boolean isNull(int index){ - return false; - } - - <#if (type.width > 8 || minor.class == "IntervalDay")> - public ${minor.javaType!type.javaType} get(int index) { - return data.slice(index * ${type.width}, ${type.width}); - } - - <#if (minor.class == "IntervalDay")> - public void get(int index, ${minor.class}Holder holder){ - final int offsetIndex = index * ${type.width}; - holder.days = data.getInt(offsetIndex); - holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - } - - public void get(int index, Nullable${minor.class}Holder holder){ - final int offsetIndex = index * ${type.width}; - holder.isSet = 1; - holder.days = data.getInt(offsetIndex); - holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - } - - @Override - public ${friendlyType} getObject(int index) { - final int offsetIndex = index * ${type.width}; - final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - final int days = data.getInt(offsetIndex); - final Period p = new Period(); - return p.plusDays(days).plusMillis(millis); - } - - public StringBuilder getAsStringBuilder(int index) { - final int offsetIndex = index * ${type.width}; - - int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - final int days = data.getInt(offsetIndex); - - final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); - - final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); - - final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); - - final String dayString = (Math.abs(days) == 1) ? " day " : " days "; - - return(new StringBuilder(). - append(days).append(dayString). - append(hours).append(":"). - append(minutes).append(":"). - append(seconds).append("."). - append(millis)); - } - - <#elseif minor.class == "Decimal"> - public void get(int index, ${minor.class}Holder holder) { - holder.start = index * ${type.width}; - holder.buffer = data; - holder.scale = scale; - holder.precision = precision; - } - - public void get(int index, Nullable${minor.class}Holder holder) { - holder.isSet = 1; - holder.start = index * ${type.width}; - holder.buffer = data; - holder.scale = scale; - holder.precision = precision; - } - - @Override - public ${friendlyType} getObject(int index) { - return DecimalUtility.getBigDecimalFromArrowBuf(data, index, scale); - } - - <#else> - public void get(int index, ${minor.class}Holder holder){ - holder.buffer = data; - holder.start = index * ${type.width}; - } - - public void get(int index, Nullable${minor.class}Holder holder){ - holder.isSet = 1; - holder.buffer = data; - holder.start = index * ${type.width}; - } - - @Override - public ${friendlyType} getObject(int index) { - return data.slice(index * ${type.width}, ${type.width}) - } - - - <#else> <#-- type.width <= 8 --> - - public ${minor.javaType!type.javaType} get(int index) { - return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - } - <#if type.width == 4> - public long getTwoAsLong(int index) { - return data.getLong(index * ${type.width}); - } - - - <#if minor.class == "DateDay" || - minor.class == "TimeSec" || - minor.class == "TimeMicro" || - minor.class == "TimeNano"> - @Override - public ${friendlyType} getObject(int index) { - return get(index); - } - - <#elseif minor.class == "DateMilli" || minor.class == "TimeMilli" || minor.class == "TimeStampMilli"> - @Override - public ${friendlyType} getObject(int index) { - org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC); - return ldt; - } - - <#elseif minor.class == "TimeStampSec"> - @Override - public ${friendlyType} getObject(int index) { - long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index)); - org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC); - return date; - } - - <#elseif minor.class == "TimeStampMicro"> - @Override - public ${friendlyType} getObject(int index) { - // value is truncated when converting microseconds to milliseconds in order to use DateTime type - long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index)); - org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC); - return date; - } - - <#elseif minor.class == "TimeStampNano"> - @Override - public ${friendlyType} getObject(int index) { - // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type - long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index)); - org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); - return date; - } - - <#elseif minor.class == "IntervalYear"> - @Override - public ${friendlyType} getObject(int index) { - - final int value = get(index); - - final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final Period p = new Period(); - return p.plusYears(years).plusMonths(months); - } - - public StringBuilder getAsStringBuilder(int index) { - int months = data.getInt(index); - - final int years = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths); - months = (months % org.apache.arrow.vector.util.DateUtility.yearsToMonths); - - final String yearString = (Math.abs(years) == 1) ? " year " : " years "; - final String monthString = (Math.abs(months) == 1) ? " month " : " months "; - - return(new StringBuilder(). - append(years).append(yearString). - append(months).append(monthString)); - } - - <#else> - @Override - public ${friendlyType} getObject(int index) { - return get(index); - } - public ${minor.javaType!type.javaType} getPrimitiveObject(int index) { - return get(index); - } - - - public void get(int index, ${minor.class}Holder holder){ - holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - } - - public void get(int index, Nullable${minor.class}Holder holder){ - holder.isSet = 1; - holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - } - - <#-- type.width --> - } - - /** - * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the - * vector are accessed by position from the logical start of the vector. Values should be pushed - * onto the vector sequentially, but may be randomly accessed. - * The width of each element is ${type.width} byte(s) - * The equivalent Java primitive is '${minor.javaType!type.javaType}' - * - * NB: this class is automatically generated from FixedValueVectorTypes.tdd using FreeMarker. - */ - public final class Mutator extends BaseDataValueVector.BaseMutator { - - private Mutator(){}; - - /** - * Set the element at the given index to the given value. Note that widths smaller than - * 32 bits are handled by the ArrowBuf interface. - * - * @param index position of the bit to set - * @param value value to set - */ - <#if (type.width > 8) || minor.class == "IntervalDay"> - public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int value) { - data.setBytes(index * ${type.width}, value, 0, ${type.width}); - } - - public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int value) { - while(index >= getValueCapacity()) { - reAlloc(); - } - data.setBytes(index * ${type.width}, value, 0, ${type.width}); - } - - <#if (minor.class == "IntervalDay")> - public void set(int index, int days, int milliseconds){ - final int offsetIndex = index * ${type.width}; - data.setInt(offsetIndex, days); - data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); - } - - protected void set(int index, ${minor.class}Holder holder){ - set(index, holder.days, holder.milliseconds); - } - - protected void set(int index, Nullable${minor.class}Holder holder){ - set(index, holder.days, holder.milliseconds); - } - - public void setSafe(int index, int days, int milliseconds){ - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, days, milliseconds); - } - - public void setSafe(int index, ${minor.class}Holder holder){ - setSafe(index, holder.days, holder.milliseconds); - } - - public void setSafe(int index, Nullable${minor.class}Holder holder){ - setSafe(index, holder.days, holder.milliseconds); - } - - <#elseif minor.class == "Decimal"> - public void set(int index, ${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); - } - - void set(int index, Nullable${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); - } - - public void setSafe(int index, Nullable${minor.class}Holder holder){ - setSafe(index, holder.start, holder.buffer); - } - public void setSafe(int index, ${minor.class}Holder holder){ - setSafe(index, holder.start, holder.buffer); - } - - public void setSafe(int index, int start, ArrowBuf buffer){ - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, start, buffer); - } - - public void set(int index, int start, ArrowBuf buffer){ - data.setBytes(index * ${type.width}, buffer, start, ${type.width}); - } - - public void set(int index, ${friendlyType} value){ - DecimalUtility.checkPrecisionAndScale(value, precision, scale); - DecimalUtility.writeBigDecimalToArrowBuf(value, data, index); - } - - public void setSafe(int index, ${friendlyType} value){ - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, value); - } - - <#else> - protected void set(int index, ${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); - } - - public void set(int index, Nullable${minor.class}Holder holder){ - set(index, holder.start, holder.buffer); - } - - public void set(int index, int start, ArrowBuf buffer){ - data.setBytes(index * ${type.width}, buffer, start, ${type.width}); - } - - public void setSafe(int index, ${minor.class}Holder holder){ - setSafe(index, holder.start, holder.buffer); - } - - public void setSafe(int index, Nullable${minor.class}Holder holder){ - setSafe(index, holder.start, holder.buffer); - } - - public void setSafe(int index, int start, ArrowBuf buffer){ - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, holder); - } - - public void set(int index, Nullable${minor.class}Holder holder){ - data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width}); - } - - - @Override - public void generateTestData(int count) { - setValueCount(count); - boolean even = true; - final int valueCount = getAccessor().getValueCount(); - for(int i = 0; i < valueCount; i++, even = !even) { - final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE; - for(int w = 0; w < ${type.width}; w++){ - data.setByte(i + w, b); - } - } - } - - <#else> <#-- type.width <= 8 --> - public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int value) { - data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value); - } - - public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int value) { - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, value); - } - - protected void set(int index, ${minor.class}Holder holder){ - data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); - } - - public void setSafe(int index, ${minor.class}Holder holder){ - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, holder); - } - - protected void set(int index, Nullable${minor.class}Holder holder){ - data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); - } - - public void setSafe(int index, Nullable${minor.class}Holder holder){ - while(index >= getValueCapacity()) { - reAlloc(); - } - set(index, holder); - } - - @Override - public void generateTestData(int size) { - setValueCount(size); - boolean even = true; - final int valueCount = getAccessor().getValueCount(); - for(int i = 0; i < valueCount; i++, even = !even) { - if(even){ - set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE); - }else{ - set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE); - } - } - } - - public void generateTestDataAlt(int size) { - setValueCount(size); - boolean even = true; - final int valueCount = getAccessor().getValueCount(); - for(int i = 0; i < valueCount; i++, even = !even) { - if(even){ - set(i, (${(minor.javaType!type.javaType)}) 1); - }else{ - set(i, (${(minor.javaType!type.javaType)}) 0); - } - } - } - - <#-- type.width --> - - @Override - public void setValueCount(int valueCount) { - final int currentValueCapacity = getValueCapacity(); - final int idx = (${type.width} * valueCount); - while(valueCount > getValueCapacity()) { - reAlloc(); - } - if (valueCount > 0 && currentValueCapacity > valueCount * 2) { - incrementAllocationMonitor(); - } else if (allocationMonitor > 0) { - allocationMonitor = 0; - } - VectorTrimmer.trim(data, idx); - data.writerIndex(valueCount * ${type.width}); - } - } -} - - <#-- type.major --> - - diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index 6afd508e68837..a5ac1b7170442 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -21,9 +21,9 @@ <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" /> <#assign index = "idx()"> <#if mode == "Single"> -<#assign containerClass = "MapVector" /> +<#assign containerClass = "NonNullableMapVector" /> <#else> -<#assign containerClass = "NullableMapVector" /> +<#assign containerClass = "MapVector" /> <#include "/@includes/license.ftl" /> @@ -51,7 +51,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter { private final Map fields = Maps.newHashMap(); public ${mode}MapWriter(${containerClass} container) { <#if mode == "Single"> - if (container instanceof NullableMapVector) { + if (container instanceof MapVector) { throw new IllegalArgumentException("Invalid container: " + container); } @@ -124,7 +124,7 @@ public MapWriter map(String name) { FieldWriter writer = fields.get(finalName); if(writer == null){ int vectorCount=container.size(); - NullableMapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class); + MapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), MapVector.class); writer = new PromotableWriter(vector, container, getNullableMapWriterFactory()); if(vectorCount != container.size()) { writer.allocate(); @@ -214,7 +214,6 @@ public void end() { <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> <#assign vectName = capName /> - <#assign vectName = "Nullable${capName}" /> <#if minor.typeParams?? > @Override diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 73165315ea556..501933f8fbc31 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -65,7 +65,7 @@ public class UnionVector implements FieldVector { MapVector internalMap; protected ArrowBuf typeBuffer; - private NullableMapVector mapVector; + private MapVector mapVector; private ListVector listVector; private FieldReader reader; @@ -174,10 +174,10 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public NullableMapVector getMap() { + public MapVector getMap() { if (mapVector == null) { int vectorCount = internalMap.size(); - mapVector = addOrGet(MinorType.MAP, NullableMapVector.class); + mapVector = addOrGet(MinorType.MAP, MapVector.class); if (internalMap.size() > vectorCount) { mapVector.allocateNew(); if (callBack != null) { @@ -195,12 +195,12 @@ public NullableMapVector getMap() { <#assign lowerCaseName = name?lower_case/> <#if !minor.typeParams?? > - private Nullable${name}Vector ${uncappedName}Vector; + private ${name}Vector ${uncappedName}Vector; - public Nullable${name}Vector get${name}Vector() { + public ${name}Vector get${name}Vector() { if (${uncappedName}Vector == null) { int vectorCount = internalMap.size(); - ${uncappedName}Vector = addOrGet(MinorType.${name?upper_case}, Nullable${name}Vector.class); + ${uncappedName}Vector = addOrGet(MinorType.${name?upper_case}, ${name}Vector.class); if (internalMap.size() > vectorCount) { ${uncappedName}Vector.allocateNew(); if (callBack != null) { diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java deleted file mode 100644 index 3934e74f11b2d..0000000000000 --- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java +++ /dev/null @@ -1,677 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.lang.Override; - -import org.apache.drill.exec.exception.OutOfMemoryException; -import org.apache.drill.exec.vector.BaseDataValueVector; -import org.apache.drill.exec.vector.BaseValueVector; -import org.apache.drill.exec.vector.VariableWidthVector; - -<@pp.dropOutputFile /> -<#list vv.types as type> -<#list type.minor as minor> - -<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> -<#assign className = "${minor.class}Vector" /> - -<#if type.major == "VarLen"> -<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" /> - -<#include "/@includes/license.ftl" /> - -package org.apache.arrow.vector; - -<#include "/@includes/vv_imports.ftl" /> - -/** - * ${minor.class}Vector implements a vector of variable width values. Elements in the vector - * are accessed by position from the logical start of the vector. A fixed width offsetVector - * is used to convert an element's position to it's offset from the start of the (0-based) - * ArrowBuf. Size is inferred by adjacent elements. - * The width of each element is ${type.width} byte(s) - * The equivalent Java primitive is '${minor.javaType!type.javaType}' - * - * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. - */ -public final class ${className} extends BaseDataValueVector implements VariableWidthVector{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); - - private static final int DEFAULT_RECORD_BYTE_COUNT = 8; - private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT; - private static final int MIN_BYTE_COUNT = 4096; - - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; - final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(OFFSETS_VECTOR_NAME, allocator); - - private final Accessor accessor; - private final Mutator mutator; - - private final UInt${type.width}Vector.Accessor oAccessor; - - private int allocationSizeInBytes = INITIAL_BYTE_COUNT; - private int allocationMonitor = 0; - - <#if minor.typeParams??> - <#list minor.typeParams as typeParam> - private final ${typeParam.type} ${typeParam.name}; - - - public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { - super(name, allocator); - this.oAccessor = offsetVector.getAccessor(); - this.accessor = new Accessor(); - this.mutator = new Mutator(); - <#list minor.typeParams as typeParam> - this.${typeParam.name} = ${typeParam.name}; - - } - <#else> - public ${className}(String name, BufferAllocator allocator) { - super(name, allocator); - this.oAccessor = offsetVector.getAccessor(); - this.accessor = new Accessor(); - this.mutator = new Mutator(); - } - - - @Override - public Field getField() { - throw new UnsupportedOperationException("internal vector"); - } - - @Override - public MinorType getMinorType() { - return MinorType.${minor.class?upper_case}; - } - - @Override - public FieldReader getReader(){ - throw new UnsupportedOperationException("internal vector"); - } - - @Override - public int getBufferSize(){ - if (getAccessor().getValueCount() == 0) { - return 0; - } - return offsetVector.getBufferSize() + data.writerIndex(); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - - final int idx = offsetVector.getAccessor().get(valueCount); - return offsetVector.getBufferSizeFor(valueCount + 1) + idx; - } - - @Override - public ArrowBuf getValidityBuffer() { - /* this operation is not supported for non-nullable vectors */ - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getDataBuffer() { - /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */ - return getBuffer(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - /* dataBuffer associated with the underlying offsetVector */ - return offsetVector.getDataBuffer(); - } - - @Override - public int getValueCapacity(){ - return Math.max(offsetVector.getValueCapacity() - 1, 0); - } - - @Override - public int getByteCapacity(){ - return data.capacity(); - } - - @Override - public int getCurrentSizeInBytes() { - return offsetVector.getAccessor().get(getAccessor().getValueCount()); - } - - /** - * Return the number of bytes contained in the current var len byte vector. - * @return the number of bytes contained in the current var len byte vector - */ - public int getVarByteLength(){ - final int valueCount = getAccessor().getValueCount(); - if(valueCount == 0) { - return 0; - } - return offsetVector.getAccessor().get(valueCount); - } - - @Override - public void clear() { - super.clear(); - offsetVector.clear(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), ArrowBuf.class); - if (clear) { - // does not make much sense but we have to retain buffers even when clear is set. refactor this interface. - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); - } - return buffers; - } - - public long getOffsetAddr(){ - return offsetVector.getDataBuffer().memoryAddress(); - } - - public UInt${type.width}Vector getOffsetVector(){ - return offsetVector; - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator){ - return new TransferImpl(name, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${className}) to); - } - - public void transferTo(${className} target){ - target.clear(); - this.offsetVector.transferTo(target.offsetVector); - target.data = data.transferOwnership(target.allocator).buffer; - target.data.writerIndex(data.writerIndex()); - clear(); - } - - public void splitAndTransferTo(int startIndex, int length, ${className} target) { - UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor(); - final int startPoint = offsetVectorAccessor.get(startIndex); - final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint; - target.clear(); - target.offsetVector.allocateNew(length + 1); - offsetVectorAccessor = this.offsetVector.getAccessor(); - final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator(); - for (int i = 0; i < length + 1; i++) { - targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint); - } - target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; - target.getMutator().setValueCount(length); -} - - protected void copyFrom(int fromIndex, int thisIndex, ${className} from){ - final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); - final int start = fromOffsetVectorAccessor.get(fromIndex); - final int end = fromOffsetVectorAccessor.get(fromIndex + 1); - final int len = end - start; - - final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); - from.data.getBytes(start, data, outputStart, len); - offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len); - } - - public boolean copyFromSafe(int fromIndex, int thisIndex, ${className} from){ - final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); - final int start = fromOffsetVectorAccessor.get(fromIndex); - final int end = fromOffsetVectorAccessor.get(fromIndex + 1); - final int len = end - start; - final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}); - - while(data.capacity() < outputStart + len) { - reAlloc(); - } - - offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len); - from.data.getBytes(start, data, outputStart, len); - return true; - } - - private class TransferImpl implements TransferPair{ - ${className} to; - - public TransferImpl(String name, BufferAllocator allocator){ - to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>, ${className}.this.${typeParam.name}); - } - - public TransferImpl(${className} to){ - this.to = to; - } - - @Override - public ${className} getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${className}.this); - } - } - - @Override - public void setInitialCapacity(final int valueCount) { - final long size = 1L * valueCount * ${type.width}; - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size"); - } - allocationSizeInBytes = (int)size; - offsetVector.setInitialCapacity(valueCount + 1); - } - - @Override - public void allocateNew() { - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating buffer."); - } - } - - @Override - public boolean allocateNewSafe() { - long curAllocationSize = allocationSizeInBytes; - if (allocationMonitor > 10) { - curAllocationSize = Math.max(MIN_BYTE_COUNT, curAllocationSize / 2); - allocationMonitor = 0; - } else if (allocationMonitor < -2) { - curAllocationSize = curAllocationSize * 2L; - allocationMonitor = 0; - } - - if (curAllocationSize > MAX_ALLOCATION_SIZE) { - return false; - } - - clear(); - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - try { - final int requestedSize = (int)curAllocationSize; - data = allocator.buffer(requestedSize); - allocationSizeInBytes = requestedSize; - offsetVector.allocateNew(); - } catch (OutOfMemoryException e) { - clear(); - return false; - } - data.readerIndex(0); - offsetVector.zeroVector(); - return true; - } - - @Override - public void allocateNew(int totalBytes, int valueCount) { - clear(); - assert totalBytes >= 0; - try { - data = allocator.buffer(totalBytes); - offsetVector.allocateNew(valueCount + 1); - } catch (RuntimeException e) { - clear(); - throw e; - } - data.readerIndex(0); - allocationSizeInBytes = totalBytes; - offsetVector.zeroVector(); - } - - @Override - public void reset() { - allocationSizeInBytes = INITIAL_BYTE_COUNT; - allocationMonitor = 0; - data.readerIndex(0); - offsetVector.zeroVector(); - super.reset(); - } - - public void reAlloc() { - long baseSize = allocationSizeInBytes; - final int currentBufferCapacity = data.capacity(); - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached."); - } - - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, data, 0, currentBufferCapacity); - data.release(); - data = newBuf; - allocationSizeInBytes = (int)newAllocationSize; - } - - public void decrementAllocationMonitor() { - if (allocationMonitor > 0) { - allocationMonitor = 0; - } - --allocationMonitor; - } - - private void incrementAllocationMonitor() { - ++allocationMonitor; - } - - @Override - public Accessor getAccessor(){ - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor { - final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor(); - public long getStartEnd(int index){ - return oAccessor.getTwoAsLong(index); - } - - public byte[] get(int index) { - assert index >= 0; - final int startIdx = oAccessor.get(index); - final int length = oAccessor.get(index + 1) - startIdx; - assert length >= 0; - final byte[] dst = new byte[length]; - data.getBytes(startIdx, dst, 0, length); - return dst; - } - - @Override - public int getValueLength(int index) { - final UInt${type.width}Vector.Accessor offsetVectorAccessor = offsetVector.getAccessor(); - return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index); - } - - public void get(int index, ${minor.class}Holder holder){ - holder.start = oAccessor.get(index); - holder.end = oAccessor.get(index + 1); - holder.buffer = data; - } - - public void get(int index, Nullable${minor.class}Holder holder){ - holder.isSet = 1; - holder.start = oAccessor.get(index); - holder.end = oAccessor.get(index + 1); - holder.buffer = data; - } - - <#switch minor.class> - <#case "VarChar"> - @Override - public ${friendlyType} getObject(int index) { - Text text = new Text(); - text.set(get(index)); - return text; - } - <#break> - <#case "Decimal"> - @Override - public ${friendlyType} getObject(int index) { - return new BigDecimal(new BigInteger(get(index)), scale); - } - <#break> - <#default> - @Override - public ${friendlyType} getObject(int index) { - return get(index); - } - - - @Override - public int getValueCount() { - return Math.max(offsetVector.getAccessor().getValueCount()-1, 0); - } - - @Override - public boolean isNull(int index){ - return false; - } - - public UInt${type.width}Vector getOffsetVector(){ - return offsetVector; - } - } - - /** - * Mutable${minor.class} implements a vector of variable width values. Elements in the vector - * are accessed by position from the logical start of the vector. A fixed width offsetVector - * is used to convert an element's position to it's offset from the start of the (0-based) - * ArrowBuf. Size is inferred by adjacent elements. - * The width of each element is ${type.width} byte(s) - * The equivalent Java primitive is '${minor.javaType!type.javaType}' - * - * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. - */ - public final class Mutator extends BaseValueVector.BaseMutator implements VariableWidthVector.VariableWidthMutator { - - /** - * Set the variable length element at the specified index to the supplied byte array. - * - * @param index position of the bit to set - * @param bytes array of bytes to write - */ - protected void set(int index, byte[] bytes) { - assert index >= 0; - final int currentOffset = offsetVector.getAccessor().get(index); - offsetVector.getMutator().set(index + 1, currentOffset + bytes.length); - data.setBytes(currentOffset, bytes, 0, bytes.length); - } - - public void setSafe(int index, byte[] bytes) { - assert index >= 0; - - final int currentOffset = offsetVector.getAccessor().get(index); - while (data.capacity() < currentOffset + bytes.length) { - reAlloc(); - } - offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length); - data.setBytes(currentOffset, bytes, 0, bytes.length); - } - - /** - * Set the variable length element at the specified index to the supplied byte array. - * - * @param index position of the bit to set - * @param bytes array of bytes to write - * @param start start index of bytes to write - * @param length length of bytes to write - */ - protected void set(int index, byte[] bytes, int start, int length) { - assert index >= 0; - final int currentOffset = offsetVector.getAccessor().get(index); - offsetVector.getMutator().set(index + 1, currentOffset + length); - data.setBytes(currentOffset, bytes, start, length); - } - - public void setSafe(int index, ByteBuffer bytes, int start, int length) { - assert index >= 0; - - int currentOffset = offsetVector.getAccessor().get(index); - - while (data.capacity() < currentOffset + length) { - reAlloc(); - } - offsetVector.getMutator().setSafe(index + 1, currentOffset + length); - data.setBytes(currentOffset, bytes, start, length); - } - - public void setSafe(int index, byte[] bytes, int start, int length) { - assert index >= 0; - - final int currentOffset = offsetVector.getAccessor().get(index); - - while (data.capacity() < currentOffset + length) { - reAlloc(); - } - offsetVector.getMutator().setSafe(index + 1, currentOffset + length); - data.setBytes(currentOffset, bytes, start, length); - } - - @Override - public void setValueLengthSafe(int index, int length) { - final int offset = offsetVector.getAccessor().get(index); - while(data.capacity() < offset + length ) { - reAlloc(); - } - offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length); - } - - - public void setSafe(int index, int start, int end, ArrowBuf buffer){ - final int len = end - start; - final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - - while(data.capacity() < outputStart + len) { - reAlloc(); - } - - offsetVector.getMutator().setSafe( index+1, outputStart + len); - buffer.getBytes(start, data, outputStart, len); - } - - public void setSafe(int index, Nullable${minor.class}Holder holder){ - assert holder.isSet == 1; - - final int start = holder.start; - final int end = holder.end; - final int len = end - start; - - int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - - while(data.capacity() < outputStart + len) { - reAlloc(); - } - - holder.buffer.getBytes(start, data, outputStart, len); - offsetVector.getMutator().setSafe( index+1, outputStart + len); - } - - public void setSafe(int index, ${minor.class}Holder holder){ - final int start = holder.start; - final int end = holder.end; - final int len = end - start; - final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - - while(data.capacity() < outputStart + len) { - reAlloc(); - } - - holder.buffer.getBytes(start, data, outputStart, len); - offsetVector.getMutator().setSafe( index+1, outputStart + len); - } - - protected void set(int index, int start, int length, ArrowBuf buffer){ - assert index >= 0; - final int currentOffset = offsetVector.getAccessor().get(index); - offsetVector.getMutator().set(index + 1, currentOffset + length); - final ArrowBuf bb = buffer.slice(start, length); - data.setBytes(currentOffset, bb); - } - - protected void set(int index, Nullable${minor.class}Holder holder){ - final int length = holder.end - holder.start; - final int currentOffset = offsetVector.getAccessor().get(index); - offsetVector.getMutator().set(index + 1, currentOffset + length); - data.setBytes(currentOffset, holder.buffer, holder.start, length); - } - - protected void set(int index, ${minor.class}Holder holder){ - final int length = holder.end - holder.start; - final int currentOffset = offsetVector.getAccessor().get(index); - offsetVector.getMutator().set(index + 1, currentOffset + length); - data.setBytes(currentOffset, holder.buffer, holder.start, length); - } - - @Override - public void setValueCount(int valueCount) { - if (valueCount == 0) { - // if no values in vector, don't try to retrieve the current value count. - offsetVector.getMutator().setValueCount(0); - } else { - final int currentByteCapacity = getByteCapacity(); - final int idx = offsetVector.getAccessor().get(valueCount); - data.writerIndex(idx); - if (currentByteCapacity > idx * 2) { - incrementAllocationMonitor(); - } else if (allocationMonitor > 0) { - allocationMonitor = 0; - } - VectorTrimmer.trim(data, idx); - offsetVector.getMutator().setValueCount(valueCount+1); - } - } - - @Override - public void generateTestData(int size){ - boolean even = true; - <#switch minor.class> - <#case "Var16Char"> - final java.nio.charset.Charset charset = Charsets.UTF_16; - <#break> - <#case "VarChar"> - <#default> - final java.nio.charset.Charset charset = Charsets.UTF_8; - - final byte[] evenValue = new String("aaaaa").getBytes(charset); - final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset); - for(int i =0; i < size; i++, even = !even){ - set(i, even ? evenValue : oddValue); - } - setValueCount(size); - } - } -} - - <#-- type.major --> - - diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java similarity index 97% rename from java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index f82077f692f3d..cc056904ae8a1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -36,12 +36,12 @@ import org.apache.arrow.vector.util.TransferPair; /** - * BaseNullableFixedWidthVector provides an abstract interface for + * BaseFixedWidthVector provides an abstract interface for * implementing vectors of fixed width values. The vectors are nullable * implying that zero or more elements in the vector could be NULL. */ -public abstract class BaseNullableFixedWidthVector extends BaseValueVector - implements FixedWidthVector, FieldVector, NullableVectorDefinitionSetter { +public abstract class BaseFixedWidthVector extends BaseValueVector + implements FixedWidthVector, FieldVector, VectorDefinitionSetter { private final byte typeWidth; protected int valueAllocationSizeInBytes; @@ -53,7 +53,7 @@ public abstract class BaseNullableFixedWidthVector extends BaseValueVector protected ArrowBuf valueBuffer; protected int valueCount; - public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator, + public BaseFixedWidthVector(final String name, final BufferAllocator allocator, FieldType fieldType, final byte typeWidth) { super(name, allocator); this.typeWidth = typeWidth; @@ -66,7 +66,7 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } else { - /* specialized handling for NullableBitVector */ + /* specialized handling for BitVector */ valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); validityAllocationSizeInBytes = valueAllocationSizeInBytes; } @@ -311,7 +311,7 @@ public void allocateNew(int valueCount) { long valueBufferSize = valueCount * typeWidth; long validityBufferSize = getValidityBufferSizeFromCount(valueCount); if (typeWidth == 0) { - /* specialized handling for NullableBitVector */ + /* specialized handling for BitVector */ valueBufferSize = validityBufferSize; } @@ -556,7 +556,7 @@ private void setReaderAndWriterIndex() { } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); if (typeWidth == 0) { - /* specialized handling for NullableBitVector */ + /* specialized handling for BitVector */ valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); } else { valueBuffer.writerIndex(valueCount * typeWidth); @@ -600,7 +600,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * for accounting and management purposes. * @param target destination vector for transfer */ - public void transferTo(BaseNullableFixedWidthVector target) { + public void transferTo(BaseFixedWidthVector target) { compareTypes(target, "transferTo"); target.clear(); target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; @@ -617,7 +617,7 @@ public void transferTo(BaseNullableFixedWidthVector target) { * @param target destination vector */ public void splitAndTransferTo(int startIndex, int length, - BaseNullableFixedWidthVector target) { + BaseFixedWidthVector target) { compareTypes(target, "splitAndTransferTo"); target.clear(); splitAndTransferValidityBuffer(startIndex, length, target); @@ -629,7 +629,7 @@ public void splitAndTransferTo(int startIndex, int length, * Data buffer can always be split and transferred using slicing. */ private void splitAndTransferValueBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target) { + BaseFixedWidthVector target) { final int startPoint = startIndex * typeWidth; final int sliceLength = length * typeWidth; target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; @@ -640,7 +640,7 @@ private void splitAndTransferValueBuffer(int startIndex, int length, * the starting position of the source index. */ private void splitAndTransferValidityBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target) { + BaseFixedWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java similarity index 98% rename from java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index b9e5442ecf666..b515586bc9f36 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -24,7 +24,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; @@ -37,8 +37,8 @@ import java.util.Collections; import java.util.List; -public abstract class BaseNullableVariableWidthVector extends BaseValueVector - implements VariableWidthVector, FieldVector, NullableVectorDefinitionSetter { +public abstract class BaseVariableWidthVector extends BaseValueVector + implements VariableWidthVector, FieldVector, VectorDefinitionSetter { private static final int DEFAULT_RECORD_BYTE_COUNT = 8; private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; @@ -57,7 +57,7 @@ public abstract class BaseNullableVariableWidthVector extends BaseValueVector protected final Field field; private boolean cleared; - public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator, + public BaseVariableWidthVector(final String name, final BufferAllocator allocator, FieldType fieldType) { super(name, allocator); valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; @@ -657,7 +657,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * for accounting and management purposes. * @param target destination vector for transfer */ - public void transferTo(BaseNullableVariableWidthVector target) { + public void transferTo(BaseVariableWidthVector target) { compareTypes(target, "transferTo"); target.clear(); target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; @@ -678,7 +678,7 @@ public void transferTo(BaseNullableVariableWidthVector target) { * @param target destination vector */ public void splitAndTransferTo(int startIndex, int length, - BaseNullableVariableWidthVector target) { + BaseVariableWidthVector target) { compareTypes(target, "splitAndTransferTo"); target.clear(); splitAndTransferValidityBuffer(startIndex, length, target); @@ -697,7 +697,7 @@ public void splitAndTransferTo(int startIndex, int length, * need to explicitly allocate the offset buffer and set the adjusted offsets * in the target vector. */ - private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { + private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseVariableWidthVector target) { final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH); final int dataLength = end - start; @@ -713,7 +713,7 @@ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullab * Transfer the validity. */ private void splitAndTransferValidityBuffer(int startIndex, int length, - BaseNullableVariableWidthVector target) { + BaseVariableWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java index 8d44c92e46664..ccf0c3067e043 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java @@ -29,35 +29,35 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableBigIntVector implements a fixed width vector (8 bytes) of + * BigIntVector implements a fixed width vector (8 bytes) of * integer values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableBigIntVector extends BaseNullableFixedWidthVector { +public class BigIntVector extends BaseFixedWidthVector { public static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** - * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * Instantiate a BigIntVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableBigIntVector(String name, BufferAllocator allocator) { + public BigIntVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), allocator); } /** - * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * Instantiate a BigIntVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + public BigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new BigIntReaderImpl(NullableBigIntVector.this); + reader = new BigIntReaderImpl(BigIntVector.this); } /** @@ -136,21 +136,21 @@ public Long getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { + public void copyFrom(int fromIndex, int thisIndex, BigIntVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that + * Same as {@link #copyFrom(int, int, BigIntVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, BigIntVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -329,22 +329,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableBigIntVector) to); + return new TransferImpl((BigIntVector) to); } private class TransferImpl implements TransferPair { - NullableBigIntVector to; + BigIntVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableBigIntVector(ref, field.getFieldType(), allocator); + to = new BigIntVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableBigIntVector to) { + public TransferImpl(BigIntVector to) { this.to = to; } @Override - public NullableBigIntVector getTo() { + public BigIntVector getTo() { return to; } @@ -360,7 +360,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); + to.copyFromSafe(fromIndex, toIndex, BigIntVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 26c817008836c..b6f7323a796b4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -18,342 +18,469 @@ package org.apache.arrow.vector; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.BaseAllocator; -import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.complex.impl.BitReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.BitHolder; import org.apache.arrow.vector.holders.NullableBitHolder; -import org.apache.arrow.vector.ipc.message.ArrowFieldNode; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; -import io.netty.buffer.ArrowBuf; - /** - * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start - * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0' - * or '1'. + * BitVector implements a fixed width (1 bit) vector of + * boolean values which could be null. Each value in the vector corresponds + * to a single bit in the underlying data stream backing the vector. */ -public final class BitVector extends BaseDataValueVector implements FixedWidthVector { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class); - - private final Accessor accessor = new Accessor(); - private final Mutator mutator = new Mutator(); - - int valueCount; - private int allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); - private int allocationMonitor = 0; +public class BitVector extends BaseFixedWidthVector { + private final FieldReader reader; + /** + * Instantiate a BitVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param allocator allocator for memory management. + */ public BitVector(String name, BufferAllocator allocator) { - super(name, allocator); + this(name, FieldType.nullable(Types.MinorType.BIT.getType()), + allocator); } - @Override - public void load(ArrowFieldNode fieldNode, ArrowBuf data) { - // When the vector is all nulls or all defined, the content of the buffer can be omitted - if (data.readableBytes() == 0 && fieldNode.getLength() != 0) { - int count = fieldNode.getLength(); - allocateNew(count); - int n = getSizeFromCount(count); - if (fieldNode.getNullCount() == 0) { - // all defined - // create an all 1s buffer - // set full bytes - int fullBytesCount = count / 8; - for (int i = 0; i < fullBytesCount; ++i) { - this.data.setByte(i, 0xFF); - } - int remainder = count % 8; - // set remaining bits - if (remainder > 0) { - byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7)); - this.data.setByte(fullBytesCount, bitMask); - } - } else if (fieldNode.getNullCount() == fieldNode.getLength()) { - // all null - // create an all 0s buffer - zeroVector(); - } else { - throw new IllegalArgumentException("The buffer can be empty only if there's no data or it's all null or all defined"); - } - this.data.writerIndex(n); - } else { - super.load(fieldNode, data); - } - this.valueCount = fieldNode.getLength(); + /** + * Instantiate a BitVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public BitVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, (byte) 0); + reader = new BitReaderImpl(BitVector.this); } + /** + * Get a reader that supports reading values from this vector + * + * @return Field Reader for this vector + */ @Override - public Field getField() { - throw new UnsupportedOperationException("internal vector"); + public FieldReader getReader() { + return reader; } + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ @Override - public MinorType getMinorType() { - return MinorType.BIT; + public Types.MinorType getMinorType() { + return Types.MinorType.BIT; } + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * + * @param valueCount desired number of elements in the vector + */ @Override - public FieldReader getReader() { - throw new UnsupportedOperationException("internal vector"); + public void setInitialCapacity(int valueCount) { + final int size = getValidityBufferSizeFromCount(valueCount); + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = size; + validityAllocationSizeInBytes = size; } + /** + * Get the current value capacity for the vector + * + * @return number of elements that vector can hold. + */ @Override - public int getBufferSize() { - return getSizeFromCount(valueCount); + public int getValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); } + /** + * Get the potential buffer size for a particular number of records. + * + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override - public int getBufferSizeFor(final int valueCount) { - return getSizeFromCount(valueCount); + public int getBufferSizeFor(final int count) { + if (count == 0) { + return 0; + } + return 2 * getValidityBufferSizeFromCount(count); } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * + * @return size of underlying buffers. + */ @Override - public ArrowBuf getValidityBuffer() { - /* this operation is not supported for non-nullable vectors */ - throw new UnsupportedOperationException(); + public int getBufferSize() { + return getBufferSizeFor(valueCount); } - @Override - public ArrowBuf getDataBuffer() { - /* we are not throwing away getBuffer() of BaseDataValueVector so use it wherever applicable */ - return getBuffer(); - } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, + validityBuffer, target.validityBuffer); + target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, + valueBuffer, target.valueBuffer); - @Override - public ArrowBuf getOffsetBuffer() { - /* this operation is not supported for fixed-width vectors */ - throw new UnsupportedOperationException(); + target.setValueCount(length); } - int getSizeFromCount(int valueCount) { - return (int) Math.ceil(valueCount / 8.0); - } + private ArrowBuf splitAndTransferBuffer(int startIndex, int length, + BaseFixedWidthVector target, + ArrowBuf sourceBuffer, ArrowBuf destBuffer) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; - @Override - public int getValueCapacity() { - return (int) Math.min((long) Integer.MAX_VALUE, data.capacity() * 8L); - } + if (length > 0) { + if (offset == 0) { + /* slice */ + if (destBuffer != null) { + destBuffer.release(); + } + destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget); + destBuffer.retain(1); + } else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + destBuffer = allocator.buffer(byteSizeTarget); + destBuffer.readerIndex(0); + destBuffer.setZero(0, destBuffer.capacity()); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); + + destBuffer.setByte(i, (b1 + b2)); + } - private int getByteIndex(int index) { - return (int) Math.floor(index / 8.0); + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, + firstByteSource + byteSizeTarget, offset); + + destBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + destBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + + return destBuffer; } - @Override - public void setInitialCapacity(final int valueCount) { - allocationSizeInBytes = getSizeFromCount(valueCount); + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + private int getBit(int index) { + final int byteIndex = index >> 3; + final byte b = valueBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); } - @Override - public void allocateNew() { - if (!allocateNewSafe()) { - throw new OutOfMemoryException(); + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); } + return getBit(index); } - @Override - public boolean allocateNewSafe() { - long curAllocationSize = allocationSizeInBytes; - if (allocationMonitor > 10) { - curAllocationSize = Math.max(8, allocationSizeInBytes / 2); - allocationMonitor = 0; - } else if (allocationMonitor < -2) { - curAllocationSize = allocationSizeInBytes * 2L; - allocationMonitor = 0; + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBitHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; } + holder.isSet = 1; + holder.value = getBit(index); + } - try { - allocateBytes(curAllocationSize); - } catch (OutOfMemoryException ex) { - return false; + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Boolean getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return new Boolean(getBit(index) != 0); } - return true; } - @Override - public void reset() { - valueCount = 0; - allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); - allocationMonitor = 0; - zeroVector(); - super.reset(); + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, BitVector from) { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + BitVectorHelper.setValidityBit(valueBuffer, thisIndex, from.getBit(fromIndex)); } /** - * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. + * Same as {@link #copyFrom(int, int, BitVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. * - * @param valueCount The number of values which can be contained within this vector. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector */ - @Override - public void allocateNew(int valueCount) { - final int size = getSizeFromCount(valueCount); - allocateBytes(size); + public void copyFromSafe(int fromIndex, int thisIndex, BitVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); } - private void allocateBytes(final long size) { - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size"); - } - final int curSize = (int) size; - clear(); - data = allocator.buffer(curSize); - zeroVector(); - allocationSizeInBytes = curSize; - } + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + /** - * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element */ - public void reAlloc() { - long baseSize = allocationSizeInBytes; - final int currentBufferCapacity = data.capacity(); - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); } - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + } - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size"); + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); } + } - final int curSize = (int) newAllocationSize; - final ArrowBuf newBuf = allocator.buffer(curSize); - newBuf.setZero(0, newBuf.capacity()); - newBuf.setBytes(0, data, 0, currentBufferCapacity); - data.release(); - data = newBuf; - allocationSizeInBytes = curSize; + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BitHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } } /** - * {@inheritDoc} + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element */ - @Override - public void zeroVector() { - data.setZero(0, data.capacity()); + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); } - public void copyFrom(int inIndex, int outIndex, BitVector from) { - this.mutator.set(outIndex, from.accessor.get(inIndex)); + /** + * Same as {@link #set(int, NullableBitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); } - public void copyFromSafe(int inIndex, int outIndex, BitVector from) { - if (outIndex >= this.getValueCapacity()) { - reAlloc(); - } - copyFrom(inIndex, outIndex, from); + /** + * Same as {@link #set(int, BitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BitHolder holder) { + handleSafe(index); + set(index, holder); } - @Override - public Mutator getMutator() { - return mutator; + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); } - @Override - public Accessor getAccessor() { - return accessor; + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } } - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return new TransferImpl(name, allocator); + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); } + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return new TransferImpl(ref, allocator); } + /** + * Construct a TransferPair with a desired target vector of the same type. + * + * @param to target vector + * @return {@link TransferPair} + */ @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((BitVector) to); } - - public void transferTo(BitVector target) { - target.clear(); - target.data = data.transferOwnership(target.allocator).buffer; - target.valueCount = valueCount; - clear(); - } - - public void splitAndTransferTo(int startIndex, int length, BitVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = getByteIndex(startIndex); - int lastByteSource = getByteIndex(valueCount - 1); - int byteSizeTarget = getSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - target.clear(); - // slice - if (target.data != null) { - target.data.release(); - } - target.data = data.slice(firstByteSource, byteSizeTarget); - target.data.retain(1); - } - else { - // Copy data - // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector. - // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte. - - target.clear(); - target.allocateNew(byteSizeTarget * 8); - - // TODO maybe do this one word at a time, rather than byte? - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + i, offset); - byte b2 = getBitsFromNextByte(this.data, firstByteSource + i + 1, offset); - - target.data.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = getBitsFromNextByte(this.data, firstByteSource + byteSizeTarget, offset); - - target.data.setByte(byteSizeTarget - 1, b1 + b2); - } - else { - byte b1 = getBitsFromCurrentByte(this.data, firstByteSource + byteSizeTarget - 1, offset); - - target.data.setByte(byteSizeTarget - 1, b1); - } - } - } - target.getMutator().setValueCount(length); - } - - private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } - - private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } - private class TransferImpl implements TransferPair { BitVector to; - public TransferImpl(String name, BufferAllocator allocator) { - this.to = new BitVector(name, allocator); + public TransferImpl(String ref, BufferAllocator allocator) { + to = new BitVector(ref, field.getFieldType(), allocator); } public TransferImpl(BitVector to) { @@ -380,269 +507,4 @@ public void copyValueSafe(int fromIndex, int toIndex) { to.copyFromSafe(fromIndex, toIndex, BitVector.this); } } - - private void decrementAllocationMonitor() { - if (allocationMonitor > 0) { - allocationMonitor = 0; - } - --allocationMonitor; - } - - private void incrementAllocationMonitor() { - ++allocationMonitor; - } - - public class Accessor extends BaseAccessor { - - /** - * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set. - * - * @param index position of the bit in the vector - * @return 1 if set, otherwise 0 - */ - public final int get(int index) { - int byteIndex = index >> 3; - byte b = data.getByte(byteIndex); - int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - @Override - public boolean isNull(int index) { - return false; - } - - @Override - public final Boolean getObject(int index) { - return new Boolean(get(index) != 0); - } - - @Override - public final int getValueCount() { - return valueCount; - } - - public final void get(int index, BitHolder holder) { - holder.value = get(index); - } - - public final void get(int index, NullableBitHolder holder) { - holder.isSet = 1; - holder.value = get(index); - } - - /** - * Get the number nulls, this correspond to the number of bits set to 0 in the vector - * - * @return the number of bits set to 0 - */ - @Override - public final int getNullCount() { - int count = 0; - int sizeInBytes = getSizeFromCount(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - byte byteValue = data.getByte(i); - // Java uses two's complement binary representation, hence 11111111_b which is -1 when converted to Int - // will have 32bits set to 1. Masking the MSB and then adding it back solves the issue. - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; - // if the valueCount is not a multiple of 8, the bits on the right were counted as null bits - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; - } - } - - /** - * MutableBit implements a vector of bit-width values. Elements in the vector are accessed by position from the - * logical start of the vector. Values should be pushed onto the vector sequentially, but may be randomly accessed. - * - * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. - */ - public class Mutator extends BaseMutator { - - private Mutator() { - } - - /** - * Set the bit at the given index to the specified value. - * - * @param index position of the bit to set - * @param value value to set (either 1 or 0) - */ - public final void set(int index, int value) { - int byteIndex = byteIndex(index); - int bitIndex = bitIndex(index); - byte currentByte = data.getByte(byteIndex); - byte bitMask = (byte) (1L << bitIndex); - if (value != 0) { - currentByte |= bitMask; - } else { - currentByte -= (bitMask & currentByte); - } - data.setByte(byteIndex, currentByte); - } - - /** - * Set the bit at the given index to 1. - * - * @param index position of the bit to set - */ - public final void setToOne(int index) { - int byteIndex = byteIndex(index); - int bitIndex = bitIndex(index); - byte currentByte = data.getByte(byteIndex); - byte bitMask = (byte) (1L << bitIndex); - currentByte |= bitMask; - data.setByte(byteIndex, currentByte); - } - - /** - * set count bits to 1 in data starting at firstBitIndex - * - * @param firstBitIndex the index of the first bit to set - * @param count the number of bits to set - */ - public void setRangeToOne(int firstBitIndex, int count) { - int starByteIndex = byteIndex(firstBitIndex); - final int lastBitIndex = firstBitIndex + count; - final int endByteIndex = byteIndex(lastBitIndex); - final int startByteBitIndex = bitIndex(firstBitIndex); - final int endBytebitIndex = bitIndex(lastBitIndex); - if (count < 8 && starByteIndex == endByteIndex) { - // handles the case where we don't have a first and a last byte - byte bitMask = 0; - for (int i = startByteBitIndex; i < endBytebitIndex; ++i) { - bitMask |= (byte) (1L << i); - } - byte currentByte = data.getByte(starByteIndex); - currentByte |= bitMask; - data.setByte(starByteIndex, currentByte); - } else { - // fill in first byte (if it's not full) - if (startByteBitIndex != 0) { - byte currentByte = data.getByte(starByteIndex); - final byte bitMask = (byte) (0xFFL << startByteBitIndex); - currentByte |= bitMask; - data.setByte(starByteIndex, currentByte); - ++starByteIndex; - } - - // fill in one full byte at a time - for (int i = starByteIndex; i < endByteIndex; i++) { - data.setByte(i, 0xFF); - } - - // fill in the last byte (if it's not full) - if (endBytebitIndex != 0) { - final int byteIndex = byteIndex(lastBitIndex - endBytebitIndex); - byte currentByte = data.getByte(byteIndex); - final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7)); - currentByte |= bitMask; - data.setByte(byteIndex, currentByte); - } - - } - } - - /** - * @param absoluteBitIndex the index of the bit in the buffer - * @return the index of the byte containing that bit - */ - private int byteIndex(int absoluteBitIndex) { - return absoluteBitIndex >> 3; - } - - /** - * @param absoluteBitIndex the index of the bit in the buffer - * @return the index of the bit inside the byte - */ - private int bitIndex(int absoluteBitIndex) { - return absoluteBitIndex & 7; - } - - public final void set(int index, BitHolder holder) { - set(index, holder.value); - } - - final void set(int index, NullableBitHolder holder) { - set(index, holder.value); - } - - public void setSafe(int index, int value) { - while (index >= getValueCapacity()) { - reAlloc(); - } - set(index, value); - } - - public void setSafeToOne(int index) { - while (index >= getValueCapacity()) { - reAlloc(); - } - setToOne(index); - } - - public void setSafe(int index, BitHolder holder) { - while (index >= getValueCapacity()) { - reAlloc(); - } - set(index, holder.value); - } - - public void setSafe(int index, NullableBitHolder holder) { - while (index >= getValueCapacity()) { - reAlloc(); - } - set(index, holder.value); - } - - @Override - public final void setValueCount(int valueCount) { - int currentValueCapacity = getValueCapacity(); - BitVector.this.valueCount = valueCount; - int idx = getSizeFromCount(valueCount); - while (valueCount > getValueCapacity()) { - reAlloc(); - } - if (valueCount > 0 && currentValueCapacity > valueCount * 2) { - incrementAllocationMonitor(); - } else if (allocationMonitor > 0) { - allocationMonitor = 0; - } - VectorTrimmer.trim(data, idx); - } - - @Override - public final void generateTestData(int values) { - boolean even = true; - for (int i = 0; i < values; i++, even = !even) { - if (even) { - set(i, 1); - } - } - setValueCount(values); - } - - public void generateTestDataAlt(int size) { - setValueCount(size); - boolean even = true; - final int valueCount = getAccessor().getValueCount(); - for (int i = 0; i < valueCount; i++, even = !even) { - if (even) { - set(i, (byte) 1); - } else { - set(i, (byte) 0); - } - } - } - } - - @Override - public void clear() { - this.valueCount = 0; - super.clear(); - } -} +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java index bb14d1ac0d41c..ed8956c1eaa3d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java @@ -30,35 +30,35 @@ import org.slf4j.Logger; /** - * NullableDateDayVector implements a fixed width (4 bytes) vector of + * DateDayVector implements a fixed width (4 bytes) vector of * date values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableDateDayVector extends BaseNullableFixedWidthVector { +public class DateDayVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** - * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * Instantiate a DateDayVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableDateDayVector(String name, BufferAllocator allocator) { + public DateDayVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), allocator); } /** - * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * Instantiate a DateDayVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + public DateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new DateDayReaderImpl(NullableDateDayVector.this); + reader = new DateDayReaderImpl(DateDayVector.this); } /** @@ -138,21 +138,21 @@ public Integer getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { + public void copyFrom(int fromIndex, int thisIndex, DateDayVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that + * Same as {@link #copyFrom(int, int, DateDayVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, DateDayVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -331,22 +331,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDateDayVector) to); + return new TransferImpl((DateDayVector) to); } private class TransferImpl implements TransferPair { - NullableDateDayVector to; + DateDayVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableDateDayVector(ref, field.getFieldType(), allocator); + to = new DateDayVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableDateDayVector to) { + public TransferImpl(DateDayVector to) { this.to = to; } @Override - public NullableDateDayVector getTo() { + public DateDayVector getTo() { return to; } @@ -362,7 +362,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); + to.copyFromSafe(fromIndex, toIndex, DateDayVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java index 97205c318317c..f21b58f85fb44 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java @@ -32,35 +32,35 @@ import org.slf4j.Logger; /** - * NullableDateMilliVector implements a fixed width vector (8 bytes) of + * DateMilliVector implements a fixed width vector (8 bytes) of * date values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableDateMilliVector extends BaseNullableFixedWidthVector { +public class DateMilliVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** - * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * Instantiate a DateMilliVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableDateMilliVector(String name, BufferAllocator allocator) { + public DateMilliVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), allocator); } /** - * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * Instantiate a DateMilliVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + public DateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new DateMilliReaderImpl(NullableDateMilliVector.this); + reader = new DateMilliReaderImpl(DateMilliVector.this); } /** @@ -143,21 +143,21 @@ public LocalDateTime getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { + public void copyFrom(int fromIndex, int thisIndex, DateMilliVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that + * Same as {@link #copyFrom(int, int, DateMilliVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, DateMilliVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -336,22 +336,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDateMilliVector) to); + return new TransferImpl((DateMilliVector) to); } private class TransferImpl implements TransferPair { - NullableDateMilliVector to; + DateMilliVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); + to = new DateMilliVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableDateMilliVector to) { + public TransferImpl(DateMilliVector to) { this.to = to; } @Override - public NullableDateMilliVector getTo() { + public DateMilliVector getTo() { return to; } @@ -367,7 +367,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); + to.copyFromSafe(fromIndex, toIndex, DateMilliVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java similarity index 91% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 5d2782a4c8460..71a74cb9f3338 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -32,11 +32,11 @@ import java.math.BigDecimal; /** - * NullableDecimalVector implements a fixed width vector (16 bytes) of + * DecimalVector implements a fixed width vector (16 bytes) of * decimal values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableDecimalVector extends BaseNullableFixedWidthVector { +public class DecimalVector extends BaseFixedWidthVector { public static final byte TYPE_WIDTH = 16; private final FieldReader reader; @@ -44,28 +44,28 @@ public class NullableDecimalVector extends BaseNullableFixedWidthVector { private final int scale; /** - * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * Instantiate a DecimalVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableDecimalVector(String name, BufferAllocator allocator, + public DecimalVector(String name, BufferAllocator allocator, int precision, int scale) { this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), allocator); } /** - * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * Instantiate a DecimalVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { + public DecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal) fieldType.getType(); - reader = new DecimalReaderImpl(NullableDecimalVector.this); + reader = new DecimalReaderImpl(DecimalVector.this); this.precision = arrowType.getPrecision(); this.scale = arrowType.getScale(); } @@ -150,21 +150,21 @@ public BigDecimal getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { + public void copyFrom(int fromIndex, int thisIndex, DecimalVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, thisIndex * TYPE_WIDTH, TYPE_WIDTH); } /** - * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that + * Same as {@link #copyFrom(int, int, DecimalVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, DecimalVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -385,23 +385,23 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDecimalVector) to); + return new TransferImpl((DecimalVector) to); } private class TransferImpl implements TransferPair { - NullableDecimalVector to; + DecimalVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, - NullableDecimalVector.this.scale); + to = new DecimalVector(ref, allocator, DecimalVector.this.precision, + DecimalVector.this.scale); } - public TransferImpl(NullableDecimalVector to) { + public TransferImpl(DecimalVector to) { this.to = to; } @Override - public NullableDecimalVector getTo() { + public DecimalVector getTo() { return to; } @@ -417,7 +417,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); + to.copyFromSafe(fromIndex, toIndex, DecimalVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java rename to java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java index 5d7128d2bdfa3..dc78bfde2a228 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java @@ -29,35 +29,35 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableFloat4Vector implements a fixed width vector (4 bytes) of + * Float4Vector implements a fixed width vector (4 bytes) of * float values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableFloat4Vector extends BaseNullableFixedWidthVector { +public class Float4Vector extends BaseFixedWidthVector { public static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** - * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * Instantiate a Float4Vector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableFloat4Vector(String name, BufferAllocator allocator) { + public Float4Vector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), allocator); } /** - * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * Instantiate a Float4Vector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + public Float4Vector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new Float4ReaderImpl(NullableFloat4Vector.this); + reader = new Float4ReaderImpl(Float4Vector.this); } /** @@ -137,21 +137,21 @@ public Float getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { + public void copyFrom(int fromIndex, int thisIndex, Float4Vector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final float value = from.valueBuffer.getFloat(fromIndex * TYPE_WIDTH); valueBuffer.setFloat(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that + * Same as {@link #copyFrom(int, int, Float4Vector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, Float4Vector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -330,22 +330,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableFloat4Vector) to); + return new TransferImpl((Float4Vector) to); } private class TransferImpl implements TransferPair { - NullableFloat4Vector to; + Float4Vector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); + to = new Float4Vector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableFloat4Vector to) { + public TransferImpl(Float4Vector to) { this.to = to; } @Override - public NullableFloat4Vector getTo() { + public Float4Vector getTo() { return to; } @@ -361,7 +361,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); + to.copyFromSafe(fromIndex, toIndex, Float4Vector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java rename to java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java index 7aa546150b319..1b410b868795d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java @@ -29,35 +29,35 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableFloat8Vector implements a fixed width vector (8 bytes) of + * Float8Vector implements a fixed width vector (8 bytes) of * double values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableFloat8Vector extends BaseNullableFixedWidthVector { +public class Float8Vector extends BaseFixedWidthVector { public static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** - * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * Instantiate a Float8Vector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableFloat8Vector(String name, BufferAllocator allocator) { + public Float8Vector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), allocator); } /** - * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * Instantiate a Float8Vector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + public Float8Vector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new Float8ReaderImpl(NullableFloat8Vector.this); + reader = new Float8ReaderImpl(Float8Vector.this); } /** @@ -137,21 +137,21 @@ public Double getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { + public void copyFrom(int fromIndex, int thisIndex, Float8Vector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final double value = from.valueBuffer.getDouble(fromIndex * TYPE_WIDTH); valueBuffer.setDouble(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that + * Same as {@link #copyFrom(int, int, Float8Vector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, Float8Vector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -330,22 +330,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableFloat8Vector) to); + return new TransferImpl((Float8Vector) to); } private class TransferImpl implements TransferPair { - NullableFloat8Vector to; + Float8Vector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); + to = new Float8Vector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableFloat8Vector to) { + public TransferImpl(Float8Vector to) { this.to = to; } @Override - public NullableFloat8Vector getTo() { + public Float8Vector getTo() { return to; } @@ -361,7 +361,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); + to.copyFromSafe(fromIndex, toIndex, Float8Vector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java index 806beb5f8fedc..89e2a02f6ac2d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java @@ -30,62 +30,62 @@ public class GenerateSampleData { public static void generateTestData(final ValueVector vector, final int valueCount) { - if (vector instanceof NullableIntVector) { - writeIntData((NullableIntVector) vector, valueCount); - } else if (vector instanceof NullableDecimalVector) { - writeDecimalData((NullableDecimalVector) vector, valueCount); - } else if (vector instanceof NullableBitVector) { - writeBooleanData((NullableBitVector) vector, valueCount); - } else if (vector instanceof NullableVarCharVector) { - writeVarCharData((NullableVarCharVector) vector, valueCount); - } else if (vector instanceof NullableVarBinaryVector) { - writeVarBinaryData((NullableVarBinaryVector) vector, valueCount); - } else if (vector instanceof NullableBigIntVector) { - writeBigIntData((NullableBigIntVector) vector, valueCount); - } else if (vector instanceof NullableFloat4Vector) { - writeFloatData((NullableFloat4Vector) vector, valueCount); - } else if (vector instanceof NullableFloat8Vector) { - writeDoubleData((NullableFloat8Vector) vector, valueCount); - } else if (vector instanceof NullableDateDayVector) { - writeDateDayData((NullableDateDayVector) vector, valueCount); - } else if (vector instanceof NullableDateMilliVector) { - writeDateMilliData((NullableDateMilliVector) vector, valueCount); - } else if (vector instanceof NullableIntervalDayVector) { - writeIntervalDayData((NullableIntervalDayVector) vector, valueCount); - } else if (vector instanceof NullableIntervalYearVector) { - writeIntervalYearData((NullableIntervalYearVector) vector, valueCount); - } else if (vector instanceof NullableSmallIntVector) { - writeSmallIntData((NullableSmallIntVector) vector, valueCount); - } else if (vector instanceof NullableTinyIntVector) { - writeTinyIntData((NullableTinyIntVector) vector, valueCount); - } else if (vector instanceof NullableTimeMicroVector) { - writeTimeMicroData((NullableTimeMicroVector) vector, valueCount); - } else if (vector instanceof NullableTimeMilliVector) { - writeTimeMilliData((NullableTimeMilliVector) vector, valueCount); - } else if (vector instanceof NullableTimeNanoVector) { - writeTimeNanoData((NullableTimeNanoVector) vector, valueCount); - } else if (vector instanceof NullableTimeSecVector) { - writeTimeSecData((NullableTimeSecVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampSecVector) { - writeTimeStampData((NullableTimeStampSecVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampMicroVector) { - writeTimeStampData((NullableTimeStampMicroVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampMilliVector) { - writeTimeStampData((NullableTimeStampMilliVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampNanoVector) { - writeTimeStampData((NullableTimeStampNanoVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampSecTZVector) { - writeTimeStampData((NullableTimeStampSecTZVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampMicroTZVector) { - writeTimeStampData((NullableTimeStampMicroTZVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampMilliTZVector) { - writeTimeStampData((NullableTimeStampMilliTZVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampNanoTZVector) { - writeTimeStampData((NullableTimeStampNanoTZVector) vector, valueCount); + if (vector instanceof IntVector) { + writeIntData((IntVector) vector, valueCount); + } else if (vector instanceof DecimalVector) { + writeDecimalData((DecimalVector) vector, valueCount); + } else if (vector instanceof BitVector) { + writeBooleanData((BitVector) vector, valueCount); + } else if (vector instanceof VarCharVector) { + writeVarCharData((VarCharVector) vector, valueCount); + } else if (vector instanceof VarBinaryVector) { + writeVarBinaryData((VarBinaryVector) vector, valueCount); + } else if (vector instanceof BigIntVector) { + writeBigIntData((BigIntVector) vector, valueCount); + } else if (vector instanceof Float4Vector) { + writeFloatData((Float4Vector) vector, valueCount); + } else if (vector instanceof Float8Vector) { + writeDoubleData((Float8Vector) vector, valueCount); + } else if (vector instanceof DateDayVector) { + writeDateDayData((DateDayVector) vector, valueCount); + } else if (vector instanceof DateMilliVector) { + writeDateMilliData((DateMilliVector) vector, valueCount); + } else if (vector instanceof IntervalDayVector) { + writeIntervalDayData((IntervalDayVector) vector, valueCount); + } else if (vector instanceof IntervalYearVector) { + writeIntervalYearData((IntervalYearVector) vector, valueCount); + } else if (vector instanceof SmallIntVector) { + writeSmallIntData((SmallIntVector) vector, valueCount); + } else if (vector instanceof TinyIntVector) { + writeTinyIntData((TinyIntVector) vector, valueCount); + } else if (vector instanceof TimeMicroVector) { + writeTimeMicroData((TimeMicroVector) vector, valueCount); + } else if (vector instanceof TimeMilliVector) { + writeTimeMilliData((TimeMilliVector) vector, valueCount); + } else if (vector instanceof TimeNanoVector) { + writeTimeNanoData((TimeNanoVector) vector, valueCount); + } else if (vector instanceof TimeSecVector) { + writeTimeSecData((TimeSecVector) vector, valueCount); + } else if (vector instanceof TimeStampSecVector) { + writeTimeStampData((TimeStampSecVector) vector, valueCount); + } else if (vector instanceof TimeStampMicroVector) { + writeTimeStampData((TimeStampMicroVector) vector, valueCount); + } else if (vector instanceof TimeStampMilliVector) { + writeTimeStampData((TimeStampMilliVector) vector, valueCount); + } else if (vector instanceof TimeStampNanoVector) { + writeTimeStampData((TimeStampNanoVector) vector, valueCount); + } else if (vector instanceof TimeStampSecTZVector) { + writeTimeStampData((TimeStampSecTZVector) vector, valueCount); + } else if (vector instanceof TimeStampMicroTZVector) { + writeTimeStampData((TimeStampMicroTZVector) vector, valueCount); + } else if (vector instanceof TimeStampMilliTZVector) { + writeTimeStampData((TimeStampMilliTZVector) vector, valueCount); + } else if (vector instanceof TimeStampNanoTZVector) { + writeTimeStampData((TimeStampNanoTZVector) vector, valueCount); } } - private static void writeTimeStampData(NullableTimeStampVector vector, int valueCount) { + private static void writeTimeStampData(TimeStampVector vector, int valueCount) { final long even = 100000; final long odd = 200000; for (int i = 0; i < valueCount; i++) { @@ -98,7 +98,7 @@ private static void writeTimeStampData(NullableTimeStampVector vector, int value vector.setValueCount(valueCount); } - private static void writeDecimalData(NullableDecimalVector vector, int valueCount) { + private static void writeDecimalData(DecimalVector vector, int valueCount) { final BigDecimal even = new BigDecimal(0.0543278923); final BigDecimal odd = new BigDecimal(2.0543278923); for (int i = 0; i < valueCount; i++) { @@ -111,7 +111,7 @@ private static void writeDecimalData(NullableDecimalVector vector, int valueCoun vector.setValueCount(valueCount); } - private static void writeIntData(NullableIntVector vector, int valueCount) { + private static void writeIntData(IntVector vector, int valueCount) { final int even = 1000; final int odd = 2000; for (int i = 0; i < valueCount; i++) { @@ -124,7 +124,7 @@ private static void writeIntData(NullableIntVector vector, int valueCount) { vector.setValueCount(valueCount); } - private static void writeBooleanData(NullableBitVector vector, int valueCount) { + private static void writeBooleanData(BitVector vector, int valueCount) { final int even = 0; final int odd = 1; for (int i = 0; i < valueCount; i++) { @@ -137,7 +137,7 @@ private static void writeBooleanData(NullableBitVector vector, int valueCount) { vector.setValueCount(valueCount); } - private static void writeIntervalYearData(NullableIntervalYearVector vector, int valueCount) { + private static void writeIntervalYearData(IntervalYearVector vector, int valueCount) { final int even = 1; final int odd = 2; for (int i = 0; i < valueCount; i++) { @@ -150,7 +150,7 @@ private static void writeIntervalYearData(NullableIntervalYearVector vector, int vector.setValueCount(valueCount); } - private static void writeIntervalDayData(NullableIntervalDayVector vector, int valueCount) { + private static void writeIntervalDayData(IntervalDayVector vector, int valueCount) { for (int i = 0; i < valueCount; i++) { if (i % 2 == 0) { vector.setSafe(i, 1, 50); @@ -161,7 +161,7 @@ private static void writeIntervalDayData(NullableIntervalDayVector vector, int v vector.setValueCount(valueCount); } - private static void writeTimeSecData(NullableTimeSecVector vector, int valueCount) { + private static void writeTimeSecData(TimeSecVector vector, int valueCount) { final int even = 500; final int odd = 900; for (int i = 0; i < valueCount; i++) { @@ -174,7 +174,7 @@ private static void writeTimeSecData(NullableTimeSecVector vector, int valueCoun vector.setValueCount(valueCount); } - private static void writeTimeMilliData(NullableTimeMilliVector vector, int valueCount) { + private static void writeTimeMilliData(TimeMilliVector vector, int valueCount) { final int even = 1000; final int odd = 2000; for (int i = 0; i < valueCount; i++) { @@ -187,7 +187,7 @@ private static void writeTimeMilliData(NullableTimeMilliVector vector, int value vector.setValueCount(valueCount); } - private static void writeTimeMicroData(NullableTimeMicroVector vector, int valueCount) { + private static void writeTimeMicroData(TimeMicroVector vector, int valueCount) { final long even = 1000000000; final long odd = 2000000000; for (int i = 0; i < valueCount; i++) { @@ -201,7 +201,7 @@ private static void writeTimeMicroData(NullableTimeMicroVector vector, int value } - private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCount) { + private static void writeTimeNanoData(TimeNanoVector vector, int valueCount) { final long even = 1000000000; final long odd = 2000000000; for (int i = 0; i < valueCount; i++) { @@ -214,7 +214,7 @@ private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCo vector.setValueCount(valueCount); } - private static void writeDateDayData(NullableDateDayVector vector, int valueCount) { + private static void writeDateDayData(DateDayVector vector, int valueCount) { final int even = 1000; final int odd = 2000; for (int i = 0; i < valueCount; i++) { @@ -227,7 +227,7 @@ private static void writeDateDayData(NullableDateDayVector vector, int valueCoun vector.setValueCount(valueCount); } - private static void writeDateMilliData(NullableDateMilliVector vector, int valueCount) { + private static void writeDateMilliData(DateMilliVector vector, int valueCount) { final long even = 1000000000; final long odd = 2000000000; for (int i = 0; i < valueCount; i++) { @@ -240,7 +240,7 @@ private static void writeDateMilliData(NullableDateMilliVector vector, int value vector.setValueCount(valueCount); } - private static void writeSmallIntData(NullableSmallIntVector vector, int valueCount) { + private static void writeSmallIntData(SmallIntVector vector, int valueCount) { final short even = 10; final short odd = 20; for (int i = 0; i < valueCount; i++) { @@ -253,7 +253,7 @@ private static void writeSmallIntData(NullableSmallIntVector vector, int valueCo vector.setValueCount(valueCount); } - private static void writeTinyIntData(NullableTinyIntVector vector, int valueCount) { + private static void writeTinyIntData(TinyIntVector vector, int valueCount) { final byte even = 1; final byte odd = 2; for (int i = 0; i < valueCount; i++) { @@ -266,7 +266,7 @@ private static void writeTinyIntData(NullableTinyIntVector vector, int valueCoun vector.setValueCount(valueCount); } - private static void writeBigIntData(NullableBigIntVector vector, int valueCount) { + private static void writeBigIntData(BigIntVector vector, int valueCount) { final long even = 1000000000; final long odd = 2000000000; for (int i = 0; i < valueCount; i++) { @@ -279,7 +279,7 @@ private static void writeBigIntData(NullableBigIntVector vector, int valueCount) vector.setValueCount(valueCount); } - private static void writeFloatData(NullableFloat4Vector vector, int valueCount) { + private static void writeFloatData(Float4Vector vector, int valueCount) { final float even = 20.3f; final float odd = 40.2f; for (int i = 0; i < valueCount; i++) { @@ -292,7 +292,7 @@ private static void writeFloatData(NullableFloat4Vector vector, int valueCount) vector.setValueCount(valueCount); } - private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) { + private static void writeDoubleData(Float8Vector vector, int valueCount) { final double even = 20.2373; final double odd = 40.2378; for (int i = 0; i < valueCount; i++) { @@ -305,7 +305,7 @@ private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) vector.setValueCount(valueCount); } - private static void writeVarBinaryData(NullableVarBinaryVector vector, int valueCount) { + private static void writeVarBinaryData(VarBinaryVector vector, int valueCount) { Charset utf8Charset = Charset.forName("UTF-8"); final byte[] even = "AAAAA1".getBytes(utf8Charset); final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); @@ -319,7 +319,7 @@ private static void writeVarBinaryData(NullableVarBinaryVector vector, int value vector.setValueCount(valueCount); } - private static void writeVarCharData(NullableVarCharVector vector, int valueCount) { + private static void writeVarCharData(VarCharVector vector, int valueCount) { Charset utf8Charset = Charset.forName("UTF-8"); final byte[] even = "AAAAA1".getBytes(utf8Charset); final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java similarity index 91% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/IntVector.java index 7c6f29ad7c559..2364310e6d5fc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java @@ -29,37 +29,37 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableIntVector implements a fixed width (4 bytes) vector of + * IntVector implements a fixed width (4 bytes) vector of * integer values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableIntVector extends BaseNullableFixedWidthVector { +public class IntVector extends BaseFixedWidthVector { public static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** - * Instantiate a NullableIntVector. This doesn't allocate any memory for + * Instantiate a IntVector. This doesn't allocate any memory for * the data in vector. * * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableIntVector(String name, BufferAllocator allocator) { + public IntVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), allocator); } /** - * Instantiate a NullableIntVector. This doesn't allocate any memory for + * Instantiate a IntVector. This doesn't allocate any memory for * the data in vector. * * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + public IntVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntReaderImpl(NullableIntVector.this); + reader = new IntReaderImpl(IntVector.this); } /** @@ -142,14 +142,14 @@ public Integer getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { + public void copyFrom(int fromIndex, int thisIndex, IntVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableIntVector)} except that + * Same as {@link #copyFrom(int, int, IntVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @@ -157,7 +157,7 @@ public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, IntVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -340,22 +340,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntVector) to); + return new TransferImpl((IntVector) to); } private class TransferImpl implements TransferPair { - NullableIntVector to; + IntVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableIntVector(ref, field.getFieldType(), allocator); + to = new IntVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableIntVector to) { + public TransferImpl(IntVector to) { this.to = to; } @Override - public NullableIntVector getTo() { + public IntVector getTo() { return to; } @@ -371,7 +371,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this); + to.copyFromSafe(fromIndex, toIndex, IntVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java similarity index 91% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java index 862f6429cb8cd..481a66f4e7b29 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java @@ -30,37 +30,37 @@ import org.joda.time.Period; /** - * NullableIntervalDayVector implements a fixed width vector (8 bytes) of + * IntervalDayVector implements a fixed width vector (8 bytes) of * interval (days and milliseconds) values which could be null. * A validity buffer (bit vector) is maintained to track which elements in the * vector are null. */ -public class NullableIntervalDayVector extends BaseNullableFixedWidthVector { +public class IntervalDayVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 8; private static final byte MILLISECOND_OFFSET = 4; private final FieldReader reader; /** - * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * Instantiate a IntervalDayVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableIntervalDayVector(String name, BufferAllocator allocator) { + public IntervalDayVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), allocator); } /** - * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * Instantiate a IntervalDayVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + public IntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); + reader = new IntervalDayReaderImpl(IntervalDayVector.this); } /** @@ -185,21 +185,21 @@ private StringBuilder getAsStringBuilderHelper(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + public void copyFrom(int fromIndex, int thisIndex, IntervalDayVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, thisIndex * TYPE_WIDTH, TYPE_WIDTH); } /** - * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that + * Same as {@link #copyFrom(int, int, IntervalDayVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, IntervalDayVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -388,22 +388,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntervalDayVector) to); + return new TransferImpl((IntervalDayVector) to); } private class TransferImpl implements TransferPair { - NullableIntervalDayVector to; + IntervalDayVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); + to = new IntervalDayVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableIntervalDayVector to) { + public TransferImpl(IntervalDayVector to) { this.to = to; } @Override - public NullableIntervalDayVector getTo() { + public IntervalDayVector getTo() { return to; } @@ -419,7 +419,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); + to.copyFromSafe(fromIndex, toIndex, IntervalDayVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java index 3fba5bfeeac00..2aa728f3d6337 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java @@ -29,35 +29,35 @@ import org.joda.time.Period; /** - * NullableIntervalYearVector implements a fixed width (4 bytes) vector of + * IntervalYearVector implements a fixed width (4 bytes) vector of * interval (years and months) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableIntervalYearVector extends BaseNullableFixedWidthVector { +public class IntervalYearVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** - * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * Instantiate a IntervalYearVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableIntervalYearVector(String name, BufferAllocator allocator) { + public IntervalYearVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), allocator); } /** - * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * Instantiate a IntervalYearVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { + public IntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); + reader = new IntervalYearReaderImpl(IntervalYearVector.this); } /** @@ -169,21 +169,21 @@ private StringBuilder getAsStringBuilderHelper(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + public void copyFrom(int fromIndex, int thisIndex, IntervalYearVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that + * Same as {@link #copyFrom(int, int, IntervalYearVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, IntervalYearVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -348,22 +348,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntervalYearVector) to); + return new TransferImpl((IntervalYearVector) to); } private class TransferImpl implements TransferPair { - NullableIntervalYearVector to; + IntervalYearVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); + to = new IntervalYearVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableIntervalYearVector to) { + public TransferImpl(IntervalYearVector to) { this.to = to; } @Override - public NullableIntervalYearVector getTo() { + public IntervalYearVector getTo() { return to; } @@ -379,7 +379,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); + to.copyFromSafe(fromIndex, toIndex, IntervalYearVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java deleted file mode 100644 index 086331e13d452..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java +++ /dev/null @@ -1,513 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector; - -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.impl.BitReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holders.BitHolder; -import org.apache.arrow.vector.holders.NullableBitHolder; -import org.apache.arrow.vector.types.Types; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.OversizedAllocationException; -import org.apache.arrow.vector.util.TransferPair; - -import java.util.ArrayList; -import java.util.List; - -/** - * NullableBitVector implements a fixed width (1 bit) vector of - * boolean values which could be null. Each value in the vector corresponds - * to a single bit in the underlying data stream backing the vector. - */ -public class NullableBitVector extends BaseNullableFixedWidthVector { - private final FieldReader reader; - - /** - * Instantiate a NullableBitVector. This doesn't allocate any memory for - * the data in vector. - * - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableBitVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.BIT.getType()), - allocator); - } - - /** - * Instantiate a NullableBitVector. This doesn't allocate any memory for - * the data in vector. - * - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, (byte) 0); - reader = new BitReaderImpl(NullableBitVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader() { - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.BIT; - } - - /** - * Sets the desired value capacity for the vector. This function doesn't - * allocate any memory for the vector. - * - * @param valueCount desired number of elements in the vector - */ - @Override - public void setInitialCapacity(int valueCount) { - final int size = getValidityBufferSizeFromCount(valueCount); - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - valueAllocationSizeInBytes = size; - validityAllocationSizeInBytes = size; - } - - /** - * Get the current value capacity for the vector - * - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity() { - return (int) (validityBuffer.capacity() * 8L); - } - - /** - * Get the potential buffer size for a particular number of records. - * - * @param count desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int count) { - if (count == 0) { - return 0; - } - return 2 * getValidityBufferSizeFromCount(count); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - return getBufferSizeFor(valueCount); - } - - /** - * Slice this vector at desired index and length and transfer the - * corresponding data to the target vector. - * - * @param startIndex start position of the split in source vector. - * @param length length of the split. - * @param target destination vector - */ - public void splitAndTransferTo(int startIndex, int length, - BaseNullableFixedWidthVector target) { - compareTypes(target, "splitAndTransferTo"); - target.clear(); - target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, - validityBuffer, target.validityBuffer); - target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, - valueBuffer, target.valueBuffer); - - target.setValueCount(length); - } - - private ArrowBuf splitAndTransferBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target, - ArrowBuf sourceBuffer, ArrowBuf destBuffer) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - /* slice */ - if (destBuffer != null) { - destBuffer.release(); - } - destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget); - destBuffer.retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - destBuffer = allocator.buffer(byteSizeTarget); - destBuffer.readerIndex(0); - destBuffer.setZero(0, destBuffer.capacity()); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); - - destBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, - firstByteSource + byteSizeTarget, offset); - - destBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, - firstByteSource + byteSizeTarget - 1, offset); - destBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - - return destBuffer; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - private int getBit(int index) { - final int byteIndex = index >> 3; - final byte b = valueBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if (isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return getBit(index); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableBitHolder holder) { - if (isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = getBit(index); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Boolean getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return new Boolean(getBit(index) != 0); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); - BitVectorHelper.setValidityBit(valueBuffer, thisIndex, from.getBit(fromIndex)); - } - - /** - * Same as {@link #copyFrom(int, int, NullableBitVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - if (value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); - } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { - if (holder.isSet < 0) { - throw new IllegalArgumentException(); - } else if (holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - if (holder.value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); - } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); - } - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, BitHolder holder) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - if (holder.value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); - } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableBitHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, BitHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, BitHolder holder) { - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - /* not really needed to set the bit to 0 as long as - * the buffer always starts from 0. - */ - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableBitVector) to); - } - - private class TransferImpl implements TransferPair { - NullableBitVector to; - - public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableBitVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableBitVector to) { - this.to = to; - } - - @Override - public NullableBitVector getTo() { - return to; - } - - @Override - public void transfer() { - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); - } - } -} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java deleted file mode 100644 index b2455e9e42b4b..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector; - -public interface NullableVector extends ValueVector { - - BitVector getValidityVector(); - - ValueVector getValuesVector(); -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java similarity index 91% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java index d5c66b115acde..859e62dbb8bbb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java @@ -29,35 +29,35 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableSmallIntVector implements a fixed width (2 bytes) vector of + * SmallIntVector implements a fixed width (2 bytes) vector of * short values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableSmallIntVector extends BaseNullableFixedWidthVector { +public class SmallIntVector extends BaseFixedWidthVector { public static final byte TYPE_WIDTH = 2; private final FieldReader reader; /** - * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * Instantiate a SmallIntVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableSmallIntVector(String name, BufferAllocator allocator) { + public SmallIntVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), allocator); } /** - * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * Instantiate a SmallIntVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + public SmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new SmallIntReaderImpl(NullableSmallIntVector.this); + reader = new SmallIntReaderImpl(SmallIntVector.this); } /** @@ -137,21 +137,21 @@ public Short getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { + public void copyFrom(int fromIndex, int thisIndex, SmallIntVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final short value = from.valueBuffer.getShort(fromIndex * TYPE_WIDTH); valueBuffer.setShort(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that + * Same as {@link #copyFrom(int, int, SmallIntVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, SmallIntVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -357,22 +357,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableSmallIntVector) to); + return new TransferImpl((SmallIntVector) to); } private class TransferImpl implements TransferPair { - NullableSmallIntVector to; + SmallIntVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); + to = new SmallIntVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableSmallIntVector to) { + public TransferImpl(SmallIntVector to) { this.to = to; } @Override - public NullableSmallIntVector getTo() { + public SmallIntVector getTo() { return to; } @@ -388,7 +388,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); + to.copyFromSafe(fromIndex, toIndex, SmallIntVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java index ca8e1aacf6e0e..604cedffd4258 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java @@ -30,36 +30,36 @@ import org.slf4j.Logger; /** - * NullableTimeMicroVector implements a fixed width vector (8 bytes) of + * TimeMicroVector implements a fixed width vector (8 bytes) of * time (microsecond resolution) values which could be null. * A validity buffer (bit vector) is maintained to track which elements in the * vector are null. */ -public class NullableTimeMicroVector extends BaseNullableFixedWidthVector { +public class TimeMicroVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** - * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * Instantiate a TimeMicroVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeMicroVector(String name, BufferAllocator allocator) { + public TimeMicroVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), allocator); } /** - * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * Instantiate a TimeMicroVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); + reader = new TimeMicroReaderImpl(TimeMicroVector.this); } /** @@ -138,21 +138,21 @@ public Long getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + public void copyFrom(int fromIndex, int thisIndex, TimeMicroVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that + * Same as {@link #copyFrom(int, int, TimeMicroVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, TimeMicroVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -331,22 +331,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeMicroVector) to); + return new TransferImpl((TimeMicroVector) to); } private class TransferImpl implements TransferPair { - NullableTimeMicroVector to; + TimeMicroVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); + to = new TimeMicroVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableTimeMicroVector to) { + public TransferImpl(TimeMicroVector to) { this.to = to; } @Override - public NullableTimeMicroVector getTo() { + public TimeMicroVector getTo() { return to; } @@ -362,7 +362,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); + to.copyFromSafe(fromIndex, toIndex, TimeMicroVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java index a28ccfe00f372..c3d100c5b6194 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java @@ -31,35 +31,35 @@ import org.slf4j.Logger; /** - * NullableTimeMilliVector implements a fixed width (4 bytes) vector of + * TimeMilliVector implements a fixed width (4 bytes) vector of * time (millisecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeMilliVector extends BaseNullableFixedWidthVector { +public class TimeMilliVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** - * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * Instantiate a TimeMilliVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeMilliVector(String name, BufferAllocator allocator) { + public TimeMilliVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), allocator); } /** - * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * Instantiate a TimeMilliVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); + reader = new TimeMilliReaderImpl(TimeMilliVector.this); } /** @@ -139,21 +139,21 @@ public LocalDateTime getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + public void copyFrom(int fromIndex, int thisIndex, TimeMilliVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that + * Same as {@link #copyFrom(int, int, TimeMilliVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, TimeMilliVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -332,22 +332,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeMilliVector) to); + return new TransferImpl((TimeMilliVector) to); } private class TransferImpl implements TransferPair { - NullableTimeMilliVector to; + TimeMilliVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); + to = new TimeMilliVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableTimeMilliVector to) { + public TransferImpl(TimeMilliVector to) { this.to = to; } @Override - public NullableTimeMilliVector getTo() { + public TimeMilliVector getTo() { return to; } @@ -363,7 +363,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); + to.copyFromSafe(fromIndex, toIndex, TimeMilliVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java index e6903c011e676..97401ec8aaa0b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java @@ -29,35 +29,35 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTimeNanoVector implements a fixed width vector (8 bytes) of + * TimeNanoVector implements a fixed width vector (8 bytes) of * time (nanosecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeNanoVector extends BaseNullableFixedWidthVector { +public class TimeNanoVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** - * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * Instantiate a TimeNanoVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeNanoVector(String name, BufferAllocator allocator) { + public TimeNanoVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), allocator); } /** - * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * Instantiate a TimeNanoVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); + reader = new TimeNanoReaderImpl(TimeNanoVector.this); } /** @@ -137,21 +137,21 @@ public Long getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + public void copyFrom(int fromIndex, int thisIndex, TimeNanoVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that + * Same as {@link #copyFrom(int, int, TimeNanoVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, TimeNanoVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -329,22 +329,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeNanoVector) to); + return new TransferImpl((TimeNanoVector) to); } private class TransferImpl implements TransferPair { - NullableTimeNanoVector to; + TimeNanoVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); + to = new TimeNanoVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableTimeNanoVector to) { + public TransferImpl(TimeNanoVector to) { this.to = to; } @Override - public NullableTimeNanoVector getTo() { + public TimeNanoVector getTo() { return to; } @@ -360,7 +360,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); + to.copyFromSafe(fromIndex, toIndex, TimeNanoVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java index 85426a36948db..a7823a916f3d3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java @@ -29,35 +29,35 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTimeSecVector implements a fixed width (4 bytes) vector of + * TimeSecVector implements a fixed width (4 bytes) vector of * time (seconds resolution) values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableTimeSecVector extends BaseNullableFixedWidthVector { +public class TimeSecVector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** - * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * Instantiate a TimeSecVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeSecVector(String name, BufferAllocator allocator) { + public TimeSecVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), allocator); } /** - * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * Instantiate a TimeSecVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeSecReaderImpl(NullableTimeSecVector.this); + reader = new TimeSecReaderImpl(TimeSecVector.this); } /** @@ -137,21 +137,21 @@ public Integer getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { + public void copyFrom(int fromIndex, int thisIndex, TimeSecVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that + * Same as {@link #copyFrom(int, int, TimeSecVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, TimeSecVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -330,22 +330,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeSecVector) to); + return new TransferImpl((TimeSecVector) to); } private class TransferImpl implements TransferPair { - NullableTimeSecVector to; + TimeSecVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); + to = new TimeSecVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableTimeSecVector to) { + public TransferImpl(TimeSecVector to) { this.to = to; } @Override - public NullableTimeSecVector getTo() { + public TimeSecVector getTo() { return to; } @@ -361,7 +361,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); + to.copyFromSafe(fromIndex, toIndex, TimeSecVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java index c7ad141549bcb..bfe330a1e40c5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java @@ -29,37 +29,37 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTimeStampMicroTZVector implements a fixed width vector (8 bytes) of + * TimeStampMicroTZVector implements a fixed width vector (8 bytes) of * timestamp (microsecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeStampMicroTZVector extends NullableTimeStampVector { +public class TimeStampMicroTZVector extends TimeStampVector { private final FieldReader reader; private final String timeZone; /** - * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampMicroTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { + public TimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), allocator); } /** - * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampMicroTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); timeZone = arrowType.getTimezone(); - reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); + reader = new TimeStampMicroTZReaderImpl(TimeStampMicroTZVector.this); } /** @@ -200,7 +200,7 @@ public void setSafe(int index, TimeStampMicroTZHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, + TimeStampMicroTZVector to = new TimeStampMicroTZVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -212,6 +212,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMicroTZVector) to); + return new TransferImpl((TimeStampMicroTZVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java index 5a34b194ce77f..85b615d8f578b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java @@ -29,34 +29,34 @@ import org.joda.time.LocalDateTime; /** - * NullableTimeStampMicroVector implements a fixed width vector (8 bytes) of + * TimeStampMicroVector implements a fixed width vector (8 bytes) of * timestamp (microsecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeStampMicroVector extends NullableTimeStampVector { +public class TimeStampMicroVector extends TimeStampVector { private final FieldReader reader; /** - * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * Instantiate a TimeStampMicroVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { + public TimeStampMicroVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), allocator); } /** - * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * Instantiate a TimeStampMicroVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); - reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); + reader = new TimeStampMicroReaderImpl(TimeStampMicroVector.this); } /** @@ -202,7 +202,7 @@ public void setSafe(int index, TimeStampMicroHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, + TimeStampMicroVector to = new TimeStampMicroVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -214,6 +214,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMicroVector) to); + return new TransferImpl((TimeStampMicroVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java index f9736aed7be62..9d68b564492b6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java @@ -29,37 +29,37 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTimeStampMilliTZVector implements a fixed width vector (8 bytes) of + * TimeStampMilliTZVector implements a fixed width vector (8 bytes) of * timestamp (millisecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeStampMilliTZVector extends NullableTimeStampVector { +public class TimeStampMilliTZVector extends TimeStampVector { private final FieldReader reader; private final String timeZone; /** - * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampMilliTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { + public TimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), allocator); } /** - * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampMilliTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); timeZone = arrowType.getTimezone(); - reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); + reader = new TimeStampMilliTZReaderImpl(TimeStampMilliTZVector.this); } /** @@ -199,7 +199,7 @@ public void setSafe(int index, TimeStampMilliTZHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, + TimeStampMilliTZVector to = new TimeStampMilliTZVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -211,6 +211,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMilliTZVector) to); + return new TransferImpl((TimeStampMilliTZVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java index fc38a6bd507de..7e8a1d0e2a854 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java @@ -29,34 +29,34 @@ import org.joda.time.LocalDateTime; /** - * NullableTimeStampMilliVector implements a fixed width vector (8 bytes) of + * TimeStampMilliVector implements a fixed width vector (8 bytes) of * timestamp (millisecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeStampMilliVector extends NullableTimeStampVector { +public class TimeStampMilliVector extends TimeStampVector { private final FieldReader reader; /** - * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * Instantiate a TimeStampMilliVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { + public TimeStampMilliVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), allocator); } /** - * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * Instantiate a TimeStampMilliVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); - reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); + reader = new TimeStampMilliReaderImpl(TimeStampMilliVector.this); } /** @@ -200,7 +200,7 @@ public void setSafe(int index, TimeStampMilliHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, + TimeStampMilliVector to = new TimeStampMilliVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -212,6 +212,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMilliVector) to); + return new TransferImpl((TimeStampMilliVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java index 8eb48462c994b..e0361820137e2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java @@ -29,37 +29,37 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTimeStampNanoTZVector implements a fixed width vector (8 bytes) of + * TimeStampNanoTZVector implements a fixed width vector (8 bytes) of * timestamp (nanosecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeStampNanoTZVector extends NullableTimeStampVector { +public class TimeStampNanoTZVector extends TimeStampVector { private final FieldReader reader; private final String timeZone; /** - * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampNanoTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { + public TimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), allocator); } /** - * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampNanoTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); timeZone = arrowType.getTimezone(); - reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); + reader = new TimeStampNanoTZReaderImpl(TimeStampNanoTZVector.this); } /** @@ -200,7 +200,7 @@ public void setSafe(int index, TimeStampNanoTZHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, + TimeStampNanoTZVector to = new TimeStampNanoTZVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -212,6 +212,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampNanoTZVector) to); + return new TransferImpl((TimeStampNanoTZVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java index 6cc5c905783d7..fdf5d26945b90 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java @@ -29,34 +29,34 @@ import org.joda.time.LocalDateTime; /** - * NullableTimeStampNanoVector implements a fixed width vector (8 bytes) of + * TimeStampNanoVector implements a fixed width vector (8 bytes) of * timestamp (nanosecond resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeStampNanoVector extends NullableTimeStampVector { +public class TimeStampNanoVector extends TimeStampVector { private final FieldReader reader; /** - * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * Instantiate a TimeStampNanoVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { + public TimeStampNanoVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), allocator); } /** - * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * Instantiate a TimeStampNanoVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); - reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); + reader = new TimeStampNanoReaderImpl(TimeStampNanoVector.this); } /** @@ -201,7 +201,7 @@ public void setSafe(int index, TimeStampNanoHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, + TimeStampNanoVector to = new TimeStampNanoVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -213,6 +213,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampNanoVector) to); + return new TransferImpl((TimeStampNanoVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java index 0593ac76dce57..201f1c317d02d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java @@ -28,37 +28,37 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTimeStampSecTZVector implements a fixed width vector (8 bytes) of + * TimeStampSecTZVector implements a fixed width vector (8 bytes) of * timestamp (seconds resolution) values which could be null. A validity buffer * (bit vector) is maintained to track which elements in the vector are null. */ -public class NullableTimeStampSecTZVector extends NullableTimeStampVector { +public class TimeStampSecTZVector extends TimeStampVector { private final FieldReader reader; private final String timeZone; /** - * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampSecTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { + public TimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), allocator); } /** - * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * Instantiate a TimeStampSecTZVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); timeZone = arrowType.getTimezone(); - reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); + reader = new TimeStampSecTZReaderImpl(TimeStampSecTZVector.this); } /** @@ -198,7 +198,7 @@ public void setSafe(int index, TimeStampSecTZHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, + TimeStampSecTZVector to = new TimeStampSecTZVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -210,6 +210,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampSecTZVector) to); + return new TransferImpl((TimeStampSecTZVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java index 1afdb30d0453e..4bcd4f7bf0026 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java @@ -29,34 +29,34 @@ import org.joda.time.LocalDateTime; /** - * NullableTimeStampSecVector implements a fixed width vector (8 bytes) of + * TimeStampSecVector implements a fixed width vector (8 bytes) of * timestamp (seconds resolution) values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableTimeStampSecVector extends NullableTimeStampVector { +public class TimeStampSecVector extends TimeStampVector { private final FieldReader reader; /** - * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * Instantiate a TimeStampSecVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTimeStampSecVector(String name, BufferAllocator allocator) { + public TimeStampSecVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), allocator); } /** - * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * Instantiate a TimeStampSecVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, fieldType, allocator); - reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); + reader = new TimeStampSecReaderImpl(TimeStampSecVector.this); } /** @@ -201,7 +201,7 @@ public void setSafe(int index, TimeStampSecHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, + TimeStampSecVector to = new TimeStampSecVector(ref, field.getFieldType(), allocator); return new TransferImpl(to); } @@ -213,6 +213,6 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampSecVector) to); + return new TransferImpl((TimeStampSecVector) to); } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java similarity index 88% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java index e1b719f66a784..4c70b819cbf37 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java @@ -24,21 +24,21 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTimeStampVector is an abstract interface for fixed width vector (8 bytes) + * TimeStampVector is an abstract interface for fixed width vector (8 bytes) * of timestamp values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector { +public abstract class TimeStampVector extends BaseFixedWidthVector { protected static final byte TYPE_WIDTH = 8; /** - * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for + * Instantiate a TimeStampVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); } @@ -69,21 +69,21 @@ public long get(int index) throws IllegalStateException { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { + public void copyFrom(int fromIndex, int thisIndex, TimeStampVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that + * Same as {@link #copyFromSafe(int, int, TimeStampVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, TimeStampVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -188,14 +188,14 @@ public static long get(final ArrowBuf buffer, final int index) { public class TransferImpl implements TransferPair { - NullableTimeStampVector to; + TimeStampVector to; - public TransferImpl(NullableTimeStampVector to) { + public TransferImpl(TimeStampVector to) { this.to = to; } @Override - public NullableTimeStampVector getTo() { + public TimeStampVector getTo() { return to; } @@ -211,7 +211,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); + to.copyFromSafe(fromIndex, toIndex, TimeStampVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java similarity index 91% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java index cadfe608538cc..306437f0fe9d6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java @@ -29,35 +29,35 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableTinyIntVector implements a fixed width (1 bytes) vector of + * TinyIntVector implements a fixed width (1 bytes) vector of * byte values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableTinyIntVector extends BaseNullableFixedWidthVector { +public class TinyIntVector extends BaseFixedWidthVector { public static final byte TYPE_WIDTH = 1; private final FieldReader reader; /** - * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * Instantiate a TinyIntVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableTinyIntVector(String name, BufferAllocator allocator) { + public TinyIntVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), allocator); } /** - * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * Instantiate a TinyIntVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + public TinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TinyIntReaderImpl(NullableTinyIntVector.this); + reader = new TinyIntReaderImpl(TinyIntVector.this); } /** @@ -137,21 +137,21 @@ public Byte getObject(int index) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { + public void copyFrom(int fromIndex, int thisIndex, TinyIntVector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH); valueBuffer.setByte(thisIndex * TYPE_WIDTH, value); } /** - * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that + * Same as {@link #copyFrom(int, int, TinyIntVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, TinyIntVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -357,22 +357,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTinyIntVector) to); + return new TransferImpl((TinyIntVector) to); } private class TransferImpl implements TransferPair { - NullableTinyIntVector to; + TinyIntVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); + to = new TinyIntVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableTinyIntVector to) { + public TransferImpl(TinyIntVector to) { this.to = to; } @Override - public NullableTinyIntVector getTo() { + public TinyIntVector getTo() { return to; } @@ -388,7 +388,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); + to.copyFromSafe(fromIndex, toIndex, TinyIntVector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java similarity index 91% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java rename to java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java index ebd9b56996cf6..6901a889a6dcf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java @@ -28,22 +28,22 @@ import org.apache.arrow.vector.util.TransferPair; /** - * NullableUInt1Vector implements a fixed width (1 bytes) vector of + * UInt1Vector implements a fixed width (1 bytes) vector of * integer values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableUInt1Vector extends BaseNullableFixedWidthVector { +public class UInt1Vector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 1; private final FieldReader reader; - public NullableUInt1Vector(String name, BufferAllocator allocator) { + public UInt1Vector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), allocator); } - public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { + public UInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt1ReaderImpl(NullableUInt1Vector.this); + reader = new UInt1ReaderImpl(UInt1Vector.this); } @Override @@ -107,13 +107,13 @@ public Byte getObject(int index) { } } - public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { + public void copyFrom(int fromIndex, int thisIndex, UInt1Vector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final byte value = from.valueBuffer.getByte(fromIndex * TYPE_WIDTH); valueBuffer.setByte(thisIndex * TYPE_WIDTH, value); } - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, UInt1Vector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -279,22 +279,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt1Vector) to); + return new TransferImpl((UInt1Vector) to); } private class TransferImpl implements TransferPair { - NullableUInt1Vector to; + UInt1Vector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); + to = new UInt1Vector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableUInt1Vector to) { + public TransferImpl(UInt1Vector to) { this.to = to; } @Override - public NullableUInt1Vector getTo() { + public UInt1Vector getTo() { return to; } @@ -310,7 +310,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); + to.copyFromSafe(fromIndex, toIndex, UInt1Vector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java similarity index 91% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java rename to java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java index bedb3f35e6bac..968ce9151fae0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java @@ -29,22 +29,22 @@ import org.slf4j.Logger; /** - * NullableUInt2Vector implements a fixed width (2 bytes) vector of + * UInt2Vector implements a fixed width (2 bytes) vector of * integer values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableUInt2Vector extends BaseNullableFixedWidthVector { +public class UInt2Vector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 2; private final FieldReader reader; - public NullableUInt2Vector(String name, BufferAllocator allocator) { + public UInt2Vector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), allocator); } - public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { + public UInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt2ReaderImpl(NullableUInt2Vector.this); + reader = new UInt2ReaderImpl(UInt2Vector.this); } @Override @@ -108,13 +108,13 @@ public Character getObject(int index) { } } - public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { + public void copyFrom(int fromIndex, int thisIndex, UInt2Vector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final char value = from.valueBuffer.getChar(fromIndex * TYPE_WIDTH); valueBuffer.setChar(thisIndex * TYPE_WIDTH, value); } - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, UInt2Vector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -280,22 +280,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt2Vector) to); + return new TransferImpl((UInt2Vector) to); } private class TransferImpl implements TransferPair { - NullableUInt2Vector to; + UInt2Vector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); + to = new UInt2Vector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableUInt2Vector to) { + public TransferImpl(UInt2Vector to) { this.to = to; } @Override - public NullableUInt2Vector getTo() { + public UInt2Vector getTo() { return to; } @@ -311,7 +311,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); + to.copyFromSafe(fromIndex, toIndex, UInt2Vector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java rename to java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java index 99b547ca4fdef..af219cb061e48 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java @@ -29,22 +29,22 @@ import org.slf4j.Logger; /** - * NullableUInt4Vector implements a fixed width (4 bytes) vector of + * UInt4Vector implements a fixed width (4 bytes) vector of * integer values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableUInt4Vector extends BaseNullableFixedWidthVector { +public class UInt4Vector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 4; private final FieldReader reader; - public NullableUInt4Vector(String name, BufferAllocator allocator) { + public UInt4Vector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), allocator); } - public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + public UInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt4ReaderImpl(NullableUInt4Vector.this); + reader = new UInt4ReaderImpl(UInt4Vector.this); } @Override @@ -108,13 +108,13 @@ public Integer getObject(int index) { } } - public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { + public void copyFrom(int fromIndex, int thisIndex, UInt4Vector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final int value = from.valueBuffer.getInt(fromIndex * TYPE_WIDTH); valueBuffer.setInt(thisIndex * TYPE_WIDTH, value); } - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, UInt4Vector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -252,22 +252,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt4Vector) to); + return new TransferImpl((UInt4Vector) to); } private class TransferImpl implements TransferPair { - NullableUInt4Vector to; + UInt4Vector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); + to = new UInt4Vector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableUInt4Vector to) { + public TransferImpl(UInt4Vector to) { this.to = to; } @Override - public NullableUInt4Vector getTo() { + public UInt4Vector getTo() { return to; } @@ -283,7 +283,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); + to.copyFromSafe(fromIndex, toIndex, UInt4Vector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java rename to java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java index e6664a09265d7..eae4a31729955 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java @@ -29,22 +29,22 @@ import org.slf4j.Logger; /** - * NullableUInt8Vector implements a fixed width vector (8 bytes) of + * UInt8Vector implements a fixed width vector (8 bytes) of * integer values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public class NullableUInt8Vector extends BaseNullableFixedWidthVector { +public class UInt8Vector extends BaseFixedWidthVector { private static final byte TYPE_WIDTH = 8; private final FieldReader reader; - public NullableUInt8Vector(String name, BufferAllocator allocator) { + public UInt8Vector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), allocator); } - public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + public UInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt8ReaderImpl(NullableUInt8Vector.this); + reader = new UInt8ReaderImpl(UInt8Vector.this); } @Override @@ -108,13 +108,13 @@ public Long getObject(int index) { } } - public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { + public void copyFrom(int fromIndex, int thisIndex, UInt8Vector from) { BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); final long value = from.valueBuffer.getLong(fromIndex * TYPE_WIDTH); valueBuffer.setLong(thisIndex * TYPE_WIDTH, value); } - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { + public void copyFromSafe(int fromIndex, int thisIndex, UInt8Vector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); } @@ -252,22 +252,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt8Vector) to); + return new TransferImpl((UInt8Vector) to); } private class TransferImpl implements TransferPair { - NullableUInt8Vector to; + UInt8Vector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); + to = new UInt8Vector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableUInt8Vector to) { + public TransferImpl(UInt8Vector to) { this.to = to; } @Override - public NullableUInt8Vector getTo() { + public UInt8Vector getTo() { return to; } @@ -283,7 +283,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); + to.copyFromSafe(fromIndex, toIndex, UInt8Vector.this); } } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java index 1dfe8912c52f9..893ad7cb594a7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java @@ -31,33 +31,33 @@ import java.nio.ByteBuffer; /** - * NullableVarBinaryVector implements a variable width vector of binary + * VarBinaryVector implements a variable width vector of binary * values which could be NULL. A validity buffer (bit vector) is maintained * to track which elements in the vector are null. */ -public class NullableVarBinaryVector extends BaseNullableVariableWidthVector { +public class VarBinaryVector extends BaseVariableWidthVector { private final FieldReader reader; /** - * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * Instantiate a VarBinaryVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableVarBinaryVector(String name, BufferAllocator allocator) { + public VarBinaryVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); } /** - * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * Instantiate a VarBinaryVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { + public VarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType); - reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); + reader = new VarBinaryReaderImpl(VarBinaryVector.this); } /** @@ -156,7 +156,7 @@ public void get(int index, NullableVarBinaryHolder holder) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + public void copyFrom(int fromIndex, int thisIndex, VarBinaryVector from) { final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); final int length = end - start; @@ -169,14 +169,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) } /** - * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that + * Same as {@link #copyFrom(int, int, VarBinaryVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, VarBinaryVector from) { final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); final int length = end - start; @@ -291,22 +291,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableVarBinaryVector) to); + return new TransferImpl((VarBinaryVector) to); } private class TransferImpl implements TransferPair { - NullableVarBinaryVector to; + VarBinaryVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); + to = new VarBinaryVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableVarBinaryVector to) { + public TransferImpl(VarBinaryVector to) { this.to = to; } @Override - public NullableVarBinaryVector getTo() { + public VarBinaryVector getTo() { return to; } @@ -322,7 +322,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); + to.copyFromSafe(fromIndex, toIndex, VarBinaryVector.this); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java similarity index 90% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java rename to java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java index 31e409cc6263f..8a38b1d455d20 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java @@ -32,33 +32,33 @@ import java.nio.ByteBuffer; /** - * NullableVarCharVector implements a variable width vector of VARCHAR + * VarCharVector implements a variable width vector of VARCHAR * values which could be NULL. A validity buffer (bit vector) is maintained * to track which elements in the vector are null. */ -public class NullableVarCharVector extends BaseNullableVariableWidthVector { +public class VarCharVector extends BaseVariableWidthVector { private final FieldReader reader; /** - * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * Instantiate a VarCharVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param allocator allocator for memory management. */ - public NullableVarCharVector(String name, BufferAllocator allocator) { + public VarCharVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); } /** - * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * Instantiate a VarCharVector. This doesn't allocate any memory for * the data in vector. * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { + public VarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType); - reader = new VarCharReaderImpl(NullableVarCharVector.this); + reader = new VarCharReaderImpl(VarCharVector.this); } /** @@ -159,7 +159,7 @@ public void get(int index, NullableVarCharHolder holder) { * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { + public void copyFrom(int fromIndex, int thisIndex, VarCharVector from) { final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); final int length = end - start; @@ -172,14 +172,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { } /** - * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that + * Same as {@link #copyFrom(int, int, VarCharVector)} except that * it handles the case when the capacity of the vector needs to be expanded * before copy. * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { + public void copyFromSafe(int fromIndex, int thisIndex, VarCharVector from) { final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); final int length = end - start; @@ -294,22 +294,22 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableVarCharVector) to); + return new TransferImpl((VarCharVector) to); } private class TransferImpl implements TransferPair { - NullableVarCharVector to; + VarCharVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new NullableVarCharVector(ref, field.getFieldType(), allocator); + to = new VarCharVector(ref, field.getFieldType(), allocator); } - public TransferImpl(NullableVarCharVector to) { + public TransferImpl(VarCharVector to) { this.to = to; } @Override - public NullableVarCharVector getTo() { + public VarCharVector getTo() { return to; } @@ -325,7 +325,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this); + to.copyFromSafe(fromIndex, toIndex, VarCharVector.this); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorDefinitionSetter.java similarity index 94% rename from java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java rename to java/vector/src/main/java/org/apache/arrow/vector/VectorDefinitionSetter.java index 1e0746aabaa61..2f45d3a5b6316 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorDefinitionSetter.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector; -public interface NullableVectorDefinitionSetter { +public interface VectorDefinitionSetter { public void setIndexDefined(int index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index db0ff86df47a9..5ded1cc6f8778 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -100,8 +100,8 @@ protected boolean supportsDirectRead() { // return the child vector's ordinal in the composite container public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name); - public NullableMapVector addOrGetMap(String name) { - return addOrGet(name, FieldType.nullable(new Struct()), NullableMapVector.class); + public MapVector addOrGetMap(String name) { + return addOrGet(name, FieldType.nullable(new Struct()), MapVector.class); } public ListVector addOrGetList(String name) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java index a76fbbe11a1fb..cd3a2ae9e6f3c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java @@ -42,14 +42,12 @@ public void populate(int lastIndex) { if (lastIndex < 0) { throw new IndexOutOfBoundsException("index cannot be negative"); } - final UInt4Vector.Accessor accessor = offsets.getAccessor(); - final UInt4Vector.Mutator mutator = offsets.getMutator(); - final int lastSet = Math.max(accessor.getValueCount() - 1, 0); - final int previousEnd = accessor.get(lastSet);//0 ? 0 : accessor.get(lastSet); + final int lastSet = Math.max(offsets.getValueCount() - 1, 0); + final int previousEnd = offsets.get(lastSet);//0 ? 0 : accessor.get(lastSet); for (int i = lastSet; i < lastIndex; i++) { - mutator.setSafe(i + 1, previousEnd); + offsets.setSafe(i + 1, previousEnd); } - mutator.setValueCount(lastIndex + 1); + offsets.setValueCount(lastIndex + 1); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 6089a67924fe0..075ae83ea4d84 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,332 +21,492 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; +import java.util.Arrays; +import java.util.Collections; import java.util.List; -import java.util.Map; -import javax.annotation.Nullable; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; -import com.google.common.primitives.Ints; +import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; - +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; +import org.apache.arrow.vector.complex.impl.NullableMapWriter; import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.ArrowType.Struct; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringHashMap; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; -public class MapVector extends AbstractMapVector { - //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); +public class MapVector extends NonNullableMapVector implements FieldVector { public static MapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); + FieldType fieldType = FieldType.nullable(Struct.INSTANCE); return new MapVector(name, allocator, fieldType, null); } - private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); - protected final FieldType fieldType; - public int valueCount; + private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); + private final NullableMapWriter writer = new NullableMapWriter(this); + + protected ArrowBuf validityBuffer; + private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @Deprecated public MapVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); + this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack); } - public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); - this.fieldType = checkNotNull(fieldType); - this.valueCount = 0; - } - - @Override - public FieldReader getReader() { - return reader; + // deprecated, use FieldType or static constructor instead + @Deprecated + public MapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { + this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack); } - transient private MapTransferPair ephPair; - - public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { - if (ephPair == null || ephPair.from != from) { - ephPair = (MapTransferPair) from.makeTransferPair(this); - } - ephPair.copyValueSafe(fromIndex, thisIndex); + public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, checkNotNull(allocator), fieldType, callBack); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); } @Override - protected boolean supportsDirectRead() { - return true; - } - - public Iterator fieldNameIterator() { - return getChildFieldNames().iterator(); + public Field getField() { + Field f = super.getField(); + FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata()); + return new Field(f.getName(), type, f.getChildren()); } @Override - public void setInitialCapacity(int numRecords) { - for (final ValueVector v : (Iterable) this) { - v.setInitialCapacity(numRecords); + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); } - } - @Override - public int getBufferSize() { - if (valueCount == 0 || size() == 0) { - return 0; - } - long buffer = 0; - for (final ValueVector v : (Iterable) this) { - buffer += v.getBufferSize(); - } + ArrowBuf bitBuffer = ownBuffers.get(0); - return (int) buffer; + validityBuffer.release(); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); + valueCount = fieldNode.getLength(); + validityAllocationSizeInBytes = validityBuffer.capacity(); } @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } + public List getFieldBuffers() { + List result = new ArrayList<>(1); + setReaderAndWriterIndex(); + result.add(validityBuffer); - long bufferSize = 0; - for (final ValueVector v : (Iterable) this) { - bufferSize += v.getBufferSizeFor(valueCount); - } + return result; + } - return (int) bufferSize; + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); } @Override - public ArrowBuf getValidityBuffer() { - throw new UnsupportedOperationException(); + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); + public NullableMapReaderImpl getReader() { + return reader; } - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); + public NullableMapWriter getWriter() { + return writer; } @Override public TransferPair getTransferPair(BufferAllocator allocator) { - return getTransferPair(name, allocator, null); + return new NullableMapTransferPair(this, new MapVector(name, allocator, fieldType, null), false); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false); + public TransferPair makeTransferPair(ValueVector to) { + return new NullableMapTransferPair(this, (MapVector) to, true); } @Override - public TransferPair makeTransferPair(ValueVector to) { - return new MapTransferPair(this, (MapVector) to); + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, null), false); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false); + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false); } - protected static class MapTransferPair implements TransferPair { - private final TransferPair[] pairs; - private final MapVector from; - private final MapVector to; + protected class NullableMapTransferPair extends MapTransferPair { - public MapTransferPair(MapVector from, MapVector to) { - this(from, to, true); - } + private MapVector target; - protected MapTransferPair(MapVector from, MapVector to, boolean allocate) { - this.from = from; - this.to = to; - this.pairs = new TransferPair[from.size()]; - this.to.ephPair = null; - - int i = 0; - FieldVector vector; - for (String child : from.getChildFieldNames()) { - int preSize = to.size(); - vector = from.getChild(child); - if (vector == null) { - continue; - } - //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector, - // the child fields may be nested fields of the top level child. For example if the structure - // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab. - // But the children member of a Materialized field is a HashSet. If the fields are added in the - // children HashSet, and the hashCode of the Materialized field includes the hash code of the - // children, the hashCode value of oa changes *after* the field has been added to the HashSet. - // (This is similar to what happens in ScanBatch where the children cannot be added till they are - // read). To take care of this, we ensure that the hashCode of the MaterializedField does not - // include the hashCode of the children but is based only on MaterializedField$key. - final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass()); - if (allocate && to.size() != preSize) { - newVector.allocateNew(); - } - pairs[i++] = vector.makeTransferPair(newVector); - } + protected NullableMapTransferPair(MapVector from, MapVector to, boolean allocate) { + super(from, to, allocate); + this.target = to; } @Override public void transfer() { - for (final TransferPair p : pairs) { - p.transfer(); - } - to.valueCount = from.valueCount; - from.clear(); - } - - @Override - public ValueVector getTo() { - return to; + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + super.transfer(); + clear(); } @Override - public void copyValueSafe(int from, int to) { - for (TransferPair p : pairs) { - p.copyValueSafe(from, to); + public void copyValueSafe(int fromIndex, int toIndex) { + while (toIndex >= target.getValidityBufferValueCapacity()) { + target.reallocValidityBuffer(); } + BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); + super.copyValueSafe(fromIndex, toIndex); } @Override public void splitAndTransfer(int startIndex, int length) { - for (TransferPair p : pairs) { - p.splitAndTransfer(startIndex, length); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + super.splitAndTransfer(startIndex, length); + } + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } } - to.setValueCount(length); } } + /** + * Get the value capacity of the internal validity buffer. + * @return number of elements that validity buffer can hold + */ + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity() { - if (size() == 0) { - return 0; - } + return Math.min(getValidityBufferValueCapacity(), + super.getValueCapacity()); + } - final Ordering natural = new Ordering() { - @Override - public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { - return Ints.compare( - checkNotNull(left).getValueCapacity(), - checkNotNull(right).getValueCapacity() - ); + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ + @Override + public ArrowBuf[] getBuffers(boolean clear) { + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), + ArrowBuf.class); + } + if (clear) { + for (ArrowBuf buffer : buffers) { + buffer.retain(); } - }; + clear(); + } - return natural.min(getChildren()).getValueCapacity(); + return buffers; } + /** + * Close the vector and release the associated buffers. + */ @Override - public Accessor getAccessor() { - throw new UnsupportedOperationException("accessor is not needed for MAP"); + public void close() { + clearValidityBuffer(); + super.close(); } + /** + * Same as {@link #close()} + */ @Override - public Mutator getMutator() { - throw new UnsupportedOperationException("mutator is not needed for MAP"); + public void clear() { + clearValidityBuffer(); + super.clear(); + } + + /** + * Release the validity buffer + */ + private void clearValidityBuffer() { + validityBuffer.release(); + validityBuffer = allocator.getEmpty(); } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override - public Object getObject(int index) { - Map vv = new JsonStringHashMap<>(); - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v != null && index < v.getValueCount()) { - Object value = v.getObject(index); - if (value != null) { - vv.put(child, value); - } + public int getBufferSize() { + if (valueCount == 0) { + return 0; + } + return super.getBufferSize() + + BitVectorHelper.getValidityBufferSize(valueCount); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } + return super.getBufferSizeFor(valueCount) + + BitVectorHelper.getValidityBufferSize(valueCount); + } + + @Override + public void setInitialCapacity(int numRecords) { + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); + super.setInitialCapacity(numRecords); + } + + @Override + public boolean allocateNewSafe() { + /* Boolean to keep track if all the memory allocations were successful + * Used in the case of composite vectors when we need to allocate multiple + * buffers for multiple vectors. If one of the allocations failed we need to + * clear all the memory that we allocated + */ + boolean success = false; + try { + clear(); + allocateValidityBuffer(validityAllocationSizeInBytes); + success = super.allocateNewSafe(); + } finally { + if (!success) { + clear(); + return false; } } - return vv; + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int) size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override - public boolean isNull(int index) { return false; } + public void reAlloc() { + /* reallocate the validity buffer */ + reallocValidityBuffer(); + super.reAlloc(); + } + + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int) newAllocationSize; + } + @Override - public int getNullCount() { return 0; } + public long getValidityBufferAddress() { + return validityBuffer.memoryAddress(); + } - public void get(int index, ComplexHolder holder) { - reader.setPosition(index); - holder.reader = reader; + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); } @Override - public int getValueCount() { - return valueCount; + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); } - public ValueVector getVectorById(int id) { - return getChildByOrdinal(id); -} + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } @Override - public void setValueCount(int valueCount) { - for (final ValueVector v : getChildren()) { - v.setValueCount(valueCount); - } - MapVector.this.valueCount = valueCount; + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); } @Override - public void clear() { - for (final ValueVector v : getChildren()) { - v.clear(); - } - valueCount = 0; + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); } @Override - public Field getField() { - List children = new ArrayList<>(); - for (ValueVector child : getChildren()) { - children.add(child.getField()); + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return super.getObject(index); } - return new Field(name, fieldType, children); } @Override - public MinorType getMinorType() { - return MinorType.MAP; + public void get(int index, ComplexHolder holder) { + holder.isSet = isSet(index); + super.get(index, holder); + } + + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + + public boolean isNull(int index) { + return isSet(index) == 0; + } + + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); } @Override - public void close() { - final Collection vectors = getChildren(); - for (final FieldVector v : vectors) { - v.close(); + public void setValueCount(int valueCount) { + assert valueCount >= 0; + while (valueCount > getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } - vectors.clear(); + super.setValueCount(valueCount); + this.valueCount = valueCount; + } + public void reset() { valueCount = 0; - - super.close(); } - public void initializeChildrenFromFields(List children) { - for (Field field : children) { - FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType()); - vector.initializeChildrenFromFields(field.getChildren()); - } + @Override + @Deprecated + public Accessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP"); } - public List getChildrenFromFields() { - return getChildren(); + @Override + @Deprecated + public Mutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP"); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java new file mode 100644 index 0000000000000..cc3ac4148a865 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java @@ -0,0 +1,352 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; + +import io.netty.buffer.ArrowBuf; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.ComplexHolder; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringHashMap; +import org.apache.arrow.vector.util.TransferPair; + +public class NonNullableMapVector extends AbstractMapVector { + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NonNullableMapVector.class); + + public static NonNullableMapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); + return new NonNullableMapVector(name, allocator, fieldType, null); + } + + private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); + protected final FieldType fieldType; + public int valueCount; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public NonNullableMapVector(String name, BufferAllocator allocator, CallBack callBack) { + this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); + } + + public NonNullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + this.fieldType = checkNotNull(fieldType); + this.valueCount = 0; + } + + @Override + public FieldReader getReader() { + return reader; + } + + transient private MapTransferPair ephPair; + + public void copyFromSafe(int fromIndex, int thisIndex, NonNullableMapVector from) { + if (ephPair == null || ephPair.from != from) { + ephPair = (MapTransferPair) from.makeTransferPair(this); + } + ephPair.copyValueSafe(fromIndex, thisIndex); + } + + @Override + protected boolean supportsDirectRead() { + return true; + } + + public Iterator fieldNameIterator() { + return getChildFieldNames().iterator(); + } + + @Override + public void setInitialCapacity(int numRecords) { + for (final ValueVector v : (Iterable) this) { + v.setInitialCapacity(numRecords); + } + } + + @Override + public int getBufferSize() { + if (valueCount == 0 || size() == 0) { + return 0; + } + long buffer = 0; + for (final ValueVector v : (Iterable) this) { + buffer += v.getBufferSize(); + } + + return (int) buffer; + } + + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } + + long bufferSize = 0; + for (final ValueVector v : (Iterable) this) { + bufferSize += v.getBufferSizeFor(valueCount); + } + + return (int) bufferSize; + } + + @Override + public ArrowBuf getValidityBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator, null); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return new MapTransferPair(this, new NonNullableMapVector(name, allocator, fieldType, callBack), false); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new MapTransferPair(this, (NonNullableMapVector) to); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new MapTransferPair(this, new NonNullableMapVector(ref, allocator, fieldType, callBack), false); + } + + protected static class MapTransferPair implements TransferPair { + private final TransferPair[] pairs; + private final NonNullableMapVector from; + private final NonNullableMapVector to; + + public MapTransferPair(NonNullableMapVector from, NonNullableMapVector to) { + this(from, to, true); + } + + protected MapTransferPair(NonNullableMapVector from, NonNullableMapVector to, boolean allocate) { + this.from = from; + this.to = to; + this.pairs = new TransferPair[from.size()]; + this.to.ephPair = null; + + int i = 0; + FieldVector vector; + for (String child : from.getChildFieldNames()) { + int preSize = to.size(); + vector = from.getChild(child); + if (vector == null) { + continue; + } + //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector, + // the child fields may be nested fields of the top level child. For example if the structure + // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab. + // But the children member of a Materialized field is a HashSet. If the fields are added in the + // children HashSet, and the hashCode of the Materialized field includes the hash code of the + // children, the hashCode value of oa changes *after* the field has been added to the HashSet. + // (This is similar to what happens in ScanBatch where the children cannot be added till they are + // read). To take care of this, we ensure that the hashCode of the MaterializedField does not + // include the hashCode of the children but is based only on MaterializedField$key. + final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass()); + if (allocate && to.size() != preSize) { + newVector.allocateNew(); + } + pairs[i++] = vector.makeTransferPair(newVector); + } + } + + @Override + public void transfer() { + for (final TransferPair p : pairs) { + p.transfer(); + } + to.valueCount = from.valueCount; + from.clear(); + } + + @Override + public ValueVector getTo() { + return to; + } + + @Override + public void copyValueSafe(int from, int to) { + for (TransferPair p : pairs) { + p.copyValueSafe(from, to); + } + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + for (TransferPair p : pairs) { + p.splitAndTransfer(startIndex, length); + } + to.setValueCount(length); + } + } + + @Override + public int getValueCapacity() { + if (size() == 0) { + return 0; + } + + final Ordering natural = new Ordering() { + @Override + public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { + return Ints.compare( + checkNotNull(left).getValueCapacity(), + checkNotNull(right).getValueCapacity() + ); + } + }; + + return natural.min(getChildren()).getValueCapacity(); + } + + @Override + public Accessor getAccessor() { + throw new UnsupportedOperationException("accessor is not needed for MAP"); + } + + @Override + public Mutator getMutator() { + throw new UnsupportedOperationException("mutator is not needed for MAP"); + } + + @Override + public Object getObject(int index) { + Map vv = new JsonStringHashMap<>(); + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); + } + } + } + return vv; + } + + @Override + public boolean isNull(int index) { return false; } + @Override + public int getNullCount() { return 0; } + + public void get(int index, ComplexHolder holder) { + reader.setPosition(index); + holder.reader = reader; + } + + @Override + public int getValueCount() { + return valueCount; + } + + public ValueVector getVectorById(int id) { + return getChildByOrdinal(id); +} + + @Override + public void setValueCount(int valueCount) { + for (final ValueVector v : getChildren()) { + v.setValueCount(valueCount); + } + NonNullableMapVector.this.valueCount = valueCount; + } + + @Override + public void clear() { + for (final ValueVector v : getChildren()) { + v.clear(); + } + valueCount = 0; + } + + @Override + public Field getField() { + List children = new ArrayList<>(); + for (ValueVector child : getChildren()) { + children.add(child.getField()); + } + return new Field(name, fieldType, children); + } + + @Override + public MinorType getMinorType() { + return MinorType.MAP; + } + + @Override + public void close() { + final Collection vectors = getChildren(); + for (final FieldVector v : vectors) { + v.close(); + } + vectors.clear(); + + valueCount = 0; + + super.close(); + } + + public void initializeChildrenFromFields(List children) { + for (Field field : children) { + FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType()); + vector.initializeChildrenFromFields(field.getChildren()); + } + } + + public List getChildrenFromFields() { + return getChildren(); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java deleted file mode 100644 index e223d1ce67449..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ /dev/null @@ -1,512 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.complex; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BaseAllocator; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; -import org.apache.arrow.vector.complex.impl.NullableMapWriter; -import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.ipc.message.ArrowFieldNode; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.ArrowType.Struct; -import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.OversizedAllocationException; -import org.apache.arrow.vector.util.TransferPair; - -public class NullableMapVector extends MapVector implements FieldVector { - - public static NullableMapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = FieldType.nullable(Struct.INSTANCE); - return new NullableMapVector(name, allocator, fieldType, null); - } - - private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); - private final NullableMapWriter writer = new NullableMapWriter(this); - - protected ArrowBuf validityBuffer; - private int validityAllocationSizeInBytes; - - // deprecated, use FieldType or static constructor instead - @Deprecated - public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack); - } - - // deprecated, use FieldType or static constructor instead - @Deprecated - public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { - this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack); - } - - public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, checkNotNull(allocator), fieldType, callBack); - this.validityBuffer = allocator.getEmpty(); - this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); - } - - @Override - public Field getField() { - Field f = super.getField(); - FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata()); - return new Field(f.getName(), type, f.getChildren()); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - if (ownBuffers.size() != 1) { - throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); - } - - ArrowBuf bitBuffer = ownBuffers.get(0); - - validityBuffer.release(); - validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); - valueCount = fieldNode.getLength(); - validityAllocationSizeInBytes = validityBuffer.capacity(); - } - - @Override - public List getFieldBuffers() { - List result = new ArrayList<>(1); - setReaderAndWriterIndex(); - result.add(validityBuffer); - - return result; - } - - private void setReaderAndWriterIndex() { - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); - } - - @Override - @Deprecated - public List getFieldInnerVectors() { - throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); - } - - @Override - public NullableMapReaderImpl getReader() { - return reader; - } - - public NullableMapWriter getWriter() { - return writer; - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new NullableMapTransferPair(this, (NullableMapVector) to, true); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false); - } - - protected class NullableMapTransferPair extends MapTransferPair { - - private NullableMapVector target; - - protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, boolean allocate) { - super(from, to, allocate); - this.target = to; - } - - @Override - public void transfer() { - target.clear(); - target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; - super.transfer(); - clear(); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - while (toIndex >= target.getValidityBufferValueCapacity()) { - target.reallocValidityBuffer(); - } - BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); - super.copyValueSafe(fromIndex, toIndex); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - target.clear(); - splitAndTransferValidityBuffer(startIndex, length, target); - super.splitAndTransfer(startIndex, length); - } - } - - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, - firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - - /** - * Get the value capacity of the internal validity buffer. - * @return number of elements that validity buffer can hold - */ - private int getValidityBufferValueCapacity() { - return (int) (validityBuffer.capacity() * 8L); - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity() { - return Math.min(getValidityBufferValueCapacity(), - super.getValueCapacity()); - } - - /** - * Return the underlying buffers associated with this vector. Note that this doesn't - * impact the reference counts for this buffer so it only should be used for in-context - * access. Also note that this buffer changes regularly thus - * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear Whether to clear vector before returning; the buffers will still be refcounted - * but the returned array will be the only reference to them - * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this - * vector instance. - */ - @Override - public ArrowBuf[] getBuffers(boolean clear) { - setReaderAndWriterIndex(); - final ArrowBuf[] buffers; - if (getBufferSize() == 0) { - buffers = new ArrowBuf[0]; - } else { - buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), - ArrowBuf.class); - } - if (clear) { - for (ArrowBuf buffer : buffers) { - buffer.retain(); - } - clear(); - } - - return buffers; - } - - /** - * Close the vector and release the associated buffers. - */ - @Override - public void close() { - clearValidityBuffer(); - super.close(); - } - - /** - * Same as {@link #close()} - */ - @Override - public void clear() { - clearValidityBuffer(); - super.clear(); - } - - /** - * Release the validity buffer - */ - private void clearValidityBuffer() { - validityBuffer.release(); - validityBuffer = allocator.getEmpty(); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - if (valueCount == 0) { - return 0; - } - return super.getBufferSize() + - BitVectorHelper.getValidityBufferSize(valueCount); - } - - /** - * Get the potential buffer size for a particular number of records. - * @param valueCount desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - return super.getBufferSizeFor(valueCount) - + BitVectorHelper.getValidityBufferSize(valueCount); - } - - @Override - public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); - super.setInitialCapacity(numRecords); - } - - @Override - public boolean allocateNewSafe() { - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - clear(); - allocateValidityBuffer(validityAllocationSizeInBytes); - success = super.allocateNewSafe(); - } finally { - if (!success) { - clear(); - return false; - } - } - return true; - } - - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); - } - - @Override - public void reAlloc() { - /* reallocate the validity buffer */ - reallocValidityBuffer(); - super.reAlloc(); - } - - private void reallocValidityBuffer() { - final int currentBufferCapacity = validityBuffer.capacity(); - long baseSize = validityAllocationSizeInBytes; - - if (baseSize < (long) currentBufferCapacity) { - baseSize = (long) currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); - newBuf.setZero(0, newBuf.capacity()); - newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); - validityBuffer.release(1); - validityBuffer = newBuf; - validityAllocationSizeInBytes = (int) newAllocationSize; - } - - @Override - public long getValidityBufferAddress() { - return validityBuffer.memoryAddress(); - } - - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getValidityBuffer() { - return validityBuffer; - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public Object getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return super.getObject(index); - } - } - - @Override - public void get(int index, ComplexHolder holder) { - holder.isSet = isSet(index); - super.get(index, holder); - } - - public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); - } - - public boolean isNull(int index) { - return isSet(index) == 0; - } - - public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - public void setIndexDefined(int index) { - while (index >= getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); - } - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - } - - public void setNull(int index) { - while (index >= getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); - } - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - while (valueCount > getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); - } - super.setValueCount(valueCount); - this.valueCount = valueCount; - } - - public void reset() { - valueCount = 0; - } - - @Override - @Deprecated - public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP"); - } - - @Override - @Deprecated - public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP"); - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java index a6960238b9165..970b90ef510fa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java @@ -20,7 +20,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StateTool; import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; import org.apache.arrow.vector.types.pojo.Field; @@ -131,7 +131,7 @@ public MapWriter directMap() { switch (mode) { case INIT: - mapRoot = nullableMapWriterFactory.build((NullableMapVector) container); + mapRoot = nullableMapWriterFactory.build((MapVector) container); mapRoot.setPosition(idx()); mode = Mode.MAP; break; @@ -152,7 +152,7 @@ public MapWriter rootAsMap() { case INIT: // TODO allow dictionaries in complex types - NullableMapVector map = container.addOrGetMap(name); + MapVector map = container.addOrGetMap(name); mapRoot = nullableMapWriterFactory.build(map); mapRoot.setPosition(idx()); mode = Mode.MAP; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java index 06b0f4d43272f..acf155af71db2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java @@ -20,17 +20,17 @@ package org.apache.arrow.vector.complex.impl; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.pojo.Field; public class NullableMapReaderImpl extends SingleMapReaderImpl { - private NullableMapVector nullableMapVector; + private MapVector nullableMapVector; public NullableMapReaderImpl(MapVector vector) { - super((NullableMapVector) vector); - this.nullableMapVector = (NullableMapVector) vector; + super((MapVector) vector); + this.nullableMapVector = (MapVector) vector; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java index d2dcb2374d0e7..06dbf784be0a5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector.complex.impl; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; public class NullableMapWriterFactory { private final boolean caseSensitive; @@ -29,7 +29,7 @@ public NullableMapWriterFactory(boolean caseSensitive) { this.caseSensitive = caseSensitive; } - public NullableMapWriter build(NullableMapVector container) { + public NullableMapWriter build(MapVector container) { return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 5bd439cac4269..9abd38d20ac62 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -23,7 +23,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.AbstractMapVector; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -94,7 +94,7 @@ private void setWriter(ValueVector v) { type = v.getMinorType(); switch (type) { case MAP: - writer = nullableMapWriterFactory.build((NullableMapVector) vector); + writer = nullableMapWriterFactory.build((MapVector) vector); break; case LIST: writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java index c77ca4e8f23ba..0341b622e0df4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java @@ -24,6 +24,7 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NonNullableMapVector; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -34,10 +35,10 @@ @SuppressWarnings("unused") public class SingleMapReaderImpl extends AbstractFieldReader { - private final MapVector vector; + private final NonNullableMapVector vector; private final Map fields = Maps.newHashMap(); - public SingleMapReaderImpl(MapVector vector) { + public SingleMapReaderImpl(NonNullableMapVector vector) { this.vector = vector; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java index 6243a2833c055..62bc271257fb2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java @@ -20,7 +20,6 @@ package org.apache.arrow.vector.complex.impl; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.reader.FieldReader; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index cb11a253058e5..04d50331b76d8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -222,7 +222,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT1 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int size = count * NullableTinyIntVector.TYPE_WIDTH; + final int size = count * TinyIntVector.TYPE_WIDTH; ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { @@ -237,7 +237,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT2 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int size = count * NullableSmallIntVector.TYPE_WIDTH; + final int size = count * SmallIntVector.TYPE_WIDTH; ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { @@ -252,7 +252,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT4 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int size = count * NullableIntVector.TYPE_WIDTH; + final int size = count * IntVector.TYPE_WIDTH; ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { @@ -267,7 +267,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT8 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int size = count * NullableBigIntVector.TYPE_WIDTH; + final int size = count * BigIntVector.TYPE_WIDTH; ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { @@ -282,7 +282,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader FLOAT4 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int size = count * NullableFloat4Vector.TYPE_WIDTH; + final int size = count * Float4Vector.TYPE_WIDTH; ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { @@ -297,7 +297,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader FLOAT8 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int size = count * NullableFloat8Vector.TYPE_WIDTH; + final int size = count * Float8Vector.TYPE_WIDTH; ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { @@ -312,7 +312,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader DECIMAL = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int size = count * NullableDecimalVector.TYPE_WIDTH; + final int size = count * DecimalVector.TYPE_WIDTH; ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 22423b844e5db..067fb25b8d86c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -160,8 +160,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE generator.writeObjectField("name", field.getName()); int valueCount = vector.getValueCount(); generator.writeObjectField("count", valueCount); - final int scale = (vector instanceof NullableDecimalVector) ? - ((NullableDecimalVector) vector).getScale() : 0; + final int scale = (vector instanceof DecimalVector) ? + ((DecimalVector) vector).getScale() : 0; for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); ArrowBuf vectorBuffer = vectorBuffers.get(v); @@ -199,86 +199,86 @@ private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, ArrowBuf offsetBuffer, FieldVector vector, final int index, final int scale) throws IOException { if (bufferType.equals(TYPE)) { - generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH)); + generator.writeNumber(buffer.getByte(index * TinyIntVector.TYPE_WIDTH)); } else if (bufferType.equals(OFFSET)) { - generator.writeNumber(buffer.getInt(index * BaseNullableVariableWidthVector.OFFSET_WIDTH)); + generator.writeNumber(buffer.getInt(index * BaseVariableWidthVector.OFFSET_WIDTH)); } else if(bufferType.equals(VALIDITY)) { generator.writeNumber(vector.isNull(index) ? 0 : 1); } else if (bufferType.equals(DATA)) { switch (vector.getMinorType()) { case TINYINT: - generator.writeNumber(NullableTinyIntVector.get(buffer, index)); + generator.writeNumber(TinyIntVector.get(buffer, index)); break; case SMALLINT: - generator.writeNumber(NullableSmallIntVector.get(buffer, index)); + generator.writeNumber(SmallIntVector.get(buffer, index)); break; case INT: - generator.writeNumber(NullableIntVector.get(buffer, index)); + generator.writeNumber(IntVector.get(buffer, index)); break; case BIGINT: - generator.writeNumber(NullableBigIntVector.get(buffer, index)); + generator.writeNumber(BigIntVector.get(buffer, index)); break; case FLOAT4: - generator.writeNumber(NullableFloat4Vector.get(buffer, index)); + generator.writeNumber(Float4Vector.get(buffer, index)); break; case FLOAT8: - generator.writeNumber(NullableFloat8Vector.get(buffer, index)); + generator.writeNumber(Float8Vector.get(buffer, index)); break; case DATEDAY: - generator.writeNumber(NullableDateDayVector.get(buffer, index)); + generator.writeNumber(DateDayVector.get(buffer, index)); break; case DATEMILLI: - generator.writeNumber(NullableDateMilliVector.get(buffer, index)); + generator.writeNumber(DateMilliVector.get(buffer, index)); break; case TIMESEC: - generator.writeNumber(NullableTimeSecVector.get(buffer, index)); + generator.writeNumber(TimeSecVector.get(buffer, index)); break; case TIMEMILLI: - generator.writeNumber(NullableTimeMilliVector.get(buffer, index)); + generator.writeNumber(TimeMilliVector.get(buffer, index)); break; case TIMEMICRO: - generator.writeNumber(NullableTimeMicroVector.get(buffer, index)); + generator.writeNumber(TimeMicroVector.get(buffer, index)); break; case TIMENANO: - generator.writeNumber(NullableTimeNanoVector.get(buffer, index)); + generator.writeNumber(TimeNanoVector.get(buffer, index)); break; case TIMESTAMPSEC: - generator.writeNumber(NullableTimeStampSecVector.get(buffer, index)); + generator.writeNumber(TimeStampSecVector.get(buffer, index)); break; case TIMESTAMPMILLI: - generator.writeNumber(NullableTimeStampMilliVector.get(buffer, index)); + generator.writeNumber(TimeStampMilliVector.get(buffer, index)); break; case TIMESTAMPMICRO: - generator.writeNumber(NullableTimeStampMicroVector.get(buffer, index)); + generator.writeNumber(TimeStampMicroVector.get(buffer, index)); break; case TIMESTAMPNANO: - generator.writeNumber(NullableTimeStampNanoVector.get(buffer, index)); + generator.writeNumber(TimeStampNanoVector.get(buffer, index)); break; case TIMESTAMPSECTZ: - generator.writeNumber(NullableTimeStampSecTZVector.get(buffer, index)); + generator.writeNumber(TimeStampSecTZVector.get(buffer, index)); break; case TIMESTAMPMILLITZ: - generator.writeNumber(NullableTimeStampMilliTZVector.get(buffer, index)); + generator.writeNumber(TimeStampMilliTZVector.get(buffer, index)); break; case TIMESTAMPMICROTZ: - generator.writeNumber(NullableTimeStampMicroTZVector.get(buffer, index)); + generator.writeNumber(TimeStampMicroTZVector.get(buffer, index)); break; case TIMESTAMPNANOTZ: - generator.writeNumber(NullableTimeStampNanoTZVector.get(buffer, index)); + generator.writeNumber(TimeStampNanoTZVector.get(buffer, index)); break; case BIT: generator.writeNumber(BitVectorHelper.get(buffer, index)); break; case VARBINARY: { assert offsetBuffer != null; - String hexString = Hex.encodeHexString(BaseNullableVariableWidthVector.get(buffer, + String hexString = Hex.encodeHexString(BaseVariableWidthVector.get(buffer, offsetBuffer, index)); generator.writeObject(hexString); break; } case VARCHAR: { assert offsetBuffer != null; - byte[] b = (BaseNullableVariableWidthVector.get(buffer, offsetBuffer, index)); + byte[] b = (BaseVariableWidthVector.get(buffer, offsetBuffer, index)); generator.writeString(new String(b, "UTF-8")); break; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index c57dd6dafe9e6..7834845aad80e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -24,41 +24,41 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.NullableBigIntVector; -import org.apache.arrow.vector.NullableBitVector; -import org.apache.arrow.vector.NullableDateDayVector; -import org.apache.arrow.vector.NullableDateMilliVector; -import org.apache.arrow.vector.NullableDecimalVector; -import org.apache.arrow.vector.NullableFloat4Vector; -import org.apache.arrow.vector.NullableFloat8Vector; -import org.apache.arrow.vector.NullableIntVector; -import org.apache.arrow.vector.NullableIntervalDayVector; -import org.apache.arrow.vector.NullableIntervalYearVector; -import org.apache.arrow.vector.NullableSmallIntVector; -import org.apache.arrow.vector.NullableTimeMicroVector; -import org.apache.arrow.vector.NullableTimeMilliVector; -import org.apache.arrow.vector.NullableTimeNanoVector; -import org.apache.arrow.vector.NullableTimeSecVector; -import org.apache.arrow.vector.NullableTimeStampMicroTZVector; -import org.apache.arrow.vector.NullableTimeStampMicroVector; -import org.apache.arrow.vector.NullableTimeStampMilliTZVector; -import org.apache.arrow.vector.NullableTimeStampMilliVector; -import org.apache.arrow.vector.NullableTimeStampNanoTZVector; -import org.apache.arrow.vector.NullableTimeStampNanoVector; -import org.apache.arrow.vector.NullableTimeStampSecTZVector; -import org.apache.arrow.vector.NullableTimeStampSecVector; -import org.apache.arrow.vector.NullableTinyIntVector; -import org.apache.arrow.vector.NullableUInt1Vector; -import org.apache.arrow.vector.NullableUInt2Vector; -import org.apache.arrow.vector.NullableUInt4Vector; -import org.apache.arrow.vector.NullableUInt8Vector; -import org.apache.arrow.vector.NullableVarBinaryVector; -import org.apache.arrow.vector.NullableVarCharVector; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.IntervalDayVector; +import org.apache.arrow.vector.IntervalYearVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeNanoVector; +import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TimeStampSecTZVector; +import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.BigIntWriterImpl; import org.apache.arrow.vector.complex.impl.BitWriterImpl; @@ -131,304 +131,304 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { MAP(Struct.INSTANCE) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback); + return new MapVector(name, allocator, fieldType, schemaChangeCallback); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new NullableMapWriter((NullableMapVector) vector); + return new NullableMapWriter((MapVector) vector); } }, TINYINT(new Int(8, true)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTinyIntVector(name, fieldType, allocator); + return new TinyIntVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TinyIntWriterImpl((NullableTinyIntVector) vector); + return new TinyIntWriterImpl((TinyIntVector) vector); } }, SMALLINT(new Int(16, true)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableSmallIntVector(name, fieldType, allocator); + return new SmallIntVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new SmallIntWriterImpl((NullableSmallIntVector) vector); + return new SmallIntWriterImpl((SmallIntVector) vector); } }, INT(new Int(32, true)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableIntVector(name, fieldType, allocator); + return new IntVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new IntWriterImpl((NullableIntVector) vector); + return new IntWriterImpl((IntVector) vector); } }, BIGINT(new Int(64, true)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableBigIntVector(name, fieldType, allocator); + return new BigIntVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new BigIntWriterImpl((NullableBigIntVector) vector); + return new BigIntWriterImpl((BigIntVector) vector); } }, DATEDAY(new Date(DateUnit.DAY)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableDateDayVector(name, fieldType, allocator); + return new DateDayVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new DateDayWriterImpl((NullableDateDayVector) vector); + return new DateDayWriterImpl((DateDayVector) vector); } }, DATEMILLI(new Date(DateUnit.MILLISECOND)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableDateMilliVector(name, fieldType, allocator); + return new DateMilliVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new DateMilliWriterImpl((NullableDateMilliVector) vector); + return new DateMilliWriterImpl((DateMilliVector) vector); } }, TIMESEC(new Time(TimeUnit.SECOND, 32)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeSecVector(name, fieldType, allocator); + return new TimeSecVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeSecWriterImpl((NullableTimeSecVector) vector); + return new TimeSecWriterImpl((TimeSecVector) vector); } }, TIMEMILLI(new Time(TimeUnit.MILLISECOND, 32)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeMilliVector(name, fieldType, allocator); + return new TimeMilliVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeMilliWriterImpl((NullableTimeMilliVector) vector); + return new TimeMilliWriterImpl((TimeMilliVector) vector); } }, TIMEMICRO(new Time(TimeUnit.MICROSECOND, 64)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeMicroVector(name, fieldType, allocator); + return new TimeMicroVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeMicroWriterImpl((NullableTimeMicroVector) vector); + return new TimeMicroWriterImpl((TimeMicroVector) vector); } }, TIMENANO(new Time(TimeUnit.NANOSECOND, 64)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeNanoVector(name, fieldType, allocator); + return new TimeNanoVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeNanoWriterImpl((NullableTimeNanoVector) vector); + return new TimeNanoWriterImpl((TimeNanoVector) vector); } }, // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampSecVector(name, fieldType, allocator); + return new TimeStampSecVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampSecWriterImpl((NullableTimeStampSecVector) vector); + return new TimeStampSecWriterImpl((TimeStampSecVector) vector); } }, // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC. TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampMilliVector(name, fieldType, allocator); + return new TimeStampMilliVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampMilliWriterImpl((NullableTimeStampMilliVector) vector); + return new TimeStampMilliWriterImpl((TimeStampMilliVector) vector); } }, // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampMicroVector(name, fieldType, allocator); + return new TimeStampMicroVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampMicroWriterImpl((NullableTimeStampMicroVector) vector); + return new TimeStampMicroWriterImpl((TimeStampMicroVector) vector); } }, // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC. TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampNanoVector(name, fieldType, allocator); + return new TimeStampNanoVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampNanoWriterImpl((NullableTimeStampNanoVector) vector); + return new TimeStampNanoWriterImpl((TimeStampNanoVector) vector); } }, INTERVALDAY(new Interval(IntervalUnit.DAY_TIME)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableIntervalDayVector(name, fieldType, allocator); + return new IntervalDayVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new IntervalDayWriterImpl((NullableIntervalDayVector) vector); + return new IntervalDayWriterImpl((IntervalDayVector) vector); } }, INTERVALYEAR(new Interval(IntervalUnit.YEAR_MONTH)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableIntervalYearVector(name, fieldType, allocator); + return new IntervalYearVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new IntervalYearWriterImpl((NullableIntervalYearVector) vector); + return new IntervalYearWriterImpl((IntervalYearVector) vector); } }, // 4 byte ieee 754 FLOAT4(new FloatingPoint(SINGLE)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableFloat4Vector(name, fieldType, allocator); + return new Float4Vector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new Float4WriterImpl((NullableFloat4Vector) vector); + return new Float4WriterImpl((Float4Vector) vector); } }, // 8 byte ieee 754 FLOAT8(new FloatingPoint(DOUBLE)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableFloat8Vector(name, fieldType, allocator); + return new Float8Vector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new Float8WriterImpl((NullableFloat8Vector) vector); + return new Float8WriterImpl((Float8Vector) vector); } }, BIT(Bool.INSTANCE) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableBitVector(name, fieldType, allocator); + return new BitVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new BitWriterImpl((NullableBitVector) vector); + return new BitWriterImpl((BitVector) vector); } }, VARCHAR(Utf8.INSTANCE) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableVarCharVector(name, fieldType, allocator); + return new VarCharVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new VarCharWriterImpl((NullableVarCharVector) vector); + return new VarCharWriterImpl((VarCharVector) vector); } }, VARBINARY(Binary.INSTANCE) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableVarBinaryVector(name, fieldType, allocator); + return new VarBinaryVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new VarBinaryWriterImpl((NullableVarBinaryVector) vector); + return new VarBinaryWriterImpl((VarBinaryVector) vector); } }, DECIMAL(null) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableDecimalVector(name, fieldType, allocator); + return new DecimalVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new DecimalWriterImpl((NullableDecimalVector) vector); + return new DecimalWriterImpl((DecimalVector) vector); } }, UINT1(new Int(8, false)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableUInt1Vector(name, fieldType, allocator); + return new UInt1Vector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new UInt1WriterImpl((NullableUInt1Vector) vector); + return new UInt1WriterImpl((UInt1Vector) vector); } }, UINT2(new Int(16, false)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableUInt2Vector(name, fieldType, allocator); + return new UInt2Vector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new UInt2WriterImpl((NullableUInt2Vector) vector); + return new UInt2WriterImpl((UInt2Vector) vector); } }, UINT4(new Int(32, false)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableUInt4Vector(name, fieldType, allocator); + return new UInt4Vector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new UInt4WriterImpl((NullableUInt4Vector) vector); + return new UInt4WriterImpl((UInt4Vector) vector); } }, UINT8(new Int(64, false)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableUInt8Vector(name, fieldType, allocator); + return new UInt8Vector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new UInt8WriterImpl((NullableUInt8Vector) vector); + return new UInt8WriterImpl((UInt8Vector) vector); } }, LIST(List.INSTANCE) { @@ -470,45 +470,45 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { TIMESTAMPSECTZ(null) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampSecTZVector(name, fieldType, allocator); + return new TimeStampSecTZVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampSecTZWriterImpl((NullableTimeStampSecTZVector) vector); + return new TimeStampSecTZWriterImpl((TimeStampSecTZVector) vector); } }, TIMESTAMPMILLITZ(null) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampMilliTZVector(name, fieldType, allocator); + return new TimeStampMilliTZVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampMilliTZWriterImpl((NullableTimeStampMilliTZVector) vector); + return new TimeStampMilliTZWriterImpl((TimeStampMilliTZVector) vector); } }, TIMESTAMPMICROTZ(null) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampMicroTZVector(name, fieldType, allocator); + return new TimeStampMicroTZVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampMicroTZWriterImpl((NullableTimeStampMicroTZVector) vector); + return new TimeStampMicroTZWriterImpl((TimeStampMicroTZVector) vector); } }, TIMESTAMPNANOTZ(null) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new NullableTimeStampNanoTZVector(name, fieldType, allocator); + return new TimeStampNanoTZVector(name, fieldType, allocator); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new TimeStampNanoTZWriterImpl((NullableTimeStampNanoTZVector) vector); + return new TimeStampNanoTZWriterImpl((TimeStampNanoTZVector) vector); } }; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index acf7c58a1337f..c6d734d18d89b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -19,7 +19,6 @@ package org.apache.arrow.vector.util; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.types.pojo.ArrowType; import java.math.BigDecimal; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java index ada341497878e..36365fa9d4063 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java @@ -55,17 +55,17 @@ public void testBitVectorCopyFromSafe() { dst.allocateNew(10); for (int i = 0; i < size; i++) { - src.getMutator().set(i, i % 2); + src.set(i, i % 2); } - src.getMutator().setValueCount(size); + src.setValueCount(size); for (int i = 0; i < size; i++) { dst.copyFromSafe(i, i, src); } - dst.getMutator().setValueCount(size); + dst.setValueCount(size); for (int i = 0; i < size; i++) { - assertEquals(src.getAccessor().getObject(i), dst.getAccessor().getObject(i)); + assertEquals(src.getObject(i), dst.getObject(i)); } } } @@ -74,25 +74,23 @@ public void testBitVectorCopyFromSafe() { public void testSplitAndTransfer() throws Exception { try (final BitVector sourceVector = new BitVector("bitvector", allocator)) { - final BitVector.Mutator sourceMutator = sourceVector.getMutator(); - final BitVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(40); /* populate the bitvector -- 010101010101010101010101..... */ for (int i = 0; i < 40; i++) { if ((i & 1) == 1) { - sourceMutator.set(i, 1); + sourceVector.set(i, 1); } else { - sourceMutator.set(i, 0); + sourceVector.set(i, 0); } } - sourceMutator.setValueCount(40); + sourceVector.setValueCount(40); /* check the vector output */ for (int i = 0; i < 40; i++) { - int result = sourceAccessor.get(i); + int result = sourceVector.get(i); if ((i & 1) == 1) { assertEquals(Integer.toString(1), Integer.toString(result)); } else { @@ -102,8 +100,6 @@ public void testSplitAndTransfer() throws Exception { try (final BitVector toVector = new BitVector("toVector", allocator)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final BitVector.Accessor toAccessor = toVector.getAccessor(); - final BitVector.Mutator toMutator = toVector.getMutator(); /* * form test cases such that we cover: @@ -123,8 +119,8 @@ public void testSplitAndTransfer() throws Exception { /* check the toVector output after doing splitAndTransfer */ for (int i = 0; i < length; i++) { - int actual = toAccessor.get(i); - int expected = sourceAccessor.get(start + i); + int actual = toVector.get(i); + int expected = sourceVector.get(start + i); assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i, expected, actual); } @@ -137,28 +133,24 @@ public void testSplitAndTransfer() throws Exception { public void testSplitAndTransfer1() throws Exception { try (final BitVector sourceVector = new BitVector("bitvector", allocator)) { - final BitVector.Mutator sourceMutator = sourceVector.getMutator(); - final BitVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(8190); /* populate the bitvector */ for (int i = 0; i < 8190; i++) { - sourceMutator.set(i, 1); + sourceVector.set(i, 1); } - sourceMutator.setValueCount(8190); + sourceVector.setValueCount(8190); /* check the vector output */ for (int i = 0; i < 8190; i++) { - int result = sourceAccessor.get(i); + int result = sourceVector.get(i); assertEquals(Integer.toString(1), Integer.toString(result)); } try (final BitVector toVector = new BitVector("toVector", allocator)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final BitVector.Accessor toAccessor = toVector.getAccessor(); - final BitVector.Mutator toMutator = toVector.getMutator(); final int[][] transferLengths = {{0, 4095}, {4095, 4095}}; @@ -170,8 +162,8 @@ public void testSplitAndTransfer1() throws Exception { /* check the toVector output after doing splitAndTransfer */ for (int i = 0; i < length; i++) { - int actual = toAccessor.get(i); - int expected = sourceAccessor.get(start + i); + int actual = toVector.get(i); + int expected = sourceVector.get(start + i); assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i, expected, actual); } @@ -184,25 +176,23 @@ public void testSplitAndTransfer1() throws Exception { public void testSplitAndTransfer2() throws Exception { try (final BitVector sourceVector = new BitVector("bitvector", allocator)) { - final BitVector.Mutator sourceMutator = sourceVector.getMutator(); - final BitVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(32); /* populate the bitvector */ for (int i = 0; i < 32; i++) { if ((i & 1) == 1) { - sourceMutator.set(i, 1); + sourceVector.set(i, 1); } else { - sourceMutator.set(i, 0); + sourceVector.set(i, 0); } } - sourceMutator.setValueCount(32); + sourceVector.setValueCount(32); /* check the vector output */ for (int i = 0; i < 32; i++) { - int result = sourceAccessor.get(i); + int result = sourceVector.get(i); if ((i & 1) == 1) { assertEquals(Integer.toString(1), Integer.toString(result)); } else { @@ -212,8 +202,6 @@ public void testSplitAndTransfer2() throws Exception { try (final BitVector toVector = new BitVector("toVector", allocator)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final BitVector.Accessor toAccessor = toVector.getAccessor(); - final BitVector.Mutator toMutator = toVector.getMutator(); final int[][] transferLengths = {{5,22}, {5,24}, {5,25}, {5,27}, {0,31}, {5,7}, {2,3}}; @@ -225,8 +213,8 @@ public void testSplitAndTransfer2() throws Exception { /* check the toVector output after doing splitAndTransfer */ for (int i = 0; i < length; i++) { - int actual = toAccessor.get(i); - int expected = sourceAccessor.get(start + i); + int actual = toVector.get(i); + int expected = sourceVector.get(start + i); assertEquals("different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i, expected, actual); } @@ -242,62 +230,56 @@ public void testReallocAfterVectorTransfer1() { int valueCapacity = vector.getValueCapacity(); assertEquals(4096, valueCapacity); - final BitVector.Mutator mutator = vector.getMutator(); - final BitVector.Accessor accessor = vector.getAccessor(); - for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - mutator.setToOne(i); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity; i++) { - int val = accessor.get(i); if ((i & 1) == 1) { - assertEquals("unexpected cleared bit at index: " + i, 1, val); + assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i)); } else { - assertEquals("unexpected set bit at index: " + i, 0, val); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger first realloc */ - mutator.setSafeToOne(valueCapacity); + vector.setSafe(valueCapacity, 1); assertEquals(valueCapacity * 2, vector.getValueCapacity()); for (int i = valueCapacity; i < valueCapacity*2; i++) { if ((i & 1) == 1) { - mutator.setToOne(i); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*2; i++) { - int val = accessor.get(i); if (((i & 1) == 1) || (i == valueCapacity)) { - assertEquals("unexpected cleared bit at index: " + i, 1, val); + assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i)); } else { - assertEquals("unexpected set bit at index: " + i, 0, val); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger second realloc */ - mutator.setSafeToOne(valueCapacity*2); + vector.setSafe(valueCapacity*2, 1); assertEquals(valueCapacity * 4, vector.getValueCapacity()); for (int i = valueCapacity*2; i < valueCapacity*4; i++) { if ((i & 1) == 1) { - mutator.setToOne(i); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*4; i++) { - int val = accessor.get(i); if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) { - assertEquals("unexpected cleared bit at index: " + i, 1, val); + assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i)); } else { - assertEquals("unexpected set bit at index: " + i, 0, val); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } @@ -305,27 +287,24 @@ public void testReallocAfterVectorTransfer1() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); final BitVector toVector = (BitVector)transferPair.getTo(); - final BitVector.Accessor toAccessor = toVector.getAccessor(); - final BitVector.Mutator toMutator = toVector.getMutator(); assertEquals(valueCapacity * 4, toVector.getValueCapacity()); /* realloc the toVector */ - toMutator.setSafeToOne(valueCapacity * 4); + toVector.setSafe(valueCapacity * 4, 1); for (int i = 0; i < toVector.getValueCapacity(); i++) { - int val = toAccessor.get(i); if (i <= valueCapacity * 4) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2) || (i == valueCapacity*4)) { - assertEquals("unexpected cleared bit at index: " + i, 1, val); + assertEquals("unexpected cleared bit at index: " + i, 1, toVector.get(i)); } else { - assertEquals("unexpected set bit at index: " + i, 0, val); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } else { - assertEquals("unexpected set bit at index: " + i, 0, val); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } @@ -335,7 +314,7 @@ public void testReallocAfterVectorTransfer1() { @Test public void testReallocAfterVectorTransfer2() { - try (final NullableBitVector vector = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) { vector.allocateNew(4096); int valueCapacity = vector.getValueCapacity(); assertEquals(4096, valueCapacity); @@ -396,7 +375,7 @@ public void testReallocAfterVectorTransfer2() { /* now transfer the vector */ TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); - final NullableBitVector toVector = (NullableBitVector)transferPair.getTo(); + final BitVector toVector = (BitVector)transferPair.getTo(); assertEquals(valueCapacity * 4, toVector.getValueCapacity()); @@ -426,81 +405,81 @@ public void testReallocAfterVectorTransfer2() { public void testBitVector() { // Create a new value vector for 1024 integers try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) { - final BitVector.Mutator m = vector.getMutator(); vector.allocateNew(1024); - m.setValueCount(1024); + vector.setValueCount(1024); // Put and set a few values - m.set(0, 1); - m.set(1, 0); - m.set(100, 0); - m.set(1022, 1); + vector.set(0, 1); + vector.set(1, 0); + vector.set(100, 0); + vector.set(1022, 1); - m.setValueCount(1024); + vector.setValueCount(1024); - final BitVector.Accessor accessor = vector.getAccessor(); - assertEquals(1, accessor.get(0)); - assertEquals(0, accessor.get(1)); - assertEquals(0, accessor.get(100)); - assertEquals(1, accessor.get(1022)); + assertEquals(1, vector.get(0)); + assertEquals(0, vector.get(1)); + assertEquals(0, vector.get(100)); + assertEquals(1, vector.get(1022)); - assertEquals(1022, accessor.getNullCount()); + assertEquals(1020, vector.getNullCount()); // test setting the same value twice - m.set(0, 1); - m.set(0, 1); - m.set(1, 0); - m.set(1, 0); - assertEquals(1, accessor.get(0)); - assertEquals(0, accessor.get(1)); + vector.set(0, 1); + vector.set(0, 1); + vector.set(1, 0); + vector.set(1, 0); + assertEquals(1, vector.get(0)); + assertEquals(0, vector.get(1)); // test toggling the values - m.set(0, 0); - m.set(1, 1); - assertEquals(0, accessor.get(0)); - assertEquals(1, accessor.get(1)); + vector.set(0, 0); + vector.set(1, 1); + assertEquals(0, vector.get(0)); + assertEquals(1, vector.get(1)); // should not change - assertEquals(1022, accessor.getNullCount()); + assertEquals(1020, vector.getNullCount()); - // Ensure unallocated space returns 0 - assertEquals(0, accessor.get(3)); + // Ensure null value + assertTrue(vector.isNull(3)); // unset the previously set bits - m.set(1, 0); - m.set(1022, 0); + vector.setNull(0); + vector.setNull(1); + vector.setNull(100); + vector.setNull(1022); // this should set all the array to 0 - assertEquals(1024, accessor.getNullCount()); + assertEquals(1024, vector.getNullCount()); // set all the array to 1 for (int i = 0; i < 1024; ++i) { - assertEquals(1024 - i, accessor.getNullCount()); - m.set(i, 1); + assertEquals(1024 - i, vector.getNullCount()); + vector.set(i, 1); } - assertEquals(0, accessor.getNullCount()); + assertEquals(0, vector.getNullCount()); vector.allocateNew(1015); - m.setValueCount(1015); + vector.setValueCount(1015); // ensure it has been zeroed - assertEquals(1015, accessor.getNullCount()); + assertEquals(1015, vector.getNullCount()); - m.set(0, 1); - m.set(1014, 1); // ensure that the last item of the last byte is allocated + vector.set(0, 1); + vector.set(1014, 1); // ensure that the last item of the last byte is allocated - assertEquals(1013, accessor.getNullCount()); + assertEquals(1013, vector.getNullCount()); vector.zeroVector(); - assertEquals(1015, accessor.getNullCount()); + assertEquals(1015, vector.getNullCount()); // set all the array to 1 for (int i = 0; i < 1015; ++i) { - assertEquals(1015 - i, accessor.getNullCount()); - m.set(i, 1); + assertEquals(1015 - i, vector.getNullCount()); + vector.set(i, 1); } - assertEquals(0, accessor.getNullCount()); + assertEquals(0, vector.getNullCount()); } } @@ -526,15 +505,17 @@ private void validateRange(int length, int start, int count) { try (BitVector bitVector = new BitVector("bits", allocator)) { bitVector.reset(); bitVector.allocateNew(length); - bitVector.getMutator().setRangeToOne(start, count); + for (int i = start; i < start + count; i++) { + bitVector.set(i, 1); + } for (int i = 0; i < start; i++) { - Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i)); + Assert.assertTrue(desc + i, bitVector.isNull(i)); } for (int i = start; i < start + count; i++) { - Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i)); + Assert.assertEquals(desc + i, 1, bitVector.get(i)); } for (int i = start + count; i < length; i++) { - Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i)); + Assert.assertTrue(desc + i, bitVector.isNull(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java index 4fe2861450a34..20f6754be2def 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java @@ -38,11 +38,11 @@ public void testTransferFixedWidth() { BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 100000, 100000); BufferAllocator childAllocator2 = allocator.newChildAllocator("child2", 100000, 100000); - NullableIntVector v1 = new NullableIntVector("v1", childAllocator1); + IntVector v1 = new IntVector("v1", childAllocator1); v1.allocateNew(); v1.setValueCount(4095); - NullableIntVector v2 = new NullableIntVector("v2", childAllocator2); + IntVector v2 = new IntVector("v2", childAllocator2); v1.makeTransferPair(v2).transfer(); @@ -58,12 +58,12 @@ public void testTransferVariableidth() { BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 100000, 100000); BufferAllocator childAllocator2 = allocator.newChildAllocator("child2", 100000, 100000); - NullableVarCharVector v1 = new NullableVarCharVector("v1", childAllocator1); + VarCharVector v1 = new VarCharVector("v1", childAllocator1); v1.allocateNew(); v1.setSafe(4094, "hello world".getBytes(), 0, 11); v1.setValueCount(4001); - NullableVarCharVector v2 = new NullableVarCharVector("v2", childAllocator2); + VarCharVector v2 = new VarCharVector("v2", childAllocator2); v1.makeTransferPair(v2).transfer(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java index 7cdfbdf452d84..87ffcafecd198 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java @@ -70,8 +70,8 @@ public void terminate() throws Exception { @Test /* NullableVarChar */ public void testCopyFromWithNulls() { - try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator); - final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) { + try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator); + final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) { vector.allocateNew(); int capacity = vector.getValueCapacity(); @@ -130,8 +130,8 @@ public void testCopyFromWithNulls() { @Test /* NullableVarChar */ public void testCopyFromWithNulls1() { - try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator); - final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) { + try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator); + final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VARCHAR, allocator)) { vector.allocateNew(); int capacity = vector.getValueCapacity(); @@ -192,10 +192,10 @@ public void testCopyFromWithNulls1() { } } - @Test /* NullableIntVector */ + @Test /* IntVector */ public void testCopyFromWithNulls2() { - try (final NullableIntVector vector1 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator); - final NullableIntVector vector2 = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final IntVector vector1 = new IntVector(EMPTY_SCHEMA_PATH, allocator); + final IntVector vector2 = new IntVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -252,10 +252,10 @@ public void testCopyFromWithNulls2() { } } - @Test /* NullableBigIntVector */ + @Test /* BigIntVector */ public void testCopyFromWithNulls3() { - try (final NullableBigIntVector vector1 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator); - final NullableBigIntVector vector2 = new NullableBigIntVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final BigIntVector vector1 = new BigIntVector(EMPTY_SCHEMA_PATH, allocator); + final BigIntVector vector2 = new BigIntVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -314,10 +314,10 @@ public void testCopyFromWithNulls3() { } } - @Test /* NullableBitVector */ + @Test /* BitVector */ public void testCopyFromWithNulls4() { - try (final NullableBitVector vector1 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator); - final NullableBitVector vector2 = new NullableBitVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final BitVector vector1 = new BitVector(EMPTY_SCHEMA_PATH, allocator); + final BitVector vector2 = new BitVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -392,10 +392,10 @@ public void testCopyFromWithNulls4() { } } - @Test /* NullableFloat4Vector */ + @Test /* Float4Vector */ public void testCopyFromWithNulls5() { - try (final NullableFloat4Vector vector1 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator); - final NullableFloat4Vector vector2 = new NullableFloat4Vector(EMPTY_SCHEMA_PATH, allocator)) { + try (final Float4Vector vector1 = new Float4Vector(EMPTY_SCHEMA_PATH, allocator); + final Float4Vector vector2 = new Float4Vector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -454,10 +454,10 @@ public void testCopyFromWithNulls5() { } } - @Test /* NullableFloat8Vector */ + @Test /* Float8Vector */ public void testCopyFromWithNulls6() { - try (final NullableFloat8Vector vector1 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator); - final NullableFloat8Vector vector2 = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) { + try (final Float8Vector vector1 = new Float8Vector(EMPTY_SCHEMA_PATH, allocator); + final Float8Vector vector2 = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -516,10 +516,10 @@ public void testCopyFromWithNulls6() { } } - @Test /* NullableIntervalDayVector */ + @Test /* IntervalDayVector */ public void testCopyFromWithNulls7() { - try (final NullableIntervalDayVector vector1 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator); - final NullableIntervalDayVector vector2 = new NullableIntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final IntervalDayVector vector1 = new IntervalDayVector(EMPTY_SCHEMA_PATH, allocator); + final IntervalDayVector vector2 = new IntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -582,10 +582,10 @@ public void testCopyFromWithNulls7() { } } - @Test /* NullableIntervalYearVector */ + @Test /* IntervalYearVector */ public void testCopyFromWithNulls8() { - try (final NullableIntervalYearVector vector1 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator); - final NullableIntervalYearVector vector2 = new NullableIntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final IntervalYearVector vector1 = new IntervalYearVector(EMPTY_SCHEMA_PATH, allocator); + final IntervalYearVector vector2 = new IntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -651,10 +651,10 @@ public void testCopyFromWithNulls8() { } } - @Test /* NullableSmallIntVector */ + @Test /* SmallIntVector */ public void testCopyFromWithNulls9() { - try (final NullableSmallIntVector vector1 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator); - final NullableSmallIntVector vector2 = new NullableSmallIntVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final SmallIntVector vector1 = new SmallIntVector(EMPTY_SCHEMA_PATH, allocator); + final SmallIntVector vector2 = new SmallIntVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -714,10 +714,10 @@ public void testCopyFromWithNulls9() { } } - @Test /* NullableTimeMicroVector */ + @Test /* TimeMicroVector */ public void testCopyFromWithNulls10() { - try (final NullableTimeMicroVector vector1 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator); - final NullableTimeMicroVector vector2 = new NullableTimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final TimeMicroVector vector1 = new TimeMicroVector(EMPTY_SCHEMA_PATH, allocator); + final TimeMicroVector vector2 = new TimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -777,10 +777,10 @@ public void testCopyFromWithNulls10() { } } - @Test /* NullableTimeMilliVector */ + @Test /* TimeMilliVector */ public void testCopyFromWithNulls11() { - try (final NullableTimeMilliVector vector1 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator); - final NullableTimeMilliVector vector2 = new NullableTimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final TimeMilliVector vector1 = new TimeMilliVector(EMPTY_SCHEMA_PATH, allocator); + final TimeMilliVector vector2 = new TimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -840,10 +840,10 @@ public void testCopyFromWithNulls11() { } } - @Test /* NullableTinyIntVector */ + @Test /* TinyIntVector */ public void testCopyFromWithNulls12() { - try (final NullableTinyIntVector vector1 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator); - final NullableTinyIntVector vector2 = new NullableTinyIntVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final TinyIntVector vector1 = new TinyIntVector(EMPTY_SCHEMA_PATH, allocator); + final TinyIntVector vector2 = new TinyIntVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -906,10 +906,10 @@ public void testCopyFromWithNulls12() { } } - @Test /* NullableDecimalVector */ + @Test /* DecimalVector */ public void testCopyFromWithNulls13() { - try (final NullableDecimalVector vector1 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16); - final NullableDecimalVector vector2 = new NullableDecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) { + try (final DecimalVector vector1 = new DecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16); + final DecimalVector vector2 = new DecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); @@ -972,10 +972,10 @@ public void testCopyFromWithNulls13() { } } - @Test /* NullableTimeStampVector */ + @Test /* TimeStampVector */ public void testCopyFromWithNulls14() { - try (final NullableTimeStampVector vector1 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator); - final NullableTimeStampVector vector2 = new NullableTimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final TimeStampVector vector1 = new TimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator); + final TimeStampVector vector2 = new TimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) { vector1.allocateNew(); assertEquals(4096, vector1.getValueCapacity()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index 4d844d6d3ca0f..a48c035718990 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -60,9 +60,9 @@ public void terminate() throws Exception { @Test public void testValuesWriteRead() { - try (NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);) { + try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(10, scale), allocator);) { - try (NullableDecimalVector oldConstructor = new NullableDecimalVector("decimal", allocator, 10, scale);) { + try (DecimalVector oldConstructor = new DecimalVector("decimal", allocator, 10, scale);) { assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType()); } @@ -85,7 +85,7 @@ public void testValuesWriteRead() { @Test public void testBigDecimalDifferentScaleAndPrecision() { - try (NullableDecimalVector decimalVector = TestUtils.newVector(NullableDecimalVector.class, "decimal", new ArrowType.Decimal(4, 2), allocator);) { + try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(4, 2), allocator);) { decimalVector.allocateNew(); // test BigDecimal with different scale diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 46a2bafa00208..ba149bcb8cb9f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector; -import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector; +import static org.apache.arrow.vector.TestUtils.newVarCharVector; import static org.junit.Assert.assertEquals; import java.nio.charset.StandardCharsets; @@ -54,8 +54,8 @@ public void terminate() throws Exception { @Test public void testEncodeStrings() { // Create a new value vector - try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator); - final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) { + try (final VarCharVector vector = newVarCharVector("foo", allocator); + final VarCharVector dictionaryVector = newVarCharVector("dict", allocator);) { vector.allocateNew(512, 5); // set some values @@ -77,9 +77,9 @@ public void testEncodeStrings() { try (final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) { // verify indices - assertEquals(NullableIntVector.class, encoded.getClass()); + assertEquals(IntVector.class, encoded.getClass()); - NullableIntVector index = ((NullableIntVector)encoded); + IntVector index = ((IntVector)encoded); assertEquals(5, index.getValueCount()); assertEquals(0, index.get(0)); assertEquals(1, index.get(1)); @@ -90,9 +90,9 @@ public void testEncodeStrings() { // now run through the decoder and verify we get the original back try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) { assertEquals(vector.getClass(), decoded.getClass()); - assertEquals(vector.getValueCount(), ((NullableVarCharVector)decoded).getValueCount()); + assertEquals(vector.getValueCount(), ((VarCharVector)decoded).getValueCount()); for (int i = 0; i < 5; i++) { - assertEquals(vector.getObject(i), ((NullableVarCharVector)decoded).getObject(i)); + assertEquals(vector.getObject(i), ((VarCharVector)decoded).getObject(i)); } } } @@ -102,8 +102,8 @@ public void testEncodeStrings() { @Test public void testEncodeLargeVector() { // Create a new value vector - try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator); - final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) { + try (final VarCharVector vector = newVarCharVector("foo", allocator); + final VarCharVector dictionaryVector = newVarCharVector("dict", allocator);) { vector.allocateNew(); int count = 10000; @@ -124,9 +124,9 @@ public void testEncodeLargeVector() { try (final ValueVector encoded = (FieldVector) DictionaryEncoder.encode(vector, dictionary)) { // verify indices - assertEquals(NullableIntVector.class, encoded.getClass()); + assertEquals(IntVector.class, encoded.getClass()); - NullableIntVector index = ((NullableIntVector) encoded); + IntVector index = ((IntVector) encoded); assertEquals(count, index.getValueCount()); for (int i = 0; i < count; ++i) { assertEquals(i % 3, index.get(i)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 2af6cd53aa0c1..50438ce116dfd 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -53,7 +53,7 @@ public void terminate() throws Exception { @Test public void testIntType() { try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { - NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); + IntVector nested = (IntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { @@ -80,7 +80,7 @@ public void testIntType() { @Test public void testFloatTypeNullable() { try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { - NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); + Float4Vector nested = (Float4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { @@ -115,7 +115,7 @@ public void testFloatTypeNullable() { public void testNestedInList() { try (ListVector vector = ListVector.empty("list", allocator)) { FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector(); - NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); + IntVector innerVector = (IntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { @@ -157,7 +157,7 @@ public void testNestedInList() { public void testTransferPair() { try (FixedSizeListVector from = new FixedSizeListVector("from", allocator, 2, null, null); FixedSizeListVector to = new FixedSizeListVector("to", allocator, 2, null, null)) { - NullableFloat4Vector nested = (NullableFloat4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); + Float4Vector nested = (Float4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); from.allocateNew(); for (int i = 0; i < 10; i++) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index f6aa86a300876..1cff7fa886e87 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -131,8 +131,8 @@ public void testSetLastSetUsage() throws Exception { ArrowBuf validityBuffer = listVector.getValidityBuffer(); ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); - /* get the underlying data vector -- NullableBigIntVector */ - NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); + /* get the underlying data vector -- BigIntVector */ + BigIntVector dataVector = (BigIntVector) listVector.getDataVector(); /* check current lastSet */ assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet())); @@ -310,7 +310,7 @@ public void testSplitAndTransfer() throws Exception { final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* get dataVector */ - NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); + BigIntVector dataVector = (BigIntVector) listVector.getDataVector(); /* check the vector output */ @@ -418,7 +418,7 @@ public void testSplitAndTransfer() throws Exception { final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer(); /* get dataVector of toVector */ - NullableBigIntVector dataVector1 = (NullableBigIntVector) toVector.getDataVector(); + BigIntVector dataVector1 = (BigIntVector) toVector.getDataVector(); for (int i = 0; i < splitLength; i++) { dataLength1 = offsetBuffer.getInt((start + i + 1) * ListVector.OFFSET_WIDTH) - diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 357df96aa2efc..e5682955bada0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -22,7 +22,7 @@ import java.util.Map; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.After; @@ -50,7 +50,7 @@ public void testFieldMetadata() throws Exception { Map metadata = new HashMap<>(); metadata.put("k1", "v1"); FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata); - try (NullableMapVector vector = new NullableMapVector("map", allocator, type, null)) { + try (MapVector vector = new MapVector("map", allocator, type, null)) { Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java index ba2ebbf05ad0d..f14dbd6637356 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java @@ -112,10 +112,10 @@ public void testVariableVectorReallocation() { try { vector.allocateNew(expectedAllocationInBytes, 10); assertTrue(expectedOffsetSize <= vector.getValueCapacity()); - assertTrue(expectedAllocationInBytes <= vector.getBuffer().capacity()); + assertTrue(expectedAllocationInBytes <= vector.getDataBuffer().capacity()); vector.reAlloc(); assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity()); - assertTrue(expectedAllocationInBytes * 2 <= vector.getBuffer().capacity()); + assertTrue(expectedAllocationInBytes * 2 <= vector.getDataBuffer().capacity()); } finally { vector.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index 7de3bcb33195f..80d5fe19700ac 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -24,7 +24,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.NullableVarCharVector; +import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.util.TransferPair; import org.junit.After; @@ -45,9 +45,9 @@ public void terminate() throws Exception { allocator.close(); } - @Test /* NullableVarCharVector */ + @Test /* VarCharVector */ public void test() throws Exception { - try(final NullableVarCharVector varCharVector = new NullableVarCharVector("myvector", allocator)) { + try(final VarCharVector varCharVector = new VarCharVector("myvector", allocator)) { varCharVector.allocateNew(10000, 1000); final int valueCount = 500; @@ -61,7 +61,7 @@ public void test() throws Exception { varCharVector.setValueCount(valueCount); final TransferPair tp = varCharVector.getTransferPair(allocator); - final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo(); + final VarCharVector newVarCharVector = (VarCharVector) tp.getTo(); final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}}; for (final int[] startLength : startLengths) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java index a148813090900..99a1d89071c71 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUtils.java @@ -25,13 +25,13 @@ public class TestUtils { - public static NullableVarCharVector newNullableVarCharVector(String name, BufferAllocator allocator) { - return (NullableVarCharVector) + public static VarCharVector newVarCharVector(String name, BufferAllocator allocator) { + return (VarCharVector) FieldType.nullable(new ArrowType.Utf8()).createNewSingleVector(name, allocator, null); } - public static NullableVarBinaryVector newNullableVarBinaryVector(String name, BufferAllocator allocator) { - return (NullableVarBinaryVector) + public static VarBinaryVector newVarBinaryVector(String name, BufferAllocator allocator) { + return (VarBinaryVector) FieldType.nullable(new ArrowType.Binary()).createNewSingleVector(name, allocator, null); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index f51a874367622..336ae1c7a0146 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -19,8 +19,8 @@ package org.apache.arrow.vector; import org.apache.arrow.vector.util.OversizedAllocationException; -import static org.apache.arrow.vector.TestUtils.newNullableVarBinaryVector; -import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector; +import static org.apache.arrow.vector.TestUtils.newVarBinaryVector; +import static org.apache.arrow.vector.TestUtils.newVarCharVector; import static org.apache.arrow.vector.TestUtils.newVector; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -88,9 +88,9 @@ public void terminate() throws Exception { * -- Float4Vector * -- Float8Vector * - * -- NullableUInt4Vector - * -- NullableIntVector - * -- NullableFloat4Vector + * -- UInt4Vector + * -- IntVector + * -- Float4Vector * * TODO: * @@ -107,28 +107,26 @@ public void testFixedType1() { boolean error = false; int initialCapacity = 0; - final UInt4Vector.Mutator mutator = vector.getMutator(); - final UInt4Vector.Accessor accessor = vector.getAccessor(); vector.allocateNew(1024); initialCapacity = vector.getValueCapacity(); assertEquals(1024, initialCapacity); // Put and set a few values - mutator.setSafe(0, 100); - mutator.setSafe(1, 101); - mutator.setSafe(100, 102); - mutator.setSafe(1022, 103); - mutator.setSafe(1023, 104); - - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); + vector.setSafe(0, 100); + vector.setSafe(1, 101); + vector.setSafe(100, 102); + vector.setSafe(1022, 103); + vector.setSafe(1023, 104); + + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); try { - mutator.set(1024, 10000); + vector.set(1024, 10000); } catch (IndexOutOfBoundsException ie) { error = true; @@ -139,7 +137,7 @@ public void testFixedType1() { } try { - accessor.get(1024); + vector.get(1024); } catch (IndexOutOfBoundsException ie) { error = true; @@ -150,18 +148,18 @@ public void testFixedType1() { } /* this should trigger a realloc() */ - mutator.setSafe(1024, 10000); + vector.setSafe(1024, 10000); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); /* check vector data after realloc */ - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); - assertEquals(10000, accessor.get(1024)); + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); + assertEquals(10000, vector.get(1024)); /* reset the vector */ vector.reset(); @@ -171,7 +169,8 @@ public void testFixedType1() { /* vector data should have been zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i)); + // TODO: test vector.get(i) is 0 after unsafe get added + assertEquals("non-zero data not expected at index: " + i, true, vector.isNull(i)); } } } @@ -179,8 +178,6 @@ public void testFixedType1() { @Test /* IntVector */ public void testFixedType2() { try (final IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) { - final IntVector.Mutator mutator = intVector.getMutator(); - final IntVector.Accessor accessor = intVector.getAccessor(); boolean error = false; int initialCapacity = 16; @@ -211,12 +208,12 @@ public void testFixedType2() { /* populate the vector */ int j = 1; for(int i = 0; i < 16; i += 2) { - mutator.set(i, j); + intVector.set(i, j); j++; } try { - mutator.set(16, 9); + intVector.set(16, 9); } catch (IndexOutOfBoundsException ie) { error = true; @@ -229,12 +226,12 @@ public void testFixedType2() { /* check vector contents */ j = 1; for(int i = 0; i < 16; i += 2) { - assertEquals("unexpected value at index: " + i, j, accessor.get(i)); + assertEquals("unexpected value at index: " + i, j, intVector.get(i)); j++; } try { - accessor.get(16); + intVector.get(16); } catch (IndexOutOfBoundsException ie) { error = true; @@ -245,7 +242,7 @@ public void testFixedType2() { } /* this should trigger a realloc() */ - mutator.setSafe(16, 9); + intVector.setSafe(16, 9); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, intVector.getValueCapacity()); @@ -253,7 +250,7 @@ public void testFixedType2() { /* vector data should still be intact after realloc */ j = 1; for(int i = 0; i <= 16; i += 2) { - assertEquals("unexpected value at index: " + i, j, accessor.get(i)); + assertEquals("unexpected value at index: " + i, j, intVector.get(i)); j++; } @@ -265,7 +262,7 @@ public void testFixedType2() { /* vector data should have been zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i)); + assertEquals("non-zero data not expected at index: " + i, true, intVector.isNull(i)); } } } @@ -273,8 +270,6 @@ public void testFixedType2() { @Test /* Float4Vector */ public void testFixedType3() { try (final Float4Vector floatVector = new Float4Vector(EMPTY_SCHEMA_PATH, allocator)) { - final Float4Vector.Mutator mutator = floatVector.getMutator(); - final Float4Vector.Accessor accessor = floatVector.getAccessor(); boolean error = false; int initialCapacity = 16; @@ -304,18 +299,18 @@ public void testFixedType3() { floatVector.zeroVector(); - /* populate the vector */ - mutator.set(0, 1.5f); - mutator.set(2, 2.5f); - mutator.set(4, 3.3f); - mutator.set(6, 4.8f); - mutator.set(8, 5.6f); - mutator.set(10, 6.6f); - mutator.set(12, 7.8f); - mutator.set(14, 8.5f); + /* populate the floatVector */ + floatVector.set(0, 1.5f); + floatVector.set(2, 2.5f); + floatVector.set(4, 3.3f); + floatVector.set(6, 4.8f); + floatVector.set(8, 5.6f); + floatVector.set(10, 6.6f); + floatVector.set(12, 7.8f); + floatVector.set(14, 8.5f); try { - mutator.set(16, 9.5f); + floatVector.set(16, 9.5f); } catch (IndexOutOfBoundsException ie) { error = true; @@ -326,17 +321,17 @@ public void testFixedType3() { } /* check vector contents */ - assertEquals(1.5f, accessor.get(0), 0); - assertEquals(2.5f, accessor.get(2), 0); - assertEquals(3.3f, accessor.get(4), 0); - assertEquals(4.8f, accessor.get(6), 0); - assertEquals(5.6f, accessor.get(8), 0); - assertEquals(6.6f, accessor.get(10), 0); - assertEquals(7.8f, accessor.get(12), 0); - assertEquals(8.5f, accessor.get(14), 0); + assertEquals(1.5f, floatVector.get(0), 0); + assertEquals(2.5f, floatVector.get(2), 0); + assertEquals(3.3f, floatVector.get(4), 0); + assertEquals(4.8f, floatVector.get(6), 0); + assertEquals(5.6f, floatVector.get(8), 0); + assertEquals(6.6f, floatVector.get(10), 0); + assertEquals(7.8f, floatVector.get(12), 0); + assertEquals(8.5f, floatVector.get(14), 0); try { - accessor.get(16); + floatVector.get(16); } catch (IndexOutOfBoundsException ie) { error = true; @@ -347,21 +342,21 @@ public void testFixedType3() { } /* this should trigger a realloc() */ - mutator.setSafe(16, 9.5f); + floatVector.setSafe(16, 9.5f); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, floatVector.getValueCapacity()); /* vector data should still be intact after realloc */ - assertEquals(1.5f, accessor.get(0), 0); - assertEquals(2.5f, accessor.get(2), 0); - assertEquals(3.3f, accessor.get(4), 0); - assertEquals(4.8f, accessor.get(6), 0); - assertEquals(5.6f, accessor.get(8), 0); - assertEquals(6.6f, accessor.get(10), 0); - assertEquals(7.8f, accessor.get(12), 0); - assertEquals(8.5f, accessor.get(14), 0); - assertEquals(9.5f, accessor.get(16), 0); + assertEquals(1.5f, floatVector.get(0), 0); + assertEquals(2.5f, floatVector.get(2), 0); + assertEquals(3.3f, floatVector.get(4), 0); + assertEquals(4.8f, floatVector.get(6), 0); + assertEquals(5.6f, floatVector.get(8), 0); + assertEquals(6.6f, floatVector.get(10), 0); + assertEquals(7.8f, floatVector.get(12), 0); + assertEquals(8.5f, floatVector.get(14), 0); + assertEquals(9.5f, floatVector.get(16), 0); /* reset the vector */ floatVector.reset(); @@ -371,7 +366,7 @@ public void testFixedType3() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i), 0); + assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i)); } } } @@ -379,8 +374,6 @@ public void testFixedType3() { @Test /* Float8Vector */ public void testFixedType4() { try (final Float8Vector floatVector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) { - final Float8Vector.Mutator mutator = floatVector.getMutator(); - final Float8Vector.Accessor accessor = floatVector.getAccessor(); boolean error = false; int initialCapacity = 16; @@ -409,17 +402,17 @@ public void testFixedType4() { assertEquals(initialCapacity, floatVector.getValueCapacity()); /* populate the vector */ - mutator.set(0, 1.55); - mutator.set(2, 2.53); - mutator.set(4, 3.36); - mutator.set(6, 4.82); - mutator.set(8, 5.67); - mutator.set(10, 6.67); - mutator.set(12, 7.87); - mutator.set(14, 8.56); + floatVector.set(0, 1.55); + floatVector.set(2, 2.53); + floatVector.set(4, 3.36); + floatVector.set(6, 4.82); + floatVector.set(8, 5.67); + floatVector.set(10, 6.67); + floatVector.set(12, 7.87); + floatVector.set(14, 8.56); try { - mutator.set(16, 9.53); + floatVector.set(16, 9.53); } catch (IndexOutOfBoundsException ie) { error = true; @@ -429,18 +422,18 @@ public void testFixedType4() { error = false; } - /* check vector contents */ - assertEquals(1.55, accessor.get(0), 0); - assertEquals(2.53, accessor.get(2), 0); - assertEquals(3.36, accessor.get(4), 0); - assertEquals(4.82, accessor.get(6), 0); - assertEquals(5.67, accessor.get(8), 0); - assertEquals(6.67, accessor.get(10), 0); - assertEquals(7.87, accessor.get(12), 0); - assertEquals(8.56, accessor.get(14), 0); + /* check floatVector contents */ + assertEquals(1.55, floatVector.get(0), 0); + assertEquals(2.53, floatVector.get(2), 0); + assertEquals(3.36, floatVector.get(4), 0); + assertEquals(4.82, floatVector.get(6), 0); + assertEquals(5.67, floatVector.get(8), 0); + assertEquals(6.67, floatVector.get(10), 0); + assertEquals(7.87, floatVector.get(12), 0); + assertEquals(8.56, floatVector.get(14), 0); try { - accessor.get(16); + floatVector.get(16); } catch (IndexOutOfBoundsException ie) { error = true; @@ -451,21 +444,21 @@ public void testFixedType4() { } /* this should trigger a realloc() */ - mutator.setSafe(16, 9.53); + floatVector.setSafe(16, 9.53); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, floatVector.getValueCapacity()); /* vector data should still be intact after realloc */ - assertEquals(1.55, accessor.get(0), 0); - assertEquals(2.53, accessor.get(2), 0); - assertEquals(3.36, accessor.get(4), 0); - assertEquals(4.82, accessor.get(6), 0); - assertEquals(5.67, accessor.get(8), 0); - assertEquals(6.67, accessor.get(10), 0); - assertEquals(7.87, accessor.get(12), 0); - assertEquals(8.56, accessor.get(14), 0); - assertEquals(9.53, accessor.get(16), 0); + assertEquals(1.55, floatVector.get(0), 0); + assertEquals(2.53, floatVector.get(2), 0); + assertEquals(3.36, floatVector.get(4), 0); + assertEquals(4.82, floatVector.get(6), 0); + assertEquals(5.67, floatVector.get(8), 0); + assertEquals(6.67, floatVector.get(10), 0); + assertEquals(7.87, floatVector.get(12), 0); + assertEquals(8.56, floatVector.get(14), 0); + assertEquals(9.53, floatVector.get(16), 0); /* reset the vector */ floatVector.reset(); @@ -475,16 +468,16 @@ public void testFixedType4() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertEquals("non-zero data not expected at index: " + i, 0, accessor.get(i), 0); + assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i)); } } } - @Test /* NullableUInt4Vector */ + @Test /* UInt4Vector */ public void testNullableFixedType1() { // Create a new value vector for 1024 integers. - try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) { + try (final UInt4Vector vector = newVector(UInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) { boolean error = false; int initialCapacity = 1024; @@ -582,10 +575,10 @@ public void testNullableFixedType1() { } } - @Test /* NullableFloat4Vector */ + @Test /* Float4Vector */ public void testNullableFixedType2() { // Create a new value vector for 1024 integers - try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) { + try (final Float4Vector vector = newVector(Float4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) { boolean error = false; int initialCapacity = 16; @@ -683,10 +676,10 @@ public void testNullableFixedType2() { } } - @Test /* NullableIntVector */ + @Test /* IntVector */ public void testNullableFixedType3() { // Create a new value vector for 1024 integers - try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { + try (final IntVector vector = newVector(IntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { boolean error = false; int initialCapacity = 1024; @@ -777,9 +770,9 @@ public void testNullableFixedType3() { } } - @Test /* NullableIntVector */ + @Test /* IntVector */ public void testNullableFixedType4() { - try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { + try (final IntVector vector = newVector(IntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { /* no memory allocation has happened yet */ assertEquals(0, vector.getValueCapacity()); @@ -870,8 +863,8 @@ public void testNullableFixedType4() { * * Covered types as of now * - * -- NullableVarCharVector - * -- NullableVarBinaryVector + * -- VarCharVector + * -- VarBinaryVector * * TODO: * @@ -879,11 +872,11 @@ public void testNullableFixedType4() { * -- VarBinaryVector */ - @Test /* NullableVarCharVector */ + @Test /* VarCharVector */ public void testNullableVarType1() { // Create a new value vector for 1024 integers. - try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final VarCharVector vector = newVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { vector.allocateNew(1024 * 10, 1024); vector.set(0, STR1); @@ -916,11 +909,11 @@ public void testNullableVarType1() { } } - @Test /* NullableVarBinaryVector */ + @Test /* VarBinaryVector */ public void testNullableVarType2() { // Create a new value vector for 1024 integers. - try (final NullableVarBinaryVector vector = newNullableVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final VarBinaryVector vector = newVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { vector.allocateNew(1024 * 10, 1024); vector.set(0, STR1); @@ -971,8 +964,6 @@ public void testNullableVarType2() { @Test /* Float8Vector */ public void testReallocAfterVectorTransfer1() { try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) { - final Float8Vector.Mutator mutator = vector.getMutator(); - final Float8Vector.Accessor accessor = vector.getAccessor(); final int initialDefaultCapacity = 4096; boolean error = false; @@ -984,7 +975,7 @@ public void testReallocAfterVectorTransfer1() { double baseValue = 100.375; for (int i = 0; i < initialDefaultCapacity; i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } /* the above setSafe calls should not have triggered a realloc as @@ -993,33 +984,33 @@ public void testReallocAfterVectorTransfer1() { assertEquals(initialDefaultCapacity, vector.getValueCapacity()); for (int i = 0; i < initialDefaultCapacity; i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); + vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity()); for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 2); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); + vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity()); for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 4); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } @@ -1036,15 +1027,12 @@ public void testReallocAfterVectorTransfer1() { toVector.reAlloc(); assertEquals(initialDefaultCapacity * 8, toVector.getValueCapacity()); - final Float8Vector.Accessor toAccessor = toVector.getAccessor(); - for (int i = 0; i < (initialDefaultCapacity * 8); i++) { - double value = toAccessor.get(i); if (i < (initialDefaultCapacity * 4)) { - assertEquals(baseValue + (double)i, value, 0); + assertEquals(baseValue + (double)i, toVector.get(i), 0); } else { - assertEquals(0, value, 0); + assertTrue(toVector.isNull(i)); } } @@ -1052,9 +1040,9 @@ public void testReallocAfterVectorTransfer1() { } } - @Test /* NullableFloat8Vector */ + @Test /* Float8Vector */ public void testReallocAfterVectorTransfer2() { - try (final NullableFloat8Vector vector = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) { + try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) { final int initialDefaultCapacity = 4096; boolean error = false; @@ -1111,7 +1099,7 @@ public void testReallocAfterVectorTransfer2() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); - NullableFloat8Vector toVector = (NullableFloat8Vector)transferPair.getTo(); + Float8Vector toVector = (Float8Vector)transferPair.getTo(); /* check toVector contents before realloc */ for (int i = 0; i < (initialDefaultCapacity * 4); i++) { @@ -1139,9 +1127,9 @@ public void testReallocAfterVectorTransfer2() { } } - @Test /* NullableVarCharVector */ + @Test /* VarCharVector */ public void testReallocAfterVectorTransfer3() { - try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) { /* 4096 values with 10 byte per record */ vector.allocateNew(4096 * 10, 4096); int valueCapacity = vector.getValueCapacity(); @@ -1223,7 +1211,7 @@ public void testReallocAfterVectorTransfer3() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); - NullableVarCharVector toVector = (NullableVarCharVector)transferPair.getTo(); + VarCharVector toVector = (VarCharVector)transferPair.getTo(); valueCapacity = toVector.getValueCapacity(); for (int i = 0; i < valueCapacity; i++) { @@ -1239,9 +1227,9 @@ public void testReallocAfterVectorTransfer3() { } } - @Test /* NullableIntVector */ + @Test /* IntVector */ public void testReallocAfterVectorTransfer4() { - try (final NullableIntVector vector = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) { /* 4096 values */ vector.allocateNew(4096); @@ -1316,7 +1304,7 @@ public void testReallocAfterVectorTransfer4() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); - NullableIntVector toVector = (NullableIntVector)transferPair.getTo(); + IntVector toVector = (IntVector)transferPair.getTo(); /* value capacity of source and target vectors should be same after * the transfer. */ @@ -1336,9 +1324,9 @@ public void testReallocAfterVectorTransfer4() { } @Test - public void testReAllocNullableFixedWidthVector() { + public void testReAllocFixedWidthVector() { // Create a new value vector for 1024 integers - try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) { + try (final Float4Vector vector = newVector(Float4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) { vector.allocateNew(1024); assertEquals(1024, vector.getValueCapacity()); @@ -1359,7 +1347,7 @@ public void testReAllocNullableFixedWidthVector() { assertEquals(104.5f, vector.get(1023), 0); assertEquals(105.5f, vector.get(2000), 0); - // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors + // Set the valueCount to be more than valueCapacity of current allocation. This is possible for ValueVectors // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the // vector vector.setValueCount(vector.getValueCapacity() + 200); @@ -1367,8 +1355,8 @@ public void testReAllocNullableFixedWidthVector() { } @Test - public void testReAllocNullableVariableWidthVector() { - try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { + public void testReAllocVariableWidthVector() { + try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { vector.allocateNew(); int initialCapacity = vector.getValueCapacity(); @@ -1392,7 +1380,7 @@ public void testReAllocNullableVariableWidthVector() { assertArrayEquals(STR2, vector.get(initialCapacity - 1)); assertArrayEquals(STR3, vector.get(initialCapacity + 200)); - // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors + // Set the valueCount to be more than valueCapacity of current allocation. This is possible for ValueVectors // as we don't call setSafe for null values, but we do call setValueCount when the current batch is processed. vector.setValueCount(vector.getValueCapacity() + 200); } @@ -1400,7 +1388,7 @@ public void testReAllocNullableVariableWidthVector() { @Test public void testFillEmptiesNotOverfill() { - try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { + try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { vector.allocateNew(); int initialCapacity = vector.getValueCapacity(); @@ -1420,8 +1408,8 @@ public void testFillEmptiesNotOverfill() { @Test public void testCopyFromWithNulls() { - try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); - final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { + try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); + final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { vector.allocateNew(); int capacity = vector.getValueCapacity(); @@ -1480,8 +1468,8 @@ public void testCopyFromWithNulls() { @Test public void testCopyFromWithNulls1() { - try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); - final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { + try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); + final VarCharVector vector2 = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { vector.allocateNew(); int capacity = vector.getValueCapacity(); @@ -1544,7 +1532,7 @@ public void testCopyFromWithNulls1() { @Test public void testSetLastSetUsage() { - try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { + try (final VarCharVector vector = new VarCharVector("myvector", allocator)) { vector.allocateNew(1024 * 10, 1024); setBytes(0, STR1, vector); @@ -1657,7 +1645,7 @@ public void testSetLastSetUsage() { @Test public void testVectorLoadUnload() { - try (final NullableVarCharVector vector1 = new NullableVarCharVector("myvector", allocator)) { + try (final VarCharVector vector1 = new VarCharVector("myvector", allocator)) { vector1.allocateNew(1024 * 10, 1024); vector1.set(0, STR1); @@ -1701,7 +1689,7 @@ public void testVectorLoadUnload() { VectorLoader vectorLoader = new VectorLoader(schemaRoot2); vectorLoader.load(recordBatch); - NullableVarCharVector vector2 = (NullableVarCharVector) schemaRoot2.getVector(fieldName); + VarCharVector vector2 = (VarCharVector) schemaRoot2.getVector(fieldName); /* * lastSet would have internally been set by VectorLoader.load() when it invokes * loadFieldBuffers. @@ -1723,7 +1711,7 @@ public void testVectorLoadUnload() { @Test public void testFillEmptiesUsage() { - try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { + try (final VarCharVector vector = new VarCharVector("myvector", allocator)) { vector.allocateNew(1024 * 10, 1024); @@ -1830,10 +1818,10 @@ public void testFillEmptiesUsage() { } } - @Test /* NullableVarCharVector */ + @Test /* VarCharVector */ public void testGetBufferAddress1() { - try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { + try (final VarCharVector vector = new VarCharVector("myvector", allocator)) { vector.allocateNew(1024 * 10, 1024); /* populate the vector */ @@ -1866,9 +1854,9 @@ public void testGetBufferAddress1() { } } - @Test /* NullableIntVector */ + @Test /* IntVector */ public void testGetBufferAddress2() { - try (final NullableIntVector vector = new NullableIntVector("myvector", allocator)) { + try (final IntVector vector = new IntVector("myvector", allocator)) { boolean error = false; vector.allocateNew(16); @@ -1905,7 +1893,7 @@ public void testGetBufferAddress2() { @Test public void testMultipleClose() { BufferAllocator vectorAllocator = allocator.newChildAllocator("vector_allocator", 0, Long.MAX_VALUE); - NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, vectorAllocator); + IntVector vector = newVector(IntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, vectorAllocator); vector.close(); vectorAllocator.close(); vector.close(); @@ -1916,7 +1904,7 @@ public void testMultipleClose() { * lastSet. The method is to test the lastSet property and that's why we load the vector * in a way that lastSet is not set automatically. */ - public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) { + public static void setBytes(int index, byte[] bytes, VarCharVector vector) { final int currentOffset = vector.offsetBuffer.getInt(index * vector.OFFSET_WIDTH); BitVectorHelper.setValidityBitToOne(vector.validityBuffer, index); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 293ffbfe19233..c0df4881f380c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -26,7 +26,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; @@ -53,14 +53,13 @@ public void terminate() throws Exception { @Test public void testFixedType() { try (final UInt4Vector vector = new UInt4Vector("", allocator)) { - final UInt4Vector.Mutator m = vector.getMutator(); vector.setInitialCapacity(512); vector.allocateNew(); assertEquals(512, vector.getValueCapacity()); try { - m.set(512, 0); + vector.set(512, 0); Assert.fail("Expected out of bounds exception"); } catch (Exception e) { // ok @@ -69,14 +68,14 @@ public void testFixedType() { vector.reAlloc(); assertEquals(1024, vector.getValueCapacity()); - m.set(512, 100); - assertEquals(100, vector.getAccessor().get(512)); + vector.set(512, 100); + assertEquals(100, vector.get(512)); } } @Test public void testNullableType() { - try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) { + try (final VarCharVector vector = new VarCharVector("", allocator)) { vector.setInitialCapacity(512); vector.allocateNew(); @@ -108,7 +107,7 @@ public void testListType() { assertEquals(1023, vector.getValueCapacity()); try { - vector.getOffsetVector().getAccessor().get(2014); + vector.getInnerValueCountAt(2014); Assert.fail("Expected out of bounds exception"); } catch (Exception e) { // ok @@ -122,8 +121,8 @@ public void testListType() { @Test public void testMapType() { - try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) { - vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class); + try (final MapVector vector = MapVector.empty("", allocator)) { + vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), IntVector.class); vector.setInitialCapacity(512); vector.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java index d53f69489d4da..28903b1170e31 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java @@ -43,12 +43,11 @@ public void terminate() throws Exception { @Test public void testFixedTypeReset() { try (final UInt4Vector vector = new UInt4Vector("", allocator)) { - final UInt4Vector.Mutator m = vector.getMutator(); vector.allocateNew(); - final int sizeBefore = vector.getAllocationSize(); + final int sizeBefore = vector.getBufferSize(); vector.reAlloc(); vector.reset(); - final int sizeAfter = vector.getAllocationSize(); + final int sizeAfter = vector.getBufferSize(); assertEquals(sizeBefore, sizeAfter); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index e61dbecf44101..439a62725e49b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -236,8 +236,8 @@ public void testLoadValidityBuffer() throws IOException { vectorLoader.load(recordBatch); - NullableIntVector intDefinedVector = (NullableIntVector) newRoot.getVector("intDefined"); - NullableIntVector intNullVector = (NullableIntVector) newRoot.getVector("intNull"); + IntVector intDefinedVector = (IntVector) newRoot.getVector("intDefined"); + IntVector intNullVector = (IntVector) newRoot.getVector("intNull"); for (int i = 0; i < count; i++) { assertFalse("#" + i, intDefinedVector.isNull(i)); assertEquals("#" + i, i, intDefinedVector.get(i)); @@ -283,7 +283,7 @@ public void testUnloadLoadDuplicates() throws IOException { FieldVector vector = field.createVector(originalVectorsAllocator); vector.allocateNew(); sources.add(vector); - NullableIntVector intVector = (NullableIntVector)vector; + IntVector intVector = (IntVector)vector; for (int i = 0; i < count; i++) { intVector.set(i, i); } @@ -302,8 +302,8 @@ public void testUnloadLoadDuplicates() throws IOException { List targets = newRoot.getFieldVectors(); Assert.assertEquals(sources.size(), targets.size()); for (int k = 0; k < sources.size(); k++) { - NullableIntVector src = (NullableIntVector) sources.get(k); - NullableIntVector tgt = (NullableIntVector) targets.get(k); + IntVector src = (IntVector) sources.get(k); + IntVector tgt = (IntVector) targets.get(k); Assert.assertEquals(src.getValueCount(), tgt.getValueCount()); for (int i = 0; i < count; i++) { Assert.assertEquals(src.get(i), tgt.get(i)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index b0d6cf555e98d..38b78424dccf4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -25,7 +25,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DirtyRootAllocator; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -55,7 +55,7 @@ public void terminate() throws Exception { public void testPromoteToUnion() throws Exception { try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator); - final NullableMapVector v = container.addOrGetMap("test"); + final MapVector v = container.addOrGetMap("test"); final PromotableWriter writer = new PromotableWriter(v, container)) { container.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 52defdc2e1b00..bd8489eb20f89 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -28,13 +28,13 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.SchemaChangeCallBack; -import org.apache.arrow.vector.NullableFloat8Vector; -import org.apache.arrow.vector.NullableFloat4Vector; -import org.apache.arrow.vector.NullableBigIntVector; -import org.apache.arrow.vector.NullableIntVector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.NonNullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.ComplexWriterImpl; import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; @@ -830,7 +830,7 @@ public void complexCopierWithList() { rootWriter.end(); writer.setValueCount(1); - NullableMapVector mapVector = (NullableMapVector) parent.getChild("root"); + MapVector mapVector = (MapVector) parent.getChild("root"); TransferPair tp = mapVector.getTransferPair(allocator); tp.splitAndTransfer(0, 1); MapVector toMapVector = (MapVector) tp.getTo(); @@ -849,7 +849,7 @@ public void testSingleMapWriter1() { /* initialize a SingleMapWriter with empty MapVector and then lazily * create all vectors with expected initialCapacity. */ - MapVector parent = MapVector.empty("parent", allocator); + NonNullableMapVector parent = NonNullableMapVector.empty("parent", allocator); SingleMapWriter singleMapWriter = new SingleMapWriter(parent); int initialCapacity = 1024; @@ -885,10 +885,10 @@ public void testSingleMapWriter1() { singleMapWriter.end(); } - NullableIntVector intVector = (NullableIntVector)parent.getChild("intField"); - NullableBigIntVector bigIntVector = (NullableBigIntVector)parent.getChild("bigIntField"); - NullableFloat4Vector float4Vector = (NullableFloat4Vector)parent.getChild("float4Field"); - NullableFloat8Vector float8Vector = (NullableFloat8Vector)parent.getChild("float8Field"); + IntVector intVector = (IntVector)parent.getChild("intField"); + BigIntVector bigIntVector = (BigIntVector)parent.getChild("bigIntField"); + Float4Vector float4Vector = (Float4Vector)parent.getChild("float4Field"); + Float8Vector float8Vector = (Float8Vector)parent.getChild("float8Field"); assertEquals(initialCapacity, singleMapWriter.getValueCapacity()); assertEquals(initialCapacity, intVector.getValueCapacity()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index 233b682c99bc6..3514acaa24278 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -28,17 +28,17 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.NullableDateMilliVector; -import org.apache.arrow.vector.NullableDecimalVector; -import org.apache.arrow.vector.NullableIntVector; -import org.apache.arrow.vector.NullableTimeMilliVector; -import org.apache.arrow.vector.NullableVarBinaryVector; -import org.apache.arrow.vector.NullableVarCharVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.ValueVector.Accessor; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.impl.ComplexWriterImpl; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -71,7 +71,7 @@ import io.netty.buffer.ArrowBuf; -import static org.apache.arrow.vector.TestUtils.newNullableVarCharVector; +import static org.apache.arrow.vector.TestUtils.newVarCharVector; /** * Helps testing the file formats @@ -172,7 +172,7 @@ private LocalDateTime makeDateTimeFromCount(int i) { return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i); } - protected void writeDateTimeData(int count, NullableMapVector parent) { + protected void writeDateTimeData(int count, MapVector parent) { Assert.assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); @@ -202,11 +202,11 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { - long dateVal = ((NullableDateMilliVector) root.getVector("date")).get(i); + long dateVal = ((DateMilliVector) root.getVector("date")).get(i); LocalDateTime dt = makeDateTimeFromCount(i); LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal); - long timeVal = ((NullableTimeMilliVector) root.getVector("time")).get(i); + long timeVal = ((TimeMilliVector) root.getVector("time")).get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); Object timestampMilliVal = root.getVector("timestamp-milli").getObject(i); Assert.assertEquals(dt, timestampMilliVal); @@ -218,7 +218,7 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) { protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) { // Define dictionaries and add to provider - NullableVarCharVector dictionary1Vector = newNullableVarCharVector("D1", bufferAllocator); + VarCharVector dictionary1Vector = newVarCharVector("D1", bufferAllocator); dictionary1Vector.allocateNewSafe(); dictionary1Vector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); dictionary1Vector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); @@ -228,7 +228,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat Dictionary dictionary1 = new Dictionary(dictionary1Vector, new DictionaryEncoding(1L, false, null)); provider.put(dictionary1); - NullableVarCharVector dictionary2Vector = newNullableVarCharVector("D2", bufferAllocator); + VarCharVector dictionary2Vector = newVarCharVector("D2", bufferAllocator); dictionary2Vector.allocateNewSafe(); dictionary2Vector.set(0, "micro".getBytes(StandardCharsets.UTF_8)); dictionary2Vector.set(1, "small".getBytes(StandardCharsets.UTF_8)); @@ -239,7 +239,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat provider.put(dictionary2); // Populate the vectors - NullableVarCharVector vector1A = newNullableVarCharVector("varcharA", bufferAllocator); + VarCharVector vector1A = newVarCharVector("varcharA", bufferAllocator); vector1A.allocateNewSafe(); vector1A.set(0, "foo".getBytes(StandardCharsets.UTF_8)); vector1A.set(1, "bar".getBytes(StandardCharsets.UTF_8)); @@ -252,7 +252,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat vector1A.close(); // Done with this vector after encoding // Write this vector using indices instead of encoding - NullableIntVector encodedVector1B = new NullableIntVector("varcharB", bufferAllocator); + IntVector encodedVector1B = new IntVector("varcharB", bufferAllocator); encodedVector1B.allocateNewSafe(); encodedVector1B.set(0, 2); // "baz" encodedVector1B.set(1, 1); // "bar" @@ -261,7 +261,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat encodedVector1B.set(5, 0); // "foo" encodedVector1B.setValueCount(6); - NullableVarCharVector vector2 = newNullableVarCharVector("sizes", bufferAllocator); + VarCharVector vector2 = newVarCharVector("sizes", bufferAllocator); vector2.allocateNewSafe(); vector2.set(1, "large".getBytes(StandardCharsets.UTF_8)); vector2.set(2, "small".getBytes(StandardCharsets.UTF_8)); @@ -327,7 +327,7 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Dictionary dictionary1 = provider.lookup(1L); Assert.assertNotNull(dictionary1); - NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary1.getVector()); + VarCharVector dictionaryVector = ((VarCharVector) dictionary1.getVector()); Assert.assertEquals(3, dictionaryVector.getValueCount()); Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); @@ -335,7 +335,7 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Dictionary dictionary2 = provider.lookup(2L); Assert.assertNotNull(dictionary2); - dictionaryVector = ((NullableVarCharVector) dictionary2.getVector()); + dictionaryVector = ((VarCharVector) dictionary2.getVector()); Assert.assertEquals(3, dictionaryVector.getValueCount()); Assert.assertEquals(new Text("micro"), dictionaryVector.getObject(0)); Assert.assertEquals(new Text("small"), dictionaryVector.getObject(1)); @@ -345,7 +345,7 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) { // Define the dictionary and add to the provider - NullableVarCharVector dictionaryVector = newNullableVarCharVector("D2", bufferAllocator); + VarCharVector dictionaryVector = newVarCharVector("D2", bufferAllocator); dictionaryVector.allocateNewSafe(); dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); @@ -395,16 +395,16 @@ protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvide Dictionary dictionary = provider.lookup(2L); Assert.assertNotNull(dictionary); - NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + VarCharVector dictionaryVector = ((VarCharVector) dictionary.getVector()); Assert.assertEquals(2, dictionaryVector.getValueCount()); Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { - NullableDecimalVector decimalVector1 = new NullableDecimalVector("decimal1", bufferAllocator, 10, 3); - NullableDecimalVector decimalVector2 = new NullableDecimalVector("decimal2", bufferAllocator, 4, 2); - NullableDecimalVector decimalVector3 = new NullableDecimalVector("decimal3", bufferAllocator, 16, 8); + DecimalVector decimalVector1 = new DecimalVector("decimal1", bufferAllocator, 10, 3); + DecimalVector decimalVector2 = new DecimalVector("decimal2", bufferAllocator, 4, 2); + DecimalVector decimalVector3 = new DecimalVector("decimal3", bufferAllocator, 16, 8); int count = 10; decimalVector1.allocateNew(count); @@ -427,9 +427,9 @@ protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { } protected void validateDecimalData(VectorSchemaRoot root) { - NullableDecimalVector decimalVector1 = (NullableDecimalVector) root.getVector("decimal1"); - NullableDecimalVector decimalVector2 = (NullableDecimalVector) root.getVector("decimal2"); - NullableDecimalVector decimalVector3 = (NullableDecimalVector) root.getVector("decimal3"); + DecimalVector decimalVector1 = (DecimalVector) root.getVector("decimal1"); + DecimalVector decimalVector2 = (DecimalVector) root.getVector("decimal2"); + DecimalVector decimalVector3 = (DecimalVector) root.getVector("decimal3"); int count = 10; Assert.assertEquals(count, root.getRowCount()); @@ -491,7 +491,7 @@ public void validateUnionData(int count, VectorSchemaRoot root) { } } - public void writeUnionData(int count, NullableMapVector parent) { + public void writeUnionData(int count, MapVector parent) { ArrowBuf varchar = allocator.buffer(3); varchar.readerIndex(0); varchar.setByte(0, 'a'); @@ -534,7 +534,7 @@ public void writeUnionData(int count, NullableMapVector parent) { varchar.release(); } - protected void writeVarBinaryData(int count, NullableMapVector parent) { + protected void writeVarBinaryData(int count, MapVector parent) { Assert.assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); @@ -577,8 +577,8 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { // ListVector lastSet should be the index of last value + 1 Assert.assertEquals(listVector.getLastSet(), count); - // NullableVarBinaryVector lastSet should be the index of last value - NullableVarBinaryVector binaryVector = (NullableVarBinaryVector) listVector.getChildrenFromFields().get(0); + // VarBinaryVector lastSet should be the index of last value + VarBinaryVector binaryVector = (VarBinaryVector) listVector.getChildrenFromFields().get(0); Assert.assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java index 4387db03655d1..0cfc9ba919d85 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java @@ -36,14 +36,14 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.NullableFloat4Vector; -import org.apache.arrow.vector.NullableIntVector; -import org.apache.arrow.vector.NullableTinyIntVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; import org.apache.arrow.vector.ipc.message.ArrowBlock; import org.apache.arrow.vector.ipc.message.ArrowBuffer; @@ -82,7 +82,7 @@ public void testWriteComplex() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); FieldVector root = parent.getChild("root"); validateComplexContent(count, new VectorSchemaRoot(root)); @@ -268,7 +268,7 @@ public void testWriteReadUnion() throws IOException { // write try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); validateUnionData(count, new VectorSchemaRoot(parent.getChild("root"))); write(parent.getChild("root"), file, stream); @@ -304,7 +304,7 @@ public void testWriteReadTiny() throws IOException { try (VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); + TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } @@ -352,7 +352,7 @@ public void testWriteReadTiny() throws IOException { private void validateTinyData(VectorSchemaRoot root) { Assert.assertEquals(16, root.getRowCount()); - NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); + TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { if (i < 8) { Assert.assertEquals((byte) (i + 1), vector.get(i)); @@ -382,7 +382,7 @@ public void testWriteReadMetadata() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) { + MapVector vector = (MapVector) field.createVector(originalVectorAllocator)) { vector.allocateNewSafe(); vector.setValueCount(0); @@ -558,10 +558,10 @@ public void testWriteReadFixedSizeList() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { + MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class); - NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); - NullableIntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), NullableIntVector.class); + Float4Vector floats = (Float4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); + IntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), IntVector.class); parent.allocateNew(); for (int i = 0; i < 10; i++) { @@ -618,7 +618,7 @@ public void testWriteReadVarBin() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeVarBinaryData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateVarBinary(count, root); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java index 7a8586a9e708e..f87a0ebd54a0f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java @@ -26,7 +26,9 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import org.apache.arrow.vector.NullableTinyIntVector; +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; @@ -65,7 +67,7 @@ public void testReadWrite() throws IOException { int numBatches = 1; root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector vector = (NullableTinyIntVector)root.getFieldVectors().get(0); + TinyIntVector vector = (TinyIntVector)root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java index 65e6cea2ec2e6..bd1ec94c8fd2b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java @@ -28,7 +28,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.NullableTinyIntVector; +import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; @@ -61,7 +61,7 @@ public void run() { writer.start(); for (int j = 0; j < numBatches; j++) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); + TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0); // Send a changing batch id first vector.set(0, j); for (int i = 1; i < 16; i++) { @@ -103,10 +103,9 @@ public boolean loadNextBatch() throws IOException { done = true; return false; } - VectorSchemaRoot root = getVectorSchemaRoot(); Assert.assertEquals(16, root.getRowCount()); - NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); + TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0); Assert.assertEquals((byte) (batchesRead - 1), vector.get(0)); for (int i = 1; i < 16; i++) { if (i < 8) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java index c3e0b795174c0..625717048bf91 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java @@ -25,7 +25,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; import org.apache.arrow.vector.types.pojo.Schema; @@ -73,7 +73,7 @@ public void testWriteComplexJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateComplexContent(root.getRowCount(), root); @@ -95,7 +95,7 @@ public void testWriteReadUnionJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); printVectors(parent.getChildrenFromFields()); @@ -127,7 +127,7 @@ public void testWriteReadDateTimeJSON() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeDateTimeData(count, parent); @@ -292,7 +292,7 @@ public void testWriteReadVarBinJSON() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeVarBinaryData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateVarBinary(count, root); From ffb37dbe431271e6f2d958f8b22004383a539a7d Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 28 Nov 2017 12:33:39 -0500 Subject: [PATCH 092/177] ARROW-1735: [C++] Test CastKernel writing into output array with non-zero offset This uncovered some bugs. I inspected the other kernels that are untested and while they look fine, at some point we may want to add some more extensive unit tests about this Author: Wes McKinney Closes #1369 from wesm/ARROW-1735 and squashes the following commits: de41d929 [Wes McKinney] Test CastKernel writing into output array with non-zero offset --- cpp/src/arrow/array.cc | 6 +-- cpp/src/arrow/array.h | 11 +++-- cpp/src/arrow/compute/compute-test.cc | 60 ++++++++++++++++++++++++++ cpp/src/arrow/compute/kernels/cast.cc | 38 ++++++++-------- cpp/src/arrow/python/arrow_to_python.h | 4 +- cpp/src/arrow/python/numpy_to_arrow.cc | 2 +- 6 files changed, 92 insertions(+), 29 deletions(-) diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index 4b1fabfdcd441..0b235cc199ca1 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -103,7 +103,7 @@ static inline std::shared_ptr SliceData(const ArrayData& data, int64_ length = std::min(data.length - offset, length); offset += data.offset; - auto new_data = data.ShallowCopy(); + auto new_data = data.Copy(); new_data->length = length; new_data->offset = offset; new_data->null_count = data.null_count != 0 ? kUnknownNullCount : 0; @@ -482,14 +482,14 @@ DictionaryArray::DictionaryArray(const std::shared_ptr& type, : dict_type_(static_cast(type.get())) { DCHECK_EQ(type->id(), Type::DICTIONARY); DCHECK_EQ(indices->type_id(), dict_type_->index_type()->id()); - auto data = indices->data()->ShallowCopy(); + auto data = indices->data()->Copy(); data->type = type; SetData(data); } void DictionaryArray::SetData(const std::shared_ptr& data) { this->Array::SetData(data); - auto indices_data = data_->ShallowCopy(); + auto indices_data = data_->Copy(); indices_data->type = dict_type_->index_type(); std::shared_ptr result; indices_ = MakeArray(indices_data); diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index ec5381d6e28a3..ebe54adcb9eaf 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -143,9 +143,14 @@ struct ARROW_EXPORT ArrayData { return *this; } - std::shared_ptr ShallowCopy() const { - return std::make_shared(*this); - } + std::shared_ptr Copy() const { return std::make_shared(*this); } + +#ifndef ARROW_NO_DEPRECATED_API + + // Deprecated since 0.8.0 + std::shared_ptr ShallowCopy() const { return Copy(); } + +#endif std::shared_ptr type; int64_t length; diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index d5158978c8de8..c73bfa309fd95 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -709,6 +709,66 @@ TEST_F(TestCast, PreallocatedMemory) { ASSERT_ARRAYS_EQUAL(*expected, *result); } +template +void CheckOffsetOutputCase(FunctionContext* ctx, const std::shared_ptr& in_type, + const vector& in_values, + const std::shared_ptr& out_type, + const vector& out_values) { + using OutTraits = TypeTraits; + + CastOptions options; + + const int64_t length = static_cast(in_values.size()); + + shared_ptr arr, expected; + ArrayFromVector(in_type, in_values, &arr); + ArrayFromVector(out_type, out_values, &expected); + + shared_ptr out_buffer; + ASSERT_OK(ctx->Allocate(OutTraits::bytes_required(length), &out_buffer)); + + std::unique_ptr kernel; + ASSERT_OK(GetCastFunction(*in_type, out_type, options, &kernel)); + + const int64_t first_half = length / 2; + + auto out_data = ArrayData::Make(out_type, length, {nullptr, out_buffer}); + auto out_second_data = out_data->Copy(); + out_second_data->offset = first_half; + + Datum out_first(out_data); + Datum out_second(out_second_data); + + // Cast each bit + ASSERT_OK(kernel->Call(ctx, Datum(arr->Slice(0, first_half)), &out_first)); + ASSERT_OK(kernel->Call(ctx, Datum(arr->Slice(first_half)), &out_second)); + + shared_ptr result = MakeArray(out_data); + + ASSERT_ARRAYS_EQUAL(*expected, *result); +} + +TEST_F(TestCast, OffsetOutputBuffer) { + // ARROW-1735 + vector v1 = {0, 10000, 2000, 1000, 0}; + vector e1 = {0, 10000, 2000, 1000, 0}; + + auto in_type = int32(); + auto out_type = int64(); + CheckOffsetOutputCase(&this->ctx_, in_type, v1, + out_type, e1); + + vector e2 = {false, true, true, true, false}; + + out_type = boolean(); + CheckOffsetOutputCase(&this->ctx_, in_type, v1, + boolean(), e2); + + vector e3 = {0, 10000, 2000, 1000, 0}; + CheckOffsetOutputCase(&this->ctx_, in_type, v1, + int16(), e3); +} + template class TestDictionaryCast : public TestCast {}; diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc index d48d669922aa6..465be958cfabf 100644 --- a/cpp/src/arrow/compute/kernels/cast.cc +++ b/cpp/src/arrow/compute/kernels/cast.cc @@ -124,12 +124,7 @@ template struct CastFunctor::value>::type> { void operator()(FunctionContext* ctx, const CastOptions& options, - const ArrayData& input, ArrayData* output) { - // Simply initialize data to 0 - auto buf = output->buffers[1]; - DCHECK_EQ(output->offset, 0); - memset(buf->mutable_data(), 0, buf->size()); - } + const ArrayData& input, ArrayData* output) {} }; template <> @@ -199,14 +194,19 @@ struct CastFunctor::v !std::is_same::value>::type> { void operator()(FunctionContext* ctx, const CastOptions& options, const ArrayData& input, ArrayData* output) { - using in_type = typename I::c_type; - DCHECK_EQ(output->offset, 0); + auto in_data = GetValues(input, 1); + internal::BitmapWriter writer(output->buffers[1]->mutable_data(), output->offset, + input.length); - const in_type* in_data = GetValues(input, 1); - uint8_t* out_data = GetMutableValues(output, 1); for (int64_t i = 0; i < input.length; ++i) { - BitUtil::SetBitTo(out_data, i, (*in_data++) != 0); + if (*in_data++ != 0) { + writer.Set(); + } else { + writer.Clear(); + } + writer.Next(); } + writer.Finish(); } }; @@ -217,7 +217,6 @@ struct CastFunctoroffset, 0); auto in_offset = input.offset; @@ -475,9 +474,10 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices, const index_c_type* in = GetValues(*indices.data(), 1); - uint8_t* out = output->buffers[1]->mutable_data(); int32_t byte_width = static_cast(*output->type).byte_width(); + + uint8_t* out = output->buffers[1]->mutable_data() + byte_width * output->offset; for (int64_t i = 0; i < indices.length(); ++i) { if (valid_bits_reader.IsSet()) { const uint8_t* value = dictionary.Value(in[i]); @@ -493,7 +493,7 @@ struct CastFunctor< typename std::enable_if::value>::type> { void operator()(FunctionContext* ctx, const CastOptions& options, const ArrayData& input, ArrayData* output) { - DictionaryArray dict_array(input.ShallowCopy()); + DictionaryArray dict_array(input.Copy()); const DictionaryType& type = static_cast(*input.type); const DataType& values_type = *type.dictionary()->type(); @@ -565,7 +565,7 @@ struct CastFunctor::value>::type> { void operator()(FunctionContext* ctx, const CastOptions& options, const ArrayData& input, ArrayData* output) { - DictionaryArray dict_array(input.ShallowCopy()); + DictionaryArray dict_array(input.Copy()); const DictionaryType& type = static_cast(*input.type); const DataType& values_type = *type.dictionary()->type(); @@ -605,12 +605,10 @@ struct CastFunctor void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary, c_type* out) { - using index_c_type = typename IndexType::c_type; - internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(), indices.length()); - const index_c_type* in = GetValues(*indices.data(), 1); + auto in = GetValues(*indices.data(), 1); for (int64_t i = 0; i < indices.length(); ++i) { if (valid_bits_reader.IsSet()) { out[i] = dictionary[in[i]]; @@ -627,7 +625,7 @@ struct CastFunctor(*input.type); const DataType& values_type = *type.dictionary()->type(); @@ -638,7 +636,7 @@ struct CastFunctor(*type.dictionary()->data(), 1); - auto out = reinterpret_cast(output->buffers[1]->mutable_data()); + auto out = GetMutableValues(output, 1); const Array& indices = *dict_array.indices(); switch (indices.type()->id()) { case Type::INT8: diff --git a/cpp/src/arrow/python/arrow_to_python.h b/cpp/src/arrow/python/arrow_to_python.h index 9440ffb32abeb..02a22f07d7e78 100644 --- a/cpp/src/arrow/python/arrow_to_python.h +++ b/cpp/src/arrow/python/arrow_to_python.h @@ -51,8 +51,8 @@ Status ReadSerializedObject(io::RandomAccessFile* src, SerializedPyObject* out); /// \brief Reconstruct SerializedPyObject from representation produced by /// SerializedPyObject::GetComponents. /// -/// \param[in] num_tensors -/// \param[in] num_buffers +/// \param[in] num_tensors number of tensors in the object +/// \param[in] num_buffers number of buffers in the object /// \param[in] data a list containing pyarrow.Buffer instances. Must be 1 + /// num_tensors * 2 + num_buffers in length /// \param[out] out the reconstructed object diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index 0c0d1a9b3fdf2..0b1124d303df1 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -765,7 +765,7 @@ Status NumPyConverter::ConvertObjectStrings() { // If we saw PyBytes, convert everything to BinaryArray if (global_have_bytes) { for (size_t i = 0; i < out_arrays_.size(); ++i) { - auto binary_data = out_arrays_[i]->data()->ShallowCopy(); + auto binary_data = out_arrays_[i]->data()->Copy(); binary_data->type = ::arrow::binary(); out_arrays_[i] = std::make_shared(binary_data); } From 155bf076204dcb6bf1f68a556e9243556f5e4ebb Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 28 Nov 2017 20:06:18 -0500 Subject: [PATCH 093/177] ARROW-1854: [Python] Use pickle to serialize numpy arrays of objects. **Just posting this for discussion.** See the preceding discussion on https://issues.apache.org/jira/browse/ARROW-1854. I think the ideal way to solve this would actually be to improve our handling of lists, which should be possible given that pickle seems to outperform us by 6x according to the benchmarks in https://issues.apache.org/jira/browse/ARROW-1854. Note that the implementation in this PR will not handle numpy arrays of user-defined classes because it will not fall back to cloudpickle when needed. cc @pcmoritz @wesm Author: Wes McKinney Author: Robert Nishihara Closes #1360 from robertnishihara/numpyobject and squashes the following commits: c37a0a08 [Wes McKinney] Fix flake 51915032 [Wes McKinney] Fix post rebase 43f2c805 [Wes McKinney] Add SerializationContext.clone method. Add pandas_serialization_context member that uses pickle for NumPy arrays with unsupported tensor types c9440231 [Wes McKinney] Use pickle.HIGHEST_PROTOCOL, convert to Buffer then memoryview for more memory-efficient transport cf719c3f [Robert Nishihara] Use pickle to serialize numpy arrays of objects. --- python/pyarrow/__init__.py | 1 + python/pyarrow/serialization.pxi | 16 ++++++++ python/pyarrow/serialization.py | 46 +++++++++++++++------- python/pyarrow/tests/test_serialization.py | 7 ++-- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index bd31b21c196de..a245fe6796023 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -125,6 +125,7 @@ localfs = LocalFileSystem.get_instance() from pyarrow.serialization import (_default_serialization_context, + pandas_serialization_context, register_default_serialization_handlers) import pyarrow.types as types diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index bb266b2f928ff..faf164b3ebd20 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -57,6 +57,22 @@ cdef class SerializationContext: self.custom_serializers = dict() self.custom_deserializers = dict() + def clone(self): + """ + Return copy of this SerializationContext + + Returns + ------- + clone : SerializationContext + """ + result = SerializationContext() + result.type_to_type_id = self.type_to_type_id.copy() + result.whitelisted_types = self.whitelisted_types.copy() + result.custom_serializers = self.custom_serializers.copy() + result.custom_deserializers = self.custom_deserializers.copy() + + return result + def register_type(self, type_, type_id, custom_serializer=None, custom_deserializer=None): """EXPERIMENTAL: Add type to the list of types we can serialize. diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py index ab25b63d5710b..08e6cce751b53 100644 --- a/python/pyarrow/serialization.py +++ b/python/pyarrow/serialization.py @@ -22,7 +22,7 @@ import numpy as np from pyarrow import serialize_pandas, deserialize_pandas -from pyarrow.lib import _default_serialization_context +from pyarrow.lib import _default_serialization_context, frombuffer try: import cloudpickle @@ -30,6 +30,28 @@ cloudpickle = pickle +# ---------------------------------------------------------------------- +# Set up serialization for numpy with dtype object (primitive types are +# handled efficiently with Arrow's Tensor facilities, see +# python_to_arrow.cc) + +def _serialize_numpy_array_list(obj): + return obj.tolist(), obj.dtype.str + + +def _deserialize_numpy_array_list(data): + return np.array(data[0], dtype=np.dtype(data[1])) + + +def _serialize_numpy_array_pickle(obj): + pickled = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + return frombuffer(pickled) + + +def _deserialize_numpy_array_pickle(data): + return pickle.loads(memoryview(data)) + + def register_default_serialization_handlers(serialization_context): # ---------------------------------------------------------------------- @@ -80,21 +102,10 @@ def _deserialize_default_dict(data): custom_serializer=cloudpickle.dumps, custom_deserializer=cloudpickle.loads) - # ---------------------------------------------------------------------- - # Set up serialization for numpy with dtype object (primitive types are - # handled efficiently with Arrow's Tensor facilities, see - # python_to_arrow.cc) - - def _serialize_numpy_array(obj): - return obj.tolist(), obj.dtype.str - - def _deserialize_numpy_array(data): - return np.array(data[0], dtype=np.dtype(data[1])) - serialization_context.register_type( np.ndarray, 'np.array', - custom_serializer=_serialize_numpy_array, - custom_deserializer=_deserialize_numpy_array) + custom_serializer=_serialize_numpy_array_list, + custom_deserializer=_deserialize_numpy_array_list) # ---------------------------------------------------------------------- # Set up serialization for pandas Series and DataFrame @@ -153,3 +164,10 @@ def _deserialize_torch_tensor(data): register_default_serialization_handlers(_default_serialization_context) + +pandas_serialization_context = _default_serialization_context.clone() + +pandas_serialization_context.register_type( + np.ndarray, 'np.array', + custom_serializer=_serialize_numpy_array_pickle, + custom_deserializer=_deserialize_numpy_array_pickle) diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index d06beeac99280..6d85621d411a7 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -212,11 +212,11 @@ def make_serialization_context(): serialization_context = make_serialization_context() -def serialization_roundtrip(value, f): +def serialization_roundtrip(value, f, ctx=serialization_context): f.seek(0) - pa.serialize_to(value, f, serialization_context) + pa.serialize_to(value, f, ctx) f.seek(0) - result = pa.deserialize_from(f, None, serialization_context) + result = pa.deserialize_from(f, None, ctx) assert_equal(value, result) _check_component_roundtrip(value) @@ -249,6 +249,7 @@ def test_primitive_serialization(large_memory_map): with pa.memory_map(large_memory_map, mode="r+") as mmap: for obj in PRIMITIVE_OBJECTS: serialization_roundtrip(obj, mmap) + serialization_roundtrip(obj, mmap, pa.pandas_serialization_context) def test_serialize_to_buffer(): From b92c435b8f64f98593267fd72ecd61d26c23ffc0 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 28 Nov 2017 20:07:11 -0500 Subject: [PATCH 094/177] ARROW-1684: [Python] Support selecting nested Parquet fields by any path prefix Author: Wes McKinney Closes #1366 from wesm/ARROW-1684 and squashes the following commits: e63e42aa [Wes McKinney] Support selecting nested Parquet fields by any path prefix --- python/pyarrow/_parquet.pxd | 1 + python/pyarrow/_parquet.pyx | 29 ++++++++++++++++---- python/pyarrow/parquet.py | 41 ++++++++++++++++++++++++---- python/pyarrow/tests/test_parquet.py | 22 +++++++++++++++ 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 7e5e575096ddd..55b66b53d160d 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -37,6 +37,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet::schema" nogil: cdef cppclass ColumnPath: c_string ToDotString() + vector[c_string] ToDotVector() cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index eca6b201b3ad0..147af217579e7 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -600,9 +600,11 @@ cdef class ParquetReader: object source CMemoryPool* allocator unique_ptr[FileReader] reader - column_idx_map FileMetaData _metadata + cdef public: + _column_idx_map + def __cinit__(self, MemoryPool memory_pool=None): self.allocator = maybe_unbox_memory_pool(memory_pool) self._metadata = None @@ -624,6 +626,23 @@ cdef class ParquetReader: check_status(OpenFile(rd_handle, self.allocator, properties, c_metadata, &self.reader)) + property column_paths: + + def __get__(self): + cdef: + FileMetaData container = self.metadata + const CFileMetaData* metadata = container._metadata + vector[c_string] path + int i = 0 + + paths = [] + for i in range(0, metadata.num_columns()): + path = (metadata.schema().Column(i) + .path().get().ToDotVector()) + paths.append([frombytes(x) for x in path]) + + return paths + @property def metadata(self): cdef: @@ -729,14 +748,14 @@ cdef class ParquetReader: const CFileMetaData* metadata = container._metadata int i = 0 - if self.column_idx_map is None: - self.column_idx_map = {} + if self._column_idx_map is None: + self._column_idx_map = {} for i in range(0, metadata.num_columns()): col_bytes = tobytes(metadata.schema().Column(i) .path().get().ToDotString()) - self.column_idx_map[col_bytes] = i + self._column_idx_map[col_bytes] = i - return self.column_idx_map[tobytes(column_name)] + return self._column_idx_map[tobytes(column_name)] def read_column(self, int column_index): cdef: diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index 37da662806366..9fb890c6e6b8d 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from collections import defaultdict import os import inspect import json @@ -54,6 +55,24 @@ def __init__(self, source, metadata=None, common_metadata=None): self.reader = ParquetReader() self.reader.open(source, metadata=metadata) self.common_metadata = common_metadata + self._nested_paths_by_prefix = self._build_nested_paths() + + def _build_nested_paths(self): + paths = self.reader.column_paths + + result = defaultdict(list) + + def _visit_piece(i, key, rest): + result[key].append(i) + + if len(rest) > 0: + nested_key = '.'.join((key, rest[0])) + _visit_piece(i, nested_key, rest[1:]) + + for i, path in enumerate(paths): + _visit_piece(i, path[0], path[1:]) + + return result @property def metadata(self): @@ -75,7 +94,9 @@ def read_row_group(self, i, columns=None, nthreads=1, Parameters ---------- columns: list - If not None, only these columns will be read from the row group. + If not None, only these columns will be read from the row group. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e' nthreads : int, default 1 Number of columns to read in parallel. If > 1, requires that the underlying file source is threadsafe @@ -100,7 +121,9 @@ def read(self, columns=None, nthreads=1, use_pandas_metadata=False): Parameters ---------- columns: list - If not None, only these columns will be read from the file. + If not None, only these columns will be read from the file. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e' nthreads : int, default 1 Number of columns to read in parallel. If > 1, requires that the underlying file source is threadsafe @@ -143,7 +166,11 @@ def _get_column_indices(self, column_names, use_pandas_metadata=False): if column_names is None: return None - indices = list(map(self.reader.column_name_idx, column_names)) + indices = [] + + for name in column_names: + if name in self._nested_paths_by_prefix: + indices.extend(self._nested_paths_by_prefix[name]) if use_pandas_metadata: file_keyvalues = self.metadata.metadata @@ -837,7 +864,9 @@ def read_table(source, columns=None, nthreads=1, metadata=None, name or directory name. For passing Python file objects or byte buffers, see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader. columns: list - If not None, only these columns will be read from the file. + If not None, only these columns will be read from the file. A column + name may be a prefix of a nested field, e.g. 'a' will select 'a.b', + 'a.c', and 'a.d.e' nthreads : int, default 1 Number of columns to read in parallel. Requires that the underlying file source is threadsafe @@ -875,7 +904,9 @@ def read_pandas(source, columns=None, nthreads=1, metadata=None): name. For passing Python file objects or byte buffers, see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader. columns: list - If not None, only these columns will be read from the file. + If not None, only these columns will be read from the file. A column + name may be a prefix of a nested field, e.g. 'a' will select 'a.b', + 'a.c', and 'a.d.e' nthreads : int, default 1 Number of columns to read in parallel. Requires that the underlying file source is threadsafe diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 274ff458fd6cc..9004fc01ae4a0 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1468,6 +1468,28 @@ def test_index_column_name_duplicate(tmpdir): tm.assert_frame_equal(result_df, dfx) +@parquet +def test_parquet_nested_convenience(tmpdir): + # ARROW-1684 + import pyarrow.parquet as pq + + df = pd.DataFrame({ + 'a': [[1, 2, 3], None, [4, 5], []], + 'b': [[1.], None, None, [6., 7.]], + }) + + path = str(tmpdir / 'nested_convenience.parquet') + + table = pa.Table.from_pandas(df, preserve_index=False) + _write_table(table, path) + + read = pq.read_table(path, columns=['a']) + tm.assert_frame_equal(read.to_pandas(), df[['a']]) + + read = pq.read_table(path, columns=['a', 'b']) + tm.assert_frame_equal(read.to_pandas(), df) + + @parquet def test_backwards_compatible_index_naming(): expected_string = b"""\ From bbbbbfb1ecd29a4d435f942cec5f01226368c71c Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 29 Nov 2017 10:43:20 -0500 Subject: [PATCH 095/177] ARROW-1844: [C++] Add initial Unique benchmarks for int64, variable-length strings I also fixed a bug this surfaced in the hash table resize (unit test coverage was not adequate) Now we have ``` $ ./release/compute-benchmark Run on (8 X 4200.16 MHz CPU s) 2017-11-28 18:33:53 Benchmark Time CPU Iterations ------------------------------------------------------------------------------------------------- BM_BuildDictionary/min_time:1.000 1352 us 1352 us 1038 2.88639GB/s BM_BuildStringDictionary/min_time:1.000 3994 us 3994 us 351 75.5809MB/s BM_UniqueInt64NoNulls/16M/50/min_time:1.000/real_time 35814 us 35816 us 39 3.49023GB/s BM_UniqueInt64NoNulls/16M/1024/min_time:1.000/real_time 119656 us 119660 us 12 1069.73MB/s BM_UniqueInt64NoNulls/16M/10k/min_time:1.000/real_time 174924 us 174930 us 8 731.747MB/s BM_UniqueInt64NoNulls/16M/1024k/min_time:1.000/real_time 448425 us 448440 us 3 285.443MB/s BM_UniqueInt64WithNulls/16M/50/min_time:1.000/real_time 49511 us 49513 us 29 2.52468GB/s BM_UniqueInt64WithNulls/16M/1024/min_time:1.000/real_time 134519 us 134523 us 10 951.541MB/s BM_UniqueInt64WithNulls/16M/10k/min_time:1.000/real_time 191331 us 191336 us 7 668.999MB/s BM_UniqueInt64WithNulls/16M/1024k/min_time:1.000/real_time 533597 us 533613 us 3 239.882MB/s BM_UniqueString10bytes/16M/50/min_time:1.000/real_time 150731 us 150736 us 9 1061.5MB/s BM_UniqueString10bytes/16M/1024/min_time:1.000/real_time 256929 us 256938 us 5 622.739MB/s BM_UniqueString10bytes/16M/10k/min_time:1.000/real_time 414412 us 414426 us 3 386.09MB/s BM_UniqueString10bytes/16M/1024k/min_time:1.000/real_time 1744253 us 1744308 us 1 91.7298MB/s BM_UniqueString100bytes/16M/50/min_time:1.000/real_time 563890 us 563909 us 2 2.77093GB/s BM_UniqueString100bytes/16M/1024/min_time:1.000/real_time 704695 us 704720 us 2 2.21727GB/s BM_UniqueString100bytes/16M/10k/min_time:1.000/real_time 995685 us 995721 us 2 1.56927GB/s BM_UniqueString100bytes/16M/1024k/min_time:1.000/real_time 3584108 us 3584230 us 1 446.415MB/s ``` We can also refactor the hash table implementations without worrying too much about whether we're making things slower Author: Wes McKinney Closes #1370 from wesm/ARROW-1844 and squashes the following commits: 638f1a11 [Wes McKinney] Decrease resize load factor to 0.5 2885c645 [Wes McKinney] Multiply bytes processed by state.iterations() f7b36194 [Wes McKinney] Add initial Unique benchmarks for int64, strings --- cpp/src/arrow/compute/compute-benchmark.cc | 127 ++++++++++++++++++++- cpp/src/arrow/compute/compute-test.cc | 4 +- cpp/src/arrow/compute/kernels/hash.cc | 4 +- 3 files changed, 129 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/compute/compute-benchmark.cc b/cpp/src/arrow/compute/compute-benchmark.cc index 974fffcd6d99d..aa7d899c864e7 100644 --- a/cpp/src/arrow/compute/compute-benchmark.cc +++ b/cpp/src/arrow/compute/compute-benchmark.cc @@ -81,8 +81,131 @@ static void BM_BuildStringDictionary( state.SetBytesProcessed(state.iterations() * total_bytes); } -BENCHMARK(BM_BuildDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond); -BENCHMARK(BM_BuildStringDictionary)->Repetitions(3)->Unit(benchmark::kMicrosecond); +template +struct HashParams { + using T = typename Type::c_type; + + double null_percent; + + void GenerateTestData(const int64_t length, const int64_t num_unique, + std::shared_ptr* arr) const { + std::vector draws; + std::vector values; + std::vector is_valid; + test::randint(length, 0, num_unique, &draws); + for (int64_t draw : draws) { + values.push_back(draw); + } + + if (this->null_percent > 0) { + test::random_is_valid(length, this->null_percent, &is_valid); + ArrayFromVector(is_valid, values, arr); + } else { + ArrayFromVector(values, arr); + } + } + + int64_t GetBytesProcessed(int64_t length) const { return length * sizeof(T); } +}; + +template <> +struct HashParams { + double null_percent; + int32_t byte_width; + void GenerateTestData(const int64_t length, const int64_t num_unique, + std::shared_ptr* arr) const { + std::vector draws; + test::randint(length, 0, num_unique, &draws); + + const int64_t total_bytes = this->byte_width * num_unique; + std::vector uniques(total_bytes); + const uint32_t seed = 0; + test::random_bytes(total_bytes, seed, uniques.data()); + + std::vector is_valid; + if (this->null_percent > 0) { + test::random_is_valid(length, this->null_percent, &is_valid); + } + + StringBuilder builder; + for (int64_t i = 0; i < length; ++i) { + if (this->null_percent == 0 || is_valid[i]) { + ABORT_NOT_OK(builder.Append(uniques.data() + this->byte_width * draws[i], + this->byte_width)); + } else { + ABORT_NOT_OK(builder.AppendNull()); + } + } + ABORT_NOT_OK(builder.Finish(arr)); + } + + int64_t GetBytesProcessed(int64_t length) const { return length * byte_width; } +}; + +template +void BenchUnique(benchmark::State& state, const ParamType& params, int64_t length, + int64_t num_unique) { + std::shared_ptr arr; + params.GenerateTestData(length, num_unique, &arr); + + FunctionContext ctx; + while (state.KeepRunning()) { + std::shared_ptr out; + ABORT_NOT_OK(Unique(&ctx, Datum(arr), &out)); + } + state.SetBytesProcessed(state.iterations() * params.GetBytesProcessed(length)); +} + +template +void BenchDictionaryEncode(benchmark::State& state, const ParamType& params, + int64_t length, int64_t num_unique) { + std::shared_ptr arr; + params.GenerateTestData(length, num_unique, &arr); + + FunctionContext ctx; + while (state.KeepRunning()) { + Datum out; + ABORT_NOT_OK(DictionaryEncode(&ctx, Datum(arr), &out)); + } + state.SetBytesProcessed(state.iterations() * params.GetBytesProcessed(length)); +} + +static void BM_UniqueInt64NoNulls(benchmark::State& state) { + BenchUnique(state, HashParams{0}, state.range(0), state.range(1)); +} + +static void BM_UniqueInt64WithNulls(benchmark::State& state) { + BenchUnique(state, HashParams{0.05}, state.range(0), state.range(1)); +} + +static void BM_UniqueString10bytes(benchmark::State& state) { + // Byte strings with 10 bytes each + BenchUnique(state, HashParams{0.05, 10}, state.range(0), state.range(1)); +} + +static void BM_UniqueString100bytes(benchmark::State& state) { + // Byte strings with 100 bytes each + BenchUnique(state, HashParams{0.05, 100}, state.range(0), state.range(1)); +} + +BENCHMARK(BM_BuildDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_BuildStringDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond); + +constexpr int64_t kHashBenchmarkLength = 1 << 24; + +#define ADD_HASH_ARGS(WHAT) \ + WHAT->Args({kHashBenchmarkLength, 50}) \ + ->Args({kHashBenchmarkLength, 1 << 10}) \ + ->Args({kHashBenchmarkLength, 10 * 1 << 10}) \ + ->Args({kHashBenchmarkLength, 1 << 20}) \ + ->MinTime(1.0) \ + ->Unit(benchmark::kMicrosecond) \ + ->UseRealTime() + +ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64NoNulls)); +ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64WithNulls)); +ADD_HASH_ARGS(BENCHMARK(BM_UniqueString10bytes)); +ADD_HASH_ARGS(BENCHMARK(BM_UniqueString100bytes)); } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index c73bfa309fd95..84af8f7c6b0a7 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -869,8 +869,8 @@ TYPED_TEST(TestHashKernelPrimitive, PrimitiveResizeTable) { return; } - const int64_t kTotalValues = 10000; - const int64_t kRepeats = 10; + const int64_t kTotalValues = 1000000; + const int64_t kRepeats = 5; vector values; vector uniques; diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc index 66c907369e3ee..750f1d36ac7d8 100644 --- a/cpp/src/arrow/compute/kernels/hash.cc +++ b/cpp/src/arrow/compute/kernels/hash.cc @@ -43,7 +43,7 @@ typedef int32_t hash_slot_t; static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits::max(); // The maximum load factor for the hash table before resizing. -static constexpr double kMaxHashTableLoad = 0.7; +static constexpr double kMaxHashTableLoad = 0.5; enum class SIMDMode : char { NOSIMD, SSE4, AVX2 }; @@ -260,7 +260,7 @@ struct HashDictionary> { COMPUTE_HASH; \ while (kHashSlotEmpty != new_hash_slots[j]) { \ ++j; \ - if (ARROW_PREDICT_FALSE(j == hash_table_size_)) { \ + if (ARROW_PREDICT_FALSE(j == new_size)) { \ j = 0; \ } \ } \ From ff8efbf046a7a1e21ca05776550d29f1445eab29 Mon Sep 17 00:00:00 2001 From: Ivan Sadikov Date: Wed, 29 Nov 2017 10:54:43 -0500 Subject: [PATCH 096/177] ARROW-1869: [JAVA] Fix LowCostIdentityHashMap name JIRA: https://issues.apache.org/jira/browse/ARROW-1869 This PR fixes spelling error in class name for `LowCostIdentityHashMap`. Follow-up for https://github.com/apache/arrow/pull/1150. Author: Ivan Sadikov Closes #1372 from sadikovi/fix-low-cost-identity-hash-map and squashes the following commits: e3529f68 [Ivan Sadikov] fix low cost identity hash map name --- .../org/apache/arrow/memory/AllocationManager.java | 2 +- ...IdentityHasMap.java => LowCostIdentityHashMap.java} | 10 +++++----- ...tityHasMap.java => TestLowCostIdentityHashMap.java} | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) rename java/memory/src/main/java/org/apache/arrow/memory/{LowCostIdentityHasMap.java => LowCostIdentityHashMap.java} (98%) rename java/memory/src/test/java/org/apache/arrow/memory/{TestLowCostIdentityHasMap.java => TestLowCostIdentityHashMap.java} (95%) diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java index 14687b54634ac..419be3429721b 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java @@ -76,7 +76,7 @@ public class AllocationManager { private final UnsafeDirectLittleEndian underlying; // ARROW-1627 Trying to minimize memory overhead caused by previously used IdentityHashMap // see JIRA for details - private final LowCostIdentityHasMap map = new LowCostIdentityHasMap<>(); + private final LowCostIdentityHashMap map = new LowCostIdentityHashMap<>(); private final ReadWriteLock lock = new ReentrantReadWriteLock(); private final AutoCloseableLock readLock = new AutoCloseableLock(lock.readLock()); private final AutoCloseableLock writeLock = new AutoCloseableLock(lock.writeLock()); diff --git a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java similarity index 98% rename from java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java rename to java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java index 1153fb5936471..fb7033815d5f4 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHasMap.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java @@ -28,7 +28,7 @@ * that provides "getKey" method * @param */ -public class LowCostIdentityHasMap> { +public class LowCostIdentityHashMap> { /* * The internal data structure to hold values. @@ -52,7 +52,7 @@ public class LowCostIdentityHasMap> { /** * Creates an Map with default expected maximum size. */ - public LowCostIdentityHasMap() { + public LowCostIdentityHashMap() { this(DEFAULT_MIN_SIZE); } @@ -63,7 +63,7 @@ public LowCostIdentityHasMap() { * The estimated maximum number of entries that will be put in * this map. */ - public LowCostIdentityHasMap(int maxSize) { + public LowCostIdentityHashMap(int maxSize) { if (maxSize >= 0) { this.size = 0; threshold = getThreshold(maxSize); @@ -96,7 +96,7 @@ private int computeElementArraySize() { private Object[] newElementArray(int s) { return new Object[s]; } - + /** * Removes all elements from this map, leaving it empty. * @@ -331,4 +331,4 @@ public V getNextValue() { } return null; } -} \ No newline at end of file +} diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java b/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java similarity index 95% rename from java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java rename to java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java index c1196147b4925..0237b38048f52 100644 --- a/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHasMap.java +++ b/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java @@ -27,11 +27,11 @@ /** * To test simplified implementation of IdentityHashMap */ -public class TestLowCostIdentityHasMap { +public class TestLowCostIdentityHashMap { @Test public void testIdentityHashMap() throws Exception { - LowCostIdentityHasMap hashMap = new LowCostIdentityHasMap<>(); + LowCostIdentityHashMap hashMap = new LowCostIdentityHashMap<>(); StringWithKey obj1 = new StringWithKey("s1key", "s1value"); StringWithKey obj2 = new StringWithKey("s2key", "s2value"); @@ -88,7 +88,7 @@ public void testIdentityHashMap() throws Exception { @Test public void testLargeMap() throws Exception { - LowCostIdentityHasMap hashMap = new LowCostIdentityHasMap<>(); + LowCostIdentityHashMap hashMap = new LowCostIdentityHashMap<>(); String [] keys = new String[200]; for (int i = 0; i < 200; i++) { From 1fd3457a77cda15ecf4068f1ced51d0683b519ab Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Wed, 29 Nov 2017 12:48:39 -0500 Subject: [PATCH 097/177] ARROW-1862: [GLib] Add GArrowDictionaryArray Author: Kouhei Sutou Closes #1365 from kou/glib-dictionary-array and squashes the following commits: 83bfa135 [Kouhei Sutou] [GLib] Add GArrowDictionaryArray --- c_glib/arrow-glib/basic-array.cpp | 3 + c_glib/arrow-glib/composite-array.cpp | 107 ++++++++++++++++++++++ c_glib/arrow-glib/composite-array.h | 21 +++++ c_glib/arrow-glib/composite-data-type.cpp | 8 ++ c_glib/test/test-dictionary-array.rb | 63 +++++++++++++ 5 files changed, 202 insertions(+) create mode 100644 c_glib/test/test-dictionary-array.rb diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp index 0698a04098050..36cf4603a55ee 100644 --- a/c_glib/arrow-glib/basic-array.cpp +++ b/c_glib/arrow-glib/basic-array.cpp @@ -2091,6 +2091,9 @@ garrow_array_new_raw(std::shared_ptr *arrow_array) case arrow::Type::type::STRUCT: type = GARROW_TYPE_STRUCT_ARRAY; break; + case arrow::Type::type::DICTIONARY: + type = GARROW_TYPE_DICTIONARY_ARRAY; + break; default: type = GARROW_TYPE_ARRAY; break; diff --git a/c_glib/arrow-glib/composite-array.cpp b/c_glib/arrow-glib/composite-array.cpp index 445103d374037..14cc46d733ee8 100644 --- a/c_glib/arrow-glib/composite-array.cpp +++ b/c_glib/arrow-glib/composite-array.cpp @@ -44,6 +44,11 @@ G_BEGIN_DECLS * or more structs. One struct has zero or more fields. If you don't * have Arrow format data, you need to use #GArrowStructArrayBuilder * to create a new array. + * + * #GArrowDictionaryArray is a class for dictionary array. It can + * store data with dictionary and indices. It's space effective than + * normal array when the array has many same values. You can convert a + * normal array to dictionary array by garrow_array_dictionary_encode(). */ G_DEFINE_TYPE(GArrowListArray, \ @@ -234,4 +239,106 @@ garrow_struct_array_get_fields(GArrowStructArray *array) return g_list_reverse(fields); } + +G_DEFINE_TYPE(GArrowDictionaryArray, \ + garrow_dictionary_array, \ + GARROW_TYPE_ARRAY) + +static void +garrow_dictionary_array_init(GArrowDictionaryArray *object) +{ +} + +static void +garrow_dictionary_array_class_init(GArrowDictionaryArrayClass *klass) +{ +} + +/** + * garrow_dictionary_array_new: + * @data_type: The data type of dictionary. + * @indices: The indices of values in dictionary. + * + * Returns: A newly created #GArrowDictionaryArray. + * + * Since: 0.8.0 + */ +GArrowDictionaryArray * +garrow_dictionary_array_new(GArrowDataType *data_type, + GArrowArray *indices) +{ + const auto arrow_data_type = garrow_data_type_get_raw(data_type); + const auto arrow_indices = garrow_array_get_raw(indices); + auto arrow_dictionary_array = + std::make_shared(arrow_data_type, + arrow_indices); + auto arrow_array = + std::static_pointer_cast(arrow_dictionary_array); + return GARROW_DICTIONARY_ARRAY(garrow_array_new_raw(&arrow_array)); +} + +/** + * garrow_dictionary_array_get_indices: + * @array: A #GArrowDictionaryArray. + * + * Returns: (transfer full): The indices of values in dictionary. + * + * Since: 0.8.0 + */ +GArrowArray * +garrow_dictionary_array_get_indices(GArrowDictionaryArray *array) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_dictionary_array = + std::static_pointer_cast(arrow_array); + auto arrow_indices = arrow_dictionary_array->indices(); + return garrow_array_new_raw(&arrow_indices); +} + +/** + * garrow_dictionary_array_get_dictionary: + * @array: A #GArrowDictionaryArray. + * + * Returns: (transfer full): The dictionary of this array. + * + * Since: 0.8.0 + */ +GArrowArray * +garrow_dictionary_array_get_dictionary(GArrowDictionaryArray *array) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_dictionary_array = + std::static_pointer_cast(arrow_array); + auto arrow_dictionary = arrow_dictionary_array->dictionary(); + return garrow_array_new_raw(&arrow_dictionary); +} + +/** + * garrow_dictionary_array_get_dictionary_data_type: + * @array: A #GArrowDictionaryArray. + * + * Returns: (transfer full): The dictionary data type of this array. + * + * Since: 0.8.0 + */ +GArrowDictionaryDataType * +garrow_dictionary_array_get_dictionary_data_type(GArrowDictionaryArray *array) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_dictionary_array = + std::static_pointer_cast(arrow_array); + auto arrow_dictionary_data_type = arrow_dictionary_array->dict_type(); + auto const_arrow_data_type = + static_cast(arrow_dictionary_data_type); + auto arrow_data_type = const_cast(const_arrow_data_type); + struct NullDeleter { + void operator()(arrow::DataType *data_type) { + } + }; + std::shared_ptr + shared_arrow_data_type(arrow_data_type, NullDeleter()); + auto data_type = garrow_data_type_new_raw(&shared_arrow_data_type); + return GARROW_DICTIONARY_DATA_TYPE(data_type); +} + G_END_DECLS diff --git a/c_glib/arrow-glib/composite-array.h b/c_glib/arrow-glib/composite-array.h index ebf955498724b..c59a61681bf42 100644 --- a/c_glib/arrow-glib/composite-array.h +++ b/c_glib/arrow-glib/composite-array.h @@ -129,4 +129,25 @@ GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array, gint i); GList *garrow_struct_array_get_fields(GArrowStructArray *array); + +#define GARROW_TYPE_DICTIONARY_ARRAY (garrow_dictionary_array_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowDictionaryArray, + garrow_dictionary_array, + GARROW, + DICTIONARY_ARRAY, + GArrowArray) +struct _GArrowDictionaryArrayClass +{ + GArrowArrayClass parent_class; +}; + +GArrowDictionaryArray * +garrow_dictionary_array_new(GArrowDataType *data_type, GArrowArray *indices); +GArrowArray * +garrow_dictionary_array_get_indices(GArrowDictionaryArray *array); +GArrowArray * +garrow_dictionary_array_get_dictionary(GArrowDictionaryArray *array); +GArrowDictionaryDataType * +garrow_dictionary_array_get_dictionary_data_type(GArrowDictionaryArray *array); + G_END_DECLS diff --git a/c_glib/arrow-glib/composite-data-type.cpp b/c_glib/arrow-glib/composite-data-type.cpp index 7ce8a978e2036..5f742e50e4028 100644 --- a/c_glib/arrow-glib/composite-data-type.cpp +++ b/c_glib/arrow-glib/composite-data-type.cpp @@ -158,6 +158,8 @@ garrow_dictionary_data_type_class_init(GArrowDictionaryDataTypeClass *klass) * @ordered: Whether dictionary contents are ordered or not. * * Returns: The newly created dictionary data type. + * + * Since: 0.8.0 */ GArrowDictionaryDataType * garrow_dictionary_data_type_new(GArrowDataType *index_data_type, @@ -177,6 +179,8 @@ garrow_dictionary_data_type_new(GArrowDataType *index_data_type, * @data_type: The #GArrowDictionaryDataType. * * Returns: (transfer full): The #GArrowDataType of index. + * + * Since: 0.8.0 */ GArrowDataType * garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_type) @@ -193,6 +197,8 @@ garrow_dictionary_data_type_get_index_data_type(GArrowDictionaryDataType *data_t * @data_type: The #GArrowDictionaryDataType. * * Returns: (transfer full): The dictionary as #GArrowArray. + * + * Since: 0.8.0 */ GArrowArray * garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type) @@ -209,6 +215,8 @@ garrow_dictionary_data_type_get_dictionary(GArrowDictionaryDataType *data_type) * @data_type: The #GArrowDictionaryDataType. * * Returns: Whether dictionary contents are ordered or not. + * + * Since: 0.8.0 */ gboolean garrow_dictionary_data_type_is_ordered(GArrowDictionaryDataType *data_type) diff --git a/c_glib/test/test-dictionary-array.rb b/c_glib/test/test-dictionary-array.rb new file mode 100644 index 0000000000000..d4f4b3443057d --- /dev/null +++ b/c_glib/test/test-dictionary-array.rb @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDictionaryArray < Test::Unit::TestCase + include Helper::Buildable + + def setup + @index_data_type = Arrow::Int32DataType.new + @dictionary = build_string_array(["C", "C++", "Ruby"]) + @ordered = false + @data_type = Arrow::DictionaryDataType.new(@index_data_type, + @dictionary, + @ordered) + end + + sub_test_case(".new") do + def test_new + indices = build_int32_array([0, 2, 2, 1, 0]) + dictionary_array = Arrow::DictionaryArray.new(@data_type, indices) + assert_equal(<<-STRING.chomp, dictionary_array.to_s) + +-- is_valid: all not null +-- dictionary: ["C", "C++", "Ruby"] +-- indices: [0, 2, 2, 1, 0] + STRING + end + end + + sub_test_case("instance methods") do + def setup + super + @indices = build_int32_array([0, 2, 2, 1, 0]) + @dictionary_array = Arrow::DictionaryArray.new(@data_type, @indices) + end + + def test_indices + assert_equal(@indices, @dictionary_array.indices) + end + + def test_dictionary + assert_equal(@dictionary, @dictionary_array.dictionary) + end + + def test_dictionary_data_type + assert_equal(@data_type, + @dictionary_array.dictionary_data_type) + end + end +end From 705d842664d6626d25843ebbb331186033126287 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Fri, 1 Dec 2017 16:14:44 -0500 Subject: [PATCH 098/177] ARROW-1874: [GLib] Add garrow_array_unique() Author: Kouhei Sutou Closes #1377 from kou/glib-unique and squashes the following commits: 4385e22d [Kouhei Sutou] Add garrow_array_unique() --- c_glib/arrow-glib/basic-array.cpp | 33 +++++++++++++++++++++++++++++++ c_glib/arrow-glib/basic-array.h | 2 ++ c_glib/test/test-unique.rb | 31 +++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 c_glib/test/test-unique.rb diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp index 36cf4603a55ee..a7afaca958354 100644 --- a/c_glib/arrow-glib/basic-array.cpp +++ b/c_glib/arrow-glib/basic-array.cpp @@ -563,6 +563,39 @@ garrow_array_cast(GArrowArray *array, return garrow_array_new_raw(&arrow_casted_array); } +/** + * garrow_array_unique: + * @array: A #GArrowArray. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (nullable) (transfer full): + * A newly created unique elements array on success, %NULL on error. + * + * Since: 0.8.0 + */ +GArrowArray * +garrow_array_unique(GArrowArray *array, + GError **error) +{ + auto arrow_array = garrow_array_get_raw(array); + auto memory_pool = arrow::default_memory_pool(); + arrow::compute::FunctionContext context(memory_pool); + std::shared_ptr arrow_unique_array; + auto status = arrow::compute::Unique(&context, + arrow::compute::Datum(arrow_array), + &arrow_unique_array); + if (!status.ok()) { + std::stringstream message; + message << "[array][unique] <"; + message << arrow_array->type()->ToString(); + message << ">"; + garrow_error_check(error, status, message.str().c_str()); + return NULL; + } + + return garrow_array_new_raw(&arrow_unique_array); +} + G_DEFINE_TYPE(GArrowNullArray, \ garrow_null_array, \ diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h index c01a04f07d8b9..613a59b071a5d 100644 --- a/c_glib/arrow-glib/basic-array.h +++ b/c_glib/arrow-glib/basic-array.h @@ -66,6 +66,8 @@ GArrowArray *garrow_array_cast (GArrowArray *array, GArrowDataType *target_data_type, GArrowCastOptions *options, GError **error); +GArrowArray *garrow_array_unique (GArrowArray *array, + GError **error); #define GARROW_TYPE_NULL_ARRAY \ (garrow_null_array_get_type()) diff --git a/c_glib/test/test-unique.rb b/c_glib/test/test-unique.rb new file mode 100644 index 0000000000000..b94ff462b4600 --- /dev/null +++ b/c_glib/test/test-unique.rb @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestUnique < Test::Unit::TestCase + include Helper::Buildable + include Helper::Omittable + + def test_int32 + assert_equal(build_int32_array([1, 3, -1, -3]), + build_int32_array([1, 3, 1, -1, -3, -1]).unique) + end + + def test_string + assert_equal(build_string_array(["Ruby", "Python"]), + build_string_array(["Ruby", "Python", "Ruby"]).unique) + end +end From ad9105e639b115e2868483b534fb3ac7a8921de3 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Fri, 1 Dec 2017 16:17:49 -0500 Subject: [PATCH 099/177] ARROW-1817: [Java] Configure JsonReader to read floating point NaN values The last upgrade of the Jackson JSON library changed behavior to no longer allow reading of "NaN" values by default. This change configures the JSON generator and parser to allow for NaN values (unquoted) alongside standard floating point numbers. A test was added for JSON writing/reading and modified the test for Arrow file and stream . Author: Bryan Cutler Closes #1375 from BryanCutler/java-JsonReader-all_non_numeric-ARROW-1817 and squashes the following commits: 4c4682a9 [Bryan Cutler] configure JsonWriter to write NaN not as strings, add test for read and write of float with NaN 1fa24f45 [Bryan Cutler] added conf for JacksonParser to allow NaN tokens --- .../arrow/vector/ipc/JsonFileReader.java | 2 ++ .../arrow/vector/ipc/JsonFileWriter.java | 2 ++ .../apache/arrow/vector/ipc/BaseFileTest.java | 33 +++++++++++-------- .../apache/arrow/vector/ipc/TestJSONFile.java | 27 +++++++++++++++ 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index 04d50331b76d8..f274b748e55dc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -69,6 +69,8 @@ public JsonFileReader(File inputFile, BufferAllocator allocator) throws JsonPars this.allocator = allocator; MappingJsonFactory jsonFactory = new MappingJsonFactory(); this.parser = jsonFactory.createParser(inputFile); + // Allow reading NaN for floating point values + this.parser.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 067fb25b8d86c..1c9e1d3809594 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -84,6 +84,8 @@ public JsonFileWriter(File outputFile, JSONWriteConfig config) throws IOExceptio prettyPrinter.indentArraysWith(NopIndenter.instance); this.generator.setPrettyPrinter(prettyPrinter); } + // Allow writing of floating point NaN values not as strings + this.generator.configure(JsonGenerator.Feature.QUOTE_NON_NUMERIC_NUMBERS, false); } public void start(Schema schema, DictionaryProvider provider) throws IOException { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index 3514acaa24278..d26385d749496 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -47,6 +47,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BigIntWriter; import org.apache.arrow.vector.complex.writer.DateMilliWriter; +import org.apache.arrow.vector.complex.writer.Float4Writer; import org.apache.arrow.vector.complex.writer.IntWriter; import org.apache.arrow.vector.complex.writer.TimeMilliWriter; import org.apache.arrow.vector.complex.writer.TimeStampMilliTZWriter; @@ -95,10 +96,28 @@ public void tearDown() { DateTimeZone.setDefault(defaultTimezone); } + protected void writeData(int count, MapVector parent) { + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + IntWriter intWriter = rootWriter.integer("int"); + BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt"); + Float4Writer float4Writer = rootWriter.float4("float"); + for (int i = 0; i < count; i++) { + intWriter.setPosition(i); + intWriter.writeInt(i); + bigIntWriter.setPosition(i); + bigIntWriter.writeBigInt(i); + float4Writer.setPosition(i); + float4Writer.writeFloat4(i == 0 ? Float.NaN : i); + } + writer.setValueCount(count); + } + protected void validateContent(int count, VectorSchemaRoot root) { for (int i = 0; i < count; i++) { Assert.assertEquals(i, root.getVector("int").getObject(i)); Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); + Assert.assertEquals(i == 0 ? Float.NaN : i, root.getVector("float").getObject(i)); } } @@ -454,20 +473,6 @@ protected void validateDecimalData(VectorSchemaRoot root) { } } - protected void writeData(int count, MapVector parent) { - ComplexWriter writer = new ComplexWriterImpl("root", parent); - MapWriter rootWriter = writer.rootAsMap(); - IntWriter intWriter = rootWriter.integer("int"); - BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt"); - for (int i = 0; i < count; i++) { - intWriter.setPosition(i); - intWriter.writeInt(i); - bigIntWriter.setPosition(i); - bigIntWriter.writeBigInt(i); - } - writer.setValueCount(count); - } - public void validateUnionData(int count, VectorSchemaRoot root) { FieldReader unionReader = root.getVector("union").getReader(); for (int i = 0; i < count; i++) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java index 625717048bf91..4f9093b8c0253 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java @@ -38,6 +38,33 @@ public class TestJSONFile extends BaseFileTest { private static final Logger LOGGER = LoggerFactory.getLogger(TestJSONFile.class); + @Test + public void testWriteRead() throws IOException { + File file = new File("target/mytest.json"); + int count = COUNT; + + // write + try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); + MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + writeData(count, parent); + writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null); + } + + // read + try ( + BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE); + JsonFileReader reader = new JsonFileReader(file, readerAllocator) + ) { + Schema schema = reader.start(); + LOGGER.debug("reading schema: " + schema); + + // initialize vectors + try (VectorSchemaRoot root = reader.read();) { + validateContent(count, root); + } + } + } + @Test public void testWriteReadComplexJSON() throws IOException { File file = new File("target/mytest_complex.json"); From 3bed4ebfb344d83e3c09a33aac51f6917fc787fe Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sun, 3 Dec 2017 16:37:22 -0500 Subject: [PATCH 100/177] ARROW-1865: [C++] Do not alter number of rows attribute when removing last column from Table Author: Wes McKinney Closes #1373 from wesm/ARROW-1865 and squashes the following commits: bc7cd336 [Wes McKinney] Removing last column from a Table does not alter number of rows. Test in C++ and Python --- cpp/src/arrow/table-test.cc | 20 ++++++++++++++++++++ cpp/src/arrow/table.cc | 3 ++- python/pyarrow/tests/test_table.py | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc index 8a2288710c896..3f1c6be3a87f6 100644 --- a/cpp/src/arrow/table-test.cc +++ b/cpp/src/arrow/table-test.cc @@ -404,6 +404,26 @@ TEST_F(TestTable, RemoveColumn) { ASSERT_TRUE(result->Equals(*expected)); } +TEST_F(TestTable, RemoveColumnEmpty) { + // ARROW-1865 + const int64_t length = 10; + + auto f0 = field("f0", int32()); + auto schema = ::arrow::schema({f0}); + auto a0 = MakeRandomArray(length); + + auto table = Table::Make(schema, {std::make_shared(f0, a0)}); + + std::shared_ptr

    empty; + ASSERT_OK(table->RemoveColumn(0, &empty)); + + ASSERT_EQ(table->num_rows(), empty->num_rows()); + + std::shared_ptr
    added; + ASSERT_OK(empty->AddColumn(0, table->column(0), &added)); + ASSERT_EQ(table->num_rows(), added->num_rows()); +} + TEST_F(TestTable, AddColumn) { const int64_t length = 10; MakeExample1(length); diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc index 129524b7e4320..2cf6c26523965 100644 --- a/cpp/src/arrow/table.cc +++ b/cpp/src/arrow/table.cc @@ -202,7 +202,8 @@ class SimpleTable : public Table { std::shared_ptr new_schema; RETURN_NOT_OK(schema_->RemoveField(i, &new_schema)); - *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i)); + *out = Table::Make(new_schema, internal::DeleteVectorElement(columns_, i), + this->num_rows()); return Status::OK(); } diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index ab012340c0ab9..e72761d32f634 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -312,6 +312,20 @@ def test_table_remove_column(): assert t2.equals(expected) +def test_table_remove_column_empty(): + # ARROW-1865 + data = [ + pa.array(range(5)), + ] + table = pa.Table.from_arrays(data, names=['a']) + + t2 = table.remove_column(0) + assert len(t2) == len(table) + + t3 = t2.add_column(0, table[0]) + assert t3.equals(table) + + def test_concat_tables(): data = [ list(range(5)), From ff4e40707048d9b3ad4744d3928d48857d32fd70 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Sun, 3 Dec 2017 16:38:39 -0500 Subject: [PATCH 101/177] ARROW-1878: [GLib] Add garrow_array_dictionary_encode() Author: Kouhei Sutou Closes #1381 from kou/glib-dictionary-encode and squashes the following commits: f2da6fcb [Kouhei Sutou] Add garrow_array_dictionary_encode() --- c_glib/arrow-glib/basic-array.cpp | 38 +++++++++++++++++++++++++ c_glib/arrow-glib/basic-array.h | 2 ++ c_glib/test/test-dictionary-encode.rb | 41 +++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 c_glib/test/test-dictionary-encode.rb diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp index a7afaca958354..3c027c0e11240 100644 --- a/c_glib/arrow-glib/basic-array.cpp +++ b/c_glib/arrow-glib/basic-array.cpp @@ -596,6 +596,44 @@ garrow_array_unique(GArrowArray *array, return garrow_array_new_raw(&arrow_unique_array); } +/** + * garrow_array_dictionary_encode: + * @array: A #GArrowArray. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (nullable) (transfer full): + * A newly created #GArrowDictionarArray for the @array on success, + * %NULL on error. + * + * Since: 0.8.0 + */ +GArrowArray * +garrow_array_dictionary_encode(GArrowArray *array, + GError **error) +{ + auto arrow_array = garrow_array_get_raw(array); + auto memory_pool = arrow::default_memory_pool(); + arrow::compute::FunctionContext context(memory_pool); + arrow::compute::Datum dictionary_encoded_datum; + auto status = + arrow::compute::DictionaryEncode(&context, + arrow::compute::Datum(arrow_array), + &dictionary_encoded_datum); + if (!status.ok()) { + std::stringstream message; + message << "[array][dictionary-encode] <"; + message << arrow_array->type()->ToString(); + message << ">"; + garrow_error_check(error, status, message.str().c_str()); + return NULL; + } + + auto arrow_dictionary_encoded_array = + arrow::MakeArray(dictionary_encoded_datum.array()); + + return garrow_array_new_raw(&arrow_dictionary_encoded_array); +} + G_DEFINE_TYPE(GArrowNullArray, \ garrow_null_array, \ diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h index 613a59b071a5d..420c30ecc7161 100644 --- a/c_glib/arrow-glib/basic-array.h +++ b/c_glib/arrow-glib/basic-array.h @@ -68,6 +68,8 @@ GArrowArray *garrow_array_cast (GArrowArray *array, GError **error); GArrowArray *garrow_array_unique (GArrowArray *array, GError **error); +GArrowArray *garrow_array_dictionary_encode(GArrowArray *array, + GError **error); #define GARROW_TYPE_NULL_ARRAY \ (garrow_null_array_get_type()) diff --git a/c_glib/test/test-dictionary-encode.rb b/c_glib/test/test-dictionary-encode.rb new file mode 100644 index 0000000000000..ea77be64ae41d --- /dev/null +++ b/c_glib/test/test-dictionary-encode.rb @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDictionaryEncode < Test::Unit::TestCase + include Helper::Buildable + include Helper::Omittable + + def test_int32 + array = build_int32_array([1, 3, 1, -1, -3, -1]) + assert_equal(<<-STRING.chomp, array.dictionary_encode.to_s) + +-- is_valid: all not null +-- dictionary: [1, 3, -1, -3] +-- indices: [0, 1, 0, 2, 3, 2] + STRING + end + + def test_string + array = build_string_array(["Ruby", "Python", "Ruby"]) + assert_equal(<<-STRING.chomp, array.dictionary_encode.to_s) + +-- is_valid: all not null +-- dictionary: ["Ruby", "Python"] +-- indices: [0, 1, 0] + STRING + end +end From 3ee75a09bb0781350f996a15828145b2171f0983 Mon Sep 17 00:00:00 2001 From: Li Jin Date: Sun, 3 Dec 2017 16:43:00 -0500 Subject: [PATCH 102/177] ARROW-1719: [Java] Remove accessor and mutator interface Author: Li Jin Closes #1374 from icexelloss/remove-accessor-mutator and squashes the following commits: 3423ad4f [Li Jin] Remove accessor and mutator interface --- .../codegen/templates/ComplexWriters.java | 6 +- .../main/codegen/templates/UnionVector.java | 12 -- .../arrow/vector/BaseDataValueVector.java | 2 +- .../arrow/vector/BaseFixedWidthVector.java | 17 +-- .../apache/arrow/vector/BaseValueVector.java | 63 ----------- .../arrow/vector/BaseVariableWidthVector.java | 12 -- .../org/apache/arrow/vector/ValueVector.java | 107 +++++------------- .../arrow/vector/VariableWidthVector.java | 12 -- .../apache/arrow/vector/VectorSchemaRoot.java | 2 +- .../org/apache/arrow/vector/ZeroVector.java | 45 -------- .../complex/BaseRepeatedValueVector.java | 12 -- .../vector/complex/FixedSizeListVector.java | 12 -- .../arrow/vector/complex/MapVector.java | 12 -- .../vector/complex/NonNullableMapVector.java | 10 -- .../vector/complex/RepeatedValueVector.java | 39 ------- .../impl/UnionFixedSizeListReader.java | 2 +- .../arrow/vector/TestVectorReAlloc.java | 2 +- .../apache/arrow/vector/ipc/BaseFileTest.java | 2 - 18 files changed, 41 insertions(+), 328 deletions(-) diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 98672d54cc38e..4a1545a173dd3 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -79,17 +79,17 @@ protected int idx() { public void write(${minor.class?cap_first}Holder h) { mutator.addSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); + vector.setValueCount(idx()+1); } public void write(${minor.class?cap_first}Holder h) { mutator.addSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); + vector.setValueCount(idx()+1); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, ); - vector.getMutator().setValueCount(idx()+1); + vector.setValueCount(idx()+1); } public void setPosition(int idx) { diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 501933f8fbc31..60cd24dcc4457 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -416,18 +416,6 @@ public void copyValueSafe(int from, int to) { } } - @Override - @Deprecated - public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from UNION"); - } - - @Override - @Deprecated - public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to UNION"); - } - @Override public FieldReader getReader() { if (reader == null) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java index 6d9eb1db03af5..80675135e3c1c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java @@ -99,7 +99,7 @@ public ArrowBuf[] getBuffers(boolean clear) { @Override public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } return data.writerIndex(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index cc056904ae8a1..77026d4e5a112 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -91,19 +91,6 @@ public BaseFixedWidthVector(final String name, final BufferAllocator allocator, * the top class as of now is not a good idea. */ - - @Override - @Deprecated - public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); - } - - @Override - @Deprecated - public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); - } - /** * Get the memory address of buffer that manages the validity * (NULL or NON-NULL nature) of elements in the vector. @@ -711,6 +698,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * * @return the number of null elements. */ + @Override public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); } @@ -721,6 +709,7 @@ public int getNullCount() { * * @return valueCount for the vector */ + @Override public int getValueCount() { return valueCount; } @@ -730,6 +719,7 @@ public int getValueCount() { * * @param valueCount value count to set */ + @Override public void setValueCount(int valueCount) { this.valueCount = valueCount; final int currentValueCapacity = getValueCapacity(); @@ -789,6 +779,7 @@ public boolean isSafe(int index) { * @param index position of element * @return true if element at given index is null, false otherwise */ + @Override public boolean isNull(int index) { return (isSet(index) == 0); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index fc0ab3e5915cc..6418ea4f8a069 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -68,42 +68,6 @@ public TransferPair getTransferPair(BufferAllocator allocator) { return getTransferPair(name, allocator); } - public abstract static class BaseAccessor implements ValueVector.Accessor { - protected BaseAccessor() { - } - - @Override - public boolean isNull(int index) { - return false; - } - - @Override - // override this in case your implementation is faster, see BitVector - public int getNullCount() { - int nullCount = 0; - for (int i = 0; i < getValueCount(); i++) { - if (isNull(i)) { - nullCount++; - } - } - return nullCount; - } - } - - public abstract static class BaseMutator implements ValueVector.Mutator { - protected BaseMutator() { - } - - @Override - public void generateTestData(int values) { - } - - //TODO: consider making mutator stateless(if possible) on another issue. - @Override - public void reset() { - } - } - @Override public Iterator iterator() { return Collections.emptyIterator(); @@ -136,33 +100,6 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) { return buffer; } - @Override - public int getValueCount() { - return getAccessor().getValueCount(); - } - - @Override - public void setValueCount(int valueCount) { - getMutator().setValueCount(valueCount); - } - - @Override - - public Object getObject(int index) { - return getAccessor().getObject(index); - } - - @Override - - public int getNullCount() { - return getAccessor().getNullCount(); - } - - @Override - public boolean isNull(int index) { - return getAccessor().isNull(index); - } - /* number of bytes for the validity buffer for the given valueCount */ protected static int getValidityBufferSizeFromCount(final int valueCount) { return (int) Math.ceil(valueCount / 8.0); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index b515586bc9f36..98820a2367b75 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -90,18 +90,6 @@ public BaseVariableWidthVector(final String name, final BufferAllocator allocato * the top class as of now is not a good idea. */ - @Override - @Deprecated - public VariableWidthMutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); - } - - @Override - @Deprecated - public VariableWidthAccessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); - } - /** * Get buffer that manages the validity (NULL or NON-NULL nature) of * elements in the vector. Consider it as a buffer for internal bit vector diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 0d3acf1646f53..e77c1b12703ec 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -135,18 +135,6 @@ public interface ValueVector extends Closeable, Iterable { */ TransferPair makeTransferPair(ValueVector target); - /** - * @return an {@link org.apache.arrow.vector.ValueVector.Accessor accessor} that is used to read from this vector - * instance. - */ - Accessor getAccessor(); - - /** - * @return an {@link org.apache.arrow.vector.ValueVector.Mutator mutator} that is used to write to this vector - * instance. - */ - Mutator getMutator(); - /** * @return a {@link org.apache.arrow.vector.complex.reader.FieldReader field reader} that supports reading values * from this vector. @@ -160,7 +148,7 @@ public interface ValueVector extends Closeable, Iterable { /** * Returns the number of bytes that is used by this vector if it holds the given number - * of values. The result will be the same as if Mutator.setValueCount() were called, followed + * of values. The result will be the same as if setValueCount() were called, followed * by calling getBufferSize(), but without any of the closing side-effects that setValueCount() * implies wrt finishing off the population of a vector. Some operations might wish to use * this to determine how much memory has been used by a vector so far, even though it is @@ -182,91 +170,56 @@ public interface ValueVector extends Closeable, Iterable { */ ArrowBuf[] getBuffers(boolean clear); - /** - * An abstraction that is used to read from this vector instance. - */ - interface Accessor { - /** - * Get the Java Object representation of the element at the specified position. Useful for testing. - * - * @param index Index of the value to get - * @return the friendly java type - */ - Object getObject(int index); - - /** - * @return the number of values that is stored in this vector. - */ - int getValueCount(); - - /** - * @param index the index to check for nullity - * @return true if the value at the given index is null, false otherwise. - */ - boolean isNull(int index); - - /** - * @return the number of null values - */ - int getNullCount(); - } - - /** - * An abstraction that is used to write into this vector instance. - */ - interface Mutator { - /** - * Sets the number of values that is stored in this vector to the given value count. - * - * @param valueCount value count to set. - */ - void setValueCount(int valueCount); - - /** - * Resets the mutator to pristine state. - */ - void reset(); - - /** - * @param values the number of values to generate - * @deprecated this has nothing to do with value vector abstraction and should be removed. - */ - @Deprecated - void generateTestData(int values); - } - /** * Gets the underlying buffer associated with validity vector * * @return buffer */ - public ArrowBuf getValidityBuffer(); + ArrowBuf getValidityBuffer(); /** * Gets the underlying buffer associated with data vector * * @return buffer */ - public ArrowBuf getDataBuffer(); + ArrowBuf getDataBuffer(); /** * Gets the underlying buffer associated with offset vector * * @return buffer */ - public ArrowBuf getOffsetBuffer(); + ArrowBuf getOffsetBuffer(); - /* temporarily add these methods here until we remove other vectors - * (non-nullable) which are under ValueVector hierarchy and still - * use the mutator/accessor interfaces. + /** + * Gets the number of values + * @return */ - public int getValueCount(); + int getValueCount(); - public void setValueCount(int valueCount); + /** + * Set number of values in the vector + * @return + */ + void setValueCount(int valueCount); - public Object getObject(int index); + /** + * Get friendly type object from the vector + * @param index + * @return + */ + Object getObject(int index); - public int getNullCount(); + /** + * Returns number of null elements in the vector + * @return + */ + int getNullCount(); - public boolean isNull(int index); + /** + * Check whether an element in the vector is null + * @param index + * @return + */ + boolean isNull(int index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java index 04c00b7c8349c..593d4dceaf782 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java @@ -35,17 +35,5 @@ public interface VariableWidthVector extends ValueVector { */ int getByteCapacity(); - VariableWidthMutator getMutator(); - - VariableWidthAccessor getAccessor(); - - interface VariableWidthAccessor extends Accessor { - int getValueLength(int index); - } - int getCurrentSizeInBytes(); - - interface VariableWidthMutator extends Mutator { - void setValueLengthSafe(int index, int length); - } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java index 196393fb9584a..3fd33d66d4ef5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java @@ -140,7 +140,7 @@ public String contentToTSVString() { for (int i = 0; i < rowCount; i++) { row.clear(); for (FieldVector v : fieldVectors) { - row.add(v.getAccessor().getObject(i)); + row.add(v.getObject(i)); } printRow(sb, row); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 0ab3a7b6843c3..962a1c947071f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -60,41 +60,6 @@ public void copyValueSafe(int from, int to) { } }; - private final Accessor defaultAccessor = new Accessor() { - @Override - public Object getObject(int index) { - return null; - } - - @Override - public int getValueCount() { - return 0; - } - - @Override - public boolean isNull(int index) { - return true; - } - - @Override - public int getNullCount() { - return 0; - } - }; - - private final Mutator defaultMutator = new Mutator() { - @Override - public void setValueCount(int valueCount) { - } - - @Override - public void reset() { - } - - @Override - public void generateTestData(int values) { - } - }; public ZeroVector() { } @@ -186,16 +151,6 @@ public TransferPair makeTransferPair(ValueVector target) { return defaultPair; } - @Override - public Accessor getAccessor() { - return defaultAccessor; - } - - @Override - public Mutator getMutator() { - return defaultMutator; - } - @Override public FieldReader getReader() { return NullReader.INSTANCE; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 4648d078949a7..a9221f2f6ea56 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -277,16 +277,4 @@ public void setValueCount(int valueCount) { offsetBuffer.getInt(valueCount * OFFSET_WIDTH); vector.setValueCount(childValueCount); } - - @Override - @Deprecated - public RepeatedAccessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from LIST."); - } - - @Override - @Deprecated - public RepeatedMutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to LIST"); - } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 774a10dbfb98f..93a8127cfd3b4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -150,18 +150,6 @@ public List getFieldInnerVectors() { throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } - @Override - @Deprecated - public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); - } - - @Override - @Deprecated - public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); - } - @Override public UnionFixedSizeListReader getReader() { return reader; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 075ae83ea4d84..72cc29f30ca14 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -497,16 +497,4 @@ public void setValueCount(int valueCount) { public void reset() { valueCount = 0; } - - @Override - @Deprecated - public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP"); - } - - @Override - @Deprecated - public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP"); - } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java index cc3ac4148a865..1cd7b70d7bfad 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java @@ -252,16 +252,6 @@ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { return natural.min(getChildren()).getValueCapacity(); } - @Override - public Accessor getAccessor() { - throw new UnsupportedOperationException("accessor is not needed for MAP"); - } - - @Override - public Mutator getMutator() { - throw new UnsupportedOperationException("mutator is not needed for MAP"); - } - @Override public Object getObject(int index) { Map vv = new JsonStringHashMap<>(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java index 91147c663f248..36401172994c5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java @@ -41,43 +41,4 @@ public interface RepeatedValueVector extends ValueVector { * @return the underlying data vector or null if none exists. */ ValueVector getDataVector(); - - @Override - RepeatedAccessor getAccessor(); - - @Override - RepeatedMutator getMutator(); - - interface RepeatedAccessor extends ValueVector.Accessor { - /** - * The result includes empty, null valued cells. - * - * @return total number of cells that vector contains. - */ - int getInnerValueCount(); - - - /** - * @param index the index of the value for which we want the size - * @return number of cells that the value at the given index contains. - */ - int getInnerValueCountAt(int index); - - /** - * @param index value index - * @return true if the value at the given index is empty, false otherwise. - */ - boolean isEmpty(int index); - } - - interface RepeatedMutator extends ValueVector.Mutator { - - /** - * Starts a new value that is a container of cells. - * - * @param index index of new value to start - * @return index into the child vector - */ - int startNewValue(int index); - } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java index 4ad2f6f5fad7a..56ae379dca210 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java @@ -80,7 +80,7 @@ public void read(int index, UnionHolder holder) { } } holder.reader = data.getReader(); - holder.isSet = vector.getAccessor().isNull(idx()) ? 0 : 1; + holder.isSet = vector.isNull(idx()) ? 0 : 1; } @Override diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index c0df4881f380c..1b13c2ed91935 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -130,7 +130,7 @@ public void testMapType() { assertEquals(512, vector.getValueCapacity()); try { - vector.getAccessor().getObject(513); + vector.getObject(513); Assert.fail("Expected out of bounds exception"); } catch (Exception e) { // ok diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index d26385d749496..1a7f71c72b483 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -34,11 +34,9 @@ import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.ValueVector.Accessor; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.impl.ComplexWriterImpl; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; From 3b438bc91ef153d5c963b676a534909d60ba3eb4 Mon Sep 17 00:00:00 2001 From: Max Risuhin Date: Sun, 3 Dec 2017 16:44:51 -0500 Subject: [PATCH 103/177] ARROW-1755: [C++] CMake option to link msvc crt statically - CMake option `ARROW_USE_STATIC_CRT` to turn on Arrow build with statically linked MSVC C/C++ runtimes (turned OFF by default). All 3rd parties should be linked with the same version of crt. - Debug and Release Appveyor's build jobs with `ARROW_USE_STATIC_CRT` option turned on - Patches for lz4 and zstd libs updated to include runtime linkage selection property used by msbuild.exe Author: Max Risuhin Closes #1333 from MaxRis/ARROW-1755 and squashes the following commits: 5c9317dc [Max Risuhin] ARROW-1755: [C++] CMake option to link msvc crt statically --- appveyor.yml | 4 + ci/msvc-build.bat | 32 + cpp/CMakeLists.txt | 4 + ...lz4_msbuild_gl_runtimelibrary_params.patch | 601 ++++++++++++++++++ ...build_wholeprogramoptimization_param.patch | 225 ------- ...std_msbuild_gl_runtimelibrary_params.patch | 528 +++++++++++++++ ...build_wholeprogramoptimization_param.patch | 199 ------ cpp/cmake_modules/SetupCxxFlags.cmake | 9 + cpp/cmake_modules/ThirdpartyToolchain.cmake | 64 +- 9 files changed, 1226 insertions(+), 440 deletions(-) create mode 100644 cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch delete mode 100644 cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch create mode 100644 cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch delete mode 100644 cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch diff --git a/appveyor.yml b/appveyor.yml index 55c58d0bf664d..e647b8b77080f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -40,6 +40,10 @@ environment: PYTHON: "3.5" ARCH: "64" CONFIGURATION: "Release" + - JOB: "Static_Crt_Build" + GENERATOR: Visual Studio 14 2015 Win64 + PYTHON: "3.5" + ARCH: "64" - JOB: "Toolchain" GENERATOR: Visual Studio 14 2015 Win64 PYTHON: "3.5" diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat index e8eb0945e516e..62ebcf364e77b 100644 --- a/ci/msvc-build.bat +++ b/ci/msvc-build.bat @@ -17,6 +17,38 @@ @echo on +if "%JOB%" == "Static_Crt_Build" ( + mkdir cpp\build-debug + pushd cpp\build-debug + + cmake -G "%GENERATOR%" ^ + -DARROW_USE_STATIC_CRT=ON ^ + -DARROW_BOOST_USE_SHARED=OFF ^ + -DCMAKE_BUILD_TYPE=Debug ^ + -DARROW_CXXFLAGS="/MP" ^ + .. || exit /B + + cmake --build . --config Debug || exit /B + popd + + mkdir cpp\build-release + pushd cpp\build-release + + cmake -G "%GENERATOR%" ^ + -DARROW_USE_STATIC_CRT=ON ^ + -DARROW_BOOST_USE_SHARED=OFF ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DARROW_CXXFLAGS="/WX /MP" ^ + .. || exit /B + + cmake --build . --config Release || exit /B + ctest -VV || exit /B + popd + + @rem Finish Static_Crt_Build build successfully + exit /B 0 +) + if "%JOB%" == "Build_Debug" ( mkdir cpp\build-debug pushd cpp\build-debug diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f4b7b29b9d3f3..dd159ced29193 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -210,6 +210,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Lz4 static lib suffix used on Windows with MSVC (default _static)") set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING "ZStd static lib suffix used on Windows with MSVC (default _static)") + + option(ARROW_USE_STATIC_CRT + "Build Arrow with statically linked CRT" + OFF) endif() endif() diff --git a/cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch b/cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch new file mode 100644 index 0000000000000..c79898d9a786c --- /dev/null +++ b/cpp/build-support/lz4_msbuild_gl_runtimelibrary_params.patch @@ -0,0 +1,601 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj +index aaf81ad..096741a 100644 +--- a/visual/VS2010/datagen/datagen.vcxproj ++++ b/visual/VS2010/datagen/datagen.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + Unicode + + + Application + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -113,6 +117,46 @@ + true + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 +diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj +index 76d12c9..34ca686 100644 +--- a/visual/VS2010/frametest/frametest.vcxproj ++++ b/visual/VS2010/frametest/frametest.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + Unicode + + + Application + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -113,6 +117,46 @@ + true + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 +diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +index c10552a..c1ca231 100644 +--- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj ++++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + Unicode + + + Application + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -117,6 +121,46 @@ + liblz4.lib;%(AdditionalDependencies) + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 +diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj +index e2d95c9..613ff3c 100644 +--- a/visual/VS2010/fullbench/fullbench.vcxproj ++++ b/visual/VS2010/fullbench/fullbench.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + Unicode + + + Application + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -113,6 +117,46 @@ + true + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 +diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj +index 85d6c9b..be8b9a1 100644 +--- a/visual/VS2010/fuzzer/fuzzer.vcxproj ++++ b/visual/VS2010/fuzzer/fuzzer.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + Unicode + + + Application + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -113,6 +117,46 @@ + true + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 +diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj +index 389f13c..e30e667 100644 +--- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj ++++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj +@@ -40,15 +40,19 @@ + + DynamicLibrary + false +- true + Unicode + + + DynamicLibrary + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -116,6 +120,46 @@ + true + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 +diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj +index a0b8000..38d2ce2 100644 +--- a/visual/VS2010/liblz4/liblz4.vcxproj ++++ b/visual/VS2010/liblz4/liblz4.vcxproj +@@ -39,15 +39,19 @@ + + StaticLibrary + false +- true + Unicode + + + StaticLibrary + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -115,6 +119,46 @@ + true + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 +diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj +index 693e121..9eb005b 100644 +--- a/visual/VS2010/lz4/lz4.vcxproj ++++ b/visual/VS2010/lz4/lz4.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + Unicode + + + Application + false +- true + Unicode + ++ ++ true ++ ++ ++ true ++ + + + +@@ -115,6 +119,46 @@ + setargv.obj;%(AdditionalDependencies) + + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + Level4 diff --git a/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch b/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch deleted file mode 100644 index ee0f8a12054bc..0000000000000 --- a/cpp/build-support/lz4_msbuild_wholeprogramoptimization_param.patch +++ /dev/null @@ -1,225 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj -index aaf81ad..67b716f 100644 ---- a/visual/VS2010/datagen/datagen.vcxproj -+++ b/visual/VS2010/datagen/datagen.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - Unicode - - - Application - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj -index 76d12c9..723571d 100644 ---- a/visual/VS2010/frametest/frametest.vcxproj -+++ b/visual/VS2010/frametest/frametest.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - Unicode - - - Application - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj -index c10552a..0c8f293 100644 ---- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj -+++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - Unicode - - - Application - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj -index e2d95c9..4cd88d0 100644 ---- a/visual/VS2010/fullbench/fullbench.vcxproj -+++ b/visual/VS2010/fullbench/fullbench.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - Unicode - - - Application - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj -index 85d6c9b..3ddc77d 100644 ---- a/visual/VS2010/fuzzer/fuzzer.vcxproj -+++ b/visual/VS2010/fuzzer/fuzzer.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - Unicode - - - Application - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj -index 389f13c..038a4d2 100644 ---- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj -+++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj -@@ -40,15 +40,19 @@ - - DynamicLibrary - false -- true - Unicode - - - DynamicLibrary - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj -index a0b8000..9aad8c2 100644 ---- a/visual/VS2010/liblz4/liblz4.vcxproj -+++ b/visual/VS2010/liblz4/liblz4.vcxproj -@@ -39,15 +39,19 @@ - - StaticLibrary - false -- true - Unicode - - - StaticLibrary - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj -index 693e121..7e63f1e 100644 ---- a/visual/VS2010/lz4/lz4.vcxproj -+++ b/visual/VS2010/lz4/lz4.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - Unicode - - - Application - false -- true - Unicode - -+ -+ true -+ -+ -+ true -+ - - - diff --git a/cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch b/cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch new file mode 100644 index 0000000000000..a168a814950e6 --- /dev/null +++ b/cpp/build-support/zstd_msbuild_gl_runtimelibrary_params.patch @@ -0,0 +1,528 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj +index bd8a213d..691d39fe 100644 +--- a/build/VS2010/datagen/datagen.vcxproj ++++ b/build/VS2010/datagen/datagen.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + MultiByte + + + Application + false +- true + MultiByte + ++ ++ true ++ ++ ++ true ++ + + + +@@ -84,6 +88,46 @@ + false + $(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + +diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj +index e697318e..a5720189 100644 +--- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj ++++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + MultiByte + + + Application + false +- true + MultiByte + ++ ++ true ++ ++ ++ true ++ + + + +@@ -84,6 +88,46 @@ + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + false + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + +diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj +index 2bff4ca4..d64fac81 100644 +--- a/build/VS2010/fullbench/fullbench.vcxproj ++++ b/build/VS2010/fullbench/fullbench.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + MultiByte + + + Application + false +- true + MultiByte + ++ ++ true ++ ++ ++ true ++ + + + +@@ -84,6 +88,46 @@ + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + false + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + +diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj +index 12a4b931..1ffc5941 100644 +--- a/build/VS2010/fuzzer/fuzzer.vcxproj ++++ b/build/VS2010/fuzzer/fuzzer.vcxproj +@@ -39,15 +39,19 @@ + + Application + false +- true + MultiByte + + + Application + false +- true + MultiByte + ++ ++ true ++ ++ ++ true ++ + + + +@@ -84,6 +88,46 @@ + false + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress;$(UniversalCRT_IncludePath); + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + +diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +index 364b3bea..1f34e956 100644 +--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj ++++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +@@ -94,15 +94,19 @@ + + DynamicLibrary + false +- true + MultiByte + + + DynamicLibrary + false +- true + MultiByte + ++ ++ true ++ ++ ++ true ++ + + + +@@ -143,6 +147,46 @@ + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + +diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj +index 6087d737..0d2a6fe0 100644 +--- a/build/VS2010/libzstd/libzstd.vcxproj ++++ b/build/VS2010/libzstd/libzstd.vcxproj +@@ -91,15 +91,19 @@ + + StaticLibrary + false +- true + MultiByte + + + StaticLibrary + false +- true + MultiByte + ++ ++ true ++ ++ ++ true ++ + + + +@@ -140,6 +144,46 @@ + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + +diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj +index 438dc617..3fb5bba8 100644 +--- a/build/VS2010/zstd/zstd.vcxproj ++++ b/build/VS2010/zstd/zstd.vcxproj +@@ -100,15 +100,19 @@ + + Application + false +- true + MultiByte + + + Application + false +- true + MultiByte + ++ ++ true ++ ++ ++ true ++ + + + +@@ -149,6 +153,46 @@ + false + $(LibraryPath); + ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ ++ ++ ++ MultiThreaded ++ ++ ++ ++ ++ MultiThreadedDebug ++ ++ ++ ++ ++ MultiThreadedDLL ++ ++ ++ ++ ++ MultiThreadedDebugDLL ++ ++ + + + diff --git a/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch b/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch deleted file mode 100644 index 8bfb928947e06..0000000000000 --- a/cpp/build-support/zstd_msbuild_wholeprogramoptimization_param.patch +++ /dev/null @@ -1,199 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/build/VS2010/datagen/datagen.vcxproj b/build/VS2010/datagen/datagen.vcxproj -index bd8a213..8e4dc89 100644 ---- a/build/VS2010/datagen/datagen.vcxproj -+++ b/build/VS2010/datagen/datagen.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - MultiByte - - - Application - false -- true - MultiByte - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj -index e697318..82cd4ab 100644 ---- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj -+++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - MultiByte - - - Application - false -- true - MultiByte - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj -index 2bff4ca..ced4047 100644 ---- a/build/VS2010/fullbench/fullbench.vcxproj -+++ b/build/VS2010/fullbench/fullbench.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - MultiByte - - - Application - false -- true - MultiByte - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj -index 12a4b93..227efd1 100644 ---- a/build/VS2010/fuzzer/fuzzer.vcxproj -+++ b/build/VS2010/fuzzer/fuzzer.vcxproj -@@ -39,15 +39,19 @@ - - Application - false -- true - MultiByte - - - Application - false -- true - MultiByte - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj -index 364b3be..b227320 100644 ---- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj -+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj -@@ -94,15 +94,19 @@ - - DynamicLibrary - false -- true - MultiByte - - - DynamicLibrary - false -- true - MultiByte - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj -index 6087d73..51a0572 100644 ---- a/build/VS2010/libzstd/libzstd.vcxproj -+++ b/build/VS2010/libzstd/libzstd.vcxproj -@@ -91,15 +91,19 @@ - - StaticLibrary - false -- true - MultiByte - - - StaticLibrary - false -- true - MultiByte - -+ -+ true -+ -+ -+ true -+ - - - -diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj -index 438dc61..834ae01 100644 ---- a/build/VS2010/zstd/zstd.vcxproj -+++ b/build/VS2010/zstd/zstd.vcxproj -@@ -100,15 +100,19 @@ - - Application - false -- true - MultiByte - - - Application - false -- true - MultiByte - -+ -+ true -+ -+ -+ true -+ - - - diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 4b1950f7a17bd..4e0ace0baea0c 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -47,6 +47,15 @@ if (MSVC) # Set desired warning level (e.g. set /W4 for more warnings) set(CXX_COMMON_FLAGS "/W3") endif() + + if (ARROW_USE_STATIC_CRT) + foreach (c_flag CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_DEBUG + CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO + CMAKE_C_FLAGS CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_DEBUG + CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO) + string(REPLACE "/MD" "-MT" ${c_flag} "${${c_flag}}") + endforeach() + endif() else() # Common flags set below with warning level set(CXX_COMMON_FLAGS "") diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 411cf7584f7f1..b706aab45265d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -130,6 +130,9 @@ endif() set(Boost_DEBUG TRUE) set(Boost_USE_MULTITHREADED ON) +if (MSVC AND ARROW_USE_STATIC_CRT) + set(Boost_USE_STATIC_RUNTIME ON) +endif() set(Boost_ADDITIONAL_VERSIONS "1.65.0" "1.65" "1.64.0" "1.64" @@ -204,11 +207,11 @@ else() else() find_package(Boost COMPONENTS system filesystem REQUIRED) if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) - set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) + set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) + set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) else() - set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) - set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) + set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) + set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) endif() set(BOOST_SYSTEM_LIBRARY boost_system_shared) set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared) @@ -222,11 +225,11 @@ else() else() find_package(Boost COMPONENTS system filesystem REQUIRED) if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) - set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) + set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) + set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) else() - set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) - set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) + set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) + set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) endif() set(BOOST_SYSTEM_LIBRARY boost_system_static) set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) @@ -272,8 +275,10 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) set(GTEST_VENDORED 1) set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX} - -Dgtest_force_shared_crt=ON -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) + if (MSVC AND NOT ARROW_USE_STATIC_CRT) + set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON) + endif() ExternalProject_Add(googletest_ep URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" @@ -320,6 +325,8 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) -DBUILD_TESTING=OFF -BUILD_CONFIG_TESTS=OFF -DINSTALL_HEADERS=ON + -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} + -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS}) ExternalProject_Add(gflags_ep @@ -434,6 +441,8 @@ if (ARROW_IPC) "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}" "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}" "-DFLATBUFFERS_BUILD_TESTS=OFF" + "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" + "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}" ${EP_LOG_OPTIONS}) set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include") @@ -550,6 +559,8 @@ if (ARROW_WITH_ZLIB) set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX} -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} + -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} -DBUILD_SHARED_LIBS=OFF) ExternalProject_Add(zlib_ep @@ -599,7 +610,9 @@ if (ARROW_WITH_SNAPPY) if (MSVC) set(SNAPPY_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${EX_C_FLAGS}" + "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" + "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}" + "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt @@ -659,7 +672,9 @@ if (ARROW_WITH_BROTLI) set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}") set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${EX_C_FLAGS}" + "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" + "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}" + "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}" -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX} -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE} -DBUILD_SHARED_LIBS=OFF) @@ -709,9 +724,17 @@ if (ARROW_WITH_LZ4) set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib") if (MSVC) + if (ARROW_USE_STATIC_CRT) + if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") + else() + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") + endif() + endif() set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") - set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) - set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_wholeprogramoptimization_param.patch) + set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 + ${LZ4_RUNTIME_LIBRARY_LINKAGE} /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) + set(LZ4_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/lz4_msbuild_gl_runtimelibrary_params.patch) else() set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh) @@ -753,9 +776,18 @@ if (ARROW_WITH_ZSTD) set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib") if (MSVC) + if (ARROW_USE_STATIC_CRT) + if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") + else() + set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") + endif() + endif() set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib") - set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ ) - set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_wholeprogramoptimization_param.patch) + set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE} + ${ZSTD_RUNTIME_LIBRARY_LINKAGE} /p:Platform=x64 /p:PlatformToolset=v140 + /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ ) + set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_gl_runtimelibrary_params.patch) else() set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a") set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh) @@ -802,7 +834,7 @@ if (ARROW_WITH_GRPC) set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_BUILD_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${EX_C_FLAGS}" + "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} -DBUILD_SHARED_LIBS=OFF) From 5dbfe301089d4f7a81fe2406ff641acd284c23ee Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 4 Dec 2017 10:49:17 +0100 Subject: [PATCH 104/177] ARROW-1863: [Python] PyObjectStringify could render bytes-like output for more types of objects Author: Phillip Cloud Closes #1385 from cpcloud/ARROW-1863 and squashes the following commits: 126d917 [Phillip Cloud] Repr when trying to convert strings ba9a8ff [Phillip Cloud] ARROW-1863: [Python] PyObjectStringify could render bytes-like output for more types of objects --- cpp/src/arrow/python/builtin_convert.cc | 3 ++- python/pyarrow/tests/test_convert_builtin.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc index fa0098bdf63e9..c716c47d2de23 100644 --- a/cpp/src/arrow/python/builtin_convert.cc +++ b/cpp/src/arrow/python/builtin_convert.cc @@ -667,7 +667,8 @@ class UTF8Converter : public TypedConverterVisitor RETURN_IF_PYERROR(); bytes_obj = obj; } else if (!PyUnicode_Check(obj)) { - PyObjectStringify stringified(obj); + OwnedRef repr(PyObject_Repr(obj)); + PyObjectStringify stringified(repr.obj()); std::stringstream ss; ss << "Non bytes/unicode value encountered: " << stringified.bytes; return Status::Invalid(ss.str()); diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py index 4c3d9e5631ce3..d7760da2f9b47 100644 --- a/python/pyarrow/tests/test_convert_builtin.py +++ b/python/pyarrow/tests/test_convert_builtin.py @@ -312,6 +312,13 @@ def test_mixed_types_fails(self): with self.assertRaises(pa.ArrowException): pa.array(data) + def test_mixed_types_with_specified_type_fails(self): + data = ['-10', '-5', {'a': 1}, '0', '5', '10'] + + type = pa.string() + with self.assertRaises(pa.ArrowInvalid): + pa.array(data, type=type) + def test_decimal(self): data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')] type = pa.decimal128(precision=7, scale=3) From 8f2dbee49dfef8b2f7bc68f8fe978f4b24fa5a79 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 4 Dec 2017 09:21:18 -0500 Subject: [PATCH 105/177] ARROW-1879: [Python] Dask integration tests are not skipped if dask is not installed Author: Phillip Cloud Closes #1384 from cpcloud/ARROW-1879 and squashes the following commits: 7b9503e9 [Phillip Cloud] ARROW-1879: [Python] Dask integration tests are not skipped if dask is not installed --- python/testing/dask_tests/test_dask_integration.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/testing/dask_tests/test_dask_integration.py b/python/testing/dask_tests/test_dask_integration.py index e678348780cd8..842c45f57d1f7 100644 --- a/python/testing/dask_tests/test_dask_integration.py +++ b/python/testing/dask_tests/test_dask_integration.py @@ -14,12 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +import pytest + from datetime import date, timedelta import csv from random import randint -import dask.dataframe as dd + import pyarrow as pa +dd = pytest.importorskip('dask.dataframe') + + def make_datafiles(tmpdir, prefix='data', num_files=20): rowcount = 5000 fieldnames = ['date', 'temperature', 'dewpoint'] @@ -37,6 +43,7 @@ def make_datafiles(tmpdir, prefix='data', num_files=20): 'dewpoint': dewpoint}) the_date += timedelta(days=1) + def test_dask_file_read(tmpdir): prefix = 'data' make_datafiles(tmpdir, prefix) From fe6f60c31875c2a42d71a975f1c18c0c81c20678 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Mon, 4 Dec 2017 17:46:12 -0500 Subject: [PATCH 106/177] ARROW-1885: [Java] Restore MapVector class names prior to ARROW-1710 Changes to MapVector were not part of the intended goal for ARROW-1710 and should be restored to `NullableMapVector` -> `MapVector` -> `AbstractMapVector` as existed prior to #1341. Author: Bryan Cutler Closes #1389 from BryanCutler/java-restore-MapVector-class-ARROW-1885 and squashes the following commits: 9ff503ab [Bryan Cutler] restore MapVector class names 3d818f0e [Bryan Cutler] renamed NonNullableMapVector to MapVector ced1705c [Bryan Cutler] renamed MapVector to NullableMapVector --- .../templates/CaseSensitiveMapWriters.java | 2 +- .../main/codegen/templates/MapWriters.java | 8 +- .../main/codegen/templates/UnionVector.java | 10 +- .../arrow/vector/BaseVariableWidthVector.java | 2 +- .../complex/AbstractContainerVector.java | 4 +- .../arrow/vector/complex/MapVector.java | 552 +++++++----------- .../vector/complex/NonNullableMapVector.java | 342 ----------- .../vector/complex/NullableMapVector.java | 500 ++++++++++++++++ .../complex/impl/ComplexWriterImpl.java | 6 +- .../complex/impl/NullableMapReaderImpl.java | 8 +- .../impl/NullableMapWriterFactory.java | 4 +- .../vector/complex/impl/PromotableWriter.java | 4 +- .../complex/impl/SingleMapReaderImpl.java | 5 +- .../org/apache/arrow/vector/types/Types.java | 6 +- .../apache/arrow/vector/TestMapVector.java | 4 +- .../arrow/vector/TestVectorReAlloc.java | 4 +- .../complex/impl/TestPromotableWriter.java | 4 +- .../complex/writer/TestComplexWriter.java | 6 +- .../apache/arrow/vector/ipc/BaseFileTest.java | 12 +- .../arrow/vector/ipc/TestArrowFile.java | 21 +- .../apache/arrow/vector/ipc/TestJSONFile.java | 15 +- 21 files changed, 758 insertions(+), 761 deletions(-) delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java index 7c652902eb8f5..5357f9b8a9d3a 100644 --- a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java +++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java @@ -23,7 +23,7 @@ <#if mode == "Single"> <#assign containerClass = "MapVector" /> <#else> -<#assign containerClass = "MapVector" /> +<#assign containerClass = "NullableMapVector" /> <#include "/@includes/license.ftl" /> diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index a5ac1b7170442..ac59e59b30888 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -21,9 +21,9 @@ <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" /> <#assign index = "idx()"> <#if mode == "Single"> -<#assign containerClass = "NonNullableMapVector" /> -<#else> <#assign containerClass = "MapVector" /> +<#else> +<#assign containerClass = "NullableMapVector" /> <#include "/@includes/license.ftl" /> @@ -51,7 +51,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter { private final Map fields = Maps.newHashMap(); public ${mode}MapWriter(${containerClass} container) { <#if mode == "Single"> - if (container instanceof MapVector) { + if (container instanceof NullableMapVector) { throw new IllegalArgumentException("Invalid container: " + container); } @@ -124,7 +124,7 @@ public MapWriter map(String name) { FieldWriter writer = fields.get(finalName); if(writer == null){ int vectorCount=container.size(); - MapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), MapVector.class); + NullableMapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class); writer = new PromotableWriter(vector, container, getNullableMapWriterFactory()); if(vectorCount != container.size()) { writer.allocate(); diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 60cd24dcc4457..9377bd0da211d 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -48,8 +48,8 @@ /** - * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each - * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, + * A vector which can hold values of different types. It does so by using a NullableMapVector which contains a vector for each + * primitive type that is stored. NullableMapVector is used in order to take advantage of its serialization/deserialization methods, * as well as the addOrGet method. * * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup @@ -65,7 +65,7 @@ public class UnionVector implements FieldVector { MapVector internalMap; protected ArrowBuf typeBuffer; - private MapVector mapVector; + private NullableMapVector mapVector; private ListVector listVector; private FieldReader reader; @@ -174,10 +174,10 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public MapVector getMap() { + public NullableMapVector getMap() { if (mapVector == null) { int vectorCount = internalMap.size(); - mapVector = addOrGet(MinorType.MAP, MapVector.class); + mapVector = addOrGet(MinorType.MAP, NullableMapVector.class); if (internalMap.size() > vectorCount) { mapVector.allocateNew(); if (callBack != null) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 98820a2367b75..fbadb35d7556a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -24,7 +24,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index 5ded1cc6f8778..db0ff86df47a9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -100,8 +100,8 @@ protected boolean supportsDirectRead() { // return the child vector's ordinal in the composite container public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name); - public MapVector addOrGetMap(String name) { - return addOrGet(name, FieldType.nullable(new Struct()), MapVector.class); + public NullableMapVector addOrGetMap(String name) { + return addOrGet(name, FieldType.nullable(new Struct()), NullableMapVector.class); } public ListVector addOrGetList(String name) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 72cc29f30ca14..e13084588ce0b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

    + * * http://www.apache.org/licenses/LICENSE-2.0 - *

    + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,480 +21,322 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; +import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.Map; -import com.google.common.collect.ObjectArrays; +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BaseAllocator; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; -import org.apache.arrow.vector.complex.impl.NullableMapWriter; +import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.ArrowType.Struct; -import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.TransferPair; -public class MapVector extends NonNullableMapVector implements FieldVector { +public class MapVector extends AbstractMapVector { + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); public static MapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = FieldType.nullable(Struct.INSTANCE); + FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); return new MapVector(name, allocator, fieldType, null); } - private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); - private final NullableMapWriter writer = new NullableMapWriter(this); - - protected ArrowBuf validityBuffer; - private int validityAllocationSizeInBytes; + private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); + protected final FieldType fieldType; + public int valueCount; // deprecated, use FieldType or static constructor instead @Deprecated public MapVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack); + this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); } - // deprecated, use FieldType or static constructor instead - @Deprecated - public MapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { - this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack); + public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + this.fieldType = checkNotNull(fieldType); + this.valueCount = 0; } - public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, checkNotNull(allocator), fieldType, callBack); - this.validityBuffer = allocator.getEmpty(); - this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + @Override + public FieldReader getReader() { + return reader; + } + + transient private MapTransferPair ephPair; + + public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { + if (ephPair == null || ephPair.from != from) { + ephPair = (MapTransferPair) from.makeTransferPair(this); + } + ephPair.copyValueSafe(fromIndex, thisIndex); } @Override - public Field getField() { - Field f = super.getField(); - FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata()); - return new Field(f.getName(), type, f.getChildren()); + protected boolean supportsDirectRead() { + return true; + } + + public Iterator fieldNameIterator() { + return getChildFieldNames().iterator(); } @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - if (ownBuffers.size() != 1) { - throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + public void setInitialCapacity(int numRecords) { + for (final ValueVector v : (Iterable) this) { + v.setInitialCapacity(numRecords); } + } - ArrowBuf bitBuffer = ownBuffers.get(0); + @Override + public int getBufferSize() { + if (valueCount == 0 || size() == 0) { + return 0; + } + long buffer = 0; + for (final ValueVector v : (Iterable) this) { + buffer += v.getBufferSize(); + } - validityBuffer.release(); - validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); - valueCount = fieldNode.getLength(); - validityAllocationSizeInBytes = validityBuffer.capacity(); + return (int) buffer; } @Override - public List getFieldBuffers() { - List result = new ArrayList<>(1); - setReaderAndWriterIndex(); - result.add(validityBuffer); + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } - return result; - } + long bufferSize = 0; + for (final ValueVector v : (Iterable) this) { + bufferSize += v.getBufferSizeFor(valueCount); + } - private void setReaderAndWriterIndex() { - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + return (int) bufferSize; } @Override - @Deprecated - public List getFieldInnerVectors() { - throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + public ArrowBuf getValidityBuffer() { + throw new UnsupportedOperationException(); } @Override - public NullableMapReaderImpl getReader() { - return reader; + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); } - public NullableMapWriter getWriter() { - return writer; + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); } @Override public TransferPair getTransferPair(BufferAllocator allocator) { - return new NullableMapTransferPair(this, new MapVector(name, allocator, fieldType, null), false); + return getTransferPair(name, allocator, null); } @Override - public TransferPair makeTransferPair(ValueVector to) { - return new NullableMapTransferPair(this, (MapVector) to, true); + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, null), false); + public TransferPair makeTransferPair(ValueVector to) { + return new MapTransferPair(this, (MapVector) to); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false); + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false); } - protected class NullableMapTransferPair extends MapTransferPair { - - private MapVector target; + protected static class MapTransferPair implements TransferPair { + private final TransferPair[] pairs; + private final MapVector from; + private final MapVector to; - protected NullableMapTransferPair(MapVector from, MapVector to, boolean allocate) { - super(from, to, allocate); - this.target = to; + public MapTransferPair(MapVector from, MapVector to) { + this(from, to, true); } - @Override - public void transfer() { - target.clear(); - target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; - super.transfer(); - clear(); + protected MapTransferPair(MapVector from, MapVector to, boolean allocate) { + this.from = from; + this.to = to; + this.pairs = new TransferPair[from.size()]; + this.to.ephPair = null; + + int i = 0; + FieldVector vector; + for (String child : from.getChildFieldNames()) { + int preSize = to.size(); + vector = from.getChild(child); + if (vector == null) { + continue; + } + //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector, + // the child fields may be nested fields of the top level child. For example if the structure + // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab. + // But the children member of a Materialized field is a HashSet. If the fields are added in the + // children HashSet, and the hashCode of the Materialized field includes the hash code of the + // children, the hashCode value of oa changes *after* the field has been added to the HashSet. + // (This is similar to what happens in ScanBatch where the children cannot be added till they are + // read). To take care of this, we ensure that the hashCode of the MaterializedField does not + // include the hashCode of the children but is based only on MaterializedField$key. + final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass()); + if (allocate && to.size() != preSize) { + newVector.allocateNew(); + } + pairs[i++] = vector.makeTransferPair(newVector); + } } @Override - public void copyValueSafe(int fromIndex, int toIndex) { - while (toIndex >= target.getValidityBufferValueCapacity()) { - target.reallocValidityBuffer(); + public void transfer() { + for (final TransferPair p : pairs) { + p.transfer(); } - BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); - super.copyValueSafe(fromIndex, toIndex); + to.valueCount = from.valueCount; + from.clear(); } @Override - public void splitAndTransfer(int startIndex, int length) { - target.clear(); - splitAndTransferValidityBuffer(startIndex, length, target); - super.splitAndTransfer(startIndex, length); + public ValueVector getTo() { + return to; } - } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, - firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } + @Override + public void copyValueSafe(int from, int to) { + for (TransferPair p : pairs) { + p.copyValueSafe(from, to); } } - } - - /** - * Get the value capacity of the internal validity buffer. - * @return number of elements that validity buffer can hold - */ - private int getValidityBufferValueCapacity() { - return (int) (validityBuffer.capacity() * 8L); - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity() { - return Math.min(getValidityBufferValueCapacity(), - super.getValueCapacity()); - } - /** - * Return the underlying buffers associated with this vector. Note that this doesn't - * impact the reference counts for this buffer so it only should be used for in-context - * access. Also note that this buffer changes regularly thus - * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear Whether to clear vector before returning; the buffers will still be refcounted - * but the returned array will be the only reference to them - * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this - * vector instance. - */ - @Override - public ArrowBuf[] getBuffers(boolean clear) { - setReaderAndWriterIndex(); - final ArrowBuf[] buffers; - if (getBufferSize() == 0) { - buffers = new ArrowBuf[0]; - } else { - buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), - ArrowBuf.class); - } - if (clear) { - for (ArrowBuf buffer : buffers) { - buffer.retain(); + @Override + public void splitAndTransfer(int startIndex, int length) { + for (TransferPair p : pairs) { + p.splitAndTransfer(startIndex, length); } - clear(); + to.setValueCount(length); } - - return buffers; } - /** - * Close the vector and release the associated buffers. - */ @Override - public void close() { - clearValidityBuffer(); - super.close(); - } - - /** - * Same as {@link #close()} - */ - @Override - public void clear() { - clearValidityBuffer(); - super.clear(); - } - - /** - * Release the validity buffer - */ - private void clearValidityBuffer() { - validityBuffer.release(); - validityBuffer = allocator.getEmpty(); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - if (valueCount == 0) { + public int getValueCapacity() { + if (size() == 0) { return 0; } - return super.getBufferSize() + - BitVectorHelper.getValidityBufferSize(valueCount); - } - /** - * Get the potential buffer size for a particular number of records. - * @param valueCount desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - return super.getBufferSizeFor(valueCount) - + BitVectorHelper.getValidityBufferSize(valueCount); - } + final Ordering natural = new Ordering() { + @Override + public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { + return Ints.compare( + checkNotNull(left).getValueCapacity(), + checkNotNull(right).getValueCapacity() + ); + } + }; - @Override - public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); - super.setInitialCapacity(numRecords); + return natural.min(getChildren()).getValueCapacity(); } @Override - public boolean allocateNewSafe() { - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - clear(); - allocateValidityBuffer(validityAllocationSizeInBytes); - success = super.allocateNewSafe(); - } finally { - if (!success) { - clear(); - return false; + public Object getObject(int index) { + Map vv = new JsonStringHashMap<>(); + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); + } } } - return true; - } - - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + return vv; } @Override - public void reAlloc() { - /* reallocate the validity buffer */ - reallocValidityBuffer(); - super.reAlloc(); - } - - private void reallocValidityBuffer() { - final int currentBufferCapacity = validityBuffer.capacity(); - long baseSize = validityAllocationSizeInBytes; - - if (baseSize < (long) currentBufferCapacity) { - baseSize = (long) currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); - newBuf.setZero(0, newBuf.capacity()); - newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); - validityBuffer.release(1); - validityBuffer = newBuf; - validityAllocationSizeInBytes = (int) newAllocationSize; - } - + public boolean isNull(int index) { return false; } @Override - public long getValidityBufferAddress() { - return validityBuffer.memoryAddress(); - } + public int getNullCount() { return 0; } - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); + public void get(int index, ComplexHolder holder) { + reader.setPosition(index); + holder.reader = reader; } @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException(); + public int getValueCount() { + return valueCount; } - @Override - public ArrowBuf getValidityBuffer() { - return validityBuffer; - } + public ValueVector getVectorById(int id) { + return getChildByOrdinal(id); +} @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); + public void setValueCount(int valueCount) { + for (final ValueVector v : getChildren()) { + v.setValueCount(valueCount); + } + MapVector.this.valueCount = valueCount; } @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); + public void clear() { + for (final ValueVector v : getChildren()) { + v.clear(); + } + valueCount = 0; } @Override - public Object getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return super.getObject(index); + public Field getField() { + List children = new ArrayList<>(); + for (ValueVector child : getChildren()) { + children.add(child.getField()); } + return new Field(name, fieldType, children); } @Override - public void get(int index, ComplexHolder holder) { - holder.isSet = isSet(index); - super.get(index, holder); + public MinorType getMinorType() { + return MinorType.MAP; } - public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); - } - - public boolean isNull(int index) { - return isSet(index) == 0; - } - - public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - public void setIndexDefined(int index) { - while (index >= getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); + @Override + public void close() { + final Collection vectors = getChildren(); + for (final FieldVector v : vectors) { + v.close(); } - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - } + vectors.clear(); - public void setNull(int index) { - while (index >= getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); - } - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + valueCount = 0; + + super.close(); } - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - while (valueCount > getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); + public void initializeChildrenFromFields(List children) { + for (Field field : children) { + FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType()); + vector.initializeChildrenFromFields(field.getChildren()); } - super.setValueCount(valueCount); - this.valueCount = valueCount; } - public void reset() { - valueCount = 0; + public List getChildrenFromFields() { + return getChildren(); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java deleted file mode 100644 index 1cd7b70d7bfad..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java +++ /dev/null @@ -1,342 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.complex; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; -import com.google.common.primitives.Ints; - -import io.netty.buffer.ArrowBuf; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringHashMap; -import org.apache.arrow.vector.util.TransferPair; - -public class NonNullableMapVector extends AbstractMapVector { - //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NonNullableMapVector.class); - - public static NonNullableMapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); - return new NonNullableMapVector(name, allocator, fieldType, null); - } - - private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); - protected final FieldType fieldType; - public int valueCount; - - // deprecated, use FieldType or static constructor instead - @Deprecated - public NonNullableMapVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); - } - - public NonNullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); - this.fieldType = checkNotNull(fieldType); - this.valueCount = 0; - } - - @Override - public FieldReader getReader() { - return reader; - } - - transient private MapTransferPair ephPair; - - public void copyFromSafe(int fromIndex, int thisIndex, NonNullableMapVector from) { - if (ephPair == null || ephPair.from != from) { - ephPair = (MapTransferPair) from.makeTransferPair(this); - } - ephPair.copyValueSafe(fromIndex, thisIndex); - } - - @Override - protected boolean supportsDirectRead() { - return true; - } - - public Iterator fieldNameIterator() { - return getChildFieldNames().iterator(); - } - - @Override - public void setInitialCapacity(int numRecords) { - for (final ValueVector v : (Iterable) this) { - v.setInitialCapacity(numRecords); - } - } - - @Override - public int getBufferSize() { - if (valueCount == 0 || size() == 0) { - return 0; - } - long buffer = 0; - for (final ValueVector v : (Iterable) this) { - buffer += v.getBufferSize(); - } - - return (int) buffer; - } - - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - - long bufferSize = 0; - for (final ValueVector v : (Iterable) this) { - bufferSize += v.getBufferSizeFor(valueCount); - } - - return (int) bufferSize; - } - - @Override - public ArrowBuf getValidityBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return getTransferPair(name, allocator, null); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new MapTransferPair(this, new NonNullableMapVector(name, allocator, fieldType, callBack), false); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new MapTransferPair(this, (NonNullableMapVector) to); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new MapTransferPair(this, new NonNullableMapVector(ref, allocator, fieldType, callBack), false); - } - - protected static class MapTransferPair implements TransferPair { - private final TransferPair[] pairs; - private final NonNullableMapVector from; - private final NonNullableMapVector to; - - public MapTransferPair(NonNullableMapVector from, NonNullableMapVector to) { - this(from, to, true); - } - - protected MapTransferPair(NonNullableMapVector from, NonNullableMapVector to, boolean allocate) { - this.from = from; - this.to = to; - this.pairs = new TransferPair[from.size()]; - this.to.ephPair = null; - - int i = 0; - FieldVector vector; - for (String child : from.getChildFieldNames()) { - int preSize = to.size(); - vector = from.getChild(child); - if (vector == null) { - continue; - } - //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector, - // the child fields may be nested fields of the top level child. For example if the structure - // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab. - // But the children member of a Materialized field is a HashSet. If the fields are added in the - // children HashSet, and the hashCode of the Materialized field includes the hash code of the - // children, the hashCode value of oa changes *after* the field has been added to the HashSet. - // (This is similar to what happens in ScanBatch where the children cannot be added till they are - // read). To take care of this, we ensure that the hashCode of the MaterializedField does not - // include the hashCode of the children but is based only on MaterializedField$key. - final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass()); - if (allocate && to.size() != preSize) { - newVector.allocateNew(); - } - pairs[i++] = vector.makeTransferPair(newVector); - } - } - - @Override - public void transfer() { - for (final TransferPair p : pairs) { - p.transfer(); - } - to.valueCount = from.valueCount; - from.clear(); - } - - @Override - public ValueVector getTo() { - return to; - } - - @Override - public void copyValueSafe(int from, int to) { - for (TransferPair p : pairs) { - p.copyValueSafe(from, to); - } - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - for (TransferPair p : pairs) { - p.splitAndTransfer(startIndex, length); - } - to.setValueCount(length); - } - } - - @Override - public int getValueCapacity() { - if (size() == 0) { - return 0; - } - - final Ordering natural = new Ordering() { - @Override - public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { - return Ints.compare( - checkNotNull(left).getValueCapacity(), - checkNotNull(right).getValueCapacity() - ); - } - }; - - return natural.min(getChildren()).getValueCapacity(); - } - - @Override - public Object getObject(int index) { - Map vv = new JsonStringHashMap<>(); - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v != null && index < v.getValueCount()) { - Object value = v.getObject(index); - if (value != null) { - vv.put(child, value); - } - } - } - return vv; - } - - @Override - public boolean isNull(int index) { return false; } - @Override - public int getNullCount() { return 0; } - - public void get(int index, ComplexHolder holder) { - reader.setPosition(index); - holder.reader = reader; - } - - @Override - public int getValueCount() { - return valueCount; - } - - public ValueVector getVectorById(int id) { - return getChildByOrdinal(id); -} - - @Override - public void setValueCount(int valueCount) { - for (final ValueVector v : getChildren()) { - v.setValueCount(valueCount); - } - NonNullableMapVector.this.valueCount = valueCount; - } - - @Override - public void clear() { - for (final ValueVector v : getChildren()) { - v.clear(); - } - valueCount = 0; - } - - @Override - public Field getField() { - List children = new ArrayList<>(); - for (ValueVector child : getChildren()) { - children.add(child.getField()); - } - return new Field(name, fieldType, children); - } - - @Override - public MinorType getMinorType() { - return MinorType.MAP; - } - - @Override - public void close() { - final Collection vectors = getChildren(); - for (final FieldVector v : vectors) { - v.close(); - } - vectors.clear(); - - valueCount = 0; - - super.close(); - } - - public void initializeChildrenFromFields(List children) { - for (Field field : children) { - FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType()); - vector.initializeChildrenFromFields(field.getChildren()); - } - } - - public List getChildrenFromFields() { - return getChildren(); - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java new file mode 100644 index 0000000000000..d887b73064643 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -0,0 +1,500 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; +import org.apache.arrow.vector.complex.impl.NullableMapWriter; +import org.apache.arrow.vector.holders.ComplexHolder; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Struct; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +public class NullableMapVector extends MapVector implements FieldVector { + + public static NullableMapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = FieldType.nullable(Struct.INSTANCE); + return new NullableMapVector(name, allocator, fieldType, null); + } + + private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); + private final NullableMapWriter writer = new NullableMapWriter(this); + + protected ArrowBuf validityBuffer; + private int validityAllocationSizeInBytes; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) { + this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack); + } + + // deprecated, use FieldType or static constructor instead + @Deprecated + public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { + this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack); + } + + public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, checkNotNull(allocator), fieldType, callBack); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + } + + @Override + public Field getField() { + Field f = super.getField(); + FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata()); + return new Field(f.getName(), type, f.getChildren()); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); + valueCount = fieldNode.getLength(); + validityAllocationSizeInBytes = validityBuffer.capacity(); + } + + @Override + public List getFieldBuffers() { + List result = new ArrayList<>(1); + setReaderAndWriterIndex(); + result.add(validityBuffer); + + return result; + } + + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + } + + @Override + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + + @Override + public NullableMapReaderImpl getReader() { + return reader; + } + + public NullableMapWriter getWriter() { + return writer; + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new NullableMapTransferPair(this, (NullableMapVector) to, true); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false); + } + + protected class NullableMapTransferPair extends MapTransferPair { + + private NullableMapVector target; + + protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, boolean allocate) { + super(from, to, allocate); + this.target = to; + } + + @Override + public void transfer() { + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + super.transfer(); + clear(); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + while (toIndex >= target.getValidityBufferValueCapacity()) { + target.reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); + super.copyValueSafe(fromIndex, toIndex); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + super.splitAndTransfer(startIndex, length); + } + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + /** + * Get the value capacity of the internal validity buffer. + * @return number of elements that validity buffer can hold + */ + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + return Math.min(getValidityBufferValueCapacity(), + super.getValueCapacity()); + } + + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ + @Override + public ArrowBuf[] getBuffers(boolean clear) { + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), + ArrowBuf.class); + } + if (clear) { + for (ArrowBuf buffer : buffers) { + buffer.retain(); + } + clear(); + } + + return buffers; + } + + /** + * Close the vector and release the associated buffers. + */ + @Override + public void close() { + clearValidityBuffer(); + super.close(); + } + + /** + * Same as {@link #close()} + */ + @Override + public void clear() { + clearValidityBuffer(); + super.clear(); + } + + /** + * Release the validity buffer + */ + private void clearValidityBuffer() { + validityBuffer.release(); + validityBuffer = allocator.getEmpty(); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + if (valueCount == 0) { + return 0; + } + return super.getBufferSize() + + BitVectorHelper.getValidityBufferSize(valueCount); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } + return super.getBufferSizeFor(valueCount) + + BitVectorHelper.getValidityBufferSize(valueCount); + } + + @Override + public void setInitialCapacity(int numRecords) { + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); + super.setInitialCapacity(numRecords); + } + + @Override + public boolean allocateNewSafe() { + /* Boolean to keep track if all the memory allocations were successful + * Used in the case of composite vectors when we need to allocate multiple + * buffers for multiple vectors. If one of the allocations failed we need to + * clear all the memory that we allocated + */ + boolean success = false; + try { + clear(); + allocateValidityBuffer(validityAllocationSizeInBytes); + success = super.allocateNewSafe(); + } finally { + if (!success) { + clear(); + return false; + } + } + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int) size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + @Override + public void reAlloc() { + /* reallocate the validity buffer */ + reallocValidityBuffer(); + super.reAlloc(); + } + + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + @Override + public long getValidityBufferAddress() { + return validityBuffer.memoryAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return super.getObject(index); + } + } + + @Override + public void get(int index, ComplexHolder holder) { + holder.isSet = isSet(index); + super.get(index, holder); + } + + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + + public boolean isNull(int index) { + return isSet(index) == 0; + } + + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + @Override + public void setValueCount(int valueCount) { + assert valueCount >= 0; + while (valueCount > getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + super.setValueCount(valueCount); + this.valueCount = valueCount; + } + + public void reset() { + valueCount = 0; + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java index 970b90ef510fa..a6960238b9165 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java @@ -20,7 +20,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.StateTool; import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; import org.apache.arrow.vector.types.pojo.Field; @@ -131,7 +131,7 @@ public MapWriter directMap() { switch (mode) { case INIT: - mapRoot = nullableMapWriterFactory.build((MapVector) container); + mapRoot = nullableMapWriterFactory.build((NullableMapVector) container); mapRoot.setPosition(idx()); mode = Mode.MAP; break; @@ -152,7 +152,7 @@ public MapWriter rootAsMap() { case INIT: // TODO allow dictionaries in complex types - MapVector map = container.addOrGetMap(name); + NullableMapVector map = container.addOrGetMap(name); mapRoot = nullableMapWriterFactory.build(map); mapRoot.setPosition(idx()); mode = Mode.MAP; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java index acf155af71db2..06b0f4d43272f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java @@ -20,17 +20,17 @@ package org.apache.arrow.vector.complex.impl; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.pojo.Field; public class NullableMapReaderImpl extends SingleMapReaderImpl { - private MapVector nullableMapVector; + private NullableMapVector nullableMapVector; public NullableMapReaderImpl(MapVector vector) { - super((MapVector) vector); - this.nullableMapVector = (MapVector) vector; + super((NullableMapVector) vector); + this.nullableMapVector = (NullableMapVector) vector; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java index 06dbf784be0a5..d2dcb2374d0e7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector.complex.impl; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; public class NullableMapWriterFactory { private final boolean caseSensitive; @@ -29,7 +29,7 @@ public NullableMapWriterFactory(boolean caseSensitive) { this.caseSensitive = caseSensitive; } - public NullableMapWriter build(MapVector container) { + public NullableMapWriter build(NullableMapVector container) { return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 9abd38d20ac62..5bd439cac4269 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -23,7 +23,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.AbstractMapVector; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -94,7 +94,7 @@ private void setWriter(ValueVector v) { type = v.getMinorType(); switch (type) { case MAP: - writer = nullableMapWriterFactory.build((MapVector) vector); + writer = nullableMapWriterFactory.build((NullableMapVector) vector); break; case LIST: writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java index 0341b622e0df4..c77ca4e8f23ba 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java @@ -24,7 +24,6 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NonNullableMapVector; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -35,10 +34,10 @@ @SuppressWarnings("unused") public class SingleMapReaderImpl extends AbstractFieldReader { - private final NonNullableMapVector vector; + private final MapVector vector; private final Map fields = Maps.newHashMap(); - public SingleMapReaderImpl(NonNullableMapVector vector) { + public SingleMapReaderImpl(MapVector vector) { this.vector = vector; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 7834845aad80e..3c5fd81d572d0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -58,7 +58,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.BigIntWriterImpl; import org.apache.arrow.vector.complex.impl.BitWriterImpl; @@ -131,12 +131,12 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { MAP(Struct.INSTANCE) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new MapVector(name, allocator, fieldType, schemaChangeCallback); + return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new NullableMapWriter((MapVector) vector); + return new NullableMapWriter((NullableMapVector) vector); } }, TINYINT(new Int(8, true)) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index e5682955bada0..357df96aa2efc 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -22,7 +22,7 @@ import java.util.Map; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.After; @@ -50,7 +50,7 @@ public void testFieldMetadata() throws Exception { Map metadata = new HashMap<>(); metadata.put("k1", "v1"); FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata); - try (MapVector vector = new MapVector("map", allocator, type, null)) { + try (NullableMapVector vector = new NullableMapVector("map", allocator, type, null)) { Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 1b13c2ed91935..f8edf8904c53e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -26,7 +26,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; @@ -121,7 +121,7 @@ public void testListType() { @Test public void testMapType() { - try (final MapVector vector = MapVector.empty("", allocator)) { + try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) { vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), IntVector.class); vector.setInitialCapacity(512); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 38b78424dccf4..b0d6cf555e98d 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -25,7 +25,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DirtyRootAllocator; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -55,7 +55,7 @@ public void terminate() throws Exception { public void testPromoteToUnion() throws Exception { try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator); - final MapVector v = container.addOrGetMap("test"); + final NullableMapVector v = container.addOrGetMap("test"); final PromotableWriter writer = new PromotableWriter(v, container)) { container.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index bd8489eb20f89..29d39aabe6b47 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -34,7 +34,7 @@ import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NonNullableMapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.ComplexWriterImpl; import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; @@ -830,7 +830,7 @@ public void complexCopierWithList() { rootWriter.end(); writer.setValueCount(1); - MapVector mapVector = (MapVector) parent.getChild("root"); + NullableMapVector mapVector = (NullableMapVector) parent.getChild("root"); TransferPair tp = mapVector.getTransferPair(allocator); tp.splitAndTransfer(0, 1); MapVector toMapVector = (MapVector) tp.getTo(); @@ -849,7 +849,7 @@ public void testSingleMapWriter1() { /* initialize a SingleMapWriter with empty MapVector and then lazily * create all vectors with expected initialCapacity. */ - NonNullableMapVector parent = NonNullableMapVector.empty("parent", allocator); + MapVector parent = MapVector.empty("parent", allocator); SingleMapWriter singleMapWriter = new SingleMapWriter(parent); int initialCapacity = 1024; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index 1a7f71c72b483..9eb55c9bbdaee 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -36,7 +36,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.impl.ComplexWriterImpl; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -94,7 +94,7 @@ public void tearDown() { DateTimeZone.setDefault(defaultTimezone); } - protected void writeData(int count, MapVector parent) { + protected void writeData(int count, NullableMapVector parent) { ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); IntWriter intWriter = rootWriter.integer("int"); @@ -119,7 +119,7 @@ protected void validateContent(int count, VectorSchemaRoot root) { } } - protected void writeComplexData(int count, MapVector parent) { + protected void writeComplexData(int count, NullableMapVector parent) { ArrowBuf varchar = allocator.buffer(3); varchar.readerIndex(0); varchar.setByte(0, 'a'); @@ -189,7 +189,7 @@ private LocalDateTime makeDateTimeFromCount(int i) { return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i); } - protected void writeDateTimeData(int count, MapVector parent) { + protected void writeDateTimeData(int count, NullableMapVector parent) { Assert.assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); @@ -494,7 +494,7 @@ public void validateUnionData(int count, VectorSchemaRoot root) { } } - public void writeUnionData(int count, MapVector parent) { + public void writeUnionData(int count, NullableMapVector parent) { ArrowBuf varchar = allocator.buffer(3); varchar.readerIndex(0); varchar.setByte(0, 'a'); @@ -537,7 +537,7 @@ public void writeUnionData(int count, MapVector parent) { varchar.release(); } - protected void writeVarBinaryData(int count, MapVector parent) { + protected void writeVarBinaryData(int count, NullableMapVector parent) { Assert.assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java index 0cfc9ba919d85..055c34e7010f3 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java @@ -42,8 +42,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.complex.FixedSizeListVector; -import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; import org.apache.arrow.vector.ipc.message.ArrowBlock; import org.apache.arrow.vector.ipc.message.ArrowBuffer; @@ -70,7 +69,7 @@ public void testWrite() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeData(count, parent); write(parent.getChild("root"), file, new ByteArrayOutputStream()); } @@ -82,7 +81,7 @@ public void testWriteComplex() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); FieldVector root = parent.getChild("root"); validateComplexContent(count, new VectorSchemaRoot(root)); @@ -98,7 +97,7 @@ public void testWriteRead() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeData(count, parent); write(parent.getChild("root"), file, stream); } @@ -154,7 +153,7 @@ public void testWriteReadComplex() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeComplexData(count, parent); write(parent.getChild("root"), file, stream); } @@ -195,7 +194,7 @@ public void testWriteReadMultipleRBs() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator); + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator); FileOutputStream fileOutputStream = new FileOutputStream(file)) { writeData(counts[0], parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); @@ -268,7 +267,7 @@ public void testWriteReadUnion() throws IOException { // write try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); validateUnionData(count, new VectorSchemaRoot(parent.getChild("root"))); write(parent.getChild("root"), file, stream); @@ -382,7 +381,7 @@ public void testWriteReadMetadata() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector vector = (MapVector) field.createVector(originalVectorAllocator)) { + NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) { vector.allocateNewSafe(); vector.setValueCount(0); @@ -558,7 +557,7 @@ public void testWriteReadFixedSizeList() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class); Float4Vector floats = (Float4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); IntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), IntVector.class); @@ -618,7 +617,7 @@ public void testWriteReadVarBin() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeVarBinaryData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateVarBinary(count, root); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java index 4f9093b8c0253..6a23a8438de48 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java @@ -24,8 +24,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; import org.apache.arrow.vector.types.pojo.Schema; @@ -45,7 +44,7 @@ public void testWriteRead() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeData(count, parent); writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null); } @@ -73,7 +72,7 @@ public void testWriteReadComplexJSON() throws IOException { // write try ( BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeComplexData(count, parent); writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null); } @@ -100,7 +99,7 @@ public void testWriteComplexJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateComplexContent(root.getRowCount(), root); @@ -122,7 +121,7 @@ public void testWriteReadUnionJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); printVectors(parent.getChildrenFromFields()); @@ -154,7 +153,7 @@ public void testWriteReadDateTimeJSON() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeDateTimeData(count, parent); @@ -319,7 +318,7 @@ public void testWriteReadVarBinJSON() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeVarBinaryData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateVarBinary(count, root); From 611a4b951e24f4f967c3d382a2027dc035fc37f0 Mon Sep 17 00:00:00 2001 From: Li Jin Date: Mon, 4 Dec 2017 17:52:48 -0500 Subject: [PATCH 107/177] ARROW-1785: [Format/C++/Java] Remove VectorLayout from serialized schemas What's here so far removes this code from the Flatbuffers schema and the C++ implementation. This logic is a little bit entangled on the Java side, I need some help from @icexelloss or @BryanCutler or @julienledem or someone else to handle the Java refactoring. It might be easiest to preserve the ArrowVectorType/TypeLayout/VectorLayout objects for now but simply remove the Flatbuffers dependency (we do need the names of the vectors in the JSON files used for integration testing) cc @trxcllnt since we may want to roll in the JS changes in this patch Author: Li Jin Author: Wes McKinney Author: Li Jin Closes #1297 from wesm/ARROW-1785 and squashes the following commits: c1e7ea9d [Li Jin] Fix comment 43ff4e3a [Li Jin] Remove Json annotation 95e8736e [Li Jin] (Refactor) Move TypeLayout and VectorLayout from ipc.message to top level. Rename VectorLayout to BufferLayout. 31cbd486 [Li Jin] Fix TypeLayout.java 890a08d7 [Li Jin] Integration test passing 2024db55 [Wes McKinney] Remove VectorLayout from Flatbuffers, C++ implementation --- cpp/src/arrow/ipc/json-internal.cc | 39 ------- cpp/src/arrow/ipc/metadata-internal.cc | 34 +----- cpp/src/arrow/type.cc | 37 ------- cpp/src/arrow/type.h | 30 ------ format/Schema.fbs | 31 +----- integration/data/simple.json | 28 ++--- integration/data/struct_example.json | 44 +------- integration/integration_test.py | 49 +-------- .../VectorLayout.java => BufferLayout.java} | 86 ++++++++------- .../vector/{ipc/message => }/TypeLayout.java | 100 ++++++++--------- .../org/apache/arrow/vector/VectorLoader.java | 7 +- .../apache/arrow/vector/VectorUnloader.java | 4 +- .../arrow/vector/ipc/JsonFileReader.java | 18 ++-- .../arrow/vector/ipc/JsonFileWriter.java | 21 ++-- .../vector/ipc/message/ArrowVectorType.java | 101 ------------------ .../apache/arrow/vector/types/pojo/Field.java | 33 +----- .../apache/arrow/vector/TestListVector.java | 10 +- .../apache/arrow/vector/TestValueVector.java | 3 - .../vector/ipc/TestArrowReaderWriter.java | 1 - .../arrow/vector/ipc/TestArrowStreamPipe.java | 3 - 20 files changed, 145 insertions(+), 534 deletions(-) rename java/vector/src/main/java/org/apache/arrow/vector/{ipc/message/VectorLayout.java => BufferLayout.java} (50%) rename java/vector/src/main/java/org/apache/arrow/vector/{ipc/message => }/TypeLayout.java (66%) delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index bfb3d282d8772..11a6956ebfd47 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -43,22 +43,6 @@ namespace ipc { namespace json { namespace internal { -static std::string GetBufferTypeName(BufferType type) { - switch (type) { - case BufferType::DATA: - return "DATA"; - case BufferType::OFFSET: - return "OFFSET"; - case BufferType::TYPE: - return "TYPE"; - case BufferType::VALIDITY: - return "VALIDITY"; - default: - break; - } - return "UNKNOWN"; -} - static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision) { switch (precision) { case FloatingPoint::HALF: @@ -174,12 +158,9 @@ class SchemaWriter { RETURN_NOT_OK(WriteDictionaryMetadata(dict_type)); const DataType& dictionary_type = *dict_type.dictionary()->type(); - const DataType& index_type = *dict_type.index_type(); RETURN_NOT_OK(WriteChildren(dictionary_type.children())); - WriteBufferLayout(index_type.GetBufferLayout()); } else { RETURN_NOT_OK(WriteChildren(type.children())); - WriteBufferLayout(type.GetBufferLayout()); } writer_->EndObject(); @@ -301,26 +282,6 @@ class SchemaWriter { return Status::OK(); } - void WriteBufferLayout(const std::vector& buffer_layout) { - writer_->Key("typeLayout"); - writer_->StartObject(); - writer_->Key("vectors"); - writer_->StartArray(); - - for (const BufferDescr& buffer : buffer_layout) { - writer_->StartObject(); - writer_->Key("type"); - writer_->String(GetBufferTypeName(buffer.type())); - - writer_->Key("typeBitWidth"); - writer_->Int(buffer.bit_width()); - - writer_->EndObject(); - } - writer_->EndArray(); - writer_->EndObject(); - } - Status WriteChildren(const std::vector>& children) { writer_->Key("children"); writer_->StartArray(); diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index 05202ea937217..af1d6c851582d 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -54,7 +54,6 @@ using DictionaryOffset = flatbuffers::Offset; using FieldOffset = flatbuffers::Offset; using KeyValueOffset = flatbuffers::Offset; using RecordBatchOffset = flatbuffers::Offset; -using VectorLayoutOffset = flatbuffers::Offset; using Offset = flatbuffers::Offset; using FBString = flatbuffers::Offset; @@ -341,34 +340,8 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data, // TODO(wesm): Convert this to visitor pattern static Status TypeToFlatbuffer(FBB& fbb, const DataType& type, std::vector* children, - std::vector* layout, flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) { - std::vector buffer_layout = type.GetBufferLayout(); - for (const BufferDescr& descr : buffer_layout) { - flatbuf::VectorType vector_type; - switch (descr.type()) { - case BufferType::OFFSET: - vector_type = flatbuf::VectorType_OFFSET; - break; - case BufferType::DATA: - vector_type = flatbuf::VectorType_DATA; - break; - case BufferType::VALIDITY: - vector_type = flatbuf::VectorType_VALIDITY; - break; - case BufferType::TYPE: - vector_type = flatbuf::VectorType_TYPE; - break; - default: - vector_type = flatbuf::VectorType_DATA; - break; - } - auto offset = flatbuf::CreateVectorLayout( - fbb, static_cast(descr.bit_width()), vector_type); - layout->push_back(offset); - } - const DataType* value_type = &type; if (type.id() == Type::DICTIONARY) { @@ -543,14 +516,11 @@ static Status FieldToFlatbuffer(FBB& fbb, const Field& field, flatbuf::Type type_enum; Offset type_offset; - Offset type_layout; std::vector children; - std::vector layout; - RETURN_NOT_OK(TypeToFlatbuffer(fbb, *field.type(), &children, &layout, &type_enum, + RETURN_NOT_OK(TypeToFlatbuffer(fbb, *field.type(), &children, &type_enum, dictionary_memo, &type_offset)); auto fb_children = fbb.CreateVector(children); - auto fb_layout = fbb.CreateVector(layout); DictionaryOffset dictionary = 0; if (field.type()->id() == Type::DICTIONARY) { @@ -560,7 +530,7 @@ static Status FieldToFlatbuffer(FBB& fbb, const Field& field, // TODO: produce the list of VectorTypes *offset = flatbuf::CreateField(fbb, fb_name, field.nullable(), type_enum, type_offset, - dictionary, fb_children, fb_layout); + dictionary, fb_children); return Status::OK(); } diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index ae17118323450..31ad53458112c 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -474,43 +474,6 @@ std::shared_ptr decimal(int32_t precision, int32_t scale) { return std::make_shared(precision, scale); } -static const BufferDescr kValidityBuffer(BufferType::VALIDITY, 1); -static const BufferDescr kOffsetBuffer(BufferType::OFFSET, 32); -static const BufferDescr kTypeBuffer(BufferType::TYPE, 32); -static const BufferDescr kBooleanBuffer(BufferType::DATA, 1); -static const BufferDescr kValues64(BufferType::DATA, 64); -static const BufferDescr kValues32(BufferType::DATA, 32); -static const BufferDescr kValues16(BufferType::DATA, 16); -static const BufferDescr kValues8(BufferType::DATA, 8); - -std::vector FixedWidthType::GetBufferLayout() const { - return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())}; -} - -std::vector NullType::GetBufferLayout() const { return {}; } - -std::vector BinaryType::GetBufferLayout() const { - return {kValidityBuffer, kOffsetBuffer, kValues8}; -} - -std::vector FixedSizeBinaryType::GetBufferLayout() const { - return {kValidityBuffer, BufferDescr(BufferType::DATA, bit_width())}; -} - -std::vector ListType::GetBufferLayout() const { - return {kValidityBuffer, kOffsetBuffer}; -} - -std::vector StructType::GetBufferLayout() const { return {kValidityBuffer}; } - -std::vector UnionType::GetBufferLayout() const { - if (mode_ == UnionMode::SPARSE) { - return {kValidityBuffer, kTypeBuffer}; - } else { - return {kValidityBuffer, kTypeBuffer, kOffsetBuffer}; - } -} - std::string Decimal128Type::ToString() const { std::stringstream s; s << "decimal(" << precision_ << ", " << scale_ << ")"; diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 8dcc1592da01f..009e07db07744 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -133,20 +133,6 @@ struct Type { }; }; -enum class BufferType : char { DATA, OFFSET, TYPE, VALIDITY }; - -class BufferDescr { - public: - BufferDescr(BufferType type, int bit_width) : type_(type), bit_width_(bit_width) {} - - BufferType type() const { return type_; } - int bit_width() const { return bit_width_; } - - private: - BufferType type_; - int bit_width_; -}; - class ARROW_EXPORT DataType { public: explicit DataType(Type::type id) : id_(id) {} @@ -176,8 +162,6 @@ class ARROW_EXPORT DataType { /// \since 0.7.0 virtual std::string name() const = 0; - virtual std::vector GetBufferLayout() const = 0; - Type::type id() const { return id_; } protected: @@ -201,8 +185,6 @@ class ARROW_EXPORT FixedWidthType : public DataType { using DataType::DataType; virtual int bit_width() const = 0; - - std::vector GetBufferLayout() const override; }; class ARROW_EXPORT PrimitiveCType : public FixedWidthType { @@ -319,8 +301,6 @@ class ARROW_EXPORT NullType : public DataType, public NoExtraMeta { std::string ToString() const override; std::string name() const override { return "null"; } - - std::vector GetBufferLayout() const override; }; class ARROW_EXPORT BooleanType : public FixedWidthType, public NoExtraMeta { @@ -425,8 +405,6 @@ class ARROW_EXPORT ListType : public NestedType { std::string ToString() const override; std::string name() const override { return "list"; } - - std::vector GetBufferLayout() const override; }; // BinaryType type is represents lists of 1-byte values. @@ -440,8 +418,6 @@ class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta { std::string ToString() const override; std::string name() const override { return "binary"; } - std::vector GetBufferLayout() const override; - protected: // Allow subclasses to change the logical type. explicit BinaryType(Type::type logical_type) : DataType(logical_type) {} @@ -461,8 +437,6 @@ class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType, public Parametri std::string ToString() const override; std::string name() const override { return "fixed_size_binary"; } - std::vector GetBufferLayout() const override; - int32_t byte_width() const { return byte_width_; } int bit_width() const override; @@ -494,8 +468,6 @@ class ARROW_EXPORT StructType : public NestedType { Status Accept(TypeVisitor* visitor) const override; std::string ToString() const override; std::string name() const override { return "struct"; } - - std::vector GetBufferLayout() const override; }; class ARROW_EXPORT DecimalType : public FixedSizeBinaryType { @@ -541,8 +513,6 @@ class ARROW_EXPORT UnionType : public NestedType { std::string name() const override { return "union"; } Status Accept(TypeVisitor* visitor) const override; - std::vector GetBufferLayout() const override; - const std::vector& type_codes() const { return type_codes_; } UnionMode::type mode() const { return mode_; } diff --git a/format/Schema.fbs b/format/Schema.fbs index 6021e92b847e7..3d739342b83c3 100644 --- a/format/Schema.fbs +++ b/format/Schema.fbs @@ -211,32 +211,6 @@ union Type { Map } -/// ---------------------------------------------------------------------- -/// The possible types of a vector - -enum VectorType: short { - /// used in List type, Dense Union and variable length primitive types (String, Binary) - OFFSET, - /// actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector - DATA, - /// Bit vector indicating if each value is null - VALIDITY, - /// Type vector used in Union type - TYPE -} - -/// ---------------------------------------------------------------------- -/// represents the physical layout of a buffer -/// buffers have fixed width slots of a given type - -table VectorLayout { - /// the width of a slot in the buffer (typically 1, 8, 16, 32 or 64) - bit_width: short; - /// the purpose of the vector - type: VectorType; -} - - /// ---------------------------------------------------------------------- /// user defined key value pairs to add custom metadata to arrow /// key namespacing is the responsibility of the user @@ -285,10 +259,7 @@ table Field { // children apply only to Nested data types like Struct, List and Union children: [Field]; - /// layout of buffers produced for this type (as derived from the Type) - /// does not include children - /// each recordbatch will return instances of those Buffers. - layout: [ VectorLayout ]; + // User-defined metadata custom_metadata: [ KeyValue ]; } diff --git a/integration/data/simple.json b/integration/data/simple.json index bc8949912a48a..6634729193b4e 100644 --- a/integration/data/simple.json +++ b/integration/data/simple.json @@ -4,36 +4,20 @@ { "name": "foo", "type": {"name": "int", "isSigned": true, "bitWidth": 32}, - "nullable": true, "children": [], - "typeLayout": { - "vectors": [ - {"type": "VALIDITY", "typeBitWidth": 1}, - {"type": "DATA", "typeBitWidth": 32} - ] - } + "nullable": true, + "children": [] }, { "name": "bar", "type": {"name": "floatingpoint", "precision": "DOUBLE"}, - "nullable": true, "children": [], - "typeLayout": { - "vectors": [ - {"type": "VALIDITY", "typeBitWidth": 1}, - {"type": "DATA", "typeBitWidth": 64} - ] - } + "nullable": true, + "children": [] }, { "name": "baz", "type": {"name": "utf8"}, - "nullable": true, "children": [], - "typeLayout": { - "vectors": [ - {"type": "VALIDITY", "typeBitWidth": 1}, - {"type": "OFFSET", "typeBitWidth": 32}, - {"type": "DATA", "typeBitWidth": 8} - ] - } + "nullable": true, + "children": [] } ] }, diff --git a/integration/data/struct_example.json b/integration/data/struct_example.json index 3ea062db7ba32..4e6cc774e3151 100644 --- a/integration/data/struct_example.json +++ b/integration/data/struct_example.json @@ -16,19 +16,7 @@ "bitWidth": 32 }, "nullable": true, - "children": [], - "typeLayout": { - "vectors": [ - { - "type": "VALIDITY", - "typeBitWidth": 1 - }, - { - "type": "DATA", - "typeBitWidth": 32 - } - ] - } + "children": [] }, { "name": "f2", @@ -36,33 +24,9 @@ "name": "utf8" }, "nullable": true, - "children": [], - "typeLayout": { - "vectors": [ - { - "type": "VALIDITY", - "typeBitWidth": 1 - }, - { - "type": "OFFSET", - "typeBitWidth": 32 - }, - { - "type": "DATA", - "typeBitWidth": 8 - } - ] - } + "children": [] } - ], - "typeLayout": { - "vectors": [ - { - "type": "VALIDITY", - "typeBitWidth": 1 - } - ] - } + ] } ] }, @@ -234,4 +198,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/integration/integration_test.py b/integration/integration_test.py index a063fb36f49d5..5468ab19de8dc 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -111,8 +111,7 @@ def get_json(self): ('name', self.name), ('type', self._get_type()), ('nullable', self.nullable), - ('children', self._get_children()), - ('typeLayout', self._get_type_layout()) + ('children', self._get_children()) ]) def _make_is_valid(self, size): @@ -158,14 +157,6 @@ class PrimitiveType(DataType): def _get_children(self): return [] - def _get_type_layout(self): - return OrderedDict([ - ('vectors', - [OrderedDict([('type', 'VALIDITY'), - ('typeBitWidth', 1)]), - OrderedDict([('type', 'DATA'), - ('typeBitWidth', self.bit_width)])])]) - class PrimitiveColumn(Column): @@ -402,14 +393,6 @@ def _get_type(self): ('scale', self.scale), ]) - def _get_type_layout(self): - return OrderedDict([ - ('vectors', - [OrderedDict([('type', 'VALIDITY'), - ('typeBitWidth', 1)]), - OrderedDict([('type', 'DATA'), - ('typeBitWidth', self.bit_width)])])]) - def generate_column(self, size, name=None): min_value, max_value = decimal_range_from_precision(self.precision) values = [random.randint(min_value, max_value) for _ in range(size)] @@ -461,16 +444,6 @@ def column_class(self): def _get_type(self): return OrderedDict([('name', 'binary')]) - def _get_type_layout(self): - return OrderedDict([ - ('vectors', - [OrderedDict([('type', 'VALIDITY'), - ('typeBitWidth', 1)]), - OrderedDict([('type', 'OFFSET'), - ('typeBitWidth', 32)]), - OrderedDict([('type', 'DATA'), - ('typeBitWidth', 8)])])]) - def generate_column(self, size, name=None): K = 7 is_valid = self._make_is_valid(size) @@ -572,14 +545,6 @@ def _get_type(self): def _get_children(self): return [self.value_type.get_json()] - def _get_type_layout(self): - return OrderedDict([ - ('vectors', - [OrderedDict([('type', 'VALIDITY'), - ('typeBitWidth', 1)]), - OrderedDict([('type', 'OFFSET'), - ('typeBitWidth', 32)])])]) - def generate_column(self, size, name=None): MAX_LIST_SIZE = 4 @@ -633,12 +598,6 @@ def _get_type(self): def _get_children(self): return [type_.get_json() for type_ in self.field_types] - def _get_type_layout(self): - return OrderedDict([ - ('vectors', - [OrderedDict([('type', 'VALIDITY'), - ('typeBitWidth', 1)])])]) - def generate_column(self, size, name=None): is_valid = self._make_is_valid(size) @@ -689,13 +648,9 @@ def get_json(self): ('id', self.dictionary.id_), ('indexType', self.index_type._get_type()), ('isOrdered', self.dictionary.ordered) - ])), - ('typeLayout', self.index_type._get_type_layout()) + ])) ]) - def _get_type_layout(self): - return self.index_type._get_type_layout() - def generate_column(self, size, name=None): if name is None: name = self.name diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java similarity index 50% rename from java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java rename to java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java index e4f2f98fde37a..f6529d8e55bba 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/VectorLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java @@ -16,33 +16,48 @@ * limitations under the License. */ -package org.apache.arrow.vector.ipc.message; +package org.apache.arrow.vector; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.flatbuffers.FlatBufferBuilder; -public class VectorLayout implements FBSerializable { +public class BufferLayout { - private static final VectorLayout VALIDITY_VECTOR = new VectorLayout(ArrowVectorType.VALIDITY, 1); - private static final VectorLayout OFFSET_VECTOR = new VectorLayout(ArrowVectorType.OFFSET, 32); - private static final VectorLayout TYPE_VECTOR = new VectorLayout(ArrowVectorType.TYPE, 32); - private static final VectorLayout BOOLEAN_VECTOR = new VectorLayout(ArrowVectorType.DATA, 1); - private static final VectorLayout VALUES_64 = new VectorLayout(ArrowVectorType.DATA, 64); - private static final VectorLayout VALUES_32 = new VectorLayout(ArrowVectorType.DATA, 32); - private static final VectorLayout VALUES_16 = new VectorLayout(ArrowVectorType.DATA, 16); - private static final VectorLayout VALUES_8 = new VectorLayout(ArrowVectorType.DATA, 8); + public enum BufferType { + DATA("DATA"), + OFFSET("OFFSET"), + VALIDITY("VALIDITY"), + TYPE("TYPE"); - public static VectorLayout typeVector() { - return TYPE_VECTOR; + final private String name; + + BufferType(String name) { + this.name = name; + } + + public String getName() { + return name; + } } - public static VectorLayout offsetVector() { - return OFFSET_VECTOR; + private static final BufferLayout VALIDITY_BUFFER = new BufferLayout(BufferType.VALIDITY, 1); + private static final BufferLayout OFFSET_BUFFER = new BufferLayout(BufferType.OFFSET, 32); + private static final BufferLayout TYPE_BUFFER = new BufferLayout(BufferType.TYPE, 32); + private static final BufferLayout BIT_BUFFER = new BufferLayout(BufferType.DATA, 1); + private static final BufferLayout VALUES_128 = new BufferLayout(BufferType.DATA, 128); + private static final BufferLayout VALUES_64 = new BufferLayout(BufferType.DATA, 64); + private static final BufferLayout VALUES_32 = new BufferLayout(BufferType.DATA, 32); + private static final BufferLayout VALUES_16 = new BufferLayout(BufferType.DATA, 16); + private static final BufferLayout VALUES_8 = new BufferLayout(BufferType.DATA, 8); + + public static BufferLayout typeBuffer() { + return TYPE_BUFFER; } - public static VectorLayout dataVector(int typeBitWidth) { + public static BufferLayout offsetBuffer() { + return OFFSET_BUFFER; + } + + public static BufferLayout dataBuffer(int typeBitWidth) { switch (typeBitWidth) { case 8: return VALUES_8; @@ -52,29 +67,30 @@ public static VectorLayout dataVector(int typeBitWidth) { return VALUES_32; case 64: return VALUES_64; + case 128: + return VALUES_128; default: throw new IllegalArgumentException("only 8, 16, 32, or 64 bits supported"); } } - public static VectorLayout booleanVector() { - return BOOLEAN_VECTOR; + public static BufferLayout booleanVector() { + return BIT_BUFFER; } - public static VectorLayout validityVector() { - return VALIDITY_VECTOR; + public static BufferLayout validityVector() { + return VALIDITY_BUFFER; } - public static VectorLayout byteVector() { - return dataVector(8); + public static BufferLayout byteVector() { + return dataBuffer(8); } private final short typeBitWidth; - private final ArrowVectorType type; + private final BufferType type; - @JsonCreator - private VectorLayout(@JsonProperty("type") ArrowVectorType type, @JsonProperty("typeBitWidth") int typeBitWidth) { + private BufferLayout(BufferType type, int typeBitWidth) { super(); this.type = Preconditions.checkNotNull(type); this.typeBitWidth = (short) typeBitWidth; @@ -83,15 +99,11 @@ private VectorLayout(@JsonProperty("type") ArrowVectorType type, @JsonProperty(" } } - public VectorLayout(org.apache.arrow.flatbuf.VectorLayout layout) { - this(new ArrowVectorType(layout.type()), layout.bitWidth()); - } - public int getTypeBitWidth() { return typeBitWidth; } - public ArrowVectorType getType() { + public BufferType getType() { return type; } @@ -116,15 +128,7 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) { return false; } - VectorLayout other = (VectorLayout) obj; + BufferLayout other = (BufferLayout) obj; return type.equals(other.type) && (typeBitWidth == other.typeBitWidth); } - - @Override - public int writeTo(FlatBufferBuilder builder) { - ; - return org.apache.arrow.flatbuf.VectorLayout.createVectorLayout(builder, typeBitWidth, type.getType()); - } - - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java similarity index 66% rename from java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java rename to java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java index 06fe9481686d7..d6f32b4b4b1d5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/TypeLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.ipc.message; +package org.apache.arrow.vector; import static java.util.Arrays.asList; @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; +import org.apache.arrow.vector.BufferLayout.BufferType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor; import org.apache.arrow.vector.types.pojo.ArrowType.Binary; @@ -41,14 +42,11 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Union; import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; /** - * The layout of vectors for a given type - * It defines its own vectors followed by the vectors for the children + * The buffer layout of vectors for a given type + * It defines its own buffers followed by the buffers for the children * if it is a nested type (Struct_, List, Union) */ public class TypeLayout { @@ -58,24 +56,24 @@ public static TypeLayout getTypeLayout(final ArrowType arrowType) { @Override public TypeLayout visit(Int type) { - return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth())); + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(type.getBitWidth())); } @Override public TypeLayout visit(Union type) { - List vectors; + List vectors; switch (type.getMode()) { case Dense: vectors = asList( // TODO: validate this - VectorLayout.validityVector(), - VectorLayout.typeVector(), - VectorLayout.offsetVector() // offset to find the vector + BufferLayout.validityVector(), + BufferLayout.typeBuffer(), + BufferLayout.offsetBuffer() // offset to find the vector ); break; case Sparse: vectors = asList( - VectorLayout.typeVector() // type of the value at the index or 0 if null + BufferLayout.typeBuffer() // type of the value at the index or 0 if null ); break; default: @@ -86,30 +84,30 @@ public TypeLayout visit(Union type) { @Override public TypeLayout visit(Struct type) { - List vectors = asList( - VectorLayout.validityVector() + List vectors = asList( + BufferLayout.validityVector() ); return new TypeLayout(vectors); } @Override public TypeLayout visit(Timestamp type) { - return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(64)); } @Override public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { - List vectors = asList( - VectorLayout.validityVector(), - VectorLayout.offsetVector() + List vectors = asList( + BufferLayout.validityVector(), + BufferLayout.offsetBuffer() ); return new TypeLayout(vectors); } @Override public TypeLayout visit(FixedSizeList type) { - List vectors = asList( - VectorLayout.validityVector() + List vectors = asList( + BufferLayout.validityVector() ); return new TypeLayout(vectors); } @@ -130,18 +128,17 @@ public TypeLayout visit(FloatingPoint type) { default: throw new UnsupportedOperationException("Unsupported Precision: " + type.getPrecision()); } - return newFixedWidthTypeLayout(VectorLayout.dataVector(bitWidth)); + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(bitWidth)); } @Override public TypeLayout visit(Decimal type) { - // TODO: check size - return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); // actually depends on the type fields + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(128)); } @Override public TypeLayout visit(Bool type) { - return newFixedWidthTypeLayout(VectorLayout.booleanVector()); + return newFixedWidthTypeLayout(BufferLayout.booleanVector()); } @Override @@ -155,39 +152,46 @@ public TypeLayout visit(Utf8 type) { } private TypeLayout newVariableWidthTypeLayout() { - return newPrimitiveTypeLayout(VectorLayout.validityVector(), VectorLayout.offsetVector(), VectorLayout.byteVector()); + return newPrimitiveTypeLayout(BufferLayout.validityVector(), BufferLayout.offsetBuffer(), BufferLayout.byteVector()); } - private TypeLayout newPrimitiveTypeLayout(VectorLayout... vectors) { + private TypeLayout newPrimitiveTypeLayout(BufferLayout... vectors) { return new TypeLayout(asList(vectors)); } - public TypeLayout newFixedWidthTypeLayout(VectorLayout dataVector) { - return newPrimitiveTypeLayout(VectorLayout.validityVector(), dataVector); + public TypeLayout newFixedWidthTypeLayout(BufferLayout dataVector) { + return newPrimitiveTypeLayout(BufferLayout.validityVector(), dataVector); } @Override public TypeLayout visit(Null type) { - return new TypeLayout(Collections.emptyList()); + return new TypeLayout(Collections.emptyList()); } @Override public TypeLayout visit(Date type) { - return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); + switch (type.getUnit()) { + case DAY: + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(32)); + case MILLISECOND: + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(64)); + default: + throw new UnsupportedOperationException("Unknown unit " + type.getUnit()); + } } @Override public TypeLayout visit(Time type) { - return newFixedWidthTypeLayout(VectorLayout.dataVector(type.getBitWidth())); + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(type.getBitWidth())); } @Override - public TypeLayout visit(Interval type) { // TODO: check size + public TypeLayout visit(Interval type) { switch (type.getUnit()) { case DAY_TIME: - return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(64)); case YEAR_MONTH: - return newFixedWidthTypeLayout(VectorLayout.dataVector(64)); + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(32)); default: throw new UnsupportedOperationException("Unknown unit " + type.getUnit()); } @@ -197,39 +201,37 @@ public TypeLayout visit(Interval type) { // TODO: check size return layout; } - private final List vectors; + private final List bufferLayouts; - @JsonCreator - public TypeLayout(@JsonProperty("vectors") List vectors) { + public TypeLayout(List bufferLayouts) { super(); - this.vectors = Preconditions.checkNotNull(vectors); + this.bufferLayouts = Preconditions.checkNotNull(bufferLayouts); } - public TypeLayout(VectorLayout... vectors) { - this(asList(vectors)); + public TypeLayout(BufferLayout... bufferLayouts) { + this(asList(bufferLayouts)); } - public List getVectors() { - return vectors; + public List getBufferLayouts() { + return bufferLayouts; } - @JsonIgnore - public List getVectorTypes() { - List types = new ArrayList<>(vectors.size()); - for (VectorLayout vector : vectors) { + public List getBufferTypes() { + List types = new ArrayList<>(bufferLayouts.size()); + for (BufferLayout vector : bufferLayouts) { types.add(vector.getType()); } return types; } public String toString() { - return vectors.toString(); + return bufferLayouts.toString(); } @Override public int hashCode() { - return vectors.hashCode(); + return bufferLayouts.hashCode(); } @Override @@ -244,7 +246,7 @@ public boolean equals(Object obj) { return false; } TypeLayout other = (TypeLayout) obj; - return vectors.equals(other.vectors); + return bufferLayouts.equals(other.bufferLayouts); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java index 2cd4099c66975..c933d149f8d00 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java @@ -26,7 +26,6 @@ import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.ipc.message.VectorLayout; import org.apache.arrow.vector.types.pojo.Field; import com.google.common.collect.Iterators; @@ -71,9 +70,9 @@ private void loadBuffers(FieldVector vector, Field field, Iterator buf checkArgument(nodes.hasNext(), "no more field nodes for for field " + field + " and vector " + vector); ArrowFieldNode fieldNode = nodes.next(); - List typeLayout = field.getTypeLayout().getVectors(); - List ownBuffers = new ArrayList<>(typeLayout.size()); - for (int j = 0; j < typeLayout.size(); j++) { + List bufferLayouts = TypeLayout.getTypeLayout(field.getType()).getBufferLayouts(); + List ownBuffers = new ArrayList<>(bufferLayouts.size()); + for (int j = 0; j < bufferLayouts.size(); j++) { ownBuffers.add(buffers.next()); } try { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 2b034894ab14d..94981ef02f657 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -22,9 +22,9 @@ import java.util.List; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.vector.BufferLayout.BufferType; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.ipc.message.ArrowVectorType; public class VectorUnloader { @@ -54,7 +54,7 @@ public ArrowRecordBatch getRecordBatch() { private void appendNodes(FieldVector vector, List nodes, List buffers) { nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1)); List fieldBuffers = vector.getFieldBuffers(); - List expectedBuffers = vector.getField().getTypeLayout().getVectorTypes(); + List expectedBuffers = TypeLayout.getTypeLayout(vector.getField().getType()).getBufferTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { throw new IllegalArgumentException(String.format( "wrong number of buffers for field %s in vector %s. found: %s", diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index f274b748e55dc..d0a9b9e18b8f2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -23,7 +23,7 @@ import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*; +import static org.apache.arrow.vector.BufferLayout.BufferType.*; import java.io.File; import java.io.IOException; @@ -42,7 +42,8 @@ import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; -import org.apache.arrow.vector.ipc.message.ArrowVectorType; +import org.apache.arrow.vector.BufferLayout.BufferType; +import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -375,7 +376,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException }; } - private ArrowBuf readIntoBuffer(BufferAllocator allocator, ArrowVectorType bufferType, + private ArrowBuf readIntoBuffer(BufferAllocator allocator, BufferType bufferType, Types.MinorType type, int count) throws IOException { ArrowBuf buf; @@ -471,7 +472,8 @@ private ArrowBuf readIntoBuffer(BufferAllocator allocator, ArrowVectorType buffe } private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException { - List vectorTypes = field.getTypeLayout().getVectorTypes(); + TypeLayout typeLayout = TypeLayout.getTypeLayout(field.getType()); + List vectorTypes = typeLayout.getBufferTypes(); ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()]; /* * The order of inner buffers is : @@ -503,15 +505,15 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json vector.setInitialCapacity(valueCount); for (int v = 0; v < vectorTypes.size(); v++) { - ArrowVectorType vectorType = vectorTypes.get(v); - nextFieldIs(vectorType.getName()); + BufferType bufferType = vectorTypes.get(v); + nextFieldIs(bufferType.getName()); int innerBufferValueCount = valueCount; - if (vectorType.equals(OFFSET)) { + if (bufferType.equals(OFFSET)) { /* offset buffer has 1 additional value capacity */ innerBufferValueCount = valueCount + 1; } - vectorBuffers[v] = readIntoBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); + vectorBuffers[v] = readIntoBuffer(allocator, bufferType, vector.getMinorType(), innerBufferValueCount); } final int nullCount = BitVectorHelper.getNullCount(vectorBuffers[0], valueCount); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 1c9e1d3809594..6eb76a7a14723 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector.ipc; -import static org.apache.arrow.vector.ipc.message.ArrowVectorType.*; +import static org.apache.arrow.vector.BufferLayout.BufferType.*; import java.io.File; import java.io.IOException; @@ -33,7 +33,8 @@ import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; -import org.apache.arrow.vector.ipc.message.ArrowVectorType; +import org.apache.arrow.vector.BufferLayout.BufferType; +import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -152,7 +153,7 @@ private void writeBatch(VectorSchemaRoot recordBatch) throws IOException { } private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOException { - List vectorTypes = field.getTypeLayout().getVectorTypes(); + List vectorTypes = TypeLayout.getTypeLayout(field.getType()).getBufferTypes(); List vectorBuffers = vector.getFieldBuffers(); if (vectorTypes.size() != vectorBuffers.size()) { throw new IllegalArgumentException("vector types and inner vector buffers are not the same size: " + vectorTypes.size() + " != " + vectorBuffers.size()); @@ -165,16 +166,16 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE final int scale = (vector instanceof DecimalVector) ? ((DecimalVector) vector).getScale() : 0; for (int v = 0; v < vectorTypes.size(); v++) { - ArrowVectorType vectorType = vectorTypes.get(v); + BufferType bufferType = vectorTypes.get(v); ArrowBuf vectorBuffer = vectorBuffers.get(v); - generator.writeArrayFieldStart(vectorType.getName()); - final int bufferValueCount = (vectorType.equals(OFFSET)) ? valueCount + 1 : valueCount; + generator.writeArrayFieldStart(bufferType.getName()); + final int bufferValueCount = (bufferType.equals(OFFSET)) ? valueCount + 1 : valueCount; for (int i = 0; i < bufferValueCount; i++) { - if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || + if (bufferType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || vector.getMinorType() == Types.MinorType.VARBINARY)) { - writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale); + writeValueToGenerator(bufferType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale); } else { - writeValueToGenerator(vectorType, vectorBuffer, null, vector, i, scale); + writeValueToGenerator(bufferType, vectorBuffer, null, vector, i, scale); } } generator.writeEndArray(); @@ -197,7 +198,7 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE generator.writeEndObject(); } - private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, + private void writeValueToGenerator(BufferType bufferType, ArrowBuf buffer, ArrowBuf offsetBuffer, FieldVector vector, final int index, final int scale) throws IOException { if (bufferType.equals(TYPE)) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java deleted file mode 100644 index 3342652bedd52..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowVectorType.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.ipc.message; - -import java.util.Map; - -import org.apache.arrow.flatbuf.VectorType; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMap.Builder; - -public class ArrowVectorType { - - public static final ArrowVectorType DATA = new ArrowVectorType(VectorType.DATA); - public static final ArrowVectorType OFFSET = new ArrowVectorType(VectorType.OFFSET); - public static final ArrowVectorType VALIDITY = new ArrowVectorType(VectorType.VALIDITY); - public static final ArrowVectorType TYPE = new ArrowVectorType(VectorType.TYPE); - - private static final Map typeByName; - - static { - ArrowVectorType[] types = {DATA, OFFSET, VALIDITY, TYPE}; - Builder builder = ImmutableMap.builder(); - for (ArrowVectorType type : types) { - builder.put(type.getName(), type); - } - typeByName = builder.build(); - } - - public static ArrowVectorType fromName(String name) { - ArrowVectorType type = typeByName.get(name); - if (type == null) { - throw new IllegalArgumentException("Unknown type " + name); - } - return type; - } - - private final short type; - - public ArrowVectorType(short type) { - this.type = type; - // validate that the type is valid - getName(); - } - - @JsonCreator - private ArrowVectorType(String name) { - this.type = fromName(name).type; - } - - public short getType() { - return type; - } - - @JsonValue - public String getName() { - try { - return VectorType.name(type); - } catch (ArrayIndexOutOfBoundsException e) { - throw new IllegalArgumentException("Unknown type " + type); - } - } - - @Override - public String toString() { - return getName(); - } - - @Override - public int hashCode() { - return type; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof ArrowVectorType) { - ArrowVectorType other = (ArrowVectorType) obj; - return type == other.type; - } - return false; - } - -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java index 574612833cd19..b1f036a34a9a6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java @@ -43,8 +43,7 @@ import org.apache.arrow.flatbuf.Type; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.ipc.message.VectorLayout; -import org.apache.arrow.vector.ipc.message.TypeLayout; +import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.types.pojo.ArrowType.Int; public class Field { @@ -60,7 +59,6 @@ public static Field nullable(String name, ArrowType type) { private final String name; private final FieldType fieldType; private final List children; - private final TypeLayout typeLayout; @JsonCreator private Field( @@ -69,16 +67,14 @@ private Field( @JsonProperty("type") ArrowType type, @JsonProperty("dictionary") DictionaryEncoding dictionary, @JsonProperty("children") List children, - @JsonProperty("typeLayout") TypeLayout typeLayout, @JsonProperty("metadata") Map metadata) { - this(name, new FieldType(nullable, type, dictionary, metadata), children, typeLayout); + this(name, new FieldType(nullable, type, dictionary, metadata), children); } private Field(String name, FieldType fieldType, List children, TypeLayout typeLayout) { this.name = name; this.fieldType = checkNotNull(fieldType); this.children = children == null ? ImmutableList.of() : ImmutableList.copyOf(children); - this.typeLayout = checkNotNull(typeLayout); } // deprecated, use FieldType or static constructor instead @@ -117,10 +113,6 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) { } dictionary = new DictionaryEncoding(dictionaryFB.id(), dictionaryFB.isOrdered(), indexType); } - ImmutableList.Builder layout = ImmutableList.builder(); - for (int i = 0; i < field.layoutLength(); ++i) { - layout.add(new VectorLayout(field.layout(i))); - } ImmutableList.Builder childrenBuilder = ImmutableList.builder(); for (int i = 0; i < field.childrenLength(); i++) { Field childField = convertField(field.children(i)); @@ -135,7 +127,7 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) { metadataBuilder.put(key == null ? "" : key, value == null ? "" : value); } Map metadata = metadataBuilder.build(); - return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()), metadata); + return new Field(name, nullable, type, dictionary, children, metadata); } /** @@ -153,19 +145,11 @@ private static Field mutateOriginalNameIfNeeded(org.apache.arrow.flatbuf.Field f originalChildField.getType(), originalChildField.getDictionary(), originalChildField.getChildren(), - originalChildField.getTypeLayout(), originalChildField.getMetadata()); } return originalChildField; } - public void validate() { - TypeLayout expectedLayout = TypeLayout.getTypeLayout(getType()); - if (!expectedLayout.equals(typeLayout)) { - throw new IllegalArgumentException("Deserialized field does not match expected vectors. expected: " + expectedLayout + " got " + typeLayout); - } - } - public int getField(FlatBufferBuilder builder) { int nameOffset = name == null ? -1 : builder.createString(name); int typeOffset = getType().getType(builder); @@ -184,12 +168,6 @@ public int getField(FlatBufferBuilder builder) { childrenData[i] = children.get(i).getField(builder); } int childrenOffset = org.apache.arrow.flatbuf.Field.createChildrenVector(builder, childrenData); - int[] buffersData = new int[typeLayout.getVectors().size()]; - for (int i = 0; i < buffersData.length; i++) { - VectorLayout vectorLayout = typeLayout.getVectors().get(i); - buffersData[i] = vectorLayout.writeTo(builder); - } - int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData); int[] metadataOffsets = new int[getMetadata().size()]; Iterator> metadataIterator = getMetadata().entrySet().iterator(); for (int i = 0; i < metadataOffsets.length; i++) { @@ -210,7 +188,6 @@ public int getField(FlatBufferBuilder builder) { org.apache.arrow.flatbuf.Field.addTypeType(builder, getType().getTypeID().getFlatbufID()); org.apache.arrow.flatbuf.Field.addType(builder, typeOffset); org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset); - org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset); org.apache.arrow.flatbuf.Field.addCustomMetadata(builder, metadataOffset); if (dictionary != null) { org.apache.arrow.flatbuf.Field.addDictionary(builder, dictionaryOffset); @@ -244,10 +221,6 @@ public List getChildren() { return children; } - public TypeLayout getTypeLayout() { - return typeLayout; - } - @JsonInclude(Include.NON_EMPTY) public Map getMetadata() { return fieldType.getMetadata(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1cff7fa886e87..1acce7e0b6654 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -170,16 +170,16 @@ public void testSetLastSetUsage() throws Exception { /* set lastset and arbitrary valuecount for list vector. * * NOTE: if we don't execute setLastSet() before setLastValueCount(), then - * the latter will corrupt the offsetVector and thus the accessor will not - * retrieve the correct values from underlying dataVector. Run the test + * the latter will corrupt the offsetBuffer and thus the accessor will not + * retrieve the correct values from underlying dataBuffer. Run the test * by commenting out next line and we should see failures from 5th assert * onwards. This is why doing setLastSet() is important before setValueCount() * once the vector has been loaded. * * Another important thing to remember is the value of lastSet itself. * Even though the listVector has elements till index 2 only, the lastSet should - * be set as 3. This is because the offsetVector has valid offsets filled till index 3. - * If we do setLastSet(2), the offsetVector at index 3 will contain incorrect value + * be set as 3. This is because the offsetBuffer has valid offsets filled till index 3. + * If we do setLastSet(2), the offsetBuffer at index 3 will contain incorrect value * after execution of setValueCount(). * * correct state of the listVector @@ -414,7 +414,7 @@ public void testSplitAndTransfer() throws Exception { transferPair.splitAndTransfer(start, splitLength); - /* get offsetVector of toVector */ + /* get offsetBuffer of toVector */ final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer(); /* get dataVector of toVector */ diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 336ae1c7a0146..601b2062ff698 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -38,7 +38,6 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.ipc.message.TypeLayout; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -711,11 +710,9 @@ public void testNullableFixedType3() { vector.setValueCount(1024); Field field = vector.getField(); - TypeLayout typeLayout = field.getTypeLayout(); List buffers = vector.getFieldBuffers(); - assertEquals(2, typeLayout.getVectors().size()); assertEquals(2, buffers.size()); ArrowBuf validityVectorBuf = buffers.get(0); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java index 49e194b51b823..bf42fbb83c84b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java @@ -105,7 +105,6 @@ public void test() throws IOException { ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { Schema readSchema = reader.getVectorSchemaRoot().getSchema(); assertEquals(schema, readSchema); - assertTrue(readSchema.getFields().get(0).getTypeLayout().getVectorTypes().toString(), readSchema.getFields().get(0).getTypeLayout().getVectors().size() > 0); // TODO: dictionaries List recordBatches = reader.getRecordBlocks(); assertEquals(1, recordBatches.size()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java index bd1ec94c8fd2b..9ec9a078f7ef0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java @@ -124,9 +124,6 @@ public boolean loadNextBatch() throws IOException { public void run() { try { assertEquals(schema, reader.getVectorSchemaRoot().getSchema()); - assertTrue( - reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectorTypes().toString(), - reader.getVectorSchemaRoot().getSchema().getFields().get(0).getTypeLayout().getVectors().size() > 0); while (!done) { assertTrue(reader.loadNextBatch() != done); } From b241eb699b6151d9b1c8809528be4c80cce26c7e Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 4 Dec 2017 22:39:44 -0500 Subject: [PATCH 108/177] ARROW-1839/ARROW-1871: [C++/Python] Add Decimal Parquet Read/Write Tests Author: Phillip Cloud Author: Wes McKinney Closes #1335 from cpcloud/ARROW-1839 and squashes the following commits: 496ec796 [Wes McKinney] Make RescaleDecimal instead an instance method of Decimal128 878d9a1d [Phillip Cloud] Fix namespace 816ff42a [Phillip Cloud] Use IsPyInteger 41c447fb [Wes McKinney] Add missing license header 3b91e550 [Wes McKinney] Rename Rescale to RescaleDecimal accc653d [Wes McKinney] Make AppendItem methods non-virtual for CRTP inlining bbc14cf4 [Phillip Cloud] int64_t for looping b6f8592f [Phillip Cloud] Generate a column for different scales afe4396a [Phillip Cloud] docstring ade030e5 [Phillip Cloud] Clarify randdecimal contract 752e905b [Phillip Cloud] Use Python C API instead of string parsing 115fbc96 [Phillip Cloud] Remove iostream include 1f6339d8 [Phillip Cloud] Clean up testing a4b1d6bb [Phillip Cloud] Add randdecimal to util 07c356ac [Phillip Cloud] Remove inference from python objects c7637f19 [Phillip Cloud] Refactor test dba8a0d9 [Phillip Cloud] Add docs 4296b5c9 [Phillip Cloud] Infer the scale by looking at every Python decimal object 98e319af [Phillip Cloud] Remove debugging 2d02b44f [Phillip Cloud] More style fixes f235ef42 [Phillip Cloud] Style fixes 97c0a6b5 [Phillip Cloud] Make sure we assign to our out variable d4a32361 [Phillip Cloud] Move rescaling to decimal.h/cc 891137fd [Phillip Cloud] Formatting deea4711 [Phillip Cloud] Refactor rescaling functionality 108e8914 [Phillip Cloud] Format c28e2c17 [Phillip Cloud] Remove test cbb2ed76 [Phillip Cloud] Rewrite infer precision and scale from python 7a2de887 [Phillip Cloud] Add rescaling b2ad3b7f [Phillip Cloud] Rename e0d6080a [Phillip Cloud] Formatting fc5412a3 [Phillip Cloud] Checkpoint [ci skip] cf3a5a26 [Phillip Cloud] Test multiple precisions and scales 853fced9 [Phillip Cloud] Make the parquet read/write from python test more robust ff6eccf8 [Phillip Cloud] Add c++ formatting test for small number and small scale/precision cd38abd4 [Phillip Cloud] Add test util 45eafe73 [Phillip Cloud] Add better test d7a0aab4 [Phillip Cloud] Make sure we do not override the inferred type eff7d2b5 [Phillip Cloud] Use array syntax 801626b4 [Phillip Cloud] Use FormatValue e519c25f [Phillip Cloud] Add Decimal parquet tests --- cpp/src/arrow/python/arrow_to_pandas.cc | 20 ++---- cpp/src/arrow/python/builtin_convert.cc | 76 +++++++++----------- cpp/src/arrow/python/helpers.cc | 77 +++++++++++++++++--- cpp/src/arrow/python/helpers.h | 11 ++- cpp/src/arrow/python/numpy-internal.h | 4 +- cpp/src/arrow/python/numpy_to_arrow.cc | 35 +++++++--- cpp/src/arrow/python/python-test.cc | 81 +++++++++++++++------ cpp/src/arrow/util/decimal-test.cc | 9 +++ cpp/src/arrow/util/decimal.cc | 64 +++++++++++++++++ cpp/src/arrow/util/decimal.h | 3 + python/pyarrow/tests/test_parquet.py | 44 +++++++++++- python/pyarrow/tests/util.py | 93 +++++++++++++++++++++++++ 12 files changed, 411 insertions(+), 106 deletions(-) create mode 100644 python/pyarrow/tests/util.py diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 096bbd55c6d41..b1825cbe69d2c 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -620,13 +620,6 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data, return Status::OK(); } -static Status RawDecimalToString(const uint8_t* bytes, int scale, std::string* result) { - DCHECK_NE(result, nullptr); - Decimal128 decimal(bytes); - *result = decimal.ToString(scale); - return Status::OK(); -} - static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data, PyObject** out_values) { PyAcquireGIL lock; @@ -637,19 +630,14 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data, PyObject* Decimal = Decimal_ref.obj(); for (int c = 0; c < data.num_chunks(); c++) { - auto* arr(static_cast(data.chunk(c).get())); - auto type(std::dynamic_pointer_cast(arr->type())); - const int scale = type->scale(); + const auto& arr(static_cast(*data.chunk(c).get())); - for (int64_t i = 0; i < arr->length(); ++i) { - if (arr->IsNull(i)) { + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { Py_INCREF(Py_None); *out_values++ = Py_None; } else { - const uint8_t* raw_value = arr->GetValue(i); - std::string decimal_string; - RETURN_NOT_OK(RawDecimalToString(raw_value, scale, &decimal_string)); - *out_values++ = internal::DecimalFromString(Decimal, decimal_string); + *out_values++ = internal::DecimalFromString(Decimal, arr.FormatValue(i)); RETURN_IF_PYERROR(); } } diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc index c716c47d2de23..08cbae7ab2da4 100644 --- a/cpp/src/arrow/python/builtin_convert.cc +++ b/cpp/src/arrow/python/builtin_convert.cc @@ -37,14 +37,6 @@ namespace arrow { namespace py { -static inline bool IsPyInteger(PyObject* obj) { -#if PYARROW_IS_PY2 - return PyLong_Check(obj) || PyInt_Check(obj); -#else - return PyLong_Check(obj); -#endif -} - Status InvalidConversion(PyObject* obj, const std::string& expected_types, std::ostream* out) { OwnedRef type(PyObject_Type(obj)); @@ -91,7 +83,7 @@ class ScalarVisitor { ++bool_count_; } else if (PyFloat_Check(obj)) { ++float_count_; - } else if (IsPyInteger(obj)) { + } else if (internal::IsPyInteger(obj)) { ++int_count_; } else if (PyDate_CheckExact(obj)) { ++date_count_; @@ -390,28 +382,26 @@ class TypedConverterVisitor : public TypedConverter { } return Status::OK(); } - - virtual Status AppendItem(const OwnedRef& item) = 0; }; class NullConverter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { return Status::Invalid("NullConverter: passed non-None value"); } }; class BoolConverter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { return typed_builder_->Append(item.obj() == Py_True); } }; class Int8Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - int64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max() || val < std::numeric_limits::min())) { @@ -426,8 +416,8 @@ class Int8Converter : public TypedConverterVisitor { class Int16Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - int64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max() || val < std::numeric_limits::min())) { @@ -442,8 +432,8 @@ class Int16Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - int64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max() || val < std::numeric_limits::min())) { @@ -458,8 +448,8 @@ class Int32Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - int64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); RETURN_IF_PYERROR(); return typed_builder_->Append(val); } @@ -467,8 +457,8 @@ class Int64Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - uint64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max())) { return Status::Invalid( @@ -482,8 +472,8 @@ class UInt8Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - uint64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max())) { return Status::Invalid( @@ -497,8 +487,8 @@ class UInt16Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - uint64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max())) { return Status::Invalid( @@ -512,8 +502,8 @@ class UInt32Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { - int64_t val = static_cast(PyLong_AsLongLong(item.obj())); + Status AppendItem(const OwnedRef& item) { + const auto val = static_cast(PyLong_AsLongLong(item.obj())); RETURN_IF_PYERROR(); return typed_builder_->Append(val); } @@ -521,13 +511,13 @@ class UInt64Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { int32_t t; if (PyDate_Check(item.obj())) { auto pydate = reinterpret_cast(item.obj()); t = static_cast(PyDate_to_s(pydate)); } else { - int64_t casted_val = static_cast(PyLong_AsLongLong(item.obj())); + const auto casted_val = static_cast(PyLong_AsLongLong(item.obj())); RETURN_IF_PYERROR(); if (casted_val > std::numeric_limits::max()) { return Status::Invalid("Integer as date32 larger than INT32_MAX"); @@ -540,7 +530,7 @@ class Date32Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { int64_t t; if (PyDate_Check(item.obj())) { auto pydate = reinterpret_cast(item.obj()); @@ -558,7 +548,7 @@ class TimestampConverter public: explicit TimestampConverter(TimeUnit::type unit) : unit_(unit) {} - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { int64_t t; if (PyDateTime_Check(item.obj())) { auto pydatetime = reinterpret_cast(item.obj()); @@ -590,7 +580,7 @@ class TimestampConverter class DoubleConverter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { double val = PyFloat_AsDouble(item.obj()); RETURN_IF_PYERROR(); return typed_builder_->Append(val); @@ -599,7 +589,7 @@ class DoubleConverter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { PyObject* bytes_obj; const char* bytes; Py_ssize_t length; @@ -627,7 +617,7 @@ class BytesConverter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { PyObject* bytes_obj; OwnedRef tmp; Py_ssize_t expected_length = @@ -654,7 +644,7 @@ class FixedWidthBytesConverter class UTF8Converter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { PyObject* bytes_obj; OwnedRef tmp; const char* bytes; @@ -689,10 +679,10 @@ class ListConverter : public TypedConverterVisitor { public: Status Init(ArrayBuilder* builder) override; - inline Status AppendItem(const OwnedRef& item) override { + Status AppendItem(const OwnedRef& item) { RETURN_NOT_OK(typed_builder_->Append()); PyObject* item_obj = item.obj(); - int64_t list_size = static_cast(PySequence_Size(item_obj)); + const auto list_size = static_cast(PySequence_Size(item_obj)); return value_converter_->AppendData(item_obj, list_size); } @@ -703,13 +693,11 @@ class ListConverter : public TypedConverterVisitor { class DecimalConverter : public TypedConverterVisitor { public: - inline Status AppendItem(const OwnedRef& item) { + Status AppendItem(const OwnedRef& item) { /// TODO(phillipc): Check for nan? - std::string string; - RETURN_NOT_OK(internal::PythonDecimalToString(item.obj(), &string)); - Decimal128 value; - RETURN_NOT_OK(Decimal128::FromString(string, &value)); + const auto& type = static_cast(*typed_builder_->type()); + RETURN_NOT_OK(internal::DecimalFromPythonDecimal(item.obj(), type, &value)); return typed_builder_->Append(value); } }; diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc index 708d9916f46d3..494f929004ae9 100644 --- a/cpp/src/arrow/python/helpers.cc +++ b/cpp/src/arrow/python/helpers.cc @@ -15,8 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/python/helpers.h" +#include + #include "arrow/python/common.h" +#include "arrow/python/helpers.h" #include "arrow/util/decimal.h" #include "arrow/util/logging.h" @@ -91,20 +93,33 @@ Status PythonDecimalToString(PyObject* python_decimal, std::string* out) { return Status::OK(); } -Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision, - int* scale) { - // Call Python's str(decimal_object) - OwnedRef str_obj(PyObject_Str(python_decimal)); +Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int32_t* precision, + int32_t* scale) { + DCHECK_NE(python_decimal, NULLPTR); + DCHECK_NE(precision, NULLPTR); + DCHECK_NE(scale, NULLPTR); + + OwnedRef as_tuple(PyObject_CallMethod(python_decimal, "as_tuple", "()")); RETURN_IF_PYERROR(); - PyObjectStringify str(str_obj.obj()); + DCHECK(PyTuple_Check(as_tuple.obj())); - const char* bytes = str.bytes; - DCHECK_NE(bytes, nullptr); + OwnedRef digits(PyObject_GetAttrString(as_tuple.obj(), "digits")); + RETURN_IF_PYERROR(); + DCHECK(PyTuple_Check(digits.obj())); - auto size = str.size; + const auto num_digits = static_cast(PyTuple_Size(digits.obj())); + RETURN_IF_PYERROR(); - std::string c_string(bytes, size); - return Decimal128::FromString(c_string, nullptr, precision, scale); + OwnedRef py_exponent(PyObject_GetAttrString(as_tuple.obj(), "exponent")); + RETURN_IF_PYERROR(); + DCHECK(IsPyInteger(py_exponent.obj())); + + const auto exponent = static_cast(PyLong_AsLong(py_exponent.obj())); + RETURN_IF_PYERROR(); + + *precision = num_digits; + *scale = -exponent; + return Status::OK(); } PyObject* DecimalFromString(PyObject* decimal_constructor, @@ -121,6 +136,46 @@ PyObject* DecimalFromString(PyObject* decimal_constructor, string_size); } +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal128* out) { + DCHECK_NE(python_decimal, NULLPTR); + DCHECK_NE(out, NULLPTR); + + std::string string; + RETURN_NOT_OK(PythonDecimalToString(python_decimal, &string)); + + int32_t inferred_precision; + int32_t inferred_scale; + + RETURN_NOT_OK( + Decimal128::FromString(string, out, &inferred_precision, &inferred_scale)); + + const int32_t precision = arrow_type.precision(); + const int32_t scale = arrow_type.scale(); + + if (ARROW_PREDICT_FALSE(inferred_precision > precision)) { + std::stringstream buf; + buf << "Decimal type with precision " << inferred_precision + << " does not fit into precision inferred from first array element: " + << precision; + return Status::Invalid(buf.str()); + } + + if (scale != inferred_scale) { + DCHECK_NE(out, NULLPTR); + RETURN_NOT_OK(out->Rescale(inferred_scale, scale, out)); + } + return Status::OK(); +} + +bool IsPyInteger(PyObject* obj) { +#if PYARROW_IS_PY2 + return PyLong_Check(obj) || PyInt_Check(obj); +#else + return PyLong_Check(obj); +#endif +} + } // namespace internal } // namespace py } // namespace arrow diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h index 719ed796e08a1..c82bdabc47614 100644 --- a/cpp/src/arrow/python/helpers.h +++ b/cpp/src/arrow/python/helpers.h @@ -29,6 +29,9 @@ #include "arrow/util/visibility.h" namespace arrow { + +class Decimal128; + namespace py { class OwnedRef; @@ -44,11 +47,15 @@ Status ImportFromModule(const OwnedRef& module, const std::string& module_name, Status PythonDecimalToString(PyObject* python_decimal, std::string* out); -Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision = NULLPTR, - int* scale = NULLPTR); +Status InferDecimalPrecisionAndScale(PyObject* python_decimal, + int32_t* precision = NULLPTR, + int32_t* scale = NULLPTR); PyObject* DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string); +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal128* out); +bool IsPyInteger(PyObject* obj); } // namespace internal } // namespace py diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h index db34d24d99da5..6c9c871a100a2 100644 --- a/cpp/src/arrow/python/numpy-internal.h +++ b/cpp/src/arrow/python/numpy-internal.h @@ -56,8 +56,8 @@ class Ndarray1DIndexer { bool is_strided() const { return stride_ == 1; } - T& operator[](size_type index) { return *(data_ + index * stride_); } - T& operator[](size_type index) const { return *(data_ + index * stride_); } + T& operator[](size_type index) { return data_[index * stride_]; } + T& operator[](size_type index) const { return data_[index * stride_]; } private: PyArrayObject* arr_; diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index 0b1124d303df1..2316a796a21c0 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -682,24 +682,41 @@ Status NumPyConverter::ConvertDecimals() { Ndarray1DIndexer objects(arr_); PyObject* object = objects[0]; - int precision; - int scale; + if (type_ == NULLPTR) { + int32_t precision; + int32_t desired_scale; - RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(object, &precision, &scale)); + int32_t tmp_precision; + int32_t tmp_scale; - type_ = std::make_shared(precision, scale); + RETURN_NOT_OK( + internal::InferDecimalPrecisionAndScale(objects[0], &precision, &desired_scale)); + + for (int64_t i = 1; i < length_; ++i) { + RETURN_NOT_OK(internal::InferDecimalPrecisionAndScale(objects[i], &tmp_precision, + &tmp_scale)); + precision = std::max(precision, tmp_precision); + + if (std::abs(desired_scale) < std::abs(tmp_scale)) { + desired_scale = tmp_scale; + } + } + + type_ = ::arrow::decimal(precision, desired_scale); + } Decimal128Builder builder(type_, pool_); RETURN_NOT_OK(builder.Resize(length_)); + const auto& decimal_type = static_cast(*type_); + PyObject* Decimal_type_object = Decimal.obj(); + for (int64_t i = 0; i < length_; ++i) { object = objects[i]; - if (PyObject_IsInstance(object, Decimal.obj())) { - std::string string; - RETURN_NOT_OK(internal::PythonDecimalToString(object, &string)); + if (PyObject_IsInstance(object, Decimal_type_object)) { Decimal128 value; - RETURN_NOT_OK(Decimal128::FromString(string, &value)); + RETURN_NOT_OK(internal::DecimalFromPythonDecimal(object, decimal_type, &value)); RETURN_NOT_OK(builder.Append(value)); } else if (PandasObjectIsNull(object)) { RETURN_NOT_OK(builder.AppendNull()); @@ -724,7 +741,7 @@ Status NumPyConverter::ConvertTimes() { Time64Builder builder(::arrow::time64(TimeUnit::MICRO), pool_); RETURN_NOT_OK(builder.Resize(length_)); - PyObject* obj; + PyObject* obj = NULLPTR; for (int64_t i = 0; i < length_; ++i) { obj = objects[i]; if (PyTime_Check(obj)) { diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc index 3b7d7d884ef09..d9919ee499188 100644 --- a/cpp/src/arrow/python/python-test.cc +++ b/cpp/src/arrow/python/python-test.cc @@ -35,34 +35,73 @@ namespace py { TEST(PyBuffer, InvalidInputObject) { PyBuffer buffer(Py_None); } -TEST(DecimalTest, TestPythonDecimalToString) { - PyAcquireGIL lock; +class DecimalTest : public ::testing::Test { + public: + DecimalTest() : lock_(), decimal_module_(), decimal_constructor_() { + auto s = internal::ImportModule("decimal", &decimal_module_); + DCHECK(s.ok()) << s.message(); + DCHECK_NE(decimal_module_.obj(), NULLPTR); - OwnedRef decimal; - OwnedRef Decimal; - ASSERT_OK(internal::ImportModule("decimal", &decimal)); - ASSERT_NE(decimal.obj(), nullptr); + s = internal::ImportFromModule(decimal_module_, "Decimal", &decimal_constructor_); + DCHECK(s.ok()) << s.message(); - ASSERT_OK(internal::ImportFromModule(decimal, "Decimal", &Decimal)); - ASSERT_NE(Decimal.obj(), nullptr); + DCHECK_NE(decimal_constructor_.obj(), NULLPTR); + } - std::string decimal_string("-39402950693754869342983"); - const char* format = "s#"; - auto c_string = decimal_string.c_str(); - ASSERT_NE(c_string, nullptr); + OwnedRef CreatePythonDecimal(const std::string& string_value) { + OwnedRef ref(internal::DecimalFromString(decimal_constructor_.obj(), string_value)); + return ref; + } - auto c_string_size = decimal_string.size(); - ASSERT_GT(c_string_size, 0); - OwnedRef pydecimal(PyObject_CallFunction(Decimal.obj(), const_cast(format), - c_string, c_string_size)); - ASSERT_NE(pydecimal.obj(), nullptr); - ASSERT_EQ(PyErr_Occurred(), nullptr); + private: + PyAcquireGIL lock_; + OwnedRef decimal_module_; + OwnedRef decimal_constructor_; +}; - PyObject* python_object = pydecimal.obj(); - ASSERT_NE(python_object, nullptr); +TEST_F(DecimalTest, TestPythonDecimalToString) { + std::string decimal_string("-39402950693754869342983"); + + OwnedRef python_object = this->CreatePythonDecimal(decimal_string); + ASSERT_NE(python_object.obj(), nullptr); std::string string_result; - ASSERT_OK(internal::PythonDecimalToString(python_object, &string_result)); + ASSERT_OK(internal::PythonDecimalToString(python_object.obj(), &string_result)); +} + +TEST_F(DecimalTest, TestInferPrecisionAndScale) { + std::string decimal_string("-394029506937548693.42983"); + OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string)); + + int32_t precision; + int32_t scale; + + ASSERT_OK( + internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale)); + + const auto expected_precision = + static_cast(decimal_string.size() - 2); // 1 for -, 1 for . + const int32_t expected_scale = 5; + + ASSERT_EQ(expected_precision, precision); + ASSERT_EQ(expected_scale, scale); +} + +TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) { + std::string decimal_string("-3.94042983E+10"); + OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string)); + + int32_t precision; + int32_t scale; + + ASSERT_OK( + internal::InferDecimalPrecisionAndScale(python_decimal.obj(), &precision, &scale)); + + const auto expected_precision = 9; + const int32_t expected_scale = -2; + + ASSERT_EQ(expected_precision, precision); + ASSERT_EQ(expected_scale, scale); } TEST(PandasConversionTest, TestObjectBlockWriteFails) { diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc index 0d0c08cc4551f..e4406747d557c 100644 --- a/cpp/src/arrow/util/decimal-test.cc +++ b/cpp/src/arrow/util/decimal-test.cc @@ -366,4 +366,13 @@ TEST(Decimal128ParseTest, WithExponentAndNullptrScale) { ASSERT_EQ(expected_value, value); } +TEST(Decimal128Test, TestSmallNumberFormat) { + Decimal128 value("0.2"); + std::string expected("0.2"); + + const int32_t scale = 1; + std::string result = value.ToString(scale); + ASSERT_EQ(expected, result); +} + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc index 447cae5c54654..e999854b10860 100644 --- a/cpp/src/arrow/util/decimal.cc +++ b/cpp/src/arrow/util/decimal.cc @@ -813,4 +813,68 @@ Decimal128 operator%(const Decimal128& left, const Decimal128& right) { return remainder; } +static const Decimal128 ScaleMultipliers[] = { + Decimal128(1), + Decimal128(10), + Decimal128(100), + Decimal128(1000), + Decimal128(10000), + Decimal128(100000), + Decimal128(1000000), + Decimal128(10000000), + Decimal128(100000000), + Decimal128(1000000000), + Decimal128(10000000000), + Decimal128(100000000000), + Decimal128(1000000000000), + Decimal128(10000000000000), + Decimal128(100000000000000), + Decimal128(1000000000000000), + Decimal128(10000000000000000), + Decimal128(100000000000000000), + Decimal128(1000000000000000000), + Decimal128("10000000000000000000"), + Decimal128("100000000000000000000"), + Decimal128("1000000000000000000000"), + Decimal128("10000000000000000000000"), + Decimal128("100000000000000000000000"), + Decimal128("1000000000000000000000000"), + Decimal128("10000000000000000000000000"), + Decimal128("100000000000000000000000000"), + Decimal128("1000000000000000000000000000"), + Decimal128("10000000000000000000000000000"), + Decimal128("100000000000000000000000000000"), + Decimal128("1000000000000000000000000000000"), + Decimal128("10000000000000000000000000000000"), + Decimal128("100000000000000000000000000000000"), + Decimal128("1000000000000000000000000000000000"), + Decimal128("10000000000000000000000000000000000"), + Decimal128("100000000000000000000000000000000000"), + Decimal128("1000000000000000000000000000000000000"), + Decimal128("10000000000000000000000000000000000000"), + Decimal128("100000000000000000000000000000000000000")}; + +Status Decimal128::Rescale(int32_t original_scale, int32_t new_scale, + Decimal128* out) const { + DCHECK_NE(out, NULLPTR); + DCHECK_NE(original_scale, new_scale); + const int32_t delta_scale = original_scale - new_scale; + const int32_t abs_delta_scale = std::abs(delta_scale); + DCHECK_GE(abs_delta_scale, 1); + DCHECK_LE(abs_delta_scale, 38); + + const Decimal128 scale_multiplier = ScaleMultipliers[abs_delta_scale]; + const Decimal128 result = *this * scale_multiplier; + + if (ARROW_PREDICT_FALSE(result < *this)) { + std::stringstream buf; + buf << "Rescaling decimal value from original scale " << original_scale + << " to new scale " << new_scale << " would cause overflow"; + return Status::Invalid(buf.str()); + } + + *out = result; + return Status::OK(); +} + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h index a0423e9fce49c..1594090a0d30e 100644 --- a/cpp/src/arrow/util/decimal.h +++ b/cpp/src/arrow/util/decimal.h @@ -126,6 +126,9 @@ class ARROW_EXPORT Decimal128 { static Status FromString(const std::string& s, Decimal128* out, int* precision = NULLPTR, int* scale = NULLPTR); + /// \brief Convert Decimal128 from one scale to another + Status Rescale(int32_t original_scale, int32_t new_scale, Decimal128* out) const; + private: int64_t high_bits_; uint64_t low_bits_; diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 9004fc01ae4a0..d17d89e24df5f 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -16,13 +16,17 @@ # under the License. from os.path import join as pjoin + import datetime +import decimal import io -import os import json +import os + import pytest from pyarrow.compat import guid, u, BytesIO, unichar, frombytes +from pyarrow.tests import util from pyarrow.filesystem import LocalFileSystem import pyarrow as pa from .pandas_examples import dataframe_with_arrays, dataframe_with_lists @@ -1564,3 +1568,41 @@ def test_backwards_compatible_index_multi_level_some_named(): t = _read_table(path) result = t.to_pandas() tm.assert_frame_equal(result, expected) + + +@pytest.mark.parametrize('precision', range(1, 39)) +def test_decimal_roundtrip(tmpdir, precision): + num_values = 10 + + columns = {} + + for scale in range(0, precision + 1): + with util.random_seed(0): + random_decimal_values = [ + util.randdecimal(precision, scale) for _ in range(num_values) + ] + column_name = 'dec_precision_{:d}_scale_{:d}'.format(precision, scale) + columns[column_name] = random_decimal_values + + expected = pd.DataFrame(columns) + filename = tmpdir.join('decimals.parquet') + string_filename = str(filename) + t = pa.Table.from_pandas(expected) + _write_table(t, string_filename) + result_table = _read_table(string_filename) + result = result_table.to_pandas() + tm.assert_frame_equal(result, expected) + + +@pytest.mark.xfail( + raises=pa.ArrowException, reason='Parquet does not support negative scale' +) +def test_decimal_roundtrip_negative_scale(tmpdir): + expected = pd.DataFrame({'decimal_num': [decimal.Decimal('1.23E4')]}) + filename = tmpdir.join('decimals.parquet') + string_filename = str(filename) + t = pa.Table.from_pandas(expected) + _write_table(t, string_filename) + result_table = _read_table(string_filename) + result = result_table.to_pandas() + tm.assert_frame_equal(result, expected) diff --git a/python/pyarrow/tests/util.py b/python/pyarrow/tests/util.py new file mode 100644 index 0000000000000..a3ba9000c2f62 --- /dev/null +++ b/python/pyarrow/tests/util.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Utility functions for testing +""" + +import decimal +import random +import contextlib + + +def randsign(): + """Randomly choose either 1 or -1. + + Returns + ------- + sign : int + """ + return random.choice((-1, 1)) + + +@contextlib.contextmanager +def random_seed(seed): + """Set the random seed inside of a context manager. + + Parameters + ---------- + seed : int + The seed to set + + Notes + ----- + This function is useful when you want to set a random seed but not affect + the random state of other functions using the random module. + """ + original_state = random.getstate() + random.seed(seed) + try: + yield + finally: + random.setstate(original_state) + + +def randdecimal(precision, scale): + """Generate a random decimal value with specified precision and scale. + + Parameters + ---------- + precision : int + The maximum number of digits to generate. Must be an integer between 1 + and 38 inclusive. + scale : int + The maximum number of digits following the decimal point. Must be an + integer greater than or equal to 0. + + Returns + ------- + decimal_value : decimal.Decimal + A random decimal.Decimal object with the specifed precision and scale. + """ + assert 1 <= precision <= 38, 'precision must be between 1 and 38 inclusive' + if scale < 0: + raise ValueError( + 'randdecimal does not yet support generating decimals with ' + 'negative scale' + ) + max_whole_value = 10 ** (precision - scale) - 1 + whole = random.randint(-max_whole_value, max_whole_value) + + if not scale: + return decimal.Decimal(whole) + + max_fractional_value = 10 ** scale - 1 + fractional = random.randint(0, max_fractional_value) + + return decimal.Decimal( + '{}.{}'.format(whole, str(fractional).rjust(scale, '0')) + ) From d64947e8c650687856bd221ea4ff15c86db0ebc1 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Tue, 5 Dec 2017 13:59:26 -0500 Subject: [PATCH 109/177] ARROW-1881: Ignore JS tags for Python packages Author: Korn, Uwe Closes #1391 from xhochy/ARROW-1881 and squashes the following commits: 1e31a74c [Korn, Uwe] ARROW-1881: Ignore JS tags for Python packages --- python/setup.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 2270cf79c2a8b..b56a4fdcace6c 100644 --- a/python/setup.py +++ b/python/setup.py @@ -395,6 +395,16 @@ def has_ext_modules(foo): install_requires.append('futures') +def parse_version(root): + from setuptools_scm import version_from_scm + import setuptools_scm.git + describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --exclude 'apache-arrow-js-*'" + version = setuptools_scm.git.parse(root, describe) + if not version: + return version_from_scm(root) + else: + return version + setup( name="pyarrow", packages=['pyarrow', 'pyarrow.tests'], @@ -413,7 +423,7 @@ def has_ext_modules(foo): 'plasma_store = pyarrow:_plasma_store_entry_point' ] }, - use_scm_version={"root": "..", "relative_to": __file__}, + use_scm_version={"root": "..", "relative_to": __file__, "parse": parse_version}, setup_requires=['setuptools_scm', 'cython >= 0.23'], install_requires=install_requires, tests_require=['pytest'], From 33b628acf535b4ea4c21504483929eb399980405 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Tue, 5 Dec 2017 22:02:49 -0500 Subject: [PATCH 110/177] ARROW-1882: [C++] Reintroduce DictionaryBuilder Readded the previous code and moved some small parts to a new common place to share logic between kernels and builder. Author: Korn, Uwe Author: Wes McKinney Closes #1388 from xhochy/ARROW-1882 and squashes the following commits: db9d67a8 [Wes McKinney] Consolidate some hash table constants in util/hash.h b387b73d [Korn, Uwe] Ensure 64but integers 4743a8bb [Korn, Uwe] Fix precision loss 4c507fed [Korn, Uwe] ninja format a1da9355 [Korn, Uwe] Reuse common hash table code ba307794 [Korn, Uwe] ARROW-1882: [C++] Reintroduce DictionaryBuilder --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/array-test.cc | 347 ++++++++++++++++++++++++++ cpp/src/arrow/builder.cc | 300 ++++++++++++++++++++++ cpp/src/arrow/builder.h | 153 ++++++++++++ cpp/src/arrow/compute/kernels/hash.cc | 61 +---- cpp/src/arrow/util/CMakeLists.txt | 1 + cpp/src/arrow/util/hash.cc | 38 +++ cpp/src/arrow/util/hash.h | 85 +++++++ 8 files changed, 927 insertions(+), 59 deletions(-) create mode 100644 cpp/src/arrow/util/hash.cc create mode 100644 cpp/src/arrow/util/hash.h diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 94705781fa42d..d645cca224480 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -38,6 +38,7 @@ set(ARROW_SRCS util/compression.cc util/cpu-info.cc util/decimal.cc + util/hash.cc util/key_value_metadata.cc ) diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc index d894df1311e9e..7ff3261ecba5e 100644 --- a/cpp/src/arrow/array-test.cc +++ b/cpp/src/arrow/array-test.cc @@ -1619,6 +1619,353 @@ TEST_F(TestAdaptiveUIntBuilder, TestAppendVector) { ASSERT_TRUE(expected_->Equals(result_)); } +// ---------------------------------------------------------------------- +// Dictionary tests + +template +class TestDictionaryBuilder : public TestBuilder {}; + +typedef ::testing::Types + PrimitiveDictionaries; + +TYPED_TEST_CASE(TestDictionaryBuilder, PrimitiveDictionaries); + +TYPED_TEST(TestDictionaryBuilder, Basic) { + DictionaryBuilder builder(default_memory_pool()); + ASSERT_OK(builder.Append(static_cast(1))); + ASSERT_OK(builder.Append(static_cast(2))); + ASSERT_OK(builder.Append(static_cast(1))); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Build expected data + NumericBuilder dict_builder; + ASSERT_OK(dict_builder.Append(static_cast(1))); + ASSERT_OK(dict_builder.Append(static_cast(2))); + std::shared_ptr dict_array; + ASSERT_OK(dict_builder.Finish(&dict_array)); + auto dtype = std::make_shared(int8(), dict_array); + + Int8Builder int_builder; + ASSERT_OK(int_builder.Append(0)); + ASSERT_OK(int_builder.Append(1)); + ASSERT_OK(int_builder.Append(0)); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + +TYPED_TEST(TestDictionaryBuilder, ArrayConversion) { + NumericBuilder builder; + // DictionaryBuilder builder; + ASSERT_OK(builder.Append(static_cast(1))); + ASSERT_OK(builder.Append(static_cast(2))); + ASSERT_OK(builder.Append(static_cast(1))); + + std::shared_ptr intermediate_result; + ASSERT_OK(builder.Finish(&intermediate_result)); + DictionaryBuilder dictionary_builder(default_memory_pool()); + ASSERT_OK(dictionary_builder.AppendArray(*intermediate_result)); + std::shared_ptr result; + ASSERT_OK(dictionary_builder.Finish(&result)); + + // Build expected data + NumericBuilder dict_builder; + ASSERT_OK(dict_builder.Append(static_cast(1))); + ASSERT_OK(dict_builder.Append(static_cast(2))); + std::shared_ptr dict_array; + ASSERT_OK(dict_builder.Finish(&dict_array)); + auto dtype = std::make_shared(int8(), dict_array); + + Int8Builder int_builder; + ASSERT_OK(int_builder.Append(0)); + ASSERT_OK(int_builder.Append(1)); + ASSERT_OK(int_builder.Append(0)); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + +TYPED_TEST(TestDictionaryBuilder, DoubleTableSize) { + using Scalar = typename TypeParam::c_type; + // Skip this test for (u)int8 + if (sizeof(Scalar) > 1) { + // Build the dictionary Array + DictionaryBuilder builder(default_memory_pool()); + // Build expected data + NumericBuilder dict_builder; + Int16Builder int_builder; + + // Fill with 1024 different values + for (int64_t i = 0; i < 1024; i++) { + ASSERT_OK(builder.Append(static_cast(i))); + ASSERT_OK(dict_builder.Append(static_cast(i))); + ASSERT_OK(int_builder.Append(static_cast(i))); + } + // Fill with an already existing value + for (int64_t i = 0; i < 1024; i++) { + ASSERT_OK(builder.Append(static_cast(1))); + ASSERT_OK(int_builder.Append(1)); + } + + // Finalize result + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Finalize expected data + std::shared_ptr dict_array; + ASSERT_OK(dict_builder.Finish(&dict_array)); + auto dtype = std::make_shared(int16(), dict_array); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); + } +} + +TEST(TestStringDictionaryBuilder, Basic) { + // Build the dictionary Array + StringDictionaryBuilder builder(default_memory_pool()); + ASSERT_OK(builder.Append("test")); + ASSERT_OK(builder.Append("test2")); + ASSERT_OK(builder.Append("test")); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Build expected data + StringBuilder str_builder; + ASSERT_OK(str_builder.Append("test")); + ASSERT_OK(str_builder.Append("test2")); + std::shared_ptr str_array; + ASSERT_OK(str_builder.Finish(&str_array)); + auto dtype = std::make_shared(int8(), str_array); + + Int8Builder int_builder; + ASSERT_OK(int_builder.Append(0)); + ASSERT_OK(int_builder.Append(1)); + ASSERT_OK(int_builder.Append(0)); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + +TEST(TestStringDictionaryBuilder, DoubleTableSize) { + // Build the dictionary Array + StringDictionaryBuilder builder(default_memory_pool()); + // Build expected data + StringBuilder str_builder; + Int16Builder int_builder; + + // Fill with 1024 different values + for (int64_t i = 0; i < 1024; i++) { + std::stringstream ss; + ss << "test" << i; + ASSERT_OK(builder.Append(ss.str())); + ASSERT_OK(str_builder.Append(ss.str())); + ASSERT_OK(int_builder.Append(static_cast(i))); + } + // Fill with an already existing value + for (int64_t i = 0; i < 1024; i++) { + ASSERT_OK(builder.Append("test1")); + ASSERT_OK(int_builder.Append(1)); + } + + // Finalize result + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Finalize expected data + std::shared_ptr str_array; + ASSERT_OK(str_builder.Finish(&str_array)); + auto dtype = std::make_shared(int16(), str_array); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + +TEST(TestFixedSizeBinaryDictionaryBuilder, Basic) { + // Build the dictionary Array + DictionaryBuilder builder(arrow::fixed_size_binary(4), + default_memory_pool()); + std::vector test{12, 12, 11, 12}; + std::vector test2{12, 12, 11, 11}; + ASSERT_OK(builder.Append(test.data())); + ASSERT_OK(builder.Append(test2.data())); + ASSERT_OK(builder.Append(test.data())); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Build expected data + FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4)); + ASSERT_OK(fsb_builder.Append(test.data())); + ASSERT_OK(fsb_builder.Append(test2.data())); + std::shared_ptr fsb_array; + ASSERT_OK(fsb_builder.Finish(&fsb_array)); + auto dtype = std::make_shared(int8(), fsb_array); + + Int8Builder int_builder; + ASSERT_OK(int_builder.Append(0)); + ASSERT_OK(int_builder.Append(1)); + ASSERT_OK(int_builder.Append(0)); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + +TEST(TestFixedSizeBinaryDictionaryBuilder, DoubleTableSize) { + // Build the dictionary Array + DictionaryBuilder builder(arrow::fixed_size_binary(4), + default_memory_pool()); + // Build expected data + FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(4)); + Int16Builder int_builder; + + // Fill with 1024 different values + for (int64_t i = 0; i < 1024; i++) { + std::vector value{12, 12, static_cast(i / 128), + static_cast(i % 128)}; + ASSERT_OK(builder.Append(value.data())); + ASSERT_OK(fsb_builder.Append(value.data())); + ASSERT_OK(int_builder.Append(static_cast(i))); + } + // Fill with an already existing value + std::vector known_value{12, 12, 0, 1}; + for (int64_t i = 0; i < 1024; i++) { + ASSERT_OK(builder.Append(known_value.data())); + ASSERT_OK(int_builder.Append(1)); + } + + // Finalize result + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Finalize expected data + std::shared_ptr fsb_array; + ASSERT_OK(fsb_builder.Finish(&fsb_array)); + auto dtype = std::make_shared(int16(), fsb_array); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + +TEST(TestFixedSizeBinaryDictionaryBuilder, InvalidTypeAppend) { + // Build the dictionary Array + DictionaryBuilder builder(arrow::fixed_size_binary(4), + default_memory_pool()); + // Build an array with different byte width + FixedSizeBinaryBuilder fsb_builder(arrow::fixed_size_binary(5)); + std::vector value{100, 1, 1, 1, 1}; + ASSERT_OK(fsb_builder.Append(value.data())); + std::shared_ptr fsb_array; + ASSERT_OK(fsb_builder.Finish(&fsb_array)); + + ASSERT_RAISES(Invalid, builder.AppendArray(*fsb_array)); +} + +TEST(TestDecimalDictionaryBuilder, Basic) { + // Build the dictionary Array + const auto& decimal_type = arrow::decimal(2, 0); + DictionaryBuilder builder(decimal_type, default_memory_pool()); + + // Test data + std::vector test{12, 12, 11, 12}; + for (const auto& value : test) { + ASSERT_OK(builder.Append(value.ToBytes().data())); + } + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Build expected data + FixedSizeBinaryBuilder decimal_builder(decimal_type); + ASSERT_OK(decimal_builder.Append(Decimal128(12).ToBytes())); + ASSERT_OK(decimal_builder.Append(Decimal128(11).ToBytes())); + + std::shared_ptr decimal_array; + ASSERT_OK(decimal_builder.Finish(&decimal_array)); + auto dtype = arrow::dictionary(int8(), decimal_array); + + Int8Builder int_builder; + ASSERT_OK(int_builder.Append({0, 0, 1, 0})); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + +TEST(TestDecimalDictionaryBuilder, DoubleTableSize) { + const auto& decimal_type = arrow::decimal(21, 0); + + // Build the dictionary Array + DictionaryBuilder builder(decimal_type, default_memory_pool()); + + // Build expected data + FixedSizeBinaryBuilder fsb_builder(decimal_type); + Int16Builder int_builder; + + // Fill with 1024 different values + for (int64_t i = 0; i < 1024; i++) { + const uint8_t bytes[] = {0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12, + 12, + static_cast(i / 128), + static_cast(i % 128)}; + ASSERT_OK(builder.Append(bytes)); + ASSERT_OK(fsb_builder.Append(bytes)); + ASSERT_OK(int_builder.Append(static_cast(i))); + } + // Fill with an already existing value + const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1}; + for (int64_t i = 0; i < 1024; i++) { + ASSERT_OK(builder.Append(known_value)); + ASSERT_OK(int_builder.Append(1)); + } + + // Finalize result + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + + // Finalize expected data + std::shared_ptr fsb_array; + ASSERT_OK(fsb_builder.Finish(&fsb_array)); + + auto dtype = std::make_shared(int16(), fsb_array); + std::shared_ptr int_array; + ASSERT_OK(int_builder.Finish(&int_array)); + + DictionaryArray expected(dtype, int_array); + ASSERT_TRUE(expected.Equals(result)); +} + // ---------------------------------------------------------------------- // List tests diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index 4d7fd5ff840ea..de132b5f6a0d1 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -34,6 +34,7 @@ #include "arrow/util/cpu-info.h" #include "arrow/util/decimal.h" #include "arrow/util/hash-util.h" +#include "arrow/util/hash.h" #include "arrow/util/logging.h" namespace arrow { @@ -805,6 +806,305 @@ Status BooleanBuilder::Append(const std::vector& values) { return Status::OK(); } +// ---------------------------------------------------------------------- +// DictionaryBuilder + +using internal::WrappedBinary; + +template +DictionaryBuilder::DictionaryBuilder(const std::shared_ptr& type, + MemoryPool* pool) + : ArrayBuilder(type, pool), + hash_slots_(nullptr), + dict_builder_(type, pool), + values_builder_(pool), + byte_width_(-1) { + if (!::arrow::CpuInfo::initialized()) { + ::arrow::CpuInfo::Init(); + } +} + +DictionaryBuilder::DictionaryBuilder(const std::shared_ptr& type, + MemoryPool* pool) + : ArrayBuilder(type, pool), values_builder_(pool) { + if (!::arrow::CpuInfo::initialized()) { + ::arrow::CpuInfo::Init(); + } +} + +DictionaryBuilder::~DictionaryBuilder() {} + +template <> +DictionaryBuilder::DictionaryBuilder( + const std::shared_ptr& type, MemoryPool* pool) + : ArrayBuilder(type, pool), + hash_slots_(nullptr), + dict_builder_(type, pool), + values_builder_(pool), + byte_width_(static_cast(*type).byte_width()) { + if (!::arrow::CpuInfo::initialized()) { + ::arrow::CpuInfo::Init(); + } +} + +template +Status DictionaryBuilder::Init(int64_t elements) { + RETURN_NOT_OK(ArrayBuilder::Init(elements)); + + // Fill the initial hash table + RETURN_NOT_OK(internal::NewHashTable(kInitialHashTableSize, pool_, &hash_table_)); + hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); + hash_table_size_ = kInitialHashTableSize; + mod_bitmask_ = kInitialHashTableSize - 1; + hash_table_load_threshold_ = + static_cast(static_cast(elements) * kMaxHashTableLoad); + + return values_builder_.Init(elements); +} + +Status DictionaryBuilder::Init(int64_t elements) { + RETURN_NOT_OK(ArrayBuilder::Init(elements)); + return values_builder_.Init(elements); +} + +template +Status DictionaryBuilder::Resize(int64_t capacity) { + if (capacity < kMinBuilderCapacity) { + capacity = kMinBuilderCapacity; + } + + if (capacity_ == 0) { + return Init(capacity); + } else { + return ArrayBuilder::Resize(capacity); + } +} + +Status DictionaryBuilder::Resize(int64_t capacity) { + if (capacity < kMinBuilderCapacity) { + capacity = kMinBuilderCapacity; + } + + if (capacity_ == 0) { + return Init(capacity); + } else { + return ArrayBuilder::Resize(capacity); + } +} + +template +Status DictionaryBuilder::FinishInternal(std::shared_ptr* out) { + std::shared_ptr dictionary; + RETURN_NOT_OK(dict_builder_.Finish(&dictionary)); + + RETURN_NOT_OK(values_builder_.FinishInternal(out)); + (*out)->type = std::make_shared((*out)->type, dictionary); + return Status::OK(); +} + +Status DictionaryBuilder::FinishInternal(std::shared_ptr* out) { + std::shared_ptr dictionary = std::make_shared(0); + + RETURN_NOT_OK(values_builder_.FinishInternal(out)); + (*out)->type = std::make_shared((*out)->type, dictionary); + return Status::OK(); +} + +template +Status DictionaryBuilder::Append(const Scalar& value) { + RETURN_NOT_OK(Reserve(1)); + // Based on DictEncoder::Put + int64_t j = HashValue(value) & mod_bitmask_; + hash_slot_t index = hash_slots_[j]; + + // Find an empty slot + while (kHashSlotEmpty != index && SlotDifferent(index, value)) { + // Linear probing + ++j; + if (j == hash_table_size_) { + j = 0; + } + index = hash_slots_[j]; + } + + if (index == kHashSlotEmpty) { + // Not in the hash table, so we insert it now + index = static_cast(dict_builder_.length()); + hash_slots_[j] = index; + RETURN_NOT_OK(AppendDictionary(value)); + + if (ARROW_PREDICT_FALSE(static_cast(dict_builder_.length()) > + hash_table_load_threshold_)) { + RETURN_NOT_OK(DoubleTableSize()); + } + } + + RETURN_NOT_OK(values_builder_.Append(index)); + + return Status::OK(); +} + +template +Status DictionaryBuilder::AppendArray(const Array& array) { + const auto& numeric_array = static_cast&>(array); + for (int64_t i = 0; i < array.length(); i++) { + if (array.IsNull(i)) { + RETURN_NOT_OK(AppendNull()); + } else { + RETURN_NOT_OK(Append(numeric_array.Value(i))); + } + } + return Status::OK(); +} + +Status DictionaryBuilder::AppendArray(const Array& array) { + for (int64_t i = 0; i < array.length(); i++) { + RETURN_NOT_OK(AppendNull()); + } + return Status::OK(); +} + +template <> +Status DictionaryBuilder::AppendArray(const Array& array) { + if (!type_->Equals(*array.type())) { + return Status::Invalid("Cannot append FixedSizeBinary array with non-matching type"); + } + + const auto& numeric_array = static_cast(array); + for (int64_t i = 0; i < array.length(); i++) { + if (array.IsNull(i)) { + RETURN_NOT_OK(AppendNull()); + } else { + RETURN_NOT_OK(Append(numeric_array.Value(i))); + } + } + return Status::OK(); +} + +template +Status DictionaryBuilder::AppendNull() { + return values_builder_.AppendNull(); +} + +Status DictionaryBuilder::AppendNull() { return values_builder_.AppendNull(); } + +template +Status DictionaryBuilder::DoubleTableSize() { +#define INNER_LOOP \ + Scalar value = GetDictionaryValue(static_cast(index)); \ + int64_t j = HashValue(value) & new_mod_bitmask; + + DOUBLE_TABLE_SIZE(, INNER_LOOP); + + return Status::OK(); +} + +template +typename DictionaryBuilder::Scalar DictionaryBuilder::GetDictionaryValue( + int64_t index) { + const Scalar* data = reinterpret_cast(dict_builder_.data()->data()); + return data[index]; +} + +template <> +const uint8_t* DictionaryBuilder::GetDictionaryValue(int64_t index) { + return dict_builder_.GetValue(index); +} + +template +int64_t DictionaryBuilder::HashValue(const Scalar& value) { + return HashUtil::Hash(&value, sizeof(Scalar), 0); +} + +template <> +int64_t DictionaryBuilder::HashValue(const Scalar& value) { + return HashUtil::Hash(value, byte_width_, 0); +} + +template +bool DictionaryBuilder::SlotDifferent(hash_slot_t index, const Scalar& value) { + const Scalar other = GetDictionaryValue(static_cast(index)); + return other != value; +} + +template <> +bool DictionaryBuilder::SlotDifferent(hash_slot_t index, + const Scalar& value) { + int32_t width = static_cast(*type_).byte_width(); + const Scalar other = GetDictionaryValue(static_cast(index)); + return memcmp(other, value, width) != 0; +} + +template +Status DictionaryBuilder::AppendDictionary(const Scalar& value) { + return dict_builder_.Append(value); +} + +#define BINARY_DICTIONARY_SPECIALIZATIONS(Type) \ + template <> \ + WrappedBinary DictionaryBuilder::GetDictionaryValue(int64_t index) { \ + int32_t v_len; \ + const uint8_t* v = dict_builder_.GetValue(static_cast(index), &v_len); \ + return WrappedBinary(v, v_len); \ + } \ + \ + template <> \ + Status DictionaryBuilder::AppendDictionary(const WrappedBinary& value) { \ + return dict_builder_.Append(value.ptr_, value.length_); \ + } \ + \ + template <> \ + Status DictionaryBuilder::AppendArray(const Array& array) { \ + const BinaryArray& binary_array = static_cast(array); \ + WrappedBinary value(nullptr, 0); \ + for (int64_t i = 0; i < array.length(); i++) { \ + if (array.IsNull(i)) { \ + RETURN_NOT_OK(AppendNull()); \ + } else { \ + value.ptr_ = binary_array.GetValue(i, &value.length_); \ + RETURN_NOT_OK(Append(value)); \ + } \ + } \ + return Status::OK(); \ + } \ + \ + template <> \ + int64_t DictionaryBuilder::HashValue(const WrappedBinary& value) { \ + return HashUtil::Hash(value.ptr_, value.length_, 0); \ + } \ + \ + template <> \ + bool DictionaryBuilder::SlotDifferent(hash_slot_t index, \ + const WrappedBinary& value) { \ + int32_t other_length; \ + const uint8_t* other_value = \ + dict_builder_.GetValue(static_cast(index), &other_length); \ + return !(other_length == value.length_ && \ + 0 == memcmp(other_value, value.ptr_, value.length_)); \ + } + +BINARY_DICTIONARY_SPECIALIZATIONS(StringType); +BINARY_DICTIONARY_SPECIALIZATIONS(BinaryType); + +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; +template class DictionaryBuilder; + // ---------------------------------------------------------------------- // Decimal128Builder diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index e59e166580ae4..ce7b8cd197da3 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -32,6 +32,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit-util.h" +#include "arrow/util/hash.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h" @@ -809,6 +810,158 @@ class ARROW_EXPORT StructBuilder : public ArrayBuilder { std::vector> field_builders_; }; +// ---------------------------------------------------------------------- +// Dictionary builder + +namespace internal { + +// TODO(ARROW-1176): Use Tensorflow's StringPiece instead of this here. +struct WrappedBinary { + WrappedBinary(const uint8_t* ptr, int32_t length) : ptr_(ptr), length_(length) {} + + const uint8_t* ptr_; + int32_t length_; +}; + +template +struct DictionaryScalar { + using type = typename T::c_type; +}; + +template <> +struct DictionaryScalar { + using type = WrappedBinary; +}; + +template <> +struct DictionaryScalar { + using type = WrappedBinary; +}; + +template <> +struct DictionaryScalar { + using type = uint8_t const*; +}; + +} // namespace internal + +/// \brief Array builder for created encoded DictionaryArray from dense array +/// data +template +class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder { + public: + using Scalar = typename internal::DictionaryScalar::type; + + ~DictionaryBuilder() {} + + DictionaryBuilder(const std::shared_ptr& type, MemoryPool* pool); + + template + explicit DictionaryBuilder( + typename std::enable_if::is_parameter_free, MemoryPool*>::type pool) + : DictionaryBuilder(TypeTraits::type_singleton(), pool) {} + + /// \brief Append a scalar value + Status Append(const Scalar& value); + + /// \brief Append a scalar null value + Status AppendNull(); + + /// \brief Append a whole dense array to the builder + Status AppendArray(const Array& array); + + Status Init(int64_t elements) override; + Status Resize(int64_t capacity) override; + Status FinishInternal(std::shared_ptr* out) override; + + protected: + Status DoubleTableSize(); + Scalar GetDictionaryValue(int64_t index); + int64_t HashValue(const Scalar& value); + bool SlotDifferent(hash_slot_t slot, const Scalar& value); + Status AppendDictionary(const Scalar& value); + + std::shared_ptr hash_table_; + int32_t* hash_slots_; + + /// Size of the table. Must be a power of 2. + int64_t hash_table_size_; + + // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j % + // hash_table_size_, but uses far fewer CPU cycles + int64_t mod_bitmask_; + + typename TypeTraits::BuilderType dict_builder_; + AdaptiveIntBuilder values_builder_; + int32_t byte_width_; + + /// Size at which we decide to resize + int64_t hash_table_load_threshold_; +}; + +template <> +class ARROW_EXPORT DictionaryBuilder : public ArrayBuilder { + public: + ~DictionaryBuilder(); + + DictionaryBuilder(const std::shared_ptr& type, MemoryPool* pool); + explicit DictionaryBuilder(MemoryPool* pool); + + /// \brief Append a scalar null value + Status AppendNull(); + + /// \brief Append a whole dense array to the builder + Status AppendArray(const Array& array); + + Status Init(int64_t elements) override; + Status Resize(int64_t capacity) override; + Status FinishInternal(std::shared_ptr* out) override; + + protected: + AdaptiveIntBuilder values_builder_; +}; + +class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder { + public: + using DictionaryBuilder::Append; + using DictionaryBuilder::DictionaryBuilder; + + Status Append(const uint8_t* value, int32_t length) { + return Append(internal::WrappedBinary(value, length)); + } + + Status Append(const char* value, int32_t length) { + return Append( + internal::WrappedBinary(reinterpret_cast(value), length)); + } + + Status Append(const std::string& value) { + return Append(internal::WrappedBinary(reinterpret_cast(value.c_str()), + static_cast(value.size()))); + } +}; + +/// \brief Dictionary array builder with convenience methods for strings +class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder { + public: + using DictionaryBuilder::Append; + using DictionaryBuilder::DictionaryBuilder; + + Status Append(const uint8_t* value, int32_t length) { + return Append(internal::WrappedBinary(value, length)); + } + + Status Append(const char* value, int32_t length) { + return Append( + internal::WrappedBinary(reinterpret_cast(value), length)); + } + + Status Append(const std::string& value) { + return Append(internal::WrappedBinary(reinterpret_cast(value.c_str()), + static_cast(value.size()))); + } +}; + // ---------------------------------------------------------------------- // Helper functions diff --git a/cpp/src/arrow/compute/kernels/hash.cc b/cpp/src/arrow/compute/kernels/hash.cc index 750f1d36ac7d8..1face78bdebfb 100644 --- a/cpp/src/arrow/compute/kernels/hash.cc +++ b/cpp/src/arrow/compute/kernels/hash.cc @@ -30,21 +30,13 @@ #include "arrow/compute/kernel.h" #include "arrow/compute/kernels/util-internal.h" #include "arrow/util/hash-util.h" +#include "arrow/util/hash.h" namespace arrow { namespace compute { namespace { -// Initially 1024 elements -static constexpr int64_t kInitialHashTableSize = 1 << 10; - -typedef int32_t hash_slot_t; -static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits::max(); - -// The maximum load factor for the hash table before resizing. -static constexpr double kMaxHashTableLoad = 0.5; - enum class SIMDMode : char { NOSIMD, SSE4, AVX2 }; #define CHECK_IMPLEMENTED(KERNEL, FUNCNAME, TYPE) \ @@ -54,17 +46,6 @@ enum class SIMDMode : char { NOSIMD, SSE4, AVX2 }; return Status::NotImplemented(ss.str()); \ } -Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr* out) { - auto hash_table = std::make_shared(pool); - - RETURN_NOT_OK(hash_table->Resize(sizeof(hash_slot_t) * size)); - int32_t* slots = reinterpret_cast(hash_table->mutable_data()); - std::fill(slots, slots + size, kHashSlotEmpty); - - *out = hash_table; - return Status::OK(); -} - // This is a slight design concession -- some hash actions have the possibility // of failure. Rather than introduce extra error checking into all actions, we // will raise an internal exception so that only the actions where errors can @@ -129,7 +110,7 @@ class HashTable { Status HashTable::Init(int64_t elements) { DCHECK_EQ(elements, BitUtil::NextPower2(elements)); - RETURN_NOT_OK(NewHashTable(elements, pool_, &hash_table_)); + RETURN_NOT_OK(internal::NewHashTable(elements, pool_, &hash_table_)); hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); hash_table_size_ = elements; hash_table_load_threshold_ = @@ -238,44 +219,6 @@ struct HashDictionary> { } \ } -#define DOUBLE_TABLE_SIZE(SETUP_CODE, COMPUTE_HASH) \ - do { \ - int64_t new_size = hash_table_size_ * 2; \ - \ - std::shared_ptr new_hash_table; \ - RETURN_NOT_OK(NewHashTable(new_size, pool_, &new_hash_table)); \ - int32_t* new_hash_slots = \ - reinterpret_cast(new_hash_table->mutable_data()); \ - int64_t new_mod_bitmask = new_size - 1; \ - \ - SETUP_CODE; \ - \ - for (int i = 0; i < hash_table_size_; ++i) { \ - hash_slot_t index = hash_slots_[i]; \ - \ - if (index == kHashSlotEmpty) { \ - continue; \ - } \ - \ - COMPUTE_HASH; \ - while (kHashSlotEmpty != new_hash_slots[j]) { \ - ++j; \ - if (ARROW_PREDICT_FALSE(j == new_size)) { \ - j = 0; \ - } \ - } \ - \ - new_hash_slots[j] = index; \ - } \ - \ - hash_table_ = new_hash_table; \ - hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); \ - hash_table_size_ = new_size; \ - hash_table_load_threshold_ = \ - static_cast(static_cast(new_size) * kMaxHashTableLoad); \ - mod_bitmask_ = new_size - 1; \ - } while (false) - template class HashTableKernel> : public HashTable { public: diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 29b18a9357ad0..42613d6a5f56b 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -34,6 +34,7 @@ install(FILES cpu-info.h decimal.h hash-util.h + hash.h key_value_metadata.h logging.h macros.h diff --git a/cpp/src/arrow/util/hash.cc b/cpp/src/arrow/util/hash.cc new file mode 100644 index 0000000000000..94ba524560faf --- /dev/null +++ b/cpp/src/arrow/util/hash.cc @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/util/hash.h" + +#include "arrow/buffer.h" +#include "arrow/status.h" + +namespace arrow { +namespace internal { + +Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr* out) { + auto hash_table = std::make_shared(pool); + + RETURN_NOT_OK(hash_table->Resize(sizeof(hash_slot_t) * size)); + int32_t* slots = reinterpret_cast(hash_table->mutable_data()); + std::fill(slots, slots + size, kHashSlotEmpty); + + *out = hash_table; + return Status::OK(); +} + +} // namespace internal +} // namespace arrow diff --git a/cpp/src/arrow/util/hash.h b/cpp/src/arrow/util/hash.h new file mode 100644 index 0000000000000..3597342716388 --- /dev/null +++ b/cpp/src/arrow/util/hash.h @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_UTIL_HASH_H +#define ARROW_UTIL_HASH_H + +#include +#include +#include + +namespace arrow { + +class Buffer; +class MemoryPool; +class Status; + +typedef int32_t hash_slot_t; +static constexpr hash_slot_t kHashSlotEmpty = std::numeric_limits::max(); + +// Initially 1024 elements +static constexpr int kInitialHashTableSize = 1 << 10; + +// The maximum load factor for the hash table before resizing. +static constexpr double kMaxHashTableLoad = 0.5; + +namespace internal { + +#define DOUBLE_TABLE_SIZE(SETUP_CODE, COMPUTE_HASH) \ + do { \ + int64_t new_size = hash_table_size_ * 2; \ + \ + std::shared_ptr new_hash_table; \ + RETURN_NOT_OK(internal::NewHashTable(new_size, pool_, &new_hash_table)); \ + int32_t* new_hash_slots = \ + reinterpret_cast(new_hash_table->mutable_data()); \ + int64_t new_mod_bitmask = new_size - 1; \ + \ + SETUP_CODE; \ + \ + for (int i = 0; i < hash_table_size_; ++i) { \ + hash_slot_t index = hash_slots_[i]; \ + \ + if (index == kHashSlotEmpty) { \ + continue; \ + } \ + \ + COMPUTE_HASH; \ + while (kHashSlotEmpty != new_hash_slots[j]) { \ + ++j; \ + if (ARROW_PREDICT_FALSE(j == new_size)) { \ + j = 0; \ + } \ + } \ + \ + new_hash_slots[j] = index; \ + } \ + \ + hash_table_ = new_hash_table; \ + hash_slots_ = reinterpret_cast(hash_table_->mutable_data()); \ + hash_table_size_ = new_size; \ + hash_table_load_threshold_ = \ + static_cast(static_cast(new_size) * kMaxHashTableLoad); \ + mod_bitmask_ = new_size - 1; \ + } while (false) + +Status NewHashTable(int64_t size, MemoryPool* pool, std::shared_ptr* out); + +} // namespace internal +} // namespace arrow + +#endif // ARROW_UTIL_HASH_H From 665da71c7d73feba6f435e96c22d90b6632a8ad1 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 6 Dec 2017 17:00:45 +0100 Subject: [PATCH 111/177] ARROW-1890: [Python] Fix mask handling for Date32 NumPy conversions Author: Uwe L. Korn Closes #1393 from xhochy/ARROW-1890 and squashes the following commits: 7ef92ea [Uwe L. Korn] ARROW-1890: [Python] Fix mask handling for Date32 NumPy conversions --- cpp/src/arrow/python/numpy_to_arrow.cc | 1 + python/pyarrow/tests/test_convert_pandas.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index 2316a796a21c0..bcaccc73434a4 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -120,6 +120,7 @@ int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) { for (int i = 0; i < length; ++i) { if (mask_values[i]) { ++null_count; + BitUtil::ClearBit(bitmap, i); } else { BitUtil::SetBit(bitmap, i); } diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index a2a1d6b6910fa..a43651ed92c22 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -572,6 +572,15 @@ def test_date_infer(self): expected['date'] = pd.to_datetime(df['date']) tm.assert_frame_equal(result, expected) + def test_date_mask(self): + arr = np.array([date(2017, 4, 3), date(2017, 4, 4)], + dtype='datetime64[D]') + mask = [True, False] + result = pa.array(arr, mask=np.array(mask)) + expected = np.array([None, date(2017, 4, 4)], dtype='datetime64[D]') + expected = pa.array(expected, from_pandas=True) + assert expected.equals(result) + def test_date_objects_typed(self): arr = np.array([ date(2017, 4, 3), From aee1a0b586a611514c65ada6cb924ef77a01721e Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Wed, 6 Dec 2017 17:01:19 +0100 Subject: [PATCH 112/177] ARROW-1889: [Python] --exclude is not available in older git versions Author: Korn, Uwe Closes #1392 from xhochy/ARROW-1889 and squashes the following commits: 05a10eb [Korn, Uwe] ARROW-1889: [Python] --exclude is not available in older git versions --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index b56a4fdcace6c..81115869b8f17 100644 --- a/python/setup.py +++ b/python/setup.py @@ -398,7 +398,7 @@ def has_ext_modules(foo): def parse_version(root): from setuptools_scm import version_from_scm import setuptools_scm.git - describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --exclude 'apache-arrow-js-*'" + describe = setuptools_scm.git.DEFAULT_DESCRIBE + " --match 'apache-arrow-[0-9]*'" version = setuptools_scm.git.parse(root, describe) if not version: return version_from_scm(root) From 712b9d2c98d82b6f3a37ba8a2944bfda6f186994 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 6 Dec 2017 14:10:23 -0500 Subject: [PATCH 113/177] ARROW-1784: [Python] Enable zero-copy serialization, deserialization of pandas.DataFrame via components This patch adds a serialization path for pandas.DataFrame (and Series) that decomposes the internal BlockManager into a dictionary structure that can be serialized to the zero-copy component representation from ARROW-1783, and then reconstructed similarly. The impact of this is that when a DataFrame has no data that requires pickling, the reconstruction is zero-copy. I will post some benchmarks to illustrate the impact of this. The performance improvements are pretty remarkable, nearly 1000x speedup on a large DataFrame. As some follow-up work, we will need to do more efficient serialization of the different pandas Index types. We should create a new JIRA for this Author: Wes McKinney Closes #1390 from wesm/ARROW-1784 and squashes the following commits: 21adbe7d [Wes McKinney] Do not test with IntervalIndex in pandas < 0.21, since manylinux1 is pinned at 0.20.1 939c02bb [Wes McKinney] Add pandas serialization test for periods, intervals 4b4c776c [Wes McKinney] Code comment, add more serialization docs for pandas / component serialization 1ac073c3 [Wes McKinney] Complete component-based serializer for pandas.DataFrame 6b01746d [Wes McKinney] Begin refactoring --- python/doc/source/index.rst | 12 +- python/doc/source/ipc.rst | 77 ++++++ python/manylinux1/README.md | 2 +- python/pyarrow/pandas_compat.py | 184 +++++++++----- python/pyarrow/serialization.pxi | 24 ++ python/pyarrow/serialization.py | 121 +++++++--- python/pyarrow/tests/test_convert_pandas.py | 255 ++++++++++++-------- 7 files changed, 468 insertions(+), 207 deletions(-) diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst index b933d2359f720..c35f20be86396 100644 --- a/python/doc/source/index.rst +++ b/python/doc/source/index.rst @@ -18,10 +18,14 @@ Apache Arrow (Python) ===================== -Arrow is a columnar in-memory analytics layer designed to accelerate big data. -It houses a set of canonical in-memory representations of flat and hierarchical -data along with multiple language-bindings for structure manipulation. It also -provides IPC and common algorithm implementations. +Apache Arrow is a cross-language development platform for in-memory data. It +specifies a standardized language-independent columnar memory format for flat +and hierarchical data, organized for efficient analytic operations on modern +hardware. It also provides computational libraries and zero-copy streaming +messaging and interprocess communication. + +The Arrow Python bindings have first-class integration with NumPy, pandas, and +built-in Python objects. This is the documentation of the Python API of Apache Arrow. For more details on the format and other language bindings see diff --git a/python/doc/source/ipc.rst b/python/doc/source/ipc.rst index 17fe84e03633d..6842cb5be9f43 100644 --- a/python/doc/source/ipc.rst +++ b/python/doc/source/ipc.rst @@ -256,6 +256,83 @@ Lastly, we use this context as an additioanl argument to ``pyarrow.serialize``: buf = pa.serialize(val, context=context).to_buffer() restored_val = pa.deserialize(buf, context=context) +The ``SerializationContext`` also has convenience methods ``serialize`` and +``deserialize``, so these are equivalent statements: + +.. code-block:: python + + buf = context.serialize(val).to_buffer() + restored_val = context.deserialize(buf) + +Component-based Serialization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For serializing Python objects containing some number of NumPy arrays, Arrow +buffers, or other data types, it may be desirable to transport their serialized +representation without having to produce an intermediate copy using the +``to_buffer`` method. To motivate this, support we have a list of NumPy arrays: + +.. ipython:: python + + import numpy as np + data = [np.random.randn(10, 10) for i in range(5)] + +The call ``pa.serialize(data)`` does not copy the memory inside each of these +NumPy arrays. This serialized representation can be then decomposed into a +dictionary containing a sequence of ``pyarrow.Buffer`` objects containing +metadata for each array and references to the memory inside the arrays. To do +this, use the ``to_components`` method: + +.. ipython:: python + + serialized = pa.serialize(data) + components = serialized.to_components() + +The particular details of the output of ``to_components`` are not too +important. The objects in the ``'data'`` field are ``pyarrow.Buffer`` objects, +which are zero-copy convertible to Python ``memoryview`` objects: + +.. ipython:: python + + memoryview(components['data'][0]) + +A memoryview can be converted back to a ``Buffer`` with ``pyarrow.frombuffer``: + +.. ipython:: python + + mv = memoryview(components['data'][0]) + buf = pa.frombuffer(mv) + +An object can be reconstructed from its component-based representation using +``deserialize_components``: + +.. ipython:: python + + restored_data = pa.deserialize_components(components) + restored_data[0] + +``deserialize_components`` is also available as a method on +``SerializationContext`` objects. + +Serializing pandas Objects +-------------------------- + +We provide a serialization context that has optimized handling of pandas +objects like ``DataFrame`` and ``Series``. This is the +``pyarrow.pandas_serialization_context`` member. Combined with component-based +serialization above, this enables zero-copy transport of pandas DataFrame +objects not containing any Python objects: + +.. ipython:: python + + import pandas as pd + df = pd.DataFrame({'a': [1, 2, 3, 4, 5]}) + context = pa.pandas_serialization_context + serialized_df = context.serialize(df) + df_components = serialized_df.to_components() + original_df = context.deserialize_components(df_components) + original_df + Feather Format -------------- diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md index a74f7a27b930a..3d462ff2f7213 100644 --- a/python/manylinux1/README.md +++ b/python/manylinux1/README.md @@ -37,7 +37,7 @@ git clone ../../ arrow # Build the native baseimage docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 . # Build the python packages -docker run --rm -t -i -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh +docker run --shm-size=2g --rm -t -i -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh # Now the new packages are located in the dist/ folder ls -l dist/ ``` diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index a50ef96e79f59..8459ec31bed54 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -20,6 +20,7 @@ import json import re +import pandas.core.internals as _int import numpy as np import pandas as pd @@ -348,25 +349,85 @@ def get_datetimetz_type(values, dtype, type_): return values, type_ +# ---------------------------------------------------------------------- +# Converting pandas.DataFrame to a dict containing only NumPy arrays or other +# objects friendly to pyarrow.serialize -def make_datetimetz(tz): + +def dataframe_to_serialized_dict(frame): + block_manager = frame._data + + blocks = [] + axes = [ax for ax in block_manager.axes] + + for block in block_manager.blocks: + values = block.values + block_data = {} + + if isinstance(block, _int.DatetimeTZBlock): + block_data['timezone'] = values.tz.zone + values = values.values + elif isinstance(block, _int.CategoricalBlock): + block_data.update(dictionary=values.categories, + ordered=values.ordered) + values = values.codes + + block_data.update( + placement=block.mgr_locs.as_array, + block=values + ) + blocks.append(block_data) + + return { + 'blocks': blocks, + 'axes': axes + } + + +def serialized_dict_to_dataframe(data): + reconstructed_blocks = [_reconstruct_block(block) + for block in data['blocks']] + + block_mgr = _int.BlockManager(reconstructed_blocks, data['axes']) + return pd.DataFrame(block_mgr) + + +def _reconstruct_block(item): + # Construct the individual blocks converting dictionary types to pandas + # categorical types and Timestamps-with-timezones types to the proper + # pandas Blocks + + block_arr = item['block'] + placement = item['placement'] + if 'dictionary' in item: + cat = pd.Categorical.from_codes(block_arr, + categories=item['dictionary'], + ordered=item['ordered']) + block = _int.make_block(cat, placement=placement, + klass=_int.CategoricalBlock, + fastpath=True) + elif 'timezone' in item: + dtype = _make_datetimetz(item['timezone']) + block = _int.make_block(block_arr, placement=placement, + klass=_int.DatetimeTZBlock, + dtype=dtype, fastpath=True) + else: + block = _int.make_block(block_arr, placement=placement) + + return block + + +def _make_datetimetz(tz): from pyarrow.compat import DatetimeTZDtype return DatetimeTZDtype('ns', tz=tz) -def backwards_compatible_index_name(raw_name, logical_name): - pattern = r'^__index_level_\d+__$' - if raw_name == logical_name and re.match(pattern, raw_name) is not None: - return None - else: - return logical_name +# ---------------------------------------------------------------------- +# Converting pyarrow.Table efficiently to pandas.DataFrame def table_to_blockmanager(options, table, memory_pool, nthreads=1, categoricals=None): - import pandas.core.internals as _int - import pyarrow.lib as lib - index_columns = [] columns = [] column_indexes = [] @@ -405,37 +466,13 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1, index_arrays.append(pd.Series(values, dtype=col_pandas.dtype)) index_names.append( - backwards_compatible_index_name(raw_name, logical_name) + _backwards_compatible_index_name(raw_name, logical_name) ) block_table = block_table.remove_column( block_table.schema.get_field_index(raw_name) ) - # Convert an arrow table to Block from the internal pandas API - result = lib.table_to_blocks(options, block_table, nthreads, memory_pool) - - # Construct the individual blocks converting dictionary types to pandas - # categorical types and Timestamps-with-timezones types to the proper - # pandas Blocks - blocks = [] - for item in result: - block_arr = item['block'] - placement = item['placement'] - if 'dictionary' in item: - cat = pd.Categorical(block_arr, - categories=item['dictionary'], - ordered=item['ordered'], fastpath=True) - block = _int.make_block(cat, placement=placement, - klass=_int.CategoricalBlock, - fastpath=True) - elif 'timezone' in item: - dtype = make_datetimetz(item['timezone']) - block = _int.make_block(block_arr, placement=placement, - klass=_int.DatetimeTZBlock, - dtype=dtype, fastpath=True) - else: - block = _int.make_block(block_arr, placement=placement) - blocks.append(block) + blocks = _table_to_blocks(options, block_table, nthreads, memory_pool) # Construct the row index if len(index_arrays) > 1: @@ -477,31 +514,7 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1, # if we're reconstructing the index if has_pandas_metadata: - - # Get levels and labels, and provide sane defaults if the index has a - # single level to avoid if/else spaghetti. - levels = getattr(columns, 'levels', None) or [columns] - labels = getattr(columns, 'labels', None) or [ - pd.RangeIndex(len(level)) for level in levels - ] - - # Convert each level to the dtype provided in the metadata - levels_dtypes = [ - (level, col_index.get('numpy_type', level.dtype)) - for level, col_index in zip_longest( - levels, column_indexes, fillvalue={} - ) - ] - new_levels = [ - _level if _level.dtype == _dtype else _level.astype(_dtype) - for _level, _dtype in levels_dtypes - ] - - columns = pd.MultiIndex( - levels=new_levels, - labels=labels, - names=columns.names - ) + columns = _reconstruct_columns_from_metadata(columns, column_indexes) # ARROW-1751: flatten a single level column MultiIndex for pandas 0.21.0 columns = _flatten_single_level_multiindex(columns) @@ -510,6 +523,55 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1, return _int.BlockManager(blocks, axes) +def _backwards_compatible_index_name(raw_name, logical_name): + # Part of table_to_blockmanager + pattern = r'^__index_level_\d+__$' + if raw_name == logical_name and re.match(pattern, raw_name) is not None: + return None + else: + return logical_name + + +def _reconstruct_columns_from_metadata(columns, column_indexes): + # Part of table_to_blockmanager + + # Get levels and labels, and provide sane defaults if the index has a + # single level to avoid if/else spaghetti. + levels = getattr(columns, 'levels', None) or [columns] + labels = getattr(columns, 'labels', None) or [ + pd.RangeIndex(len(level)) for level in levels + ] + + # Convert each level to the dtype provided in the metadata + levels_dtypes = [ + (level, col_index.get('numpy_type', level.dtype)) + for level, col_index in zip_longest( + levels, column_indexes, fillvalue={} + ) + ] + new_levels = [ + _level if _level.dtype == _dtype else _level.astype(_dtype) + for _level, _dtype in levels_dtypes + ] + + return pd.MultiIndex( + levels=new_levels, + labels=labels, + names=columns.names + ) + + +def _table_to_blocks(options, block_table, nthreads, memory_pool): + # Part of table_to_blockmanager + + # Convert an arrow table to Block from the internal pandas API + result = pa.lib.table_to_blocks(options, block_table, nthreads, + memory_pool) + + # Defined above + return [_reconstruct_block(item) for item in result] + + def _flatten_single_level_multiindex(index): if isinstance(index, pd.MultiIndex) and index.nlevels == 1: levels, = index.levels diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index faf164b3ebd20..cbc5e3b8408e4 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -152,6 +152,30 @@ cdef class SerializationContext: obj.__dict__.update(serialized_obj) return obj + def serialize(self, obj): + """ + Call pyarrow.serialize and pass this SerializationContext + """ + return serialize(obj, context=self) + + def serialize_to(self, object value, sink): + """ + Call pyarrow.serialize_to and pass this SerializationContext + """ + return serialize_to(value, sink, context=self) + + def deserialize(self, what): + """ + Call pyarrow.deserialize and pass this SerializationContext + """ + return deserialize(what, context=self) + + def deserialize_components(self, what): + """ + Call pyarrow.deserialize_components and pass this SerializationContext + """ + return deserialize_components(what, context=self) + _default_serialization_context = SerializationContext() diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py index 08e6cce751b53..b6d2b0258bd08 100644 --- a/python/pyarrow/serialization.py +++ b/python/pyarrow/serialization.py @@ -43,15 +43,19 @@ def _deserialize_numpy_array_list(data): return np.array(data[0], dtype=np.dtype(data[1])) -def _serialize_numpy_array_pickle(obj): - pickled = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) +def _pickle_to_buffer(x): + pickled = pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL) return frombuffer(pickled) -def _deserialize_numpy_array_pickle(data): +def _load_pickle_from_buffer(data): return pickle.loads(memoryview(data)) +_serialize_numpy_array_pickle = _pickle_to_buffer +_deserialize_numpy_array_pickle = _load_pickle_from_buffer + + def register_default_serialization_handlers(serialization_context): # ---------------------------------------------------------------------- @@ -107,38 +111,6 @@ def _deserialize_default_dict(data): custom_serializer=_serialize_numpy_array_list, custom_deserializer=_deserialize_numpy_array_list) - # ---------------------------------------------------------------------- - # Set up serialization for pandas Series and DataFrame - - try: - import pandas as pd - - def _serialize_pandas_series(obj): - return serialize_pandas(pd.DataFrame({obj.name: obj})) - - def _deserialize_pandas_series(data): - deserialized = deserialize_pandas(data) - return deserialized[deserialized.columns[0]] - - def _serialize_pandas_dataframe(obj): - return serialize_pandas(obj) - - def _deserialize_pandas_dataframe(data): - return deserialize_pandas(data) - - serialization_context.register_type( - pd.Series, 'pd.Series', - custom_serializer=_serialize_pandas_series, - custom_deserializer=_deserialize_pandas_series) - - serialization_context.register_type( - pd.DataFrame, 'pd.DataFrame', - custom_serializer=_serialize_pandas_dataframe, - custom_deserializer=_deserialize_pandas_dataframe) - except ImportError: - # no pandas - pass - # ---------------------------------------------------------------------- # Set up serialization for pytorch tensors @@ -165,8 +137,87 @@ def _deserialize_torch_tensor(data): register_default_serialization_handlers(_default_serialization_context) + +# ---------------------------------------------------------------------- +# pandas-specific serialization matters + + pandas_serialization_context = _default_serialization_context.clone() + +def _register_pandas_arrow_handlers(context): + try: + import pandas as pd + except ImportError: + return + + def _serialize_pandas_series(obj): + return serialize_pandas(pd.DataFrame({obj.name: obj})) + + def _deserialize_pandas_series(data): + deserialized = deserialize_pandas(data) + return deserialized[deserialized.columns[0]] + + def _serialize_pandas_dataframe(obj): + return serialize_pandas(obj) + + def _deserialize_pandas_dataframe(data): + return deserialize_pandas(data) + + context.register_type( + pd.Series, 'pd.Series', + custom_serializer=_serialize_pandas_series, + custom_deserializer=_deserialize_pandas_series) + + context.register_type( + pd.DataFrame, 'pd.DataFrame', + custom_serializer=_serialize_pandas_dataframe, + custom_deserializer=_deserialize_pandas_dataframe) + + +def _register_custom_pandas_handlers(context): + # ARROW-1784, faster path for pandas-only visibility + + try: + import pandas as pd + except ImportError: + return + + import pyarrow.pandas_compat as pdcompat + + def _serialize_pandas_dataframe(obj): + return pdcompat.dataframe_to_serialized_dict(obj) + + def _deserialize_pandas_dataframe(data): + return pdcompat.serialized_dict_to_dataframe(data) + + def _serialize_pandas_series(obj): + return _serialize_pandas_dataframe(pd.DataFrame({obj.name: obj})) + + def _deserialize_pandas_series(data): + deserialized = _deserialize_pandas_dataframe(data) + return deserialized[deserialized.columns[0]] + + context.register_type( + pd.Series, 'pd.Series', + custom_serializer=_serialize_pandas_series, + custom_deserializer=_deserialize_pandas_series) + + context.register_type( + pd.Index, 'pd.Index', + custom_serializer=_pickle_to_buffer, + custom_deserializer=_load_pickle_from_buffer) + + context.register_type( + pd.DataFrame, 'pd.DataFrame', + custom_serializer=_serialize_pandas_dataframe, + custom_deserializer=_deserialize_pandas_dataframe) + + +_register_pandas_arrow_handlers(_default_serialization_context) +_register_custom_pandas_handlers(pandas_serialization_context) + + pandas_serialization_context.register_type( np.ndarray, 'np.array', custom_serializer=_serialize_numpy_array_pickle, diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index a43651ed92c22..e94ee4608ec5a 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -60,70 +60,73 @@ def _alltypes_example(size=100): }) -class TestPandasConversion(object): +def _check_pandas_roundtrip(df, expected=None, nthreads=1, + expected_schema=None, + check_dtype=True, schema=None, + preserve_index=False, + as_batch=False): + klass = pa.RecordBatch if as_batch else pa.Table + table = klass.from_pandas(df, schema=schema, + preserve_index=preserve_index, + nthreads=nthreads) + + result = table.to_pandas(nthreads=nthreads) + if expected_schema: + assert table.schema.equals(expected_schema) + if expected is None: + expected = df + tm.assert_frame_equal(result, expected, check_dtype=check_dtype, + check_index_type=('equiv' if preserve_index + else False)) + + +def _check_series_roundtrip(s, type_=None): + arr = pa.array(s, from_pandas=True, type=type_) + + result = pd.Series(arr.to_pandas(), name=s.name) + if patypes.is_timestamp(arr.type) and arr.type.tz is not None: + result = (result.dt.tz_localize('utc') + .dt.tz_convert(arr.type.tz)) + + tm.assert_series_equal(s, result) + + +def _check_array_roundtrip(values, expected=None, mask=None, + type=None): + arr = pa.array(values, from_pandas=True, mask=mask, type=type) + result = arr.to_pandas() + + values_nulls = pd.isnull(values) + if mask is None: + assert arr.null_count == values_nulls.sum() + else: + assert arr.null_count == (mask | values_nulls).sum() + + if mask is None: + tm.assert_series_equal(pd.Series(result), pd.Series(values), + check_names=False) + else: + expected = pd.Series(np.ma.masked_array(values, mask=mask)) + tm.assert_series_equal(pd.Series(result), expected, + check_names=False) + + +def _check_array_from_pandas_roundtrip(np_array): + arr = pa.array(np_array, from_pandas=True) + result = arr.to_pandas() + npt.assert_array_equal(result, np_array) - def setUp(self): - pass - - def tearDown(self): - pass - - def _check_pandas_roundtrip(self, df, expected=None, nthreads=1, - expected_schema=None, - check_dtype=True, schema=None, - preserve_index=False, - as_batch=False): - klass = pa.RecordBatch if as_batch else pa.Table - table = klass.from_pandas(df, schema=schema, - preserve_index=preserve_index, - nthreads=nthreads) - - result = table.to_pandas(nthreads=nthreads) - if expected_schema: - assert table.schema.equals(expected_schema) - if expected is None: - expected = df - tm.assert_frame_equal(result, expected, check_dtype=check_dtype, - check_index_type=('equiv' if preserve_index - else False)) - - def _check_series_roundtrip(self, s, type_=None): - arr = pa.array(s, from_pandas=True, type=type_) - - result = pd.Series(arr.to_pandas(), name=s.name) - if patypes.is_timestamp(arr.type) and arr.type.tz is not None: - result = (result.dt.tz_localize('utc') - .dt.tz_convert(arr.type.tz)) - - tm.assert_series_equal(s, result) - - def _check_array_roundtrip(self, values, expected=None, mask=None, - type=None): - arr = pa.array(values, from_pandas=True, mask=mask, type=type) - result = arr.to_pandas() - - values_nulls = pd.isnull(values) - if mask is None: - assert arr.null_count == values_nulls.sum() - else: - assert arr.null_count == (mask | values_nulls).sum() - - if mask is None: - tm.assert_series_equal(pd.Series(result), pd.Series(values), - check_names=False) - else: - expected = pd.Series(np.ma.masked_array(values, mask=mask)) - tm.assert_series_equal(pd.Series(result), expected, - check_names=False) + +class TestPandasConversion(object): def test_all_none_objects(self): df = pd.DataFrame({'a': [None, None, None]}) - self._check_pandas_roundtrip(df) + _check_pandas_roundtrip(df) def test_all_none_category(self): df = pd.DataFrame({'a': [None, None, None]}) df['a'] = df['a'].astype('category') - self._check_pandas_roundtrip(df) + _check_pandas_roundtrip(df) def test_non_string_columns(self): df = pd.DataFrame({0: [1, 2, 3]}) @@ -133,14 +136,14 @@ def test_non_string_columns(self): def test_column_index_names_are_preserved(self): df = pd.DataFrame({'data': [1, 2, 3]}) df.columns.names = ['a'] - self._check_pandas_roundtrip(df, preserve_index=True) + _check_pandas_roundtrip(df, preserve_index=True) def test_multiindex_columns(self): columns = pd.MultiIndex.from_arrays([ ['one', 'two'], ['X', 'Y'] ]) df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns) - self._check_pandas_roundtrip(df, preserve_index=True) + _check_pandas_roundtrip(df, preserve_index=True) def test_multiindex_columns_with_dtypes(self): columns = pd.MultiIndex.from_arrays( @@ -151,11 +154,11 @@ def test_multiindex_columns_with_dtypes(self): names=['level_1', 'level_2'], ) df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')], columns=columns) - self._check_pandas_roundtrip(df, preserve_index=True) + _check_pandas_roundtrip(df, preserve_index=True) def test_integer_index_column(self): df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')]) - self._check_pandas_roundtrip(df, preserve_index=True) + _check_pandas_roundtrip(df, preserve_index=True) def test_categorical_column_index(self): # I *really* hope no one uses category dtypes for single level column @@ -203,7 +206,7 @@ def test_categorical_row_index(self): df['a'] = df.a.astype('category') df = df.set_index('a') - self._check_pandas_roundtrip(df, preserve_index=True) + _check_pandas_roundtrip(df, preserve_index=True) def test_float_no_nulls(self): data = {} @@ -218,7 +221,7 @@ def test_float_no_nulls(self): df = pd.DataFrame(data) schema = pa.schema(fields) - self._check_pandas_roundtrip(df, expected_schema=schema) + _check_pandas_roundtrip(df, expected_schema=schema) def test_zero_copy_success(self): result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True) @@ -312,8 +315,8 @@ def test_float_object_nulls(self): expected = pd.DataFrame({'floats': pd.to_numeric(arr)}) field = pa.field('floats', pa.float64()) schema = pa.schema([field]) - self._check_pandas_roundtrip(df, expected=expected, - expected_schema=schema) + _check_pandas_roundtrip(df, expected=expected, + expected_schema=schema) def test_int_object_nulls(self): arr = np.array([None, 1, np.int64(3)] * 5, dtype=object) @@ -321,8 +324,8 @@ def test_int_object_nulls(self): expected = pd.DataFrame({'ints': pd.to_numeric(arr)}) field = pa.field('ints', pa.int64()) schema = pa.schema([field]) - self._check_pandas_roundtrip(df, expected=expected, - expected_schema=schema) + _check_pandas_roundtrip(df, expected=expected, + expected_schema=schema) def test_integer_no_nulls(self): data = OrderedDict() @@ -347,7 +350,7 @@ def test_integer_no_nulls(self): df = pd.DataFrame(data) schema = pa.schema(fields) - self._check_pandas_roundtrip(df, expected_schema=schema) + _check_pandas_roundtrip(df, expected_schema=schema) def test_integer_with_nulls(self): # pandas requires upcast to float dtype @@ -395,7 +398,7 @@ def test_boolean_no_nulls(self): df = pd.DataFrame({'bools': np.random.randn(num_values) > 0}) field = pa.field('bools', pa.bool_()) schema = pa.schema([field]) - self._check_pandas_roundtrip(df, expected_schema=schema) + _check_pandas_roundtrip(df, expected_schema=schema) def test_boolean_nulls(self): # pandas requires upcast to object dtype @@ -425,7 +428,7 @@ def test_boolean_object_nulls(self): df = pd.DataFrame({'bools': arr}) field = pa.field('bools', pa.bool_()) schema = pa.schema([field]) - self._check_pandas_roundtrip(df, expected_schema=schema) + _check_pandas_roundtrip(df, expected_schema=schema) def test_all_nulls_cast_numeric(self): arr = np.array([None], dtype=object) @@ -445,7 +448,7 @@ def test_unicode(self): field = pa.field('strings', pa.string()) schema = pa.schema([field]) - self._check_pandas_roundtrip(df, expected_schema=schema) + _check_pandas_roundtrip(df, expected_schema=schema) def test_bytes_to_binary(self): values = [u('qux'), b'foo', None, 'bar', 'qux', np.nan] @@ -456,7 +459,7 @@ def test_bytes_to_binary(self): values2 = [b'qux', b'foo', None, b'bar', b'qux', np.nan] expected = pd.DataFrame({'strings': values2}) - self._check_pandas_roundtrip(df, expected) + _check_pandas_roundtrip(df, expected) @pytest.mark.large_memory def test_bytes_exceed_2gb(self): @@ -499,7 +502,7 @@ def test_timestamps_notimezone_no_nulls(self): }) field = pa.field('datetime64', pa.timestamp('ns')) schema = pa.schema([field]) - self._check_pandas_roundtrip( + _check_pandas_roundtrip( df, expected_schema=schema, ) @@ -514,7 +517,7 @@ def test_timestamps_notimezone_nulls(self): }) field = pa.field('datetime64', pa.timestamp('ns')) schema = pa.schema([field]) - self._check_pandas_roundtrip( + _check_pandas_roundtrip( df, expected_schema=schema, ) @@ -529,9 +532,9 @@ def test_timestamps_with_timezone(self): }) df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern') .to_frame()) - self._check_pandas_roundtrip(df) + _check_pandas_roundtrip(df) - self._check_series_roundtrip(df['datetime64']) + _check_series_roundtrip(df['datetime64']) # drop-in a null and ns instead of ms df = pd.DataFrame({ @@ -545,7 +548,7 @@ def test_timestamps_with_timezone(self): df['datetime64'] = (df['datetime64'].dt.tz_localize('US/Eastern') .to_frame()) - self._check_pandas_roundtrip(df) + _check_pandas_roundtrip(df) def test_datetime64_to_date32(self): # ARROW-1718 @@ -647,13 +650,13 @@ def test_timedelta(self): def test_column_of_arrays(self): df, schema = dataframe_with_arrays() - self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema) + _check_pandas_roundtrip(df, schema=schema, expected_schema=schema) table = pa.Table.from_pandas(df, schema=schema, preserve_index=False) assert table.schema.equals(schema) for column in df.columns: field = schema.field_by_name(column) - self._check_array_roundtrip(df[column], type=field.type) + _check_array_roundtrip(df[column], type=field.type) def test_column_of_arrays_to_py(self): # Test regression in ARROW-1199 not caught in above test @@ -674,13 +677,13 @@ def test_column_of_arrays_to_py(self): def test_column_of_lists(self): df, schema = dataframe_with_lists() - self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema) + _check_pandas_roundtrip(df, schema=schema, expected_schema=schema) table = pa.Table.from_pandas(df, schema=schema, preserve_index=False) assert table.schema.equals(schema) for column in df.columns: field = schema.field_by_name(column) - self._check_array_roundtrip(df[column], type=field.type) + _check_array_roundtrip(df[column], type=field.type) def test_column_of_lists_chunked(self): # ARROW-1357 @@ -732,7 +735,7 @@ def test_column_of_lists_strided(self): arr = df['int64'].values[::3] assert arr.strides[0] != 8 - self._check_array_roundtrip(arr) + _check_array_roundtrip(arr) def test_nested_lists_all_none(self): data = np.array([[None, None], None], dtype=object) @@ -751,8 +754,8 @@ def test_nested_lists_all_none(self): def test_threaded_conversion(self): df = _alltypes_example() - self._check_pandas_roundtrip(df, nthreads=2) - self._check_pandas_roundtrip(df, nthreads=2, as_batch=True) + _check_pandas_roundtrip(df, nthreads=2) + _check_pandas_roundtrip(df, nthreads=2, as_batch=True) def test_category(self): repeats = 5 @@ -770,7 +773,7 @@ def test_category(self): 'strings': v1 * repeats, 'strings2': v1 * repeats, 'strings3': v3 * repeats}) - self._check_pandas_roundtrip(df) + _check_pandas_roundtrip(df) arrays = [ pd.Categorical(v1 * repeats), @@ -778,7 +781,7 @@ def test_category(self): pd.Categorical(v3 * repeats) ] for values in arrays: - self._check_array_roundtrip(values) + _check_array_roundtrip(values) def test_mixed_types_fails(self): data = pd.DataFrame({'a': ['a', 1, 2.0]}) @@ -825,9 +828,9 @@ def test_strided_data_import(self): df = pd.DataFrame(case, columns=columns) col = df['a'] - self._check_pandas_roundtrip(df) - self._check_array_roundtrip(col) - self._check_array_roundtrip(col, mask=strided_mask) + _check_pandas_roundtrip(df) + _check_array_roundtrip(col) + _check_array_roundtrip(col, mask=strided_mask) def test_decimal_32_from_pandas(self): expected = pd.DataFrame({ @@ -987,11 +990,6 @@ def test_arrow_time_to_pandas(self): tm.assert_frame_equal(df, expected_df) - def _check_array_from_pandas_roundtrip(self, np_array): - arr = pa.array(np_array, from_pandas=True) - result = arr.to_pandas() - npt.assert_array_equal(result, np_array) - def test_numpy_datetime64_columns(self): datetime64_ns = np.array([ '2007-07-13T01:23:34.123456789', @@ -999,7 +997,7 @@ def test_numpy_datetime64_columns(self): '2006-01-13T12:34:56.432539784', '2010-08-13T05:46:57.437699912'], dtype='datetime64[ns]') - self._check_array_from_pandas_roundtrip(datetime64_ns) + _check_array_from_pandas_roundtrip(datetime64_ns) datetime64_us = np.array([ '2007-07-13T01:23:34.123456', @@ -1007,7 +1005,7 @@ def test_numpy_datetime64_columns(self): '2006-01-13T12:34:56.432539', '2010-08-13T05:46:57.437699'], dtype='datetime64[us]') - self._check_array_from_pandas_roundtrip(datetime64_us) + _check_array_from_pandas_roundtrip(datetime64_us) datetime64_ms = np.array([ '2007-07-13T01:23:34.123', @@ -1015,7 +1013,7 @@ def test_numpy_datetime64_columns(self): '2006-01-13T12:34:56.432', '2010-08-13T05:46:57.437'], dtype='datetime64[ms]') - self._check_array_from_pandas_roundtrip(datetime64_ms) + _check_array_from_pandas_roundtrip(datetime64_ms) datetime64_s = np.array([ '2007-07-13T01:23:34', @@ -1023,7 +1021,7 @@ def test_numpy_datetime64_columns(self): '2006-01-13T12:34:56', '2010-08-13T05:46:57'], dtype='datetime64[s]') - self._check_array_from_pandas_roundtrip(datetime64_s) + _check_array_from_pandas_roundtrip(datetime64_s) def test_numpy_datetime64_day_unit(self): datetime64_d = np.array([ @@ -1032,7 +1030,7 @@ def test_numpy_datetime64_day_unit(self): '2006-01-15', '2010-08-19'], dtype='datetime64[D]') - self._check_array_from_pandas_roundtrip(datetime64_d) + _check_array_from_pandas_roundtrip(datetime64_d) def test_all_nones(self): def _check_series(s): @@ -1079,8 +1077,8 @@ def test_partial_schema(self): pa.field('c', pa.int64()) ]) - self._check_pandas_roundtrip(df, schema=partial_schema, - expected_schema=expected_schema) + _check_pandas_roundtrip(df, schema=partial_schema, + expected_schema=expected_schema) def test_structarray(self): ints = pa.array([None, 2, 3], type=pa.int64()) @@ -1115,7 +1113,7 @@ def test_infer_lists(self): pa.field('nested_strs', pa.list_(pa.list_(pa.string()))) ]) - self._check_pandas_roundtrip(df, expected_schema=expected_schema) + _check_pandas_roundtrip(df, expected_schema=expected_schema) def test_infer_numpy_array(self): data = OrderedDict([ @@ -1129,7 +1127,7 @@ def test_infer_numpy_array(self): pa.field('ints', pa.list_(pa.int64())) ]) - self._check_pandas_roundtrip(df, expected_schema=expected_schema) + _check_pandas_roundtrip(df, expected_schema=expected_schema) def test_metadata_with_mixed_types(self): df = pd.DataFrame({'data': [b'some_bytes', u'some_unicode']}) @@ -1184,12 +1182,12 @@ def test_table_str_to_categorical(self): def test_table_batch_empty_dataframe(self): df = pd.DataFrame({}) - self._check_pandas_roundtrip(df) - self._check_pandas_roundtrip(df, as_batch=True) + _check_pandas_roundtrip(df) + _check_pandas_roundtrip(df, as_batch=True) df2 = pd.DataFrame({}, index=[0, 1, 2]) - self._check_pandas_roundtrip(df2, preserve_index=True) - self._check_pandas_roundtrip(df2, as_batch=True, preserve_index=True) + _check_pandas_roundtrip(df2, preserve_index=True) + _check_pandas_roundtrip(df2, as_batch=True, preserve_index=True) def test_array_from_pandas_date_with_mask(self): m = np.array([True, False, True]) @@ -1231,6 +1229,51 @@ def test_array_from_pandas_typed_array_with_mask(self, t, data, expected): type=pa.list_(t())).equals(result) +def _fully_loaded_dataframe_example(): + from distutils.version import LooseVersion + + index = pd.MultiIndex.from_arrays([ + pd.date_range('2000-01-01', periods=5).repeat(2), + np.tile(np.array(['foo', 'bar'], dtype=object), 5) + ]) + + c1 = pd.date_range('2000-01-01', periods=10) + data = { + 0: c1, + 1: c1.tz_localize('utc'), + 2: c1.tz_localize('US/Eastern'), + 3: c1[::2].tz_localize('utc').repeat(2).astype('category'), + 4: ['foo', 'bar'] * 5, + 5: pd.Series(['foo', 'bar'] * 5).astype('category').values, + 6: [True, False] * 5, + 7: np.random.randn(10), + 8: np.random.randint(0, 100, size=10), + 9: pd.period_range('2013', periods=10, freq='M') + } + + if LooseVersion(pd.__version__) >= '0.21': + # There is an issue with pickling IntervalIndex in pandas 0.20.x + data[10] = pd.interval_range(start=1, freq=1, periods=10) + + return pd.DataFrame(data, index=index) + + +def _check_serialize_components_roundtrip(df): + ctx = pa.pandas_serialization_context + + components = ctx.serialize(df).to_components() + deserialized = ctx.deserialize_components(components) + + tm.assert_frame_equal(df, deserialized) + + +def test_serialize_deserialize_pandas(): + # ARROW-1784, serialize and deserialize DataFrame by decomposing + # BlockManager + df = _fully_loaded_dataframe_example() + _check_serialize_components_roundtrip(df) + + def _pytime_from_micros(val): microseconds = val % 1000000 val //= 1000000 From aad07824a41c4721992ee8ae2ce4a6c3a154e3ec Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 6 Dec 2017 14:11:20 -0500 Subject: [PATCH 114/177] ARROW-1857: [Python] Add switch for boost linkage with static parquet in wheels Author: Uwe L. Korn Closes #1395 from xhochy/ARROW-1857 and squashes the following commits: 71d97bc7 [Uwe L. Korn] ARROW-1857: [Python] Add switch for boost linkage with static parquet in wheels --- python/CMakeLists.txt | 9 ++++++++- python/setup.py | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 00a4ee68737c6..cbbb464d05a76 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -64,6 +64,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") option(PYARROW_PARQUET_USE_SHARED "Rely on parquet shared libraries where relevant" ON) + option(PYARROW_BOOST_USE_SHARED + "Rely on boost shared libraries on linking static parquet" + OFF) option(PYARROW_BUILD_PLASMA "Build the PyArrow Plasma integration" OFF) @@ -306,7 +309,11 @@ if (PYARROW_BUILD_PARQUET) parquet_shared) else() find_package(Thrift) - set(Boost_USE_STATIC_LIBS ON) + if (PYARROW_BOOST_USE_SHARED) + set(Boost_USE_STATIC_LIBS OFF) + else() + set(Boost_USE_STATIC_LIBS ON) + endif() find_package(Boost COMPONENTS regex REQUIRED) ADD_THIRDPARTY_LIB(boost_regex STATIC_LIB ${Boost_REGEX_LIBRARY_RELEASE}) diff --git a/python/setup.py b/python/setup.py index 81115869b8f17..32e76ab05d836 100644 --- a/python/setup.py +++ b/python/setup.py @@ -105,6 +105,8 @@ def initialize_options(self): os.environ.get('PYARROW_WITH_PARQUET', '0')) self.with_static_parquet = strtobool( os.environ.get('PYARROW_WITH_STATIC_PARQUET', '0')) + self.with_static_boost = strtobool( + os.environ.get('PYARROW_WITH_STATIC_BOOST', '1')) self.with_plasma = strtobool( os.environ.get('PYARROW_WITH_PLASMA', '0')) self.bundle_arrow_cpp = strtobool( @@ -149,6 +151,8 @@ def _run_cmake(self): cmake_options.append('-DPYARROW_BUILD_PARQUET=on') if self.with_static_parquet: cmake_options.append('-DPYARROW_PARQUET_USE_SHARED=off') + if not self.with_static_boost: + cmake_options.append('-DPYARROW_BOOST_USE_SHARED=on') if self.with_plasma: cmake_options.append('-DPYARROW_BUILD_PLASMA=on') From fb800135aacd91d17f34ba7e85cbf4e6e52d34e3 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 6 Dec 2017 15:59:42 -0500 Subject: [PATCH 115/177] ARROW-1892: [Python] Support binaries in lists Author: Uwe L. Korn Closes #1394 from xhochy/ARROW-1892 and squashes the following commits: 1dbf4b25 [Uwe L. Korn] ARROW-1892: [Python] Support binaries in lists --- cpp/src/arrow/python/arrow_to_pandas.cc | 3 + cpp/src/arrow/python/numpy_to_arrow.cc | 100 ++++++++++++++++++++++++ python/pyarrow/tests/pandas_examples.py | 8 ++ 3 files changed, 111 insertions(+) diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index b1825cbe69d2c..1a1b71125f039 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -100,6 +100,7 @@ static inline bool ListTypeSupported(const DataType& type) { case Type::UINT64: case Type::FLOAT: case Type::DOUBLE: + case Type::BINARY: case Type::STRING: case Type::TIMESTAMP: // The above types are all supported. @@ -695,6 +696,7 @@ class ObjectBlock : public PandasBlock { CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP) CONVERTLISTSLIKE_CASE(FloatType, FLOAT) CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE) + CONVERTLISTSLIKE_CASE(BinaryType, BINARY) CONVERTLISTSLIKE_CASE(StringType, STRING) CONVERTLISTSLIKE_CASE(ListType, LIST) default: { @@ -1642,6 +1644,7 @@ class ArrowDeserializer { CONVERTVALUES_LISTSLIKE_CASE(TimestampType, TIMESTAMP) CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT) CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE) + CONVERTVALUES_LISTSLIKE_CASE(BinaryType, BINARY) CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING) CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL) CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST) diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index bcaccc73434a4..798822c1b895c 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -146,6 +146,52 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int np_type) { } // namespace +/// Append as many string objects from NumPy arrays to a `StringBuilder` as we +/// can fit +/// +/// \param[in] offset starting offset for appending +/// \param[out] end_offset ending offset where we stopped appending. Will +/// be length of arr if fully consumed +/// \param[out] have_bytes true if we encountered any PyBytes object +static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask, + int64_t offset, BinaryBuilder* builder, + int64_t* end_offset, bool* have_bytes) { + PyObject* obj; + + Ndarray1DIndexer objects(arr); + Ndarray1DIndexer mask_values; + + bool have_mask = false; + if (mask != nullptr) { + mask_values.Init(mask); + have_mask = true; + } + + for (; offset < objects.size(); ++offset) { + OwnedRef tmp_obj; + obj = objects[offset]; + if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) { + RETURN_NOT_OK(builder->AppendNull()); + continue; + } else if (!PyBytes_Check(obj)) { + std::stringstream ss; + ss << "Error converting to Python objects to bytes: "; + RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss)); + return Status::Invalid(ss.str()); + } + + const int32_t length = static_cast(PyBytes_GET_SIZE(obj)); + if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) { + break; + } + RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length)); + } + + // If we consumed the whole array, this will be the length of arr + *end_offset = offset; + return Status::OK(); +} + /// Append as many string objects from NumPy arrays to a `StringBuilder` as we /// can fit /// @@ -1193,6 +1239,59 @@ inline Status NumPyConverter::ConvertTypedLists( return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item); } +template <> +inline Status NumPyConverter::ConvertTypedLists( + const std::shared_ptr& type, ListBuilder* builder, PyObject* list) { + PyAcquireGIL lock; + // TODO: If there are bytes involed, convert to Binary representation + bool have_bytes = false; + + Ndarray1DIndexer mask_values; + + bool have_mask = false; + if (mask_ != nullptr) { + mask_values.Init(mask_); + have_mask = true; + } + + auto value_builder = static_cast(builder->value_builder()); + + auto foreach_item = [&](PyObject* object, bool mask) { + if (mask || PandasObjectIsNull(object)) { + return builder->AppendNull(); + } else if (PyArray_Check(object)) { + auto numpy_array = reinterpret_cast(object); + RETURN_NOT_OK(builder->Append(true)); + + // TODO(uwe): Support more complex numpy array structures + RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT)); + + int64_t offset = 0; + RETURN_NOT_OK(AppendObjectBinaries(numpy_array, nullptr, 0, value_builder, &offset, + &have_bytes)); + if (offset < PyArray_SIZE(numpy_array)) { + return Status::Invalid("Array cell value exceeded 2GB"); + } + return Status::OK(); + } else if (PyList_Check(object)) { + int64_t size; + std::shared_ptr inferred_type; + RETURN_NOT_OK(builder->Append(true)); + RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type)); + if (inferred_type->id() != Type::NA && inferred_type->id() != Type::BINARY) { + std::stringstream ss; + ss << inferred_type->ToString() << " cannot be converted to BINARY."; + return Status::TypeError(ss.str()); + } + return AppendPySequence(object, size, inferred_type, value_builder); + } else { + return Status::TypeError("Unsupported Python type for list items"); + } + }; + + return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item); +} + template <> inline Status NumPyConverter::ConvertTypedLists( const std::shared_ptr& type, ListBuilder* builder, PyObject* list) { @@ -1267,6 +1366,7 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr& type, LIST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType) LIST_CASE(FLOAT, NPY_FLOAT, FloatType) LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType) + LIST_CASE(BINARY, NPY_OBJECT, BinaryType) LIST_CASE(STRING, NPY_OBJECT, StringType) case Type::LIST: { const auto& list_type = static_cast(*type); diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py index c145e96342668..f11da3c6ce945 100644 --- a/python/pyarrow/tests/pandas_examples.py +++ b/python/pyarrow/tests/pandas_examples.py @@ -110,6 +110,14 @@ def dataframe_with_lists(include_index=False): [0.], np.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.] * 2)[::2], ] + fields.append(pa.field('bytes_list', pa.list_(pa.binary()))) + arrays['bytes_list'] = [ + [b"1", b"f"], + None, + [b"1"], + [b"1", b"2", b"3"], + [], + ] fields.append(pa.field('str_list', pa.list_(pa.string()))) arrays['str_list'] = [ [u"1", u"ä"], From 1d519d8351c0b26ccb3214116eaf3aa684e4bba0 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Wed, 6 Dec 2017 16:40:33 -0500 Subject: [PATCH 116/177] ARROW-1487: [C++] Implement casts from List to List, where a cast function is defined from any A to B This closes [ARROW-1487](https://issues.apache.org/jira/browse/ARROW-1487). Author: Licht-T Author: Wes McKinney Closes #1383 from Licht-T/feature-list-to-list-cast and squashes the following commits: 53a21a5f [Wes McKinney] Fix cpplint warning 0903e5fc [Wes McKinney] Remove unneeded macro 87a00207 [Wes McKinney] Implement ListCastKernel a8046d08 [Licht-T] Implement child cast kernel checking before cast evaluation cca90681 [Licht-T] TST: Add test for casting from List to List 058aad07 [Licht-T] ENH: Implement List to List cast --- cpp/src/arrow/compute/compute-test.cc | 41 ++++++++++++++++ cpp/src/arrow/compute/kernels/cast.cc | 67 ++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc index 84af8f7c6b0a7..3fc15018630da 100644 --- a/cpp/src/arrow/compute/compute-test.cc +++ b/cpp/src/arrow/compute/compute-test.cc @@ -801,6 +801,47 @@ TYPED_TEST(TestDictionaryCast, Basic) { this->CheckPass(*plain_array, *dict_array, dict_array->type(), options); }*/ +TEST_F(TestCast, ListToList) { + CastOptions options; + std::shared_ptr offsets; + + vector offsets_values = {0, 1, 2, 5, 7, 7, 8, 10}; + std::vector offsets_is_valid = {true, true, true, true, false, true, true, true}; + ArrayFromVector(offsets_is_valid, offsets_values, &offsets); + + shared_ptr int32_plain_array = + TestBase::MakeRandomArray::ArrayType>(10, 2); + std::shared_ptr int32_list_array; + ASSERT_OK( + ListArray::FromArrays(*offsets, *int32_plain_array, pool_, &int32_list_array)); + + std::shared_ptr int64_plain_array; + ASSERT_OK(Cast(&this->ctx_, *int32_plain_array, int64(), options, &int64_plain_array)); + std::shared_ptr int64_list_array; + ASSERT_OK( + ListArray::FromArrays(*offsets, *int64_plain_array, pool_, &int64_list_array)); + + std::shared_ptr float64_plain_array; + ASSERT_OK( + Cast(&this->ctx_, *int32_plain_array, float64(), options, &float64_plain_array)); + std::shared_ptr float64_list_array; + ASSERT_OK( + ListArray::FromArrays(*offsets, *float64_plain_array, pool_, &float64_list_array)); + + this->CheckPass(*int32_list_array, *int64_list_array, int64_list_array->type(), + options); + this->CheckPass(*int32_list_array, *float64_list_array, float64_list_array->type(), + options); + this->CheckPass(*int64_list_array, *int32_list_array, int32_list_array->type(), + options); + this->CheckPass(*int64_list_array, *float64_list_array, float64_list_array->type(), + options); + this->CheckPass(*float64_list_array, *int32_list_array, int32_list_array->type(), + options); + this->CheckPass(*float64_list_array, *int64_list_array, int64_list_array->type(), + options); +} + // ---------------------------------------------------------------------- // Dictionary tests diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc index 465be958cfabf..afa05485f65b4 100644 --- a/cpp/src/arrow/compute/kernels/cast.cc +++ b/cpp/src/arrow/compute/kernels/cast.cc @@ -460,6 +460,49 @@ struct CastFunctor { } }; +// ---------------------------------------------------------------------- +// List to List + +class ListCastKernel : public UnaryKernel { + public: + ListCastKernel(std::unique_ptr child_caster, + const std::shared_ptr& out_type) + : child_caster_(std::move(child_caster)), out_type_(out_type) {} + + Status Call(FunctionContext* ctx, const Datum& input, Datum* out) override { + DCHECK_EQ(Datum::ARRAY, input.kind()); + + const ArrayData& in_data = *input.array(); + DCHECK_EQ(Type::LIST, in_data.type->id()); + ArrayData* result; + + if (in_data.offset != 0) { + return Status::NotImplemented( + "Casting sliced lists (non-zero offset) not yet implemented"); + } + + if (out->kind() == Datum::NONE) { + out->value = ArrayData::Make(out_type_, in_data.length); + } + + result = out->array().get(); + + // Copy buffers from parent + result->buffers = in_data.buffers; + + Datum casted_child; + RETURN_NOT_OK(child_caster_->Call(ctx, Datum(in_data.child_data[0]), &casted_child)); + result->child_data.push_back(casted_child.array()); + + RETURN_IF_ERROR(ctx); + return Status::OK(); + } + + private: + std::unique_ptr child_caster_; + std::shared_ptr out_type_; +}; + // ---------------------------------------------------------------------- // Dictionary to other things @@ -895,7 +938,6 @@ GET_CAST_FUNCTION(DATE64_CASES, Date64Type); GET_CAST_FUNCTION(TIME32_CASES, Time32Type); GET_CAST_FUNCTION(TIME64_CASES, Time64Type); GET_CAST_FUNCTION(TIMESTAMP_CASES, TimestampType); - GET_CAST_FUNCTION(DICTIONARY_CASES, DictionaryType); #define CAST_FUNCTION_CASE(InType) \ @@ -903,6 +945,26 @@ GET_CAST_FUNCTION(DICTIONARY_CASES, DictionaryType); *kernel = Get##InType##CastFunc(out_type, options); \ break +namespace { + +Status GetListCastFunc(const DataType& in_type, const std::shared_ptr& out_type, + const CastOptions& options, std::unique_ptr* kernel) { + if (out_type->id() != Type::LIST) { + // Kernel will be null + return Status::OK(); + } + const DataType& in_value_type = *static_cast(in_type).value_type(); + std::shared_ptr out_value_type = + static_cast(*out_type).value_type(); + std::unique_ptr child_caster; + RETURN_NOT_OK(GetCastFunction(in_value_type, out_value_type, options, &child_caster)); + *kernel = + std::unique_ptr(new ListCastKernel(std::move(child_caster), out_type)); + return Status::OK(); +} + +} // namespace + Status GetCastFunction(const DataType& in_type, const std::shared_ptr& out_type, const CastOptions& options, std::unique_ptr* kernel) { switch (in_type.id()) { @@ -924,6 +986,9 @@ Status GetCastFunction(const DataType& in_type, const std::shared_ptr& CAST_FUNCTION_CASE(Time64Type); CAST_FUNCTION_CASE(TimestampType); CAST_FUNCTION_CASE(DictionaryType); + case Type::LIST: + RETURN_NOT_OK(GetListCastFunc(in_type, out_type, options, kernel)); + break; default: break; } From ad454e868bcdf8e9ebe161e0050bbea2f1f2a88a Mon Sep 17 00:00:00 2001 From: vkorukanti Date: Wed, 6 Dec 2017 18:34:50 -0500 Subject: [PATCH 117/177] ARROW-1877: [Java] Fix incorrect equals method in JsonStringArrayList Currently it uses containsAll which could return wrong results. Ex. e1: [true, true, false], e2: [true, false, false]. Remove the equals method and fallback to super class method which has the correct implementation. Author: vkorukanti Closes #1380 from vkorukanti/ARROW-1877 and squashes the following commits: 194629ed [vkorukanti] removed `equals` from `JsonStringHashMap` 4732ddf6 [vkorukanti] ARROW-1877: Fix incorrect equals method in JsonStringArrayList --- .../vector/util/JsonStringArrayList.java | 16 ---------- .../arrow/vector/util/JsonStringHashMap.java | 31 ------------------- 2 files changed, 47 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java index 480bd76d445b0..b6db29a7fc64e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringArrayList.java @@ -19,7 +19,6 @@ package org.apache.arrow.vector.util; import java.util.ArrayList; -import java.util.List; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -40,21 +39,6 @@ public JsonStringArrayList(int size) { super(size); } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof List)) { - return false; - } - List other = (List) obj; - return this.size() == other.size() && this.containsAll(other); - } - @Override public final String toString() { try { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java index 6455389d582b9..cdb44fbeeaf3b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/JsonStringHashMap.java @@ -19,7 +19,6 @@ package org.apache.arrow.vector.util; import java.util.LinkedHashMap; -import java.util.Map; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -36,36 +35,6 @@ public class JsonStringHashMap extends LinkedHashMap { mapper = new ObjectMapper(); } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof Map)) { - return false; - } - Map other = (Map) obj; - if (this.size() != other.size()) { - return false; - } - for (K key : this.keySet()) { - if (this.get(key) == null) { - if (other.get(key) == null) { - continue; - } else { - return false; - } - } - if (!this.get(key).equals(other.get(key))) { - return false; - } - } - return true; - } - @Override public final String toString() { try { From 7ed4ca7209f8826fbdaf08188e2695803e3043bf Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Wed, 6 Dec 2017 23:18:40 -0500 Subject: [PATCH 118/177] ARROW-1867: [Java] Add missing methods to BitVector from legacy vector class After the ValueVector refactor, the BitVector class was missing the methods setToOne and setRangeToOne from the legacy vector class. This PR adds them back and restores the BItVector tests to use these APIs. Author: Bryan Cutler Closes #1396 from BryanCutler/java-addmissing-BitVector-methods-ARROW-1867 and squashes the following commits: 16142162 [Bryan Cutler] added methods setToOne and setRangeToOne from legacy BitVector --- .../org/apache/arrow/vector/BitVector.java | 66 +++++++++++++++++++ .../apache/arrow/vector/BitVectorHelper.java | 14 ++++ .../apache/arrow/vector/TestBitVector.java | 16 ++--- 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index b6f7323a796b4..3887da4a618f0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -444,6 +444,72 @@ public void setSafe(int index, int isSet, int value) { set(index, isSet, value); } + /** + * Set the element at the given index to one. + * + * @param index position of element + */ + public void setToOne(int index) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } + + /** + * Same as {@link #setToOne(int)} except that it handles the case when + * index is greater than or equal to current value capacity of the vector. + * + * @param index position of the element + */ + public void setSafeToOne(int index) { + handleSafe(index); + setToOne(index); + } + + /** + * Set count bits to 1 in data starting at firstBitIndex + * + * @param firstBitIndex the index of the first bit to set + * @param count the number of bits to set + */ + public void setRangeToOne(int firstBitIndex, int count) { + int startByteIndex = BitVectorHelper.byteIndex(firstBitIndex); + final int lastBitIndex = firstBitIndex + count; + final int endByteIndex = BitVectorHelper.byteIndex(lastBitIndex); + final int startByteBitIndex = BitVectorHelper.bitIndex(firstBitIndex); + final int endBytebitIndex = BitVectorHelper.bitIndex(lastBitIndex); + if (count < 8 && startByteIndex == endByteIndex) { + // handles the case where we don't have a first and a last byte + byte bitMask = 0; + for (int i = startByteBitIndex; i < endBytebitIndex; ++i) { + bitMask |= (byte) (1L << i); + } + BitVectorHelper.setBitMaskedByte(validityBuffer, startByteIndex, bitMask); + BitVectorHelper.setBitMaskedByte(valueBuffer, startByteIndex, bitMask); + } else { + // fill in first byte (if it's not full) + if (startByteBitIndex != 0) { + final byte bitMask = (byte) (0xFFL << startByteBitIndex); + BitVectorHelper.setBitMaskedByte(validityBuffer, startByteIndex, bitMask); + BitVectorHelper.setBitMaskedByte(valueBuffer, startByteIndex, bitMask); + ++startByteIndex; + } + + // fill in one full byte at a time + for (int i = startByteIndex; i < endByteIndex; i++) { + validityBuffer.setByte(i, 0xFF); + valueBuffer.setByte(i, 0xFF); + } + + // fill in the last byte (if it's not full) + if (endBytebitIndex != 0) { + final int byteIndex = BitVectorHelper.byteIndex(lastBitIndex - endBytebitIndex); + final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7)); + BitVectorHelper.setBitMaskedByte(validityBuffer, byteIndex, bitMask); + BitVectorHelper.setBitMaskedByte(valueBuffer, byteIndex, bitMask); + } + } + } + /****************************************************************** * * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 2d4db85c583bf..8322a1ac8fc40 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -198,4 +198,18 @@ public static ArrowBuf loadValidityBuffer(final ArrowFieldNode fieldNode, return newBuffer; } + + /** + * Set the byte of the given index in the data buffer by applying a bit mask to + * the current byte at that index. + * + * @param data + * @param byteIndex + * @param bitMask + */ + static void setBitMaskedByte(ArrowBuf data, int byteIndex, byte bitMask) { + byte currentByte = data.getByte(byteIndex); + currentByte |= bitMask; + data.setByte(byteIndex, currentByte); + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java index 36365fa9d4063..a59e5cdd48fce 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java @@ -232,7 +232,7 @@ public void testReallocAfterVectorTransfer1() { for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - vector.set(i, 1); + vector.setToOne(i); } } @@ -246,12 +246,12 @@ public void testReallocAfterVectorTransfer1() { } /* trigger first realloc */ - vector.setSafe(valueCapacity, 1); + vector.setSafeToOne(valueCapacity); assertEquals(valueCapacity * 2, vector.getValueCapacity()); for (int i = valueCapacity; i < valueCapacity*2; i++) { if ((i & 1) == 1) { - vector.set(i, 1); + vector.setToOne(i); } } @@ -265,12 +265,12 @@ public void testReallocAfterVectorTransfer1() { } /* trigger second realloc */ - vector.setSafe(valueCapacity*2, 1); + vector.setSafeToOne(valueCapacity*2); assertEquals(valueCapacity * 4, vector.getValueCapacity()); for (int i = valueCapacity*2; i < valueCapacity*4; i++) { if ((i & 1) == 1) { - vector.set(i, 1); + vector.setToOne(i); } } @@ -291,7 +291,7 @@ public void testReallocAfterVectorTransfer1() { assertEquals(valueCapacity * 4, toVector.getValueCapacity()); /* realloc the toVector */ - toVector.setSafe(valueCapacity * 4, 1); + toVector.setSafeToOne(valueCapacity * 4); for (int i = 0; i < toVector.getValueCapacity(); i++) { if (i <= valueCapacity * 4) { @@ -505,9 +505,7 @@ private void validateRange(int length, int start, int count) { try (BitVector bitVector = new BitVector("bits", allocator)) { bitVector.reset(); bitVector.allocateNew(length); - for (int i = start; i < start + count; i++) { - bitVector.set(i, 1); - } + bitVector.setRangeToOne(start, count); for (int i = 0; i < start; i++) { Assert.assertTrue(desc + i, bitVector.isNull(i)); } From ad30138a0ec9be3dfb179d1e9425a4502d556085 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 7 Dec 2017 10:02:36 -0500 Subject: [PATCH 119/177] ARROW-1891: [Python] Always use NumPy NaT sentinels to mark nulls when converting to array This code path could benefit from some refactoring, but this is enough to get things working properly, and we can refactor later. Author: Wes McKinney Closes #1399 from wesm/ARROW-1891 and squashes the following commits: 718c70c8 [Wes McKinney] Null sentinels always possible whether or not from_pandas=True when dealing with datetime64 values 7af1bf54 [Wes McKinney] Failing test case --- cpp/src/arrow/python/numpy_to_arrow.cc | 11 ++++++++--- python/pyarrow/tests/test_array.py | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index 798822c1b895c..f21b40ed3c246 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -421,7 +421,11 @@ class NumPyConverter { using traits = internal::arrow_traits; const bool null_sentinels_possible = - (use_pandas_null_sentinels_ && traits::supports_nulls); + // NumPy has a NaT type + (ArrowType::type_id == Type::TIMESTAMP || ArrowType::type_id == Type::DATE32) || + + // Observing pandas's null sentinels + ((use_pandas_null_sentinels_ && traits::supports_nulls)); if (mask_ != nullptr || null_sentinels_possible) { RETURN_NOT_OK(InitNullBitmap()); @@ -631,8 +635,6 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d auto date_dtype = reinterpret_cast(dtype_->c_metadata); if (dtype_->type_num == NPY_DATETIME) { - const int64_t null_count = ValuesToBitmap(arr_, null_bitmap_data_); - // If we have inbound datetime64[D] data, this needs to be downcasted // separately here from int64_t to int32_t, because this data is not // supported in compute::Cast @@ -642,6 +644,9 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d Status s = StaticCastBuffer(**data, length_, pool_, data); RETURN_NOT_OK(s); } else { + // TODO(wesm): This is redundant, and recomputed in VisitNative() + const int64_t null_count = ValuesToBitmap(arr_, null_bitmap_data_); + RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); if (!input_type->Equals(*type_)) { RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count, diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index a4d781a3354b0..92562da14150c 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -495,6 +495,14 @@ def test_array_conversions_no_sentinel_values(): assert arr3.null_count == 0 +def test_array_from_numpy_datetimeD(): + arr = np.array([None, datetime.date(2017, 4, 4)], dtype='datetime64[D]') + + result = pa.array(arr) + expected = pa.array([None, datetime.date(2017, 4, 4)], type=pa.date32()) + assert result.equals(expected) + + def test_array_from_numpy_ascii(): arr = np.array(['abcde', 'abc', ''], dtype='|S5') From aaa978b9c072d3c436838ad2a35b0f6bde1891b6 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 7 Dec 2017 11:00:56 -0500 Subject: [PATCH 120/177] ARROW-1893: [Python] Convert memoryview to bytes when loading from pickle in Python 2.7 It seems somewhere in the 2.7.x series, Python 2.7 acquired the ability to load from memoryview. To be on the safe side, we'll always convert memoryview to bytes. Here's a related workaround from IPython: https://github.com/ipython/ipython_genutils/blob/master/ipython_genutils/py3compat.py#L153 Author: Wes McKinney Closes #1398 from wesm/ARROW-1893 and squashes the following commits: cbe69134 [Wes McKinney] Not all versions of Python 2.7 can load pickles directly from memoryview --- python/pyarrow/compat.py | 4 +++- python/pyarrow/serialization.py | 13 +++++++++---- python/pyarrow/tests/test_parquet.py | 20 +++++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py index 866cbdd96d063..1b19ca0e4029b 100644 --- a/python/pyarrow/compat.py +++ b/python/pyarrow/compat.py @@ -70,7 +70,7 @@ class Categorical(ClassPlaceholder): if PY2: - import cPickle + import cPickle as builtin_pickle try: from cdecimal import Decimal @@ -107,6 +107,8 @@ def frombytes(o): def unichar(s): return unichr(s) else: + import pickle as builtin_pickle + unicode_type = str def lzip(*x): return list(zip(*x)) diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py index b6d2b0258bd08..3059dfc1bfff2 100644 --- a/python/pyarrow/serialization.py +++ b/python/pyarrow/serialization.py @@ -16,18 +16,19 @@ # under the License. from collections import OrderedDict, defaultdict +import six import sys -import pickle import numpy as np from pyarrow import serialize_pandas, deserialize_pandas +from pyarrow.compat import builtin_pickle from pyarrow.lib import _default_serialization_context, frombuffer try: import cloudpickle except ImportError: - cloudpickle = pickle + cloudpickle = builtin_pickle # ---------------------------------------------------------------------- @@ -44,12 +45,16 @@ def _deserialize_numpy_array_list(data): def _pickle_to_buffer(x): - pickled = pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL) + pickled = builtin_pickle.dumps(x, protocol=builtin_pickle.HIGHEST_PROTOCOL) return frombuffer(pickled) def _load_pickle_from_buffer(data): - return pickle.loads(memoryview(data)) + as_memoryview = memoryview(data) + if six.PY2: + return builtin_pickle.loads(as_memoryview.tobytes()) + else: + return builtin_pickle.loads(as_memoryview) _serialize_numpy_array_pickle = _pickle_to_buffer diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index d17d89e24df5f..2543e7d17befd 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1570,19 +1570,21 @@ def test_backwards_compatible_index_multi_level_some_named(): tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize('precision', range(1, 39)) -def test_decimal_roundtrip(tmpdir, precision): +def test_decimal_roundtrip(tmpdir): num_values = 10 columns = {} - for scale in range(0, precision + 1): - with util.random_seed(0): - random_decimal_values = [ - util.randdecimal(precision, scale) for _ in range(num_values) - ] - column_name = 'dec_precision_{:d}_scale_{:d}'.format(precision, scale) - columns[column_name] = random_decimal_values + for precision in range(1, 39): + for scale in range(0, precision + 1): + with util.random_seed(0): + random_decimal_values = [ + util.randdecimal(precision, scale) + for _ in range(num_values) + ] + column_name = ('dec_precision_{:d}_scale_{:d}' + .format(precision, scale)) + columns[column_name] = random_decimal_values expected = pd.DataFrame(columns) filename = tmpdir.join('decimals.parquet') From 64af87bdee0626c2bb19e0e0fd097c475c57c8d1 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 7 Dec 2017 15:54:24 -0500 Subject: [PATCH 121/177] ARROW-1884: [C++] Exclude integration test JSON reader/writer classes from public API These were showing up in our Doxygen docs and may mislead users reading the public API into thinking these classes do something that they do not (they don't read general JSON) Author: Wes McKinney Closes #1400 from wesm/ARROW-1884 and squashes the following commits: 9b299144 [Wes McKinney] Move integration JSON reader/writer into internal namespace, do not export symbols --- cpp/src/arrow/ipc/ipc-json-test.cc | 10 ++++--- cpp/src/arrow/ipc/json-integration-test.cc | 31 +++++++++++----------- cpp/src/arrow/ipc/json-internal.cc | 7 +++-- cpp/src/arrow/ipc/json-internal.h | 4 +-- cpp/src/arrow/ipc/json.cc | 13 +++++---- cpp/src/arrow/ipc/json.h | 8 ++++-- 6 files changed, 42 insertions(+), 31 deletions(-) diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc index e496826f96b9b..12fa4bf3ed7af 100644 --- a/cpp/src/arrow/ipc/ipc-json-test.cc +++ b/cpp/src/arrow/ipc/ipc-json-test.cc @@ -39,6 +39,7 @@ namespace arrow { namespace ipc { +namespace internal { namespace json { void TestSchemaRoundTrip(const Schema& schema) { @@ -46,7 +47,7 @@ void TestSchemaRoundTrip(const Schema& schema) { rj::Writer writer(sb); writer.StartObject(); - ASSERT_OK(internal::WriteSchema(schema, &writer)); + ASSERT_OK(WriteSchema(schema, &writer)); writer.EndObject(); std::string json_schema = sb.GetString(); @@ -55,7 +56,7 @@ void TestSchemaRoundTrip(const Schema& schema) { d.Parse(json_schema); std::shared_ptr out; - if (!internal::ReadSchema(d, default_memory_pool(), &out).ok()) { + if (!ReadSchema(d, default_memory_pool(), &out).ok()) { FAIL() << "Unable to read JSON schema: " << json_schema; } @@ -70,7 +71,7 @@ void TestArrayRoundTrip(const Array& array) { rj::StringBuffer sb; rj::Writer writer(sb); - ASSERT_OK(internal::WriteArray(name, array, &writer)); + ASSERT_OK(WriteArray(name, array, &writer)); std::string array_as_json = sb.GetString(); @@ -82,7 +83,7 @@ void TestArrayRoundTrip(const Array& array) { } std::shared_ptr out; - ASSERT_OK(internal::ReadArray(default_memory_pool(), d, array.type(), &out)); + ASSERT_OK(ReadArray(default_memory_pool(), d, array.type(), &out)); // std::cout << array_as_json << std::endl; CompareArraysDetailed(0, *out, array); @@ -415,5 +416,6 @@ TEST_P(TestJsonRoundTrip, RoundTrip) { INSTANTIATE_TEST_CASE_P(TestJsonRoundTrip, TestJsonRoundTrip, BATCH_CASES()); } // namespace json +} // namespace internal } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc index f362d97015978..37778fa25166d 100644 --- a/cpp/src/arrow/ipc/json-integration-test.cc +++ b/cpp/src/arrow/ipc/json-integration-test.cc @@ -50,8 +50,7 @@ DEFINE_bool(verbose, true, "Verbose output"); namespace fs = boost::filesystem; namespace arrow { - -class Buffer; +namespace ipc { bool file_exists(const char* path) { std::ifstream handle(path); @@ -73,16 +72,15 @@ static Status ConvertJsonToArrow(const std::string& json_path, std::shared_ptr json_buffer; RETURN_NOT_OK(in_file->Read(file_size, &json_buffer)); - std::unique_ptr reader; - RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &reader)); + std::unique_ptr reader; + RETURN_NOT_OK(internal::json::JsonReader::Open(json_buffer, &reader)); if (FLAGS_verbose) { std::cout << "Found schema: " << reader->schema()->ToString() << std::endl; } - std::shared_ptr writer; - RETURN_NOT_OK( - ipc::RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer)); + std::shared_ptr writer; + RETURN_NOT_OK(RecordBatchFileWriter::Open(out_file.get(), reader->schema(), &writer)); for (int i = 0; i < reader->num_record_batches(); ++i) { std::shared_ptr batch; @@ -101,15 +99,15 @@ static Status ConvertArrowToJson(const std::string& arrow_path, RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &in_file)); RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file)); - std::shared_ptr reader; - RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(in_file.get(), &reader)); + std::shared_ptr reader; + RETURN_NOT_OK(RecordBatchFileReader::Open(in_file.get(), &reader)); if (FLAGS_verbose) { std::cout << "Found schema: " << reader->schema()->ToString() << std::endl; } - std::unique_ptr writer; - RETURN_NOT_OK(ipc::JsonWriter::Open(reader->schema(), &writer)); + std::unique_ptr writer; + RETURN_NOT_OK(internal::json::JsonWriter::Open(reader->schema(), &writer)); for (int i = 0; i < reader->num_record_batches(); ++i) { std::shared_ptr batch; @@ -134,15 +132,15 @@ static Status ValidateArrowVsJson(const std::string& arrow_path, std::shared_ptr json_buffer; RETURN_NOT_OK(json_file->Read(file_size, &json_buffer)); - std::unique_ptr json_reader; - RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &json_reader)); + std::unique_ptr json_reader; + RETURN_NOT_OK(internal::json::JsonReader::Open(json_buffer, &json_reader)); // Construct Arrow reader std::shared_ptr arrow_file; RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file)); - std::shared_ptr arrow_reader; - RETURN_NOT_OK(ipc::RecordBatchFileReader::Open(arrow_file.get(), &arrow_reader)); + std::shared_ptr arrow_reader; + RETURN_NOT_OK(RecordBatchFileReader::Open(arrow_file.get(), &arrow_reader)); auto json_schema = json_reader->schema(); auto arrow_schema = arrow_reader->schema(); @@ -399,6 +397,7 @@ TEST_F(TestJSONIntegration, ErrorStates) { ASSERT_RAISES(Invalid, RunCommand(json_path, "", "VALIDATE")); } +} // namespace ipc } // namespace arrow int main(int argc, char** argv) { @@ -407,7 +406,7 @@ int main(int argc, char** argv) { int ret = 0; if (FLAGS_integration) { - arrow::Status result = arrow::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode); + arrow::Status result = arrow::ipc::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode); if (!result.ok()) { std::cout << "Error message: " << result.ToString() << std::endl; ret = 1; diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index 11a6956ebfd47..4088a8f20e6a0 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -40,8 +40,11 @@ namespace arrow { namespace ipc { -namespace json { namespace internal { +namespace json { + +using ::arrow::ipc::DictionaryMemo; +using ::arrow::ipc::DictionaryTypeMap; static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision) { switch (precision) { @@ -1463,7 +1466,7 @@ Status ReadArray(MemoryPool* pool, const rj::Value& json_array, const Schema& sc return ReadArray(pool, json_array, result->type(), array); } -} // namespace internal } // namespace json +} // namespace internal } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/json-internal.h b/cpp/src/arrow/ipc/json-internal.h index 506fe682949e6..92afc14447dc7 100644 --- a/cpp/src/arrow/ipc/json-internal.h +++ b/cpp/src/arrow/ipc/json-internal.h @@ -92,8 +92,8 @@ using RjObject = rj::Value::ConstObject; namespace arrow { namespace ipc { -namespace json { namespace internal { +namespace json { Status WriteSchema(const Schema& schema, RjWriter* writer); Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer); @@ -111,8 +111,8 @@ Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, const Schema& schema, std::shared_ptr* array); -} // namespace internal } // namespace json +} // namespace internal } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc index ea2947d5d4c36..394563c53c09d 100644 --- a/cpp/src/arrow/ipc/json.cc +++ b/cpp/src/arrow/ipc/json.cc @@ -33,6 +33,8 @@ using std::size_t; namespace arrow { namespace ipc { +namespace internal { +namespace json { // ---------------------------------------------------------------------- // Writer implementation @@ -45,7 +47,7 @@ class JsonWriter::JsonWriterImpl { Status Start() { writer_->StartObject(); - RETURN_NOT_OK(json::internal::WriteSchema(*schema_, writer_.get())); + RETURN_NOT_OK(json::WriteSchema(*schema_, writer_.get())); // Record batches writer_->Key("batches"); @@ -63,7 +65,7 @@ class JsonWriter::JsonWriterImpl { Status WriteRecordBatch(const RecordBatch& batch) { DCHECK_EQ(batch.num_columns(), schema_->num_fields()); - return json::internal::WriteRecordBatch(batch, writer_.get()); + return json::WriteRecordBatch(batch, writer_.get()); } private: @@ -106,7 +108,7 @@ class JsonReader::JsonReaderImpl { return Status::IOError("JSON parsing failed"); } - RETURN_NOT_OK(json::internal::ReadSchema(doc_, pool_, &schema_)); + RETURN_NOT_OK(json::ReadSchema(doc_, pool_, &schema_)); auto it = doc_.FindMember("batches"); RETURN_NOT_ARRAY("batches", it, doc_); @@ -120,8 +122,7 @@ class JsonReader::JsonReaderImpl { DCHECK_LT(i, static_cast(record_batches_->GetArray().Size())) << "i out of bounds"; - return json::internal::ReadRecordBatch(record_batches_->GetArray()[i], schema_, pool_, - batch); + return json::ReadRecordBatch(record_batches_->GetArray()[i], schema_, pool_, batch); } std::shared_ptr schema() const { return schema_; } @@ -164,5 +165,7 @@ Status JsonReader::ReadRecordBatch(int i, std::shared_ptr* batch) c return impl_->ReadRecordBatch(i, batch); } +} // namespace json +} // namespace internal } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/json.h b/cpp/src/arrow/ipc/json.h index 51f30f0c109f3..674c3745ed413 100644 --- a/cpp/src/arrow/ipc/json.h +++ b/cpp/src/arrow/ipc/json.h @@ -34,12 +34,14 @@ class RecordBatch; class Schema; namespace ipc { +namespace internal { +namespace json { /// \class JsonWriter /// \brief Write the JSON representation of an Arrow record batch file or stream /// /// This is used for integration testing -class ARROW_EXPORT JsonWriter { +class JsonWriter { public: ~JsonWriter(); @@ -72,7 +74,7 @@ class ARROW_EXPORT JsonWriter { /// \brief Read the JSON representation of an Arrow record batch file or stream /// /// This is used for integration testing -class ARROW_EXPORT JsonReader { +class JsonReader { public: ~JsonReader(); @@ -113,6 +115,8 @@ class ARROW_EXPORT JsonReader { std::unique_ptr impl_; }; +} // namespace json +} // namespace internal } // namespace ipc } // namespace arrow From 2994815b7274f22f8a14867fcb6e6b061a2412eb Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 7 Dec 2017 20:06:47 -0500 Subject: [PATCH 122/177] ARROW-1873: [Python] Catch more possible Python/OOM errors in to_pandas conversion path I also ran into a gnarly method dispatching bug ARROW-1904 while working on this. I will address that deprecation in a separate patch Author: Wes McKinney Closes #1404 from wesm/ARROW-1873 and squashes the following commits: 10001c6a [Wes McKinney] Add various Python error checks for possible OOM. More centralized handling of array offsets --- cpp/src/arrow/python/arrow_to_pandas.cc | 106 +++++++++++++----------- 1 file changed, 58 insertions(+), 48 deletions(-) diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 1a1b71125f039..08ce37cda6351 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -240,19 +240,14 @@ class PandasBlock { block_arr = PyArray_SimpleNewFromDescr(1, block_dims, descr); } - if (block_arr == NULL) { - // TODO(wesm): propagating Python exception - return Status::OK(); - } + RETURN_IF_PYERROR(); PyArray_ENABLEFLAGS(reinterpret_cast(block_arr), NPY_ARRAY_OWNDATA); npy_intp placement_dims[1] = {num_columns_}; PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - if (placement_arr == NULL) { - // TODO(wesm): propagating Python exception - return Status::OK(); - } + + RETURN_IF_PYERROR(); block_arr_.reset(block_arr); placement_arr_.reset(placement_arr); @@ -282,12 +277,19 @@ class PandasBlock { ARROW_DISALLOW_COPY_AND_ASSIGN(PandasBlock); }; +template +inline const T* GetPrimitiveValues(const Array& arr) { + const auto& prim_arr = static_cast(arr); + const T* raw_values = reinterpret_cast(prim_arr.values()->data()); + return raw_values + arr.offset(); +} + template inline void ConvertIntegerWithNulls(PandasOptions options, const ChunkedArray& data, double* out_values) { for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = static_cast(*data.chunk(c)); - auto in_values = reinterpret_cast(arr.raw_values()); + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); // Upcast to double, set NaN as appropriate for (int i = 0; i < arr.length(); ++i) { @@ -300,8 +302,8 @@ template inline void ConvertIntegerNoNullsSameType(PandasOptions options, const ChunkedArray& data, T* out_values) { for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = static_cast(*data.chunk(c)); - auto in_values = reinterpret_cast(arr.raw_values()); + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); memcpy(out_values, in_values, sizeof(T) * arr.length()); out_values += arr.length(); } @@ -311,8 +313,8 @@ template inline void ConvertIntegerNoNullsCast(PandasOptions options, const ChunkedArray& data, OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = static_cast(*data.chunk(c)); - auto in_values = reinterpret_cast(arr.raw_values()); + const auto& arr = *data.chunk(c); + const InType* in_values = GetPrimitiveValues(arr); for (int64_t i = 0; i < arr.length(); ++i) { *out_values = in_values[i]; } @@ -323,14 +325,13 @@ static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray& PyObject** out_values) { PyAcquireGIL lock; for (int c = 0; c < data.num_chunks(); c++) { - const std::shared_ptr arr = data.chunk(c); - auto bool_arr = static_cast(arr.get()); + const auto& arr = static_cast(*data.chunk(c)); - for (int64_t i = 0; i < arr->length(); ++i) { - if (bool_arr->IsNull(i)) { + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { Py_INCREF(Py_None); *out_values++ = Py_None; - } else if (bool_arr->Value(i)) { + } else if (arr.Value(i)) { // True Py_INCREF(Py_True); *out_values++ = Py_True; @@ -347,10 +348,9 @@ static Status ConvertBooleanWithNulls(PandasOptions options, const ChunkedArray& static void ConvertBooleanNoNulls(PandasOptions options, const ChunkedArray& data, uint8_t* out_values) { for (int c = 0; c < data.num_chunks(); c++) { - const std::shared_ptr arr = data.chunk(c); - auto bool_arr = static_cast(arr.get()); - for (int64_t i = 0; i < arr->length(); ++i) { - *out_values++ = static_cast(bool_arr->Value(i)); + const auto& arr = static_cast(*data.chunk(c)); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = static_cast(arr.Value(i)); } } } @@ -361,17 +361,17 @@ inline Status ConvertBinaryLike(PandasOptions options, const ChunkedArray& data, using ArrayType = typename TypeTraits::ArrayType; PyAcquireGIL lock; for (int c = 0; c < data.num_chunks(); c++) { - auto arr = static_cast(data.chunk(c).get()); + const auto& arr = static_cast(*data.chunk(c)); const uint8_t* data_ptr; int32_t length; const bool has_nulls = data.null_count() > 0; - for (int64_t i = 0; i < arr->length(); ++i) { - if (has_nulls && arr->IsNull(i)) { + for (int64_t i = 0; i < arr.length(); ++i) { + if (has_nulls && arr.IsNull(i)) { Py_INCREF(Py_None); *out_values = Py_None; } else { - data_ptr = arr->GetValue(i, &length); + data_ptr = arr.GetValue(i, &length); *out_values = WrapBytes::Wrap(data_ptr, length); if (*out_values == nullptr) { PyErr_Clear(); @@ -530,13 +530,25 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptrvalue_offset(i) + chunk_offset); PyObject* end = PyLong_FromLongLong(arr->value_offset(i + 1) + chunk_offset); PyObject* slice = PySlice_New(start, end, NULL); + Py_XDECREF(start); + Py_XDECREF(end); + + if (ARROW_PREDICT_FALSE(slice == nullptr)) { + // Fall out of loop, will return from RETURN_IF_PYERROR + break; + } *out_values = PyObject_GetItem(numpy_array, slice); - Py_DECREF(start); - Py_DECREF(end); - Py_DECREF(slice); + + if (*out_values == nullptr) { + // Fall out of loop, will return from RETURN_IF_PYERROR + break; + } + + Py_XDECREF(slice); } ++out_values; } + RETURN_IF_PYERROR(); chunk_offset += arr->values()->length(); } @@ -548,14 +560,12 @@ inline Status ConvertListsLike(PandasOptions options, const std::shared_ptr inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) { for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = static_cast(*data.chunk(c)); - auto in_values = reinterpret_cast(arr.raw_values()); - - const uint8_t* valid_bits = arr.null_bitmap_data(); + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); if (arr.null_count() > 0) { for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = BitUtil::BitNotSet(valid_bits, i) ? na_value : in_values[i]; + *out_values++ = arr.IsNull(i) ? na_value : in_values[i]; } } else { memcpy(out_values, in_values, sizeof(T) * arr.length()); @@ -568,8 +578,8 @@ template inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_value, OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = static_cast(*data.chunk(c)); - auto in_values = reinterpret_cast(arr.raw_values()); + const auto& arr = *data.chunk(c); + const InType* in_values = GetPrimitiveValues(arr); for (int64_t i = 0; i < arr.length(); ++i) { *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]); @@ -577,11 +587,11 @@ inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_valu } } -template +template inline void ConvertDatetimeNanos(const ChunkedArray& data, int64_t* out_values) { for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = static_cast(*data.chunk(c)); - auto in_values = reinterpret_cast(arr.raw_values()); + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); for (int64_t i = 0; i < arr.length(); ++i) { *out_values++ = arr.IsNull(i) ? kPandasTimestampNull @@ -631,7 +641,7 @@ static Status ConvertDecimals(PandasOptions options, const ChunkedArray& data, PyObject* Decimal = Decimal_ref.obj(); for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr(static_cast(*data.chunk(c).get())); + const auto& arr = static_cast(*data.chunk(c)); for (int64_t i = 0; i < arr.length(); ++i) { if (arr.IsNull(i)) { @@ -856,7 +866,7 @@ class BoolBlock : public PandasBlock { uint8_t* out_buffer = reinterpret_cast(block_data_) + rel_placement * num_rows_; - ConvertBooleanNoNulls(options_, *col->data().get(), out_buffer); + ConvertBooleanNoNulls(options_, *col->data(), out_buffer); placement_data_[rel_placement] = abs_placement; return Status::OK(); } @@ -884,7 +894,7 @@ class DatetimeBlock : public PandasBlock { int64_t* out_buffer = reinterpret_cast(block_data_) + rel_placement * num_rows_; - const ChunkedArray& data = *col.get()->data(); + const ChunkedArray& data = *col->data(); if (type == Type::DATE32) { // Convert from days since epoch to datetime64[ns] @@ -1089,7 +1099,7 @@ class CategoricalBlock : public PandasBlock { Status AllocateNDArrayFromIndices(int npy_type, const PrimitiveArray& indices) { npy_intp block_dims[1] = {num_rows_}; - auto in_values = reinterpret_cast(indices.raw_values()); + const T* in_values = GetPrimitiveValues(indices); void* data = const_cast(in_values); PyAcquireGIL lock; @@ -1420,6 +1430,7 @@ class ArrowDeserializer { PyAcquireGIL lock; result_ = NewArray1DFromType(col_->type().get(), type, col_->length(), nullptr); + RETURN_IF_PYERROR(); arr_ = reinterpret_cast(result_); return Status::OK(); } @@ -1429,8 +1440,7 @@ class ArrowDeserializer { std::shared_ptr arr) { typedef typename internal::arrow_traits::T T; - const auto& prim_arr = static_cast(*arr); - auto in_values = reinterpret_cast(prim_arr.raw_values()); + const T* in_values = GetPrimitiveValues(*arr); // Zero-Copy. We can pass the data pointer directly to NumPy. void* data = const_cast(in_values); @@ -1528,8 +1538,8 @@ class ArrowDeserializer { constexpr int64_t kShift = traits::npy_shift; for (int c = 0; c < data_.num_chunks(); c++) { - const auto& arr = static_cast(*data_.chunk(c)); - auto in_values = reinterpret_cast(arr.raw_values()); + const auto& arr = *data_.chunk(c); + const c_type* in_values = GetPrimitiveValues(arr); for (int64_t i = 0; i < arr.length(); ++i) { *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]) / kShift; From 543808d5b2b64df008b57efec2ab9057a9bdc723 Mon Sep 17 00:00:00 2001 From: Shixiong Zhu Date: Thu, 7 Dec 2017 20:20:07 -0500 Subject: [PATCH 123/177] ARROW-1864: [Java] Upgrade Netty to 4.1.17 Upgrade Netty to 4.1.17 since the Netty community will deprecate 4.0.x soon. This PR includes the following changes: - Bump Netty version. - Implement new ByteBuf APIs added in Netty 4.1.x: a bunch of get/setXXXLE methods. They are the opposite of get/setXXX method regarding byte order. E.g., as ArrowBuf is little endian, `setInt` will put an `int` to the buffer in little endian byte order, while `setIntLE` will put `int` in big byte endian order. The method naming seems confusing anyway, and I opened a Netty issue: https://github.com/netty/netty/issues/7465. The user can call these new methods to get or set multi-byte integers in big endian byte order. - Make ArrowByteBufAllocator overwrite AbstractByteBufAllocator. Author: Shixiong Zhu Closes #1376 from zsxwing/ARROW-1864 and squashes the following commits: 96a93e18 [Shixiong Zhu] extend AbstractByteBufAllocator; add javadoc for new methods bb973335 [Shixiong Zhu] Add comment for calculateNewCapacity 555f88ae [Shixiong Zhu] Add methods back 5e09cca6 [Shixiong Zhu] Upgrade Netty to 4.1.x --- .../main/java/io/netty/buffer/ArrowBuf.java | 199 +++++++++++++++++- .../netty/buffer/MutableWrappedByteBuf.java | 116 +++++++++- .../arrow/memory/ArrowByteBufAllocator.java | 15 +- java/pom.xml | 2 +- .../arrow/vector/util/MapWithOrdinal.java | 4 +- 5 files changed, 320 insertions(+), 16 deletions(-) diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java index e2bbe35480b66..23f5d65fbb550 100644 --- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java +++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java @@ -23,6 +23,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.channels.FileChannel; import java.nio.channels.GatheringByteChannel; import java.nio.channels.ScatteringByteChannel; import java.nio.charset.Charset; @@ -493,6 +494,16 @@ public ArrowBuf retain() { return retain(1); } + @Override + public ByteBuf touch() { + return this; + } + + @Override + public ByteBuf touch(Object hint) { + return this; + } + @Override public long getLong(int index) { chk(index, 8); @@ -505,6 +516,17 @@ public float getFloat(int index) { return Float.intBitsToFloat(getInt(index)); } + /** + * Gets a 64-bit long integer at the specified absolute {@code index} in + * this buffer in Big Endian Byte Order. + */ + @Override + public long getLongLE(int index) { + chk(index, 8); + final long v = PlatformDependent.getLong(addr(index)); + return Long.reverseBytes(v); + } + @Override public double getDouble(int index) { return Double.longBitsToDouble(getLong(index)); @@ -527,6 +549,17 @@ public int getInt(int index) { return v; } + /** + * Gets a 32-bit integer at the specified absolute {@code index} in + * this buffer in Big Endian Byte Order. + */ + @Override + public int getIntLE(int index) { + chk(index, 4); + final int v = PlatformDependent.getInt(addr(index)); + return Integer.reverseBytes(v); + } + @Override public int getUnsignedShort(int index) { return getShort(index) & 0xFFFF; @@ -535,10 +568,44 @@ public int getUnsignedShort(int index) { @Override public short getShort(int index) { chk(index, 2); - short v = PlatformDependent.getShort(addr(index)); + final short v = PlatformDependent.getShort(addr(index)); return v; } + /** + * Gets a 16-bit short integer at the specified absolute {@code index} in + * this buffer in Big Endian Byte Order. + */ + @Override + public short getShortLE(int index) { + final short v = PlatformDependent.getShort(addr(index)); + return Short.reverseBytes(v); + } + + /** + * Gets an unsigned 24-bit medium integer at the specified absolute + * {@code index} in this buffer. + */ + @Override + public int getUnsignedMedium(int index) { + chk(index, 3); + final long addr = addr(index); + return (PlatformDependent.getByte(addr) & 0xff) << 16 | + (PlatformDependent.getShort(addr + 1) & 0xffff); + } + + /** + * Gets an unsigned 24-bit medium integer at the specified absolute {@code index} in + * this buffer in Big Endian Byte Order. + */ + @Override + public int getUnsignedMediumLE(int index) { + chk(index, 3); + final long addr = addr(index); + return (PlatformDependent.getByte(addr) & 0xff) | + (Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) << 8; + } + @Override public ArrowBuf setShort(int index, int value) { chk(index, 2); @@ -546,6 +613,44 @@ public ArrowBuf setShort(int index, int value) { return this; } + /** + * Sets the specified 16-bit short integer at the specified absolute {@code index} + * in this buffer with Big Endian byte order. + */ + @Override + public ByteBuf setShortLE(int index, int value) { + chk(index, 2); + PlatformDependent.putShort(addr(index), Short.reverseBytes((short) value)); + return this; + } + + /** + * Sets the specified 24-bit medium integer at the specified absolute + * {@code index} in this buffer. + */ + @Override + public ByteBuf setMedium(int index, int value) { + chk(index, 3); + final long addr = addr(index); + PlatformDependent.putByte(addr, (byte) (value >>> 16)); + PlatformDependent.putShort(addr + 1, (short) value); + return this; + } + + + /** + * Sets the specified 24-bit medium integer at the specified absolute {@code index} + * in this buffer with Big Endian byte order. + */ + @Override + public ByteBuf setMediumLE(int index, int value) { + chk(index, 3); + final long addr = addr(index); + PlatformDependent.putByte(addr, (byte) value); + PlatformDependent.putShort(addr + 1, Short.reverseBytes((short) (value >>> 8))); + return this; + } + @Override public ArrowBuf setInt(int index, int value) { chk(index, 4); @@ -553,6 +658,17 @@ public ArrowBuf setInt(int index, int value) { return this; } + /** + * Sets the specified 32-bit integer at the specified absolute {@code index} + * in this buffer with Big Endian byte order. + */ + @Override + public ByteBuf setIntLE(int index, int value) { + chk(index, 4); + PlatformDependent.putInt(addr(index), Integer.reverseBytes(value)); + return this; + } + @Override public ArrowBuf setLong(int index, long value) { chk(index, 8); @@ -560,6 +676,17 @@ public ArrowBuf setLong(int index, long value) { return this; } + /** + * Sets the specified 64-bit long integer at the specified absolute {@code index} + * in this buffer with Big Endian byte order. + */ + @Override + public ByteBuf setLongLE(int index, long value) { + chk(index, 8); + PlatformDependent.putLong(addr(index), Long.reverseBytes(value)); + return this; + } + @Override public ArrowBuf setChar(int index, int value) { chk(index, 2); @@ -668,16 +795,46 @@ protected short _getShort(int index) { return getShort(index); } + /** @see {@link #getShortLE(int)} */ + @Override + protected short _getShortLE(int index) { + return getShortLE(index); + } + @Override protected int _getInt(int index) { return getInt(index); } + /** @see {@link #getIntLE(int)} */ + @Override + protected int _getIntLE(int index) { + return getIntLE(index); + } + + /** @see {@link #getUnsignedMedium(int)} */ + @Override + protected int _getUnsignedMedium(int index) { + return getUnsignedMedium(index); + } + + /** @see {@link #getUnsignedMediumLE(int)} */ + @Override + protected int _getUnsignedMediumLE(int index) { + return getUnsignedMediumLE(index); + } + @Override protected long _getLong(int index) { return getLong(index); } + /** @see {@link #getLongLE(int)} */ + @Override + protected long _getLongLE(int index) { + return getLongLE(index); + } + @Override protected void _setByte(int index, int value) { setByte(index, value); @@ -688,21 +845,45 @@ protected void _setShort(int index, int value) { setShort(index, value); } + /** @see {@link #setShortLE(int, int)} */ + @Override + protected void _setShortLE(int index, int value) { + setShortLE(index, value); + } + @Override protected void _setMedium(int index, int value) { setMedium(index, value); } + /** @see {@link #setMediumLE(int, int)} */ + @Override + protected void _setMediumLE(int index, int value) { + setMediumLE(index, value); + } + @Override protected void _setInt(int index, int value) { setInt(index, value); } + /** @see {@link #setIntLE(int, int)} */ + @Override + protected void _setIntLE(int index, int value) { + setIntLE(index, value); + } + @Override protected void _setLong(int index, long value) { setLong(index, value); } + /** @see {@link #setLongLE(int, long)} */ + @Override + public void _setLongLE(int index, long value) { + setLongLE(index, value); + } + @Override public ArrowBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) { udle.getBytes(index + offset, dst, dstIndex, length); @@ -716,16 +897,13 @@ public ArrowBuf getBytes(int index, OutputStream out, int length) throws IOExcep } @Override - protected int _getUnsignedMedium(int index) { - final long addr = addr(index); - return (PlatformDependent.getByte(addr) & 0xff) << 16 | - (PlatformDependent.getByte(addr + 1) & 0xff) << 8 | - PlatformDependent.getByte(addr + 2) & 0xff; + public int getBytes(int index, GatheringByteChannel out, int length) throws IOException { + return udle.getBytes(index + offset, out, length); } @Override - public int getBytes(int index, GatheringByteChannel out, int length) throws IOException { - return udle.getBytes(index + offset, out, length); + public int getBytes(int index, FileChannel out, long position, int length) throws IOException { + return udle.getBytes(index + offset, out, position, length); } @Override @@ -776,6 +954,11 @@ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOEx return udle.setBytes(index + offset, in, length); } + @Override + public int setBytes(int index, FileChannel in, long position, int length) throws IOException { + return udle.setBytes(index + offset, in, position, length); + } + @Override public byte getByte(int index) { chk(index, 1); diff --git a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java index a5683adccbc32..f0bc84cdc2db2 100644 --- a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java +++ b/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java @@ -23,9 +23,12 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.channels.FileChannel; import java.nio.channels.GatheringByteChannel; import java.nio.channels.ScatteringByteChannel; +import io.netty.util.ByteProcessor; + /** * This is basically a complete copy of DuplicatedByteBuf. We copy because we want to override * some behaviors and make @@ -128,6 +131,16 @@ protected short _getShort(int index) { return buffer.getShort(index); } + @Override + public short getShortLE(int index) { + return buffer.getShortLE(index); + } + + @Override + protected short _getShortLE(int index) { + return buffer.getShortLE(index); + } + @Override public int getUnsignedMedium(int index) { return _getUnsignedMedium(index); @@ -138,6 +151,16 @@ protected int _getUnsignedMedium(int index) { return buffer.getUnsignedMedium(index); } + @Override + public int getUnsignedMediumLE(int index) { + return buffer.getUnsignedMediumLE(index); + } + + @Override + protected int _getUnsignedMediumLE(int index) { + return buffer.getUnsignedMediumLE(index); + } + @Override public int getInt(int index) { return _getInt(index); @@ -148,6 +171,16 @@ protected int _getInt(int index) { return buffer.getInt(index); } + @Override + public int getIntLE(int index) { + return buffer.getIntLE(index); + } + + @Override + protected int _getIntLE(int index) { + return buffer.getIntLE(index); + } + @Override public long getLong(int index) { return _getLong(index); @@ -158,6 +191,16 @@ protected long _getLong(int index) { return buffer.getLong(index); } + @Override + public long getLongLE(int index) { + return buffer.getLongLE(index); + } + + @Override + protected long _getLongLE(int index) { + return buffer.getLongLE(index); + } + @Override public abstract ByteBuf copy(int index, int length); @@ -206,6 +249,17 @@ protected void _setShort(int index, int value) { buffer.setShort(index, value); } + @Override + public ByteBuf setShortLE(int index, int value) { + buffer.setShortLE(index, value); + return this; + } + + @Override + protected void _setShortLE(int index, int value) { + buffer.setShortLE(index, value); + } + @Override public ByteBuf setMedium(int index, int value) { _setMedium(index, value); @@ -217,6 +271,17 @@ protected void _setMedium(int index, int value) { buffer.setMedium(index, value); } + @Override + public ByteBuf setMediumLE(int index, int value) { + buffer.setMediumLE(index, value); + return this; + } + + @Override + protected void _setMediumLE(int index, int value) { + buffer.setMediumLE(index, value); + } + @Override public ByteBuf setInt(int index, int value) { _setInt(index, value); @@ -228,6 +293,17 @@ protected void _setInt(int index, int value) { buffer.setInt(index, value); } + @Override + public ByteBuf setIntLE(int index, int value) { + buffer.setIntLE(index, value); + return this; + } + + @Override + protected void _setIntLE(int index, int value) { + buffer.setIntLE(index, value); + } + @Override public ByteBuf setLong(int index, long value) { _setLong(index, value); @@ -239,6 +315,17 @@ protected void _setLong(int index, long value) { buffer.setLong(index, value); } + @Override + public ByteBuf setLongLE(int index, long value) { + buffer.setLongLE(index, value); + return this; + } + + @Override + protected void _setLongLE(int index, long value) { + buffer.setLongLE(index, value); + } + @Override public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) { buffer.setBytes(index, src, srcIndex, length); @@ -257,6 +344,12 @@ public ByteBuf setBytes(int index, ByteBuffer src) { return this; } + @Override + public int setBytes(int index, FileChannel in, long position, int length) + throws IOException { + return buffer.setBytes(index, in, position, length); + } + @Override public ByteBuf getBytes(int index, OutputStream out, int length) throws IOException { @@ -282,6 +375,13 @@ public int setBytes(int index, ScatteringByteChannel in, int length) return buffer.setBytes(index, in, length); } + + @Override + public int getBytes(int index, FileChannel out, long position, int length) + throws IOException { + return buffer.getBytes(index, out, position, length); + } + @Override public int nioBufferCount() { return buffer.nioBufferCount(); @@ -298,12 +398,12 @@ public ByteBuffer internalNioBuffer(int index, int length) { } @Override - public int forEachByte(int index, int length, ByteBufProcessor processor) { + public int forEachByte(int index, int length, ByteProcessor processor) { return buffer.forEachByte(index, length, processor); } @Override - public int forEachByteDesc(int index, int length, ByteBufProcessor processor) { + public int forEachByteDesc(int index, int length, ByteProcessor processor) { return buffer.forEachByteDesc(index, length, processor); } @@ -312,6 +412,18 @@ public final int refCnt() { return unwrap().refCnt(); } + @Override + public final ByteBuf touch() { + unwrap().touch(); + return this; + } + + @Override + public final ByteBuf touch(Object hint) { + unwrap().touch(hint); + return this; + } + @Override public final ByteBuf retain() { unwrap().retain(); diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java index b8b5283423c82..94102992139d8 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java @@ -18,8 +18,8 @@ package org.apache.arrow.memory; +import io.netty.buffer.AbstractByteBufAllocator; import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.ExpandableByteBuf; @@ -32,7 +32,7 @@ * otherwise non-expandable * ArrowBufs to be expandable. */ -public class ArrowByteBufAllocator implements ByteBufAllocator { +public class ArrowByteBufAllocator extends AbstractByteBufAllocator { private static final int DEFAULT_BUFFER_SIZE = 4096; private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16; @@ -142,8 +142,17 @@ public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) { throw fail(); } + @Override + protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) { + throw fail(); + } + + @Override + protected ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity) { + return buffer(initialCapacity, maxCapacity); + } + private RuntimeException fail() { throw new UnsupportedOperationException("Allocator doesn't support heap-based memory."); } - } diff --git a/java/pom.xml b/java/pom.xml index c479d651f6b5f..162c53460cc93 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -32,7 +32,7 @@ 4.11 1.7.25 18.0 - 4.0.49.Final + 4.1.17.Final 2.7.9 2.7.1 1.2.0-3f79e055 diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java index 6d3b390379a56..b863fa8af86fd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java @@ -134,9 +134,9 @@ public Set keySet() { @Override public Collection values() { - return Lists.newArrayList(Iterables.transform(secondary.entries(), new Function, V>() { + return Lists.newArrayList(Iterables.transform(secondary.entries(), new Function, V>() { @Override - public V apply(IntObjectMap.Entry entry) { + public V apply(IntObjectMap.PrimitiveEntry entry) { return Preconditions.checkNotNull(entry).value(); } })); From fe33155099e9893470fc938b458b92d776983ef0 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 8 Dec 2017 11:41:45 +0100 Subject: [PATCH 124/177] ARROW-1901: [Python] Support recursive mkdir for DaskFilesystem Author: Uwe L. Korn Closes #1401 from xhochy/ARROW-1901 and squashes the following commits: 4a86b4b [Uwe L. Korn] ARROW-1901: [Python] Support recursive mkdir for DaskFilesystem --- python/pyarrow/filesystem.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py index 926df0e30e565..ff7809575620a 100644 --- a/python/pyarrow/filesystem.py +++ b/python/pyarrow/filesystem.py @@ -275,8 +275,11 @@ def exists(self, path): return self.fs.exists(path) @implements(FileSystem.mkdir) - def mkdir(self, path): - return self.fs.mkdir(path) + def mkdir(self, path, create_parents=True): + if create_parents: + return self.fs.mkdirs(path) + else: + return self.fs.mkdir(path) @implements(FileSystem.open) def open(self, path, mode='rb'): From fcf46d9d947eb3a31c63be3e65118cd69cdd2976 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 8 Dec 2017 11:42:19 +0100 Subject: [PATCH 125/177] =?UTF-8?q?ARROW-1902:=20[Python]=C2=A0Remove=20mk?= =?UTF-8?q?dir=20race=20condition=20from=20write=5Fto=5Fdataset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author: Uwe L. Korn Closes #1402 from xhochy/ARROW-1902 and squashes the following commits: 199e101 [Uwe L. Korn] ARROW-1902: [Python] Remove mkdir race condition from write_to_dataset --- python/pyarrow/parquet.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index 9fb890c6e6b8d..d9f1bd2c36430 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -1013,7 +1013,10 @@ def write_to_dataset(table, root_path, partition_cols=None, fs = _ensure_filesystem(filesystem) if fs._isfilestore() and not fs.exists(root_path): - fs.mkdir(root_path) + try: + fs.mkdir(root_path) + except OSError: + assert fs.exists(root_path) if partition_cols is not None and len(partition_cols) > 0: df = table.to_pandas() From 74b2ebed6e19f21680d7b49a43fc0fa101c7b6fb Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 9 Dec 2017 23:39:49 -0500 Subject: [PATCH 126/177] ARROW-1908: [Python] Construction of arrow table from pandas DataFrame with duplicate column names crashes Author: Phillip Cloud Closes #1405 from cpcloud/ARROW-1908 and squashes the following commits: 86b2b21e [Phillip Cloud] ARROW-1908: [Python] Construction of arrow table from pandas DataFrame with duplicate column names crashes --- cpp/src/arrow/python/builtin_convert.cc | 32 +++++++++++++-------- python/pyarrow/pandas_compat.py | 6 ++++ python/pyarrow/tests/test_convert_pandas.py | 5 ++++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc index 08cbae7ab2da4..cd88d557d4830 100644 --- a/cpp/src/arrow/python/builtin_convert.cc +++ b/cpp/src/arrow/python/builtin_convert.cc @@ -148,15 +148,14 @@ static constexpr int MAX_NESTING_LEVELS = 32; // SeqVisitor is used to infer the type. class SeqVisitor { public: - SeqVisitor() : max_nesting_level_(0), max_observed_level_(0) { - memset(nesting_histogram_, 0, MAX_NESTING_LEVELS * sizeof(int)); + SeqVisitor() : max_nesting_level_(0), max_observed_level_(0), nesting_histogram_() { + std::fill(nesting_histogram_, nesting_histogram_ + MAX_NESTING_LEVELS, 0); } // co-recursive with VisitElem Status Visit(PyObject* obj, int level = 0) { - if (level > max_nesting_level_) { - max_nesting_level_ = level; - } + max_nesting_level_ = std::max(max_nesting_level_, level); + // Loop through either a sequence or an iterator. if (PySequence_Check(obj)) { Py_ssize_t size = PySequence_Size(obj); @@ -165,18 +164,26 @@ class SeqVisitor { if (PyArray_Check(obj)) { auto array = reinterpret_cast(obj); auto ptr = reinterpret_cast(PyArray_GETPTR1(array, i)); + ref.reset(PyArray_GETITEM(array, ptr)); + RETURN_IF_PYERROR(); + RETURN_NOT_OK(VisitElem(ref, level)); } else { ref.reset(PySequence_GetItem(obj, i)); + RETURN_IF_PYERROR(); RETURN_NOT_OK(VisitElem(ref, level)); } } } else if (PyObject_HasAttrString(obj, "__iter__")) { - OwnedRef iter = OwnedRef(PyObject_GetIter(obj)); - PyObject* item; + OwnedRef iter(PyObject_GetIter(obj)); + RETURN_IF_PYERROR(); + + PyObject* item = NULLPTR; while ((item = PyIter_Next(iter.obj()))) { - OwnedRef ref = OwnedRef(item); + RETURN_IF_PYERROR(); + + OwnedRef ref(item); RETURN_NOT_OK(VisitElem(ref, level)); } } else { @@ -242,6 +249,7 @@ class SeqVisitor { // Visits a specific element (inner part of the loop). Status VisitElem(const OwnedRef& item_ref, int level) { + DCHECK_NE(item_ref.obj(), NULLPTR); if (PyList_Check(item_ref.obj())) { RETURN_NOT_OK(Visit(item_ref.obj(), level + 1)); } else if (PyDict_Check(item_ref.obj())) { @@ -323,7 +331,7 @@ class SeqConverter { virtual Status AppendData(PyObject* seq, int64_t size) = 0; - virtual ~SeqConverter() {} + virtual ~SeqConverter() = default; protected: ArrayBuilder* builder_; @@ -459,13 +467,13 @@ class UInt8Converter : public TypedConverterVisitor(PyLong_AsLongLong(item.obj())); + RETURN_IF_PYERROR(); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max())) { return Status::Invalid( "Cannot coerce values to array type that would " "lose data"); } - RETURN_IF_PYERROR(); return typed_builder_->Append(static_cast(val)); } }; @@ -474,13 +482,13 @@ class UInt16Converter : public TypedConverterVisitor(PyLong_AsLongLong(item.obj())); + RETURN_IF_PYERROR(); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max())) { return Status::Invalid( "Cannot coerce values to array type that would " "lose data"); } - RETURN_IF_PYERROR(); return typed_builder_->Append(static_cast(val)); } }; @@ -489,13 +497,13 @@ class UInt32Converter : public TypedConverterVisitor(PyLong_AsLongLong(item.obj())); + RETURN_IF_PYERROR(); if (ARROW_PREDICT_FALSE(val > std::numeric_limits::max())) { return Status::Invalid( "Cannot coerce values to array type that would " "lose data"); } - RETURN_IF_PYERROR(); return typed_builder_->Append(static_cast(val)); } }; diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 8459ec31bed54..48384ad478251 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -281,6 +281,12 @@ def dataframe_to_arrays(df, schema, preserve_index, nthreads=1): columns_to_convert = [] convert_types = [] + + if not df.columns.is_unique: + raise ValueError( + 'Duplicate column names found: {}'.format(list(df.columns)) + ) + for name in df.columns: col = df[name] if not isinstance(name, six.string_types): diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index e94ee4608ec5a..1231844e357d1 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -227,6 +227,11 @@ def test_zero_copy_success(self): result = pa.array([0, 1, 2]).to_pandas(zero_copy_only=True) npt.assert_array_equal(result, [0, 1, 2]) + def test_duplicate_column_names_does_not_crash(self): + df = pd.DataFrame([(1, 'a'), (2, 'b')], columns=list('aa')) + with pytest.raises(ValueError): + pa.Table.from_pandas(df) + def test_dictionary_indices_boundscheck(self): # ARROW-1658. No validation of indices leads to segfaults in pandas indices = [[0, 1], [0, -1]] From 501d60e918bd4d10c429ab34e0b8e8a87dffb732 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 10 Dec 2017 19:33:55 +0100 Subject: [PATCH 127/177] ARROW-1895/ARROW-1897: [Python] Add field_name to pandas index metadata Author: Phillip Cloud Author: Wes McKinney Closes #1397 from cpcloud/ARROW-1895 and squashes the following commits: 1293b24 [Wes McKinney] Extra metadata is None in py2 3f7760f [Phillip Cloud] Fix py2 test 3bc30fd [Phillip Cloud] Operator precedence 891671b [Phillip Cloud] Unicode bytes difference 3c41905 [Phillip Cloud] Cleaner construction cf52001 [Phillip Cloud] Use field_name to map arrow table field names to pandas names 37dca10 [Phillip Cloud] Fix string vs unicode in column_indexes and add field_name as well f570871 [Phillip Cloud] Use categorical codes instead of object 20bf15a [Phillip Cloud] ARROW-1895: [Python] Add field_name to pandas index metadata --- python/pyarrow/pandas_compat.py | 87 +++++++++++++++------ python/pyarrow/tests/test_convert_pandas.py | 65 +++++++++++++-- 2 files changed, 124 insertions(+), 28 deletions(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 48384ad478251..668048fd645dd 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -110,7 +110,11 @@ def get_logical_type_from_numpy(pandas_collection): except KeyError: if hasattr(pandas_collection.dtype, 'tz'): return 'datetimetz' - return infer_dtype(pandas_collection) + result = infer_dtype(pandas_collection) + + if result == 'string': + return 'bytes' if PY2 else 'unicode' + return result def get_extension_dtype_info(column): @@ -122,7 +126,7 @@ def get_extension_dtype_info(column): 'num_categories': len(cats.categories), 'ordered': cats.ordered, } - physical_dtype = 'object' + physical_dtype = str(cats.codes.dtype) elif hasattr(dtype, 'tz'): metadata = {'timezone': str(dtype.tz)} physical_dtype = 'datetime64[ns]' @@ -132,7 +136,7 @@ def get_extension_dtype_info(column): return physical_dtype, metadata -def get_column_metadata(column, name, arrow_type): +def get_column_metadata(column, name, arrow_type, field_name): """Construct the metadata for a given column Parameters @@ -140,6 +144,10 @@ def get_column_metadata(column, name, arrow_type): column : pandas.Series or pandas.Index name : str arrow_type : pyarrow.DataType + field_name : str + Equivalent to `name` when `column` is a `Series`, otherwise if `column` + is a pandas Index then `field_name` will not be the same as `name`. + This is the name of the field in the arrow Table's schema. Returns ------- @@ -164,6 +172,7 @@ def get_column_metadata(column, name, arrow_type): return { 'name': name, + 'field_name': str(field_name), 'pandas_type': logical_type, 'numpy_type': string_dtype, 'metadata': extra_metadata, @@ -193,10 +202,14 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types): index_types = types[ncolumns - len(index_levels):] column_metadata = [ - get_column_metadata(df[col_name], name=sanitized_name, - arrow_type=arrow_type) - for col_name, sanitized_name, arrow_type in - zip(df.columns, column_names, df_types) + get_column_metadata( + df[col_name], + name=sanitized_name, + arrow_type=arrow_type, + field_name=sanitized_name + ) for col_name, sanitized_name, arrow_type in zip( + df.columns, column_names, df_types + ) ] if preserve_index: @@ -204,9 +217,13 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types): index_level_name, range(len(index_levels)) )) index_column_metadata = [ - get_column_metadata(level, name=level.name, arrow_type=arrow_type) - for i, (level, arrow_type) in enumerate( - zip(index_levels, index_types) + get_column_metadata( + level, + name=level.name, + arrow_type=arrow_type, + field_name=field_name, + ) for i, (level, arrow_type, field_name) in enumerate( + zip(index_levels, index_types, index_column_names) ) ] @@ -214,9 +231,16 @@ def construct_metadata(df, column_names, index_levels, preserve_index, types): for level in getattr(df.columns, 'levels', [df.columns]): string_dtype, extra_metadata = get_extension_dtype_info(level) + + pandas_type = get_logical_type_from_numpy(level) + if pandas_type == 'unicode': + assert not extra_metadata + extra_metadata = {'encoding': 'UTF-8'} + column_index = { 'name': level.name, - 'pandas_type': get_logical_type_from_numpy(level), + 'field_name': level.name, + 'pandas_type': pandas_type, 'numpy_type': string_dtype, 'metadata': extra_metadata, } @@ -442,7 +466,6 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1, schema = table.schema row_count = table.num_rows metadata = schema.metadata - columns_metadata = None has_pandas_metadata = metadata is not None and b'pandas' in metadata @@ -452,13 +475,36 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1, columns = pandas_metadata['columns'] column_indexes = pandas_metadata.get('column_indexes', []) table = _add_any_metadata(table, pandas_metadata) - columns_metadata = pandas_metadata.get('columns', None) block_table = table + index_columns_set = frozenset(index_columns) + + # 0. 'field_name' is the name of the column in the arrow Table + # 1. 'name' is the user-facing name of the column, that is, it came from + # pandas + # 2. 'field_name' and 'name' differ for index columns + # 3. We fall back on c['name'] for backwards compatibility + logical_index_names = [ + c['name'] for c in columns + if c.get('field_name', c['name']) in index_columns_set + ] + + # There must be the same number of field names and physical names + # (fields in the arrow Table) + assert len(logical_index_names) == len(index_columns_set) + + # It can never be the case in a released version of pyarrow that + # c['name'] is None *and* 'field_name' is not a key in the column metadata, + # because the change to allow c['name'] to be None and the change to add + # 'field_name' are in the same release (0.8.0) + assert all( + (c['name'] is None and 'field_name' in c) or c['name'] is not None + for c in columns + ) + # Build up a list of index columns and names while removing those columns # from the original table - logical_index_names = [c['name'] for c in columns[-len(index_columns):]] for raw_name, logical_name in zip(index_columns, logical_index_names): i = schema.get_field_index(raw_name) if i != -1: @@ -489,15 +535,12 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1, index = pd.RangeIndex(row_count) column_strings = [x.name for x in block_table.itercolumns()] - if columns_metadata is not None: - columns_name_dict = dict( - (str(x['name']), x['name']) - for x in columns_metadata - ) + if columns: + columns_name_dict = { + c.get('field_name', str(c['name'])): c['name'] for c in columns + } columns_values = [ - columns_name_dict[y] - if y in columns_name_dict.keys() else y - for y in column_strings + columns_name_dict.get(name, name) for name in column_strings ] else: columns_values = column_strings diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index 1231844e357d1..97bbb6a175b0f 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -30,7 +30,7 @@ import pandas as pd import pandas.util.testing as tm -from pyarrow.compat import u +from pyarrow.compat import u, PY2 import pyarrow as pa import pyarrow.types as patypes @@ -160,9 +160,41 @@ def test_integer_index_column(self): df = pd.DataFrame([(1, 'a'), (2, 'b'), (3, 'c')]) _check_pandas_roundtrip(df, preserve_index=True) + def test_index_metadata_field_name(self): + # test None case, and strangely named non-index columns + df = pd.DataFrame( + [(1, 'a', 3.1), (2, 'b', 2.2), (3, 'c', 1.3)], + index=pd.MultiIndex.from_arrays( + [['c', 'b', 'a'], [3, 2, 1]], + names=[None, 'foo'] + ), + columns=['a', None, '__index_level_0__'], + ) + t = pa.Table.from_pandas(df, preserve_index=True) + raw_metadata = t.schema.metadata + + js = json.loads(raw_metadata[b'pandas'].decode('utf8')) + + col1, col2, col3, idx0, foo = js['columns'] + + assert col1['name'] == 'a' + assert col1['name'] == col1['field_name'] + + assert col2['name'] is None + assert col2['field_name'] == 'None' + + assert col3['name'] == '__index_level_0__' + assert col3['name'] == col3['field_name'] + + idx0_name, foo_name = js['index_columns'] + assert idx0_name == '__index_level_0__' + assert idx0['field_name'] == idx0_name + assert idx0['name'] is None + + assert foo_name == '__index_level_1__' + assert foo['name'] == 'foo' + def test_categorical_column_index(self): - # I *really* hope no one uses category dtypes for single level column - # indexes df = pd.DataFrame( [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)], columns=pd.Index(list('def'), dtype='category') @@ -174,15 +206,36 @@ def test_categorical_column_index(self): column_indexes, = js['column_indexes'] assert column_indexes['name'] is None assert column_indexes['pandas_type'] == 'categorical' - assert column_indexes['numpy_type'] == 'object' + assert column_indexes['numpy_type'] == 'int8' md = column_indexes['metadata'] assert md['num_categories'] == 3 assert md['ordered'] is False + def test_string_column_index(self): + df = pd.DataFrame( + [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)], + columns=pd.Index(list('def'), name='stringz') + ) + t = pa.Table.from_pandas(df, preserve_index=True) + raw_metadata = t.schema.metadata + js = json.loads(raw_metadata[b'pandas'].decode('utf8')) + + column_indexes, = js['column_indexes'] + assert column_indexes['name'] == 'stringz' + assert column_indexes['name'] == column_indexes['field_name'] + assert column_indexes['pandas_type'] == ('bytes' if PY2 else 'unicode') + assert column_indexes['numpy_type'] == 'object' + + md = column_indexes['metadata'] + + if not PY2: + assert len(md) == 1 + assert md['encoding'] == 'UTF-8' + else: + assert md is None or 'encoding' not in md + def test_datetimetz_column_index(self): - # I *really* hope no one uses category dtypes for single level column - # indexes df = pd.DataFrame( [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)], columns=pd.date_range( From 84f18c579f4092ba21e1d077c4a1823432022010 Mon Sep 17 00:00:00 2001 From: Lewis John McGibbney Date: Sun, 10 Dec 2017 15:25:39 -0800 Subject: [PATCH 128/177] ARROW-1910: [C++] Use c_glib Brewfile in README for installing dependencies on macOS (#1407) --- cpp/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/README.md b/cpp/README.md index 2034d29fae324..92e87cb436ada 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -42,7 +42,7 @@ sudo apt-get install cmake \ On OS X, you can use [Homebrew][1]: ```shell -brew update && brew bundle --file=cpp/Brewfile +brew update && brew bundle --file=c_glib/Brewfile ``` If you are developing on Windows, see the [Windows developer guide][2]. From 97678c106054c4d9a738ddcb1c895a6ab7ca9d31 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 10 Dec 2017 18:40:09 -0500 Subject: [PATCH 129/177] ARROW-1883: [Python] Fix handling of metadata in to_pandas when not all columns are present This closes [ARROW-1883](https://issues.apache.org/jira/browse/ARROW-1883). So basically what I did in `_add_any_metadata` was replacing `col = table[i]` with: ``` idx = schema.get_field_index(raw_name) if idx != -1: col = table[idx] ``` to check that the column is actually present in the schema. However, that involved some more code to get to `raw_name` (the name how the column is present in the schema), as this does not always match the name in `pandas_metadata['column'][..]['name']`. Not sure if there is a better way to get that name. (or if it would be better to filter `pandas_metadata` earlier on, instead of checking when actually trying to process the metadata of that column) Author: Joris Van den Bossche Closes #1386 from jorisvandenbossche/parquet-column-selection and squashes the following commits: 3f605ef3 [Joris Van den Bossche] add test for compatibility for arrow 0.7.1 written parquet files ea891b25 [Joris Van den Bossche] Use field_name if available f6bdd1d3 [Joris Van den Bossche] Merge remote-tracking branch 'upstream/master' into parquet-column-selection 5df51f18 [Joris Van den Bossche] Add additional tests 306eaba3 [Joris Van den Bossche] ARROW-1883: [Python] Fix handling of metadata in to_pandas when not all columns are present --- python/pyarrow/pandas_compat.py | 39 +++++++++++++----- .../v0.7.1.column-metadata-handling.parquet | Bin 0 -> 2012 bytes python/pyarrow/tests/test_convert_pandas.py | 31 ++++++++++++++ python/pyarrow/tests/test_parquet.py | 23 +++++++++++ 4 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 python/pyarrow/tests/data/v0.7.1.column-metadata-handling.parquet diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 668048fd645dd..b5d395fe5700b 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -641,19 +641,36 @@ def _add_any_metadata(table, pandas_metadata): schema = table.schema + index_columns = pandas_metadata['index_columns'] + n_index_levels = len(index_columns) + n_columns = len(pandas_metadata['columns']) - n_index_levels + # Add time zones for i, col_meta in enumerate(pandas_metadata['columns']): - if col_meta['pandas_type'] == 'datetimetz': - col = table[i] - converted = col.to_pandas() - tz = col_meta['metadata']['timezone'] - tz_aware_type = pa.timestamp('ns', tz=tz) - with_metadata = pa.Array.from_pandas(converted.values, - type=tz_aware_type) - - field = pa.field(schema[i].name, tz_aware_type) - modified_columns[i] = pa.Column.from_array(field, - with_metadata) + + raw_name = col_meta.get('field_name') + if not raw_name: + # deal with metadata written with arrow < 0.8 + raw_name = col_meta['name'] + if i >= n_columns: + # index columns + raw_name = index_columns[i - n_columns] + if raw_name is None: + raw_name = 'None' + + idx = schema.get_field_index(raw_name) + if idx != -1: + if col_meta['pandas_type'] == 'datetimetz': + col = table[idx] + converted = col.to_pandas() + tz = col_meta['metadata']['timezone'] + tz_aware_type = pa.timestamp('ns', tz=tz) + with_metadata = pa.Array.from_pandas(converted.values, + type=tz_aware_type) + + field = pa.field(schema[idx].name, tz_aware_type) + modified_columns[idx] = pa.Column.from_array(field, + with_metadata) if len(modified_columns) > 0: columns = [] diff --git a/python/pyarrow/tests/data/v0.7.1.column-metadata-handling.parquet b/python/pyarrow/tests/data/v0.7.1.column-metadata-handling.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d48041f518d55ce67b24862732e29960932d6ff9 GIT binary patch literal 2012 zcmchYO>5LZ7{_PQY|O3&UvNSef-GSTi`~ZE?u#!7OU0rG(WUj)mXOVkx{^)SO|sZ7 zy@&@qLDkRVMYL(JkK=FT>R{C*=X=07zm;rQH$Q?eCV+}WmM zzY(W=#VaVoC|itj4{Gmb2<8`#a3;wV5~7T=`iFB399+J0a61LS8i&s=k$Lp>MV`!Y zR<?M@mLTw}xuwSQ?Iw*wY(Y7UYVixV5KD6kZVT3$}jR)8^%7!sv z*(gKEex#3R8JD$dKo&gf3GSLGL;CU(N1K~;Xofzu=u;RA{R3#0M8~d>Lyy)HxX1?v zBq6J0*N*kG(T=yfwc~9-?WnDzo$TpK-vvnkSw$9lrqeb(`Aur5Mjd3icF=V^^+s+r zim1h$8f!0gokXKrV}Sc!$8|`BTGAI5^pf7jgVJ23T+tt2EthJQ%5|zSAv{!_pxfIp z{EZ%_WgW+_S7}Zc`)1qpO%ikh+dkYQmg$(?u+&tyin(dMN49JJtKvj1(`+Sa5-jX@ zy0~6lb-ZG3Dd@XBeC1X@@H}jL>r?bYmF8}`P6#=bOy!w*?8TV8ze$xCiIqg+ww_|k zpV~0luE6P7ZW~WZhNqk)%LxAK3&tC3YkdpI#GU@L0Q>V+ua_J5SFWsA^z!`5{l(_W P-3JfBdmex$`LF*5ATPr- literal 0 HcmV?d00001 diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index 97bbb6a175b0f..7609d3488c6c0 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -1286,6 +1286,37 @@ def test_array_from_pandas_typed_array_with_mask(self, t, data, expected): assert pa.Array.from_pandas(expected, type=pa.list_(t())).equals(result) + def test_table_column_subset_metadata(self): + # ARROW-1883 + df = pd.DataFrame({ + 'a': [1, 2, 3], + 'b': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')}) + table = pa.Table.from_pandas(df) + + table_subset = table.remove_column(1) + result = table_subset.to_pandas() + tm.assert_frame_equal(result, df[['a']]) + + table_subset2 = table_subset.remove_column(1) + result = table_subset2.to_pandas() + tm.assert_frame_equal(result, df[['a']]) + + # non-default index + for index in [ + pd.Index(['a', 'b', 'c'], name='index'), + pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')]: + df = pd.DataFrame({'a': [1, 2, 3], + 'b': [.1, .2, .3]}, index=index) + table = pa.Table.from_pandas(df) + + table_subset = table.remove_column(1) + result = table_subset.to_pandas() + tm.assert_frame_equal(result, df[['a']]) + + table_subset2 = table_subset.remove_column(1) + result = table_subset2.to_pandas() + tm.assert_frame_equal(result, df[['a']].reset_index(drop=True)) + def _fully_loaded_dataframe_example(): from distutils.version import LooseVersion diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 2543e7d17befd..79e24d8d4e6eb 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1570,6 +1570,29 @@ def test_backwards_compatible_index_multi_level_some_named(): tm.assert_frame_equal(result, expected) +@parquet +def test_backwards_compatible_column_metadata_handling(): + expected = pd.DataFrame( + {'a': [1, 2, 3], 'b': [.1, .2, .3], + 'c': pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')}) + expected.index = pd.MultiIndex.from_arrays( + [['a', 'b', 'c'], + pd.date_range("2017-01-01", periods=3, tz='Europe/Brussels')], + names=['index', None]) + + path = os.path.join( + os.path.dirname(__file__), 'data', + 'v0.7.1.column-metadata-handling.parquet' + ) + t = _read_table(path) + result = t.to_pandas() + tm.assert_frame_equal(result, expected) + + t = _read_table(path, columns=['a']) + result = t.to_pandas() + tm.assert_frame_equal(result, expected[['a']].reset_index(drop=True)) + + def test_decimal_roundtrip(tmpdir): num_values = 10 From 20bbbcba5da268186ae91c5fd002938a4bd59db3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 11 Dec 2017 10:06:25 -0500 Subject: [PATCH 130/177] ARROW-1906: [Python] Do not override user-supplied type in pyarrow.array when converting DatetimeTZ pandas data cc @BryanCutler Author: Wes McKinney Closes #1411 from wesm/ARROW-1906 and squashes the following commits: 49392e03 [Wes McKinney] Trust supplied user type in pyarrow.array when converting datetime+tz pandas data --- python/pyarrow/pandas_compat.py | 3 ++- python/pyarrow/tests/test_array.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index b5d395fe5700b..f08049f6adba0 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -369,7 +369,8 @@ def get_datetimetz_type(values, dtype, type_): if values.dtype.type != np.datetime64: return values, type_ - if isinstance(dtype, DatetimeTZDtype): + if isinstance(dtype, DatetimeTZDtype) and type_ is None: + # If no user type passed, construct a tz-aware timestamp type tz = dtype.tz unit = dtype.unit type_ = pa.timestamp(unit, tz) diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index 92562da14150c..fa38c9257854e 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -331,8 +331,12 @@ def test_cast_timestamp_unit(): s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York') us_with_tz = pa.timestamp('us', tz='America/New_York') + arr = pa.Array.from_pandas(s_nyc, type=us_with_tz) + # ARROW-1906 + assert arr.type == us_with_tz + arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us')) assert arr[0].as_py() == s_nyc[0] From abb934176ff3424663bab40b62afcc0eacf4ad2b Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 11 Dec 2017 10:09:16 -0500 Subject: [PATCH 131/177] ARROW-1905: [Python] Add more comprehensive list of exact type checking functions to pyarrow.types cc @BryanCutler Author: Wes McKinney Closes #1410 from wesm/ARROW-1905 and squashes the following commits: 93bddc9f [Wes McKinney] Add new functions to API, fix docstring dcd0829f [Wes McKinney] Add more exact type checking functions --- python/doc/source/api.rst | 15 ++++ python/pyarrow/tests/test_types.py | 23 ++++++ python/pyarrow/types.py | 117 +++++++++++++++++++++++++++-- 3 files changed, 149 insertions(+), 6 deletions(-) diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst index 636f41d67bc6a..2d3e39c6954a1 100644 --- a/python/doc/source/api.rst +++ b/python/doc/source/api.rst @@ -71,7 +71,18 @@ Type checking functions is_integer is_signed_integer is_unsigned_integer + is_int8 + is_int16 + is_int32 + is_int64 + is_uint8 + is_uint16 + is_uint32 + is_uint64 is_floating + is_float16 + is_float32 + is_float64 is_decimal is_list is_struct @@ -80,7 +91,11 @@ Type checking functions is_temporal is_timestamp is_date + is_date32 + is_date64 is_time + is_time32 + is_time64 is_null is_binary is_unicode diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py index e352e35a3a3a0..68dc499cf48b4 100644 --- a/python/pyarrow/tests/test_types.py +++ b/python/pyarrow/tests/test_types.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import pytest + import pyarrow as pa import pyarrow.types as types @@ -161,3 +163,24 @@ def test_types_hashable(): assert hash(type_) == hash(type_) in_dict[type_] = i assert in_dict[type_] == i + + +@pytest.mark.parametrize('t,check_func', [ + (pa.date32(), types.is_date32), + (pa.date64(), types.is_date64), + (pa.time32('s'), types.is_time32), + (pa.time64('ns'), types.is_time64), + (pa.int8(), types.is_int8), + (pa.int16(), types.is_int16), + (pa.int32(), types.is_int32), + (pa.int64(), types.is_int64), + (pa.uint8(), types.is_uint8), + (pa.uint16(), types.is_uint16), + (pa.uint32(), types.is_uint32), + (pa.uint64(), types.is_uint64), + (pa.float16(), types.is_float16), + (pa.float32(), types.is_float32), + (pa.float64(), types.is_float64) +]) +def test_exact_primitive_types(t, check_func): + assert check_func(t) diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py index 440d7eb09daa9..24557148a6363 100644 --- a/python/pyarrow/types.py +++ b/python/pyarrow/types.py @@ -32,6 +32,13 @@ _NESTED_TYPES = {lib.Type_LIST, lib.Type_STRUCT, lib.Type_UNION, lib.Type_MAP} +def is_null(t): + """ + Return True if value is an instance of a null type + """ + return t.id == lib.Type_NA + + def is_boolean(t): """ Return True if value is an instance of a boolean type @@ -41,25 +48,81 @@ def is_boolean(t): def is_integer(t): """ - Return True if value is an instance of an integer type + Return True if value is an instance of any integer type """ return t.id in _INTEGER_TYPES def is_signed_integer(t): """ - Return True if value is an instance of a signed integer type + Return True if value is an instance of any signed integer type """ return t.id in _SIGNED_INTEGER_TYPES def is_unsigned_integer(t): """ - Return True if value is an instance of an unsigned integer type + Return True if value is an instance of any unsigned integer type """ return t.id in _UNSIGNED_INTEGER_TYPES +def is_int8(t): + """ + Return True if value is an instance of an int8 type + """ + return t.id == lib.Type_INT8 + + +def is_int16(t): + """ + Return True if value is an instance of an int16 type + """ + return t.id == lib.Type_INT16 + + +def is_int32(t): + """ + Return True if value is an instance of an int32 type + """ + return t.id == lib.Type_INT32 + + +def is_int64(t): + """ + Return True if value is an instance of an int64 type + """ + return t.id == lib.Type_INT64 + + +def is_uint8(t): + """ + Return True if value is an instance of an uint8 type + """ + return t.id == lib.Type_UINT8 + + +def is_uint16(t): + """ + Return True if value is an instance of an uint16 type + """ + return t.id == lib.Type_UINT16 + + +def is_uint32(t): + """ + Return True if value is an instance of an uint32 type + """ + return t.id == lib.Type_UINT32 + + +def is_uint64(t): + """ + Return True if value is an instance of an uint64 type + """ + return t.id == lib.Type_UINT64 + + def is_floating(t): """ Return True if value is an instance of a floating point numeric type @@ -67,6 +130,27 @@ def is_floating(t): return t.id in _FLOATING_TYPES +def is_float16(t): + """ + Return True if value is an instance of an float16 (half-precision) type + """ + return t.id == lib.Type_HALF_FLOAT + + +def is_float32(t): + """ + Return True if value is an instance of an float32 (single precision) type + """ + return t.id == lib.Type_FLOAT + + +def is_float64(t): + """ + Return True if value is an instance of an float64 (double precision) type + """ + return t.id == lib.Type_DOUBLE + + def is_list(t): """ Return True if value is an instance of a list type @@ -117,11 +201,18 @@ def is_time(t): return t.id in _TIME_TYPES -def is_null(t): +def is_time32(t): """ - Return True if value is an instance of a null type + Return True if value is an instance of a time32 type """ - return t.id == lib.Type_NA + return t.id == lib.Type_TIME32 + + +def is_time64(t): + """ + Return True if value is an instance of a time64 type + """ + return t.id == lib.Type_TIME64 def is_binary(t): @@ -159,6 +250,20 @@ def is_date(t): return t.id in _DATE_TYPES +def is_date32(t): + """ + Return True if value is an instance of a date32 (days) type + """ + return t.id == lib.Type_DATE32 + + +def is_date64(t): + """ + Return True if value is an instance of a date64 (milliseconds) type + """ + return t.id == lib.Type_DATE64 + + def is_map(t): """ Return True if value is an instance of a map logical type From 6a7ce328d585fb77121dc09bc55928ca6a0ce69d Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 11 Dec 2017 10:42:00 -0500 Subject: [PATCH 132/177] ARROW-1904: [C++] Deprecate PrimitiveArray::raw_values I was mistaken about this method's handling of the offset parameter, but it doesn't work correctly for boolean data (also a subclass of `PrimitiveArray`), so I think it's better to remove this method altogether Author: Wes McKinney Closes #1409 from wesm/ARROW-1904 and squashes the following commits: eb735555 [Wes McKinney] clang-format fa88b4c9 [Wes McKinney] Deprecate PrimitiveArray::raw_values, add implementation for FixedSizeBinary --- cpp/src/arrow/array.cc | 4 ++++ cpp/src/arrow/array.h | 8 ++++++++ cpp/src/arrow/compare.cc | 12 ++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index 0b235cc199ca1..144fbcd05c205 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -139,11 +139,15 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr& type, int64_t le SetData(ArrayData::Make(type, length, {null_bitmap, data}, null_count, offset)); } +#ifndef ARROW_NO_DEPRECATED_API + const uint8_t* PrimitiveArray::raw_values() const { return raw_values_ + offset() * static_cast(*type()).bit_width() / CHAR_BIT; } +#endif + template NumericArray::NumericArray(const std::shared_ptr& data) : PrimitiveArray(data) { diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index ebe54adcb9eaf..6721c7d2717f3 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -334,9 +334,15 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray { /// Does not account for any slice offset std::shared_ptr values() const { return data_->buffers[1]; } +#ifndef ARROW_NO_DEPRECATED_API + /// \brief Return pointer to start of raw data + /// + /// \note Deprecated since 0.8.0 const uint8_t* raw_values() const; +#endif + protected: PrimitiveArray() {} @@ -567,6 +573,8 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray { int32_t byte_width() const { return byte_width_; } + const uint8_t* raw_values() const { return raw_values_ + data_->offset * byte_width_; } + protected: inline void SetData(const std::shared_ptr& data) { this->PrimitiveArray::SetData(data); diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index 72ef122e3481e..9f07fa7eff826 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -312,8 +312,16 @@ static bool IsEqualPrimitive(const PrimitiveArray& left, const PrimitiveArray& r const auto& size_meta = dynamic_cast(*left.type()); const int byte_width = size_meta.bit_width() / CHAR_BIT; - const uint8_t* left_data = left.values() ? left.raw_values() : nullptr; - const uint8_t* right_data = right.values() ? right.raw_values() : nullptr; + const uint8_t* left_data = nullptr; + const uint8_t* right_data = nullptr; + + if (left.values()) { + left_data = left.values()->data() + left.offset() * byte_width; + } + + if (right.values()) { + right_data = right.values()->data() + right.offset() * byte_width; + } if (left.null_count() > 0) { for (int64_t i = 0; i < left.length(); ++i) { From 0095c4e7a151c79a50255e190ddbc44c7f307510 Mon Sep 17 00:00:00 2001 From: lmeyerov Date: Tue, 12 Dec 2017 15:08:14 -0500 Subject: [PATCH 133/177] ARROW-1911: [JS] Add Graphistry to Arrow JS proof points Realized Graphistry's use wasn't listed, which matters as we're the most enterprise-y user right now. Also made the GOAI reference more clear. Author: lmeyerov Closes #1412 from lmeyerov/patch-1 and squashes the following commits: ff5f7375 [lmeyerov] fix(more precise description) 1fe8dad2 [lmeyerov] docs(fix backlink information architecture) 3a80c4d8 [lmeyerov] docs(tweak js readme) d78f414c [lmeyerov] docs(normalize and sort implementation order) 4968dc93 [lmeyerov] docs(graphistry and goai in main powered by) af8c0383 [lmeyerov] docs(include JS impl in main readme.md list) e70e534d [lmeyerov] docs(add getting involved to develop.md) a7e8d017 [lmeyerov] docs(add and cleanup orgs) 29629260 [lmeyerov] [JS] Add Graphistry to Arrow JS projects --- README.md | 5 +++-- js/DEVELOP.md | 24 ++++++++++++++++++++++++ js/README.md | 35 ++++++++++++++++++++++++++--------- site/powered_by.md | 9 +++++++++ 4 files changed, 62 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9dda25de91178..15a9ffea2a7a3 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,9 @@ Arrow is a set of technologies that enable big-data systems to process and move Initial implementations include: - [The Arrow Format](https://github.com/apache/arrow/tree/master/format) - - [Java implementation](https://github.com/apache/arrow/tree/master/java) - [C++ implementation](https://github.com/apache/arrow/tree/master/cpp) + - [Java implementation](https://github.com/apache/arrow/tree/master/java) + - [JavaScript implementation](https://github.com/apache/arrow/tree/master/js) - [Python interface to C++ libraries](https://github.com/apache/arrow/tree/master/python) Arrow is an [Apache Software Foundation](www.apache.org) project. Learn more at @@ -110,4 +111,4 @@ Thank you in advance for your contributions! [1]: mailto:dev-subscribe@arrow.apache.org [2]: https://github.com/apache/arrow/tree/master/format [3]: https://issues.apache.org/jira/browse/ARROW -[4]: https://github.com/apache/arrow \ No newline at end of file +[4]: https://github.com/apache/arrow diff --git a/js/DEVELOP.md b/js/DEVELOP.md index 9f586e1b39bcb..1dd999a9efbb6 100644 --- a/js/DEVELOP.md +++ b/js/DEVELOP.md @@ -17,6 +17,30 @@ under the License. --> +# Getting Involved +Even if you do not plan to contribute to Apache Arrow itself or Arrow +integrations in other projects, we'd be happy to have you involved: + +* Join the mailing list: send an email to + [dev-subscribe@arrow.apache.org][1]. Share your ideas and use cases for the + project. +* [Follow our activity on JIRA][3] +* [Learn the format][2] +* Contribute code to one of the reference implementations + +We prefer to receive contributions in the form of GitHub pull requests. Please send pull requests against the [github.com/apache/arrow][4] repository. + +If you are looking for some ideas on what to contribute, check out the [JIRA +issues][3] for the Apache Arrow project. Comment on the issue and/or contact +[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/) +with your questions and ideas. + +If you’d like to report a bug but don’t have time to fix it, you can still post +it on JIRA, or email the mailing list +[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/) + + + # The npm scripts * `npm run clean` - cleans targets diff --git a/js/README.md b/js/README.md index 9b1753f68335e..e58d335cd0d42 100644 --- a/js/README.md +++ b/js/README.md @@ -22,7 +22,7 @@ [![Build Status](https://travis-ci.org/apache/arrow.svg?branch=master)](https://travis-ci.org/apache/arrow) [![Coverage Status](https://coveralls.io/repos/github/apache/arrow/badge.svg)](https://coveralls.io/github/apache/arrow) -Arrow is a set of technologies that enable big-data systems to process and transfer data quickly. +Arrow is a set of technologies that enable big data systems to process and transfer data quickly. ## install [apache-arrow from npm](https://www.npmjs.com/package/apache-arrow) @@ -32,14 +32,9 @@ Arrow is a set of technologies that enable big-data systems to process and trans # Powering Columnar In-Memory Analytics -Apache Arrow is a columnar memory layout specification for encoding vectors and table-like containers of flat and nested data. The Arrow spec aligns columnar data in memory to minimize cache misses and take advantage of the latest SIMD (Single input multiple data) and GPU operations on modern processors. +[Apache Arrow](https://github.com/apache/arrow) is a columnar memory layout specification for encoding vectors and table-like containers of flat and nested data. The Arrow spec aligns columnar data in memory to minimize cache misses and take advantage of the latest SIMD (Single input multiple data) and GPU operations on modern processors. -Apache Arrow is the emerging standard for large in-memory columnar data ([Spark](https://spark.apache.org/), [Pandas](http://wesmckinney.com/blog/pandas-and-apache-arrow/), [Drill](https://drill.apache.org/), ...). By standardizing on a common binary interchange format, big data systems can reduce the costs and friction associated with cross-system communication. - -# Related Projects - -* [GoAI](http://gpuopenanalytics.com/) -- Arrow-powered GPU analytics -* [rxjs-mapd](https://github.com/graphistry/rxjs-mapd) -- A MapD Core node-driver that returns query results as Arrow columns +Apache Arrow is the emerging standard for large in-memory columnar data ([Spark](https://spark.apache.org/), [Pandas](http://wesmckinney.com/blog/pandas-and-apache-arrow/), [Drill](https://drill.apache.org/), [Graphistry](https://www.graphistry.com), ...). By standardizing on a common binary interchange format, big data systems can reduce the costs and friction associated with cross-system communication. # Usage @@ -202,6 +197,28 @@ The JS community is a diverse group with a varied list of target environments an If you think we missed a compilation target and it's a blocker for adoption, please open an issue. +# People + +Full list of broader Apache Arrow [committers](https://arrow.apache.org/committers/). + +* Brian Hulette, CCRi, _contributor_ +* Paul Taylor, Graphistry, Inc., _committer_ + +# Powered By Apache Arrow in JS + +Full list of broader Apache Arrow [projects & organizations](https://github.com/apache/arrow/blob/master/site/powered_by.md). + +## Open Source Projects + +* [Apache Arrow](https://arrow.apache.org) -- Parent project for Powering Columnar In-Memory Analytics, including affiliated open source projects +* [rxjs-mapd](https://github.com/graphistry/rxjs-mapd) -- A MapD Core node-driver that returns query results as Arrow columns + +## Companies & Organizations + +* [CCRi](http://www.ccri.com/) -- Commonwealth Computer Research Inc, or CCRi, is a Central Virginia based data science and software engineering company +* [GOAI](http://gpuopenanalytics.com/) -- GPU Open Analytics Initiative standardizes on Arrow as part of creating common data frameworks that enable developers and statistical researchers to accelerate data science on GPUs +* [Graphistry, Inc.](https://www.graphistry.com/) - An end-to-end GPU accelerated visual investigation platform used by teams for security, anti-fraud, and related investigations. Graphistry uses Arrow in its NodeJS GPU backend and client libraries, and is an early contributing member to GOAI and Arrow\[JS\] working to bring these technologies to the enterprise. + # License [Apache 2.0](https://github.com/apache/arrow/blob/master/LICENSE) @@ -209,4 +226,4 @@ If you think we missed a compilation target and it's a blocker for adoption, ple [1]: mailto:dev-subscribe@arrow.apache.org [2]: https://github.com/apache/arrow/tree/master/format [3]: https://issues.apache.org/jira/browse/ARROW -[4]: https://github.com/apache/arrow \ No newline at end of file +[4]: https://github.com/apache/arrow diff --git a/site/powered_by.md b/site/powered_by.md index 38320c0ec91b0..56f6e2bcee41a 100644 --- a/site/powered_by.md +++ b/site/powered_by.md @@ -103,6 +103,13 @@ short description of your use case. Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI, Python, R, and more (all backed by Apache Arrow). +* **[GOAI][19]:** Open GPU-Accelerated Analytics Initiative for Arrow-powered + analytics across GPU tools and vendors +* **[Graphistry][18]:** Supercharged Visual Investigation Platform used by + teams for security, anti-fraud, and related investigations. The Graphistry + team uses Arrow in its NodeJS GPU backend and client libraries, and is an + early contributing member to GOAI and Arrow\[JS\] focused on bringing these + technologies to the enterprise. * **[Quilt Data][13]:** Quilt is a data package manager, designed to make managing data as easy as managing code. It supports Parquet format via pyarrow for data access. @@ -124,3 +131,5 @@ short description of your use case. [15]: https://github.com/dask/dask [16]: https://red-data-tools.github.io/ [17]: https://github.com/red-data-tools/red-arrow/ +[18]: https://www.graphistry.com +[19]: http://gpuopenanalytics.com From 16ddb127c5e6e51d80aa24d94dffded393db78a9 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Tue, 12 Dec 2017 15:08:54 -0500 Subject: [PATCH 134/177] ARROW-1914: [C++] Fix build dependency for GPU support build "make -j" may cause build error: [100%] Built target gflags_nothreads_static Install the project... -- Install configuration: "RELEASE" -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-config.cmake -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-config-version.cmake -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-targets.cmake -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/cmake/gflags/gflags-targets-release.cmake -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/bin/gflags_completions.sh -- Installing: /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/build/gflags_ep-prefix/src/gflags_ep/lib/pkgconfig/gflags.pc -- Installing: /home/kou/.cmake/packages/gflags/fb801def37c922433975cbfefb3aa08d [ 26%] Completed 'gflags_ep' [ 55%] Building C object CMakeFiles/brotlienc.dir/enc/literal_cost.c.o [ 26%] Built target gflags_ep Scanning dependencies of target arrow_gpu_objlib [ 26%] Building CXX object src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/cuda_arrow_ipc.cc.o [ 59%] Building C object CMakeFiles/brotlienc.dir/enc/memory.c.o /tmp/arrow-0.8.0.GLyu7/apache-arrow-0.8.0/cpp/src/arrow/gpu/cuda_arrow_ipc.cc:26:10: fatal error: arrow/ipc/Message_generated.h: No such file or directory #include "arrow/ipc/Message_generated.h" ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ compilation terminated. src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/build.make:62: recipe for target 'src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/cuda_arrow_ipc.cc.o' failed make[2]: *** [src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/cuda_arrow_ipc.cc.o] Error 1 CMakeFiles/Makefile2:2108: recipe for target 'src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/all' failed make[1]: *** [src/arrow/gpu/CMakeFiles/arrow_gpu_objlib.dir/all] Error 2 make[1]: *** Waiting for unfinished jobs.... Author: Kouhei Sutou Closes #1414 from kou/cpp-fix-gpu-build-dependency and squashes the following commits: 8fe28c7a [Kouhei Sutou] [C++] Fix build dependency for GPU support build --- cpp/src/arrow/gpu/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt index 3ddf2c7974f0e..be474131e3c72 100644 --- a/cpp/src/arrow/gpu/CMakeLists.txt +++ b/cpp/src/arrow/gpu/CMakeLists.txt @@ -32,9 +32,6 @@ set(ARROW_GPU_SRCS cuda_memory.cc ) -add_custom_target(arrow_gpu_sources DEPENDS ${ARROW_GPU_SRCS}) -add_dependencies(arrow_gpu_sources metadata_fbs) - set(ARROW_GPU_SHARED_LINK_LIBS arrow_shared ${CUDA_LIBRARIES} @@ -43,6 +40,7 @@ set(ARROW_GPU_SHARED_LINK_LIBS ADD_ARROW_LIB(arrow_gpu SOURCES ${ARROW_GPU_SRCS} + DEPENDENCIES metadata_fbs SHARED_LINK_FLAGS "" SHARED_LINK_LIBS ${ARROW_GPU_SHARED_LINK_LIBS} STATIC_LINK_LIBS "" From 4f1bc97033ca0c7c60e91fce463a65283e7570d2 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 12 Dec 2017 15:58:00 -0500 Subject: [PATCH 135/177] ARROW-1916: [Java] Include java/dev/checkstyle in git archive for source releases Author: Wes McKinney Closes #1416 from wesm/ARROW-1916 and squashes the following commits: 58402637 [Wes McKinney] Include java/dev/checkstyle in git archive for source releases --- java/.gitattributes | 1 - 1 file changed, 1 deletion(-) diff --git a/java/.gitattributes b/java/.gitattributes index cb02d822677ed..596615322fb3e 100644 --- a/java/.gitattributes +++ b/java/.gitattributes @@ -1,3 +1,2 @@ .gitattributes export-ignore .gitignore export-ignore -/dev export-ignore From 8d006dc44194b5e4f4c7848ebdd7f916dff2439c Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Tue, 12 Dec 2017 17:17:20 -0500 Subject: [PATCH 136/177] ARROW-1915: [Python] Add missing parquet decorator to decimal tests In `test_parquet.py` add the `@parquet` decorator to 2 decimal tests so they can be skipped. Author: Bryan Cutler Closes #1415 from BryanCutler/python-parquet-decimal-tests-decorator-ARROW-1915 and squashes the following commits: 822ec433 [Bryan Cutler] add missing parquet decorator to decimal tests --- python/pyarrow/tests/test_parquet.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 79e24d8d4e6eb..fc8c8f0c98da9 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1593,6 +1593,7 @@ def test_backwards_compatible_column_metadata_handling(): tm.assert_frame_equal(result, expected[['a']].reset_index(drop=True)) +@parquet def test_decimal_roundtrip(tmpdir): num_values = 10 @@ -1619,6 +1620,7 @@ def test_decimal_roundtrip(tmpdir): tm.assert_frame_equal(result, expected) +@parquet @pytest.mark.xfail( raises=pa.ArrowException, reason='Parquet does not support negative scale' ) From e4770b286eb5e08697dfceb1ff59bef672a29ed7 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 12 Dec 2017 17:18:18 -0500 Subject: [PATCH 137/177] ARROW-1917: Fixes to enable verify-release-candidate.sh to work for 0.8.0 Author: Wes McKinney Closes #1417 from wesm/ARROW-1917 and squashes the following commits: e394f301 [Wes McKinney] Disable integration portion of JS. Run integration tests earlier 363c4bea [Wes McKinney] Set GI_TYPELIB_PATH when testing glib in verify-release-candidate-sh 420a5716 [Wes McKinney] [Java] Change scm git link in pom.xml --- dev/release/verify-release-candidate.sh | 25 +++++++++++++++---------- java/pom.xml | 4 ++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 7c5e7fd0abaaf..f33211e26a3f6 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -165,7 +165,9 @@ test_glib() { make -j$NPROC make install - NO_MAKE=yes test/run-test.sh + GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0 \ + NO_MAKE=yes \ + test/run-test.sh popd } @@ -174,15 +176,17 @@ test_js() { pushd js npm install # clean, lint, and build JS source - npm run clean:all - npm run lint - npm run build + npx run-s clean:all lint build + npm run test + # create initial integration test data - npm run create:testdata + # npm run create:testdata + # run once to write the snapshots - npm test -- -t ts -u --integration + # npm test -- -t ts -u --integration + # run again to test all builds against the snapshots - npm test -- --integration + # npm test -- --integration popd } @@ -238,13 +242,14 @@ fetch_archive $DIST_NAME tar xvzf ${DIST_NAME}.tar.gz cd ${DIST_NAME} +test_package_java setup_miniconda test_and_install_cpp +test_integration +test_glib install_parquet_cpp test_python -test_glib -test_package_java -test_integration + test_js echo 'Release candidate looks good!' diff --git a/java/pom.xml b/java/pom.xml index 162c53460cc93..922048c018b7a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -41,8 +41,8 @@ - scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git - scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git + scm:git:https://github.com/apache/arrow.git + scm:git:https://github.com/apache/arrow.git https://github.com/apache/arrow apache-arrow-0.7.1 From 3afb8f4e2df14fb9a512d2a82b6aeff27d013df6 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 12 Dec 2017 17:19:42 -0500 Subject: [PATCH 138/177] [Release] Update CHANGELOG.md for 0.8.0 Change-Id: If8489283f73f5aa8947f742958cdb4a0b5a76084 --- CHANGELOG.md | 290 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 485afc1497ebe..153159cb779e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,295 @@ under the License. --> +# Apache Arrow 0.8.0 (12 December 2017) + +## Bug + +* ARROW-1282 - Large memory reallocation by Arrow causes hang in jemalloc +* ARROW-1341 - [C++] Deprecate arrow::MakeTable in favor of new ctor from ARROW-1334 +* ARROW-1347 - [JAVA] List null type should use consistent name for inner field +* ARROW-1398 - [Python] No support reading columns of type decimal(19,4) +* ARROW-1409 - [Format] Use for "page" attribute in Buffer in metadata +* ARROW-1540 - [C++] Fix valgrind warnings in cuda-test if possible +* ARROW-1541 - [C++] Race condition with arrow\_gpu +* ARROW-1543 - [C++] row\_wise\_conversion example doesn't correspond to ListBuilder constructor arguments +* ARROW-1555 - [Python] write\_to\_dataset on s3 +* ARROW-1584 - [PYTHON] serialize\_pandas on empty dataframe +* ARROW-1585 - serialize\_pandas round trip fails on integer columns +* ARROW-1586 - [PYTHON] serialize\_pandas roundtrip loses columns name +* ARROW-1609 - Plasma: Build fails with Xcode 9.0 +* ARROW-1615 - CXX flags for development more permissive than Travis CI builds +* ARROW-1617 - [Python] Do not use symlinks in python/cmake\_modules +* ARROW-1620 - Python: Download Boost in manylinux1 build from bintray +* ARROW-1624 - [C++] Follow up fixes / tweaks to compiler warnings for Plasma / LLVM 4.0, add to readme +* ARROW-1625 - [Serialization] Support OrderedDict properly +* ARROW-1629 - [C++] Fix problematic code paths identified by infer tool +* ARROW-1633 - [Python] numpy "unicode" arrays not understood +* ARROW-1640 - Resolve OpenSSL issues in Travis CI +* ARROW-1647 - [Plasma] Potential bug when reading/writing messages. +* ARROW-1653 - [Plasma] Use static cast to avoid compiler warning. +* ARROW-1656 - [C++] Endianness Macro is Incorrect on Windows And Mac +* ARROW-1657 - [C++] Multithreaded Read Test Failing on Arch Linux +* ARROW-1658 - [Python] Out of bounds dictionary indices causes segfault after converting to pandas +* ARROW-1663 - [Java] Follow up on ARROW-1347 and make schema backward compatible +* ARROW-1670 - [Python] Speed up deserialization code path +* ARROW-1672 - [Python] Failure to write Feather bytes column +* ARROW-1673 - [Python] NumPy boolean arrays get converted to uint8 arrays on NdarrayToTensor roundtrip +* ARROW-1676 - [C++] Correctly truncate oversized validity bitmaps when writing Feather format +* ARROW-1678 - [Python] Incorrect serialization of numpy.float16 +* ARROW-1680 - [Python] Timestamp unit change not done in from\_pandas() conversion +* ARROW-1686 - Documentation generation script creates "apidocs" directory under site/java +* ARROW-1693 - [JS] Error reading dictionary-encoded integration test files +* ARROW-1695 - [Serialization] Fix reference counting of numpy arrays created in custom serialializer +* ARROW-1698 - [JS] File reader attempts to load the same dictionary batch more than once +* ARROW-1704 - [GLib] Go example in test suite is broken +* ARROW-1708 - [JS] Linter problem breaks master build +* ARROW-1709 - [C++] Decimal.ToString is incorrect for negative scale +* ARROW-1711 - [Python] flake8 checks still not failing builds +* ARROW-1714 - [Python] No named pd.Series name serialized as u'None' +* ARROW-1720 - [Python] Segmentation fault while trying to access an out-of-bound chunk +* ARROW-1723 - Windows: \_\_declspec(dllexport) specified when building arrow static library +* ARROW-1730 - [Python] Incorrect result from pyarrow.array when passing timestamp type +* ARROW-1732 - [Python] RecordBatch.from\_pandas fails on DataFrame with no columns when preserve\_index=False +* ARROW-1735 - [C++] Cast kernels cannot write into sliced output array +* ARROW-1738 - [Python] Wrong datetime conversion when pa.array with unit +* ARROW-1739 - [Python] Fix usages of assertRaises causing broken build +* ARROW-1742 - C++: clang-format is not detected correct on OSX anymore +* ARROW-1743 - [Python] Table to\_pandas fails when index contains categorical column +* ARROW-1745 - Compilation failure on Mac OS in plasma tests +* ARROW-1749 - [C++] Handle range of Decimal128 values that require 39 digits to be displayed +* ARROW-1751 - [Python] Pandas 0.21.0 introduces a breaking API change for MultiIndex construction +* ARROW-1754 - [Python] Fix buggy Parquet roundtrip when an index name is the same as a column name +* ARROW-1756 - [Python] Observed int32 overflow in Feather write/read path +* ARROW-1762 - [C++] unittest failure for language environment +* ARROW-1764 - [Python] Add -c conda-forge for Windows dev installation instructions +* ARROW-1766 - [GLib] Fix failing builds on OSX +* ARROW-1768 - [Python] Fix suppressed exception in ParquetWriter.\_\_del\_\_ +* ARROW-1770 - [GLib] Fix GLib compiler warning +* ARROW-1771 - [C++] ARROW-1749 Breaks Public API test in parquet-cpp +* ARROW-1776 - [C++[ arrow::gpu::CudaContext::bytes\_allocated() isn't defined +* ARROW-1778 - [Python] Link parquet-cpp statically, privately in manylinux1 wheels +* ARROW-1781 - [CI] OSX Builds on Travis-CI time out often +* ARROW-1788 - Plasma store crashes when trying to abort objects for disconnected client +* ARROW-1791 - Integration tests generate date[DAY] values outside of reasonable range +* ARROW-1793 - [Integration] fix a typo for README.md +* ARROW-1800 - [C++] Fix and simplify random\_decimals +* ARROW-1805 - [Python] ignore non-parquet files when exploring dataset +* ARROW-1811 - [C++/Python] Rename all Decimal based APIs to Decimal128 +* ARROW-1812 - Plasma store modifies hash table while iterating during client disconnect +* ARROW-1829 - [Plasma] Clean up eviction policy bookkeeping +* ARROW-1830 - [Python] Error when loading all the files in a dictionary +* ARROW-1836 - [C++] Fix C4996 warning from arrow/util/variant.h on MSVC builds +* ARROW-1840 - [Website] The installation command failed on Windows10 anaconda environment. +* ARROW-1845 - [Python] Expose Decimal128Type +* ARROW-1852 - [Plasma] Make retrieving manager file descriptor const +* ARROW-1853 - [Plasma] Fix off-by-one error in retry processing +* ARROW-1863 - [Python] PyObjectStringify could render bytes-like output for more types of objects +* ARROW-1865 - [C++] Adding a column to an empty Table fails +* ARROW-1869 - Fix typo in LowCostIdentityHashMap +* ARROW-1871 - [Python/C++] Appending Python Decimals with different scales requires rescaling +* ARROW-1873 - [Python] Segmentation fault when loading total 2GB of parquet files +* ARROW-1877 - Incorrect comparison in JsonStringArrayList.equals +* ARROW-1879 - [Python] Dask integration tests are not skipped if dask is not installed +* ARROW-1881 - [Python] setuptools\_scm picks up JS version tags +* ARROW-1882 - [C++] Reintroduce DictionaryBuilder +* ARROW-1883 - [Python] BUG: Table.to\_pandas metadata checking fails if columns are not present +* ARROW-1889 - [Python] --exclude is not available in older git versions +* ARROW-1890 - [Python] Masking for date32 arrays not working +* ARROW-1891 - [Python] NaT date32 values are only converted to nulls if from\_pandas is used +* ARROW-1892 - [Python] Unknown list item type: binary +* ARROW-1893 - [Python] test\_primitive\_serialization fails on Python 2.7.3 +* ARROW-1895 - [Python] Add field\_name to pandas index metadata +* ARROW-1897 - [Python] Incorrect numpy\_type for pandas metadata of Categoricals +* ARROW-1904 - [C++] Deprecate PrimitiveArray::raw\_values +* ARROW-1906 - [Python] Creating a pyarrow.Array with timestamp of different unit is not casted +* ARROW-1908 - [Python] Construction of arrow table from pandas DataFrame with duplicate column names crashes +* ARROW-1910 - CPP README Brewfile link incorrect +* ARROW-1914 - [C++] make -j may fail to build with -DARROW\_GPU=on +* ARROW-1915 - [Python] Parquet tests should be optional +* ARROW-1916 - [Java] Do not exclude java/dev/checkstyle from source releases +* ARROW-1917 - [GLib] Must set GI\_TYPELIB\_PATH in verify-release-candidate.sh +* ARROW-226 - [C++] libhdfs: feedback to help determining cause of failure in opening file path +* ARROW-641 - [C++] Do not build/run io-hdfs-test if ARROW\_HDFS=off + +## Improvement + +* ARROW-1087 - [Python] add get\_include to expose directory containing header files +* ARROW-1134 - [C++] Allow C++/CLI projects to build with Arrow​ +* ARROW-1178 - [Python] Create alternative to Table.from\_pandas that yields a list of RecordBatch objects with a given chunk size +* ARROW-1226 - [C++] Improve / correct doxygen function documentation in arrow::ipc +* ARROW-1371 - [Website] Add "Powered By" page to the website +* ARROW-1455 - [Python] Add Dockerfile for validating Dask integration outside of usual CI +* ARROW-1488 - [C++] Implement ArrayBuilder::Finish in terms of internal::ArrayData +* ARROW-1498 - [GitHub] Add CONTRIBUTING.md and ISSUE\_TEMPLATE.md +* ARROW-1503 - [Python] Add serialization callbacks for pandas objects in pyarrow.serialize +* ARROW-1522 - [C++] Support pyarrow.Buffer as built-in type in pyarrow.serialize +* ARROW-1523 - [C++] Add helper data struct with methods for reading a validity bitmap possibly having a non-zero offset +* ARROW-1524 - [C++] More graceful solution for handling non-zero offsets on inputs and outputs in compute library +* ARROW-1525 - [C++] Change functions in arrow/compare.h to not return Status +* ARROW-1526 - [Python] Unit tests to exercise code path in PARQUET-1100 +* ARROW-1535 - [Python] Enable sdist source tarballs to build assuming that Arrow C++ libraries are available on the host system +* ARROW-1538 - [C++] Support Ubuntu 14.04 in .deb packaging automation +* ARROW-1539 - [C++] Remove functions deprecated as of 0.7.0 and prior releases +* ARROW-1556 - [C++] Incorporate AssertArraysEqual function from PARQUET-1100 patch +* ARROW-1588 - [C++/Format] Harden Decimal Format +* ARROW-1593 - [PYTHON] serialize\_pandas should pass through the preserve\_index keyword +* ARROW-1594 - [Python] Enable multi-threaded conversions in Table.from\_pandas +* ARROW-1600 - [C++] Zero-copy Buffer constructor from std::string +* ARROW-1602 - [C++] Add IsValid/IsNotNull method to arrow::Array +* ARROW-1603 - [C++] Add BinaryArray method to get a value as a std::string +* ARROW-1604 - [Python] Support common type aliases in cast(...) and various type= arguments +* ARROW-1605 - [Python] pyarrow.array should be able to yield smaller integer types without an explicit cast +* ARROW-1607 - [C++] Implement DictionaryBuilder for Decimals +* ARROW-1613 - [Java] ArrowReader should not close the input ReadChannel +* ARROW-1616 - [Python] Add "write" method to RecordBatchStreamWriter that dispatches to write\_table/write\_back as appropriate +* ARROW-1626 - Add make targets to run the inter-procedural static analysis tool called "infer". +* ARROW-1627 - [JAVA] Reduce heap usage(Phase 2) - memory footprint in AllocationManager.BufferLedger +* ARROW-1630 - [Serialization] Support Python datetime objects +* ARROW-1635 - Add release management guide for PMCs +* ARROW-1641 - [C++] Do not include in public headers +* ARROW-1651 - [JS] Lazy row accessor in Table +* ARROW-1652 - [JS] Separate Vector into BatchVector and CompositeVector +* ARROW-1654 - [Python] pa.DataType cannot be pickled +* ARROW-1662 - Move OSX Dependency management into brew bundle Brewfiles +* ARROW-1665 - [Serialization] Support more custom datatypes in the default serialization context +* ARROW-1666 - [GLib] Enable gtk-doc on Travis CI Mac environment +* ARROW-1671 - [C++] Change arrow::MakeArray to not return Status +* ARROW-1675 - [Python] Use RecordBatch.from\_pandas in FeatherWriter.write +* ARROW-1677 - [Blog] Add blog post on Ray and Arrow Python serialization +* ARROW-1679 - [GLib] Add garrow\_record\_batch\_reader\_read\_next() +* ARROW-1683 - [Python] Restore "TimestampType" to pyarrow namespace +* ARROW-1684 - [Python] Simplify user API for reading nested Parquet columns +* ARROW-1689 - [Python] Categorical Indices Should Be Zero-Copy +* ARROW-1691 - [Java] Conform Java Decimal type implementation to format decisions in ARROW-1588 +* ARROW-1701 - [Serialization] Support zero copy PyTorch Tensor serialization +* ARROW-1702 - Update jemalloc in manylinux1 build +* ARROW-1703 - [C++] Vendor exact version of jemalloc we depend on +* ARROW-1707 - Update dev README after movement to GitBox +* ARROW-1716 - [Format/JSON] Use string integer value for Decimals in JSON +* ARROW-1721 - [Python] Support null mask in places where it isn't supported in numpy\_to\_arrow.cc +* ARROW-1724 - [Packaging] Support Ubuntu 17.10 +* ARROW-1725 - [Packaging] Upload .deb for Ubuntu 17.10 +* ARROW-1726 - [GLib] Add setup description to verify C GLib build +* ARROW-1727 - [Format] Expand Arrow streaming format to permit new dictionaries and deltas / additions to existing dictionaries +* ARROW-1728 - [C++] Run clang-format checks in Travis CI +* ARROW-1737 - [GLib] Use G\_DECLARE\_DERIVABLE\_TYPE +* ARROW-1746 - [Python] Add build dependencies for Arch Linux +* ARROW-1747 - [C++] Don't export symbols of statically linked libraries +* ARROW-1750 - [C++] Remove the need for arrow/util/random.h +* ARROW-1753 - [Python] Provide for matching subclasses with register\_type in serialization context +* ARROW-1755 - [C++] Add build options for MSVC to use static runtime libraries +* ARROW-1758 - [Python] Remove pickle=True option for object serialization +* ARROW-1763 - [Python] DataType should be hashable +* ARROW-1765 - [Doc] Use dependencies from conda in C++ docker build +* ARROW-1785 - [Format/C++/Java] Remove VectorLayout metadata from Flatbuffers metadata +* ARROW-1787 - [Python] Support reading parquet files into DataFrames in a backward compatible way +* ARROW-1794 - [C++/Python] Rename DecimalArray to Decimal128Array +* ARROW-1801 - [Docs] Update install instructions to use red-data-tools repos +* ARROW-1808 - [C++] Make RecordBatch interface virtual to permit record batches that lazy-materialize columns +* ARROW-1809 - [GLib] Use .xml instead of .sgml for GTK-Doc main file +* ARROW-1810 - [Plasma] Remove test shell scripts +* ARROW-1817 - Configure JsonFileReader to read NaN for floats +* ARROW-1826 - [JAVA] Avoid branching at cell level (copyFrom) +* ARROW-1828 - [C++] Implement hash kernel specialization for BooleanType +* ARROW-1834 - [Doc] Build documentation in separate build folders +* ARROW-1838 - [C++] Use compute::Datum uniformly for input argument to kernels +* ARROW-1841 - [JS] Update text-encoding-utf-8 and tslib for node ESModules support +* ARROW-1849 - [GLib] Add input checks to GArrowRecordBatch +* ARROW-1850 - [C++] Use const void* in Writable::Write instead of const uint8\_t* +* ARROW-1854 - [Python] Improve performance of serializing object dtype ndarrays +* ARROW-1855 - [GLib] Add workaround for build failure on macOS +* ARROW-1864 - [Java] Upgrade Netty to 4.1.x +* ARROW-1884 - [C++] Make JsonReader/JsonWriter classes internal APIs +* ARROW-1901 - [Python] Support recursive mkdir for DaskFilesystem +* ARROW-1902 - [Python] Remove mkdir race condition from write\_to\_dataset +* ARROW-1905 - [Python] Add more functions for checking exact types in pyarrow.types +* ARROW-1911 - Add Graphistry to Arrow JS proof points +* ARROW-905 - [Docs] Add Dockerfile for reproducible documentation generation +* ARROW-942 - Support integration testing on Python 2.7 +* ARROW-950 - [Site] Add Google Analytics tag + +## New Feature + +* ARROW-1032 - [JS] Support custom\_metadata +* ARROW-1047 - [Java] Add generalized stream writer and reader interfaces that are decoupled from IO / message framing +* ARROW-1114 - [C++] Create Record Batch Builder class as a reusable and efficient way to transpose row-by-row data to columns +* ARROW-1250 - [Python] Define API for user type checking of array types +* ARROW-1482 - [C++] Implement casts between date32 and date64 +* ARROW-1483 - [C++] Implement casts between time32 and time64 +* ARROW-1484 - [C++] Implement (safe and unsafe) casts between timestamps and times of different units +* ARROW-1486 - [C++] Decide if arrow::RecordBatch needs to be copyable +* ARROW-1487 - [C++] Implement casts from List to List, where a cast function is defined from any A to B +* ARROW-1559 - [C++] Kernel implementations for "unique" (compute distinct elements of array) +* ARROW-1573 - [C++] Implement stateful kernel function that uses DictionaryBuilder to compute dictionary indices +* ARROW-1575 - [Python] Add pyarrow.column factory function +* ARROW-1577 - [JS] Package release script for NPM modules +* ARROW-1631 - [C++] Add GRPC to ThirdpartyToolchain.cmake +* ARROW-1637 - [C++] IPC round-trip for null type +* ARROW-1648 - C++: Add cast from Dictionary[NullType] to NullType +* ARROW-1649 - C++: Print number of nulls in PrettyPrint for NullArray +* ARROW-1667 - [GLib] Support Meson +* ARROW-1685 - [GLib] Add GArrowTableReader +* ARROW-1690 - [GLib] Add garrow\_array\_is\_valid() +* ARROW-1697 - [GitHub] Add ISSUE\_TEMPLATE.md +* ARROW-1718 - [Python] Implement casts from timestamp to date32/date64 and support in Array.from\_pandas +* ARROW-1734 - C++/Python: Add cast function on Column-level +* ARROW-1736 - [GLib] Add GArrowCastOptions:allow-time-truncate +* ARROW-1748 - [GLib] Add GArrowRecordBatchBuilder +* ARROW-1752 - [Packaging] Add GPU packages for Debian and Ubuntu +* ARROW-1767 - [C++] Support file reads and writes over 2GB on Windows +* ARROW-1772 - [C++] Add public-api-test module in style of parquet-cpp +* ARROW-1773 - [C++] Add casts from date/time types to compatible signed integers +* ARROW-1775 - Ability to abort created but unsealed Plasma objects +* ARROW-1777 - [C++] Add static ctor ArrayData::Make for nicer syntax in places +* ARROW-1782 - [Python] Expose compressors as pyarrow.compress, pyarrow.decompress +* ARROW-1783 - [Python] Convert SerializedPyObject to/from sequence of component buffers with minimal memory allocation / copying +* ARROW-1784 - [Python] Read and write pandas.DataFrame in pyarrow.serialize by decomposing the BlockManager rather than coercing to Arrow format +* ARROW-1802 - [GLib] Add Arrow GPU support +* ARROW-1806 - [GLib] Add garrow\_record\_batch\_writer\_write\_table() +* ARROW-1844 - [C++] Basic benchmark suite for hash kernels +* ARROW-1857 - [Python] Add switch for boost linkage with static parquet in wheels +* ARROW-1859 - [GLib] Add GArrowDictionaryDataType +* ARROW-1862 - [GLib] Add GArrowDictionaryArray +* ARROW-1874 - [GLib] Add garrow\_array\_unique() +* ARROW-1878 - [GLib] Add garrow\_array\_dictionary\_encode() +* ARROW-480 - [Python] Add accessors for Parquet column statistics +* ARROW-504 - [Python] Add adapter to write pandas.DataFrame in user-selected chunk size to streaming format +* ARROW-507 - [C++/Python] Construct List container from offsets and values subarrays +* ARROW-541 - [JS] Implement JavaScript-compatible implementation +* ARROW-571 - [Python] Add APIs to build Parquet files incrementally from Arrow tables +* ARROW-587 - Add JIRA fix version to merge tool +* ARROW-609 - [C++] Function for casting from days since UNIX epoch to int64 date +* ARROW-838 - [Python] Efficient construction of arrays from non-pandas 1D NumPy arrays +* ARROW-972 - [Python] Add test cases and basic APIs for UnionArray + +## Sub-task + +* ARROW-1471 - [JAVA] Document requirements and non/requirements for ValueVector updates +* ARROW-1472 - [JAVA] Design updated ValueVector Object Hierarchy +* ARROW-1473 - [JAVA] Create Prototype Code Hierarchy (Implementation Phase 1) +* ARROW-1474 - [JAVA] ValueVector hierarchy (Implementation Phase 2) +* ARROW-1476 - [JAVA] Implement final ValueVector updates +* ARROW-1710 - [Java] Remove non-nullable vectors in new vector class hierarchy +* ARROW-1717 - [Java] Remove public static helper method in vector classes for JSONReader/Writer +* ARROW-1719 - [Java] Remove accessor/mutator +* ARROW-1779 - [Java] Integration test breaks without zeroing out validity vectors +* ARROW-1819 - [Java] Remove legacy vector classes +* ARROW-1867 - [Java] Add BitVector APIs from old vector class +* ARROW-1885 - [Java] Restore previous MapVector class names + +## Task + +* ARROW-1369 - Support boolean types in the javascript arrow reader library +* ARROW-1818 - Examine Java Dependencies +* ARROW-1827 - [Java] Add checkstyle config file and header file + +## Test + +* ARROW-1549 - [JS] Integrate auto-generated Arrow test files +* ARROW-1821 - Add integration test case to explicitly check for optional validity buffer +* ARROW-1839 - [C++/Python] Add Decimal Parquet Read/Write Tests + # Apache Arrow 0.7.1 (27 September 2017) ## Bug @@ -1286,3 +1575,4 @@ * ARROW-83 - Add basic test infrastructure for DecimalType + From 1d689e5a3e417d3a4b3de16fea7a6fe195c8f8df Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 12 Dec 2017 17:22:29 -0500 Subject: [PATCH 139/177] [maven-release-plugin] prepare release apache-arrow-0.8.0 Change-Id: Ia6278e52e2669880cbe32d79bd2fcda48951d3fa --- java/format/pom.xml | 2 +- java/memory/pom.xml | 2 +- java/pom.xml | 4 ++-- java/tools/pom.xml | 2 +- java/vector/pom.xml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/format/pom.xml b/java/format/pom.xml index cd2b6c0de6710..b05e47b95701f 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 0.8.0-SNAPSHOT + 0.8.0 arrow-format diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 7efc8e6aa470c..5917634531f97 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 0.8.0-SNAPSHOT + 0.8.0 arrow-memory Arrow Memory diff --git a/java/pom.xml b/java/pom.xml index 922048c018b7a..2c7078f8963ea 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-java-root - 0.8.0-SNAPSHOT + 0.8.0 pom Apache Arrow Java Root POM @@ -44,7 +44,7 @@ scm:git:https://github.com/apache/arrow.git scm:git:https://github.com/apache/arrow.git https://github.com/apache/arrow - apache-arrow-0.7.1 + apache-arrow-0.8.0 diff --git a/java/tools/pom.xml b/java/tools/pom.xml index f3c86688134dc..9e9d80843d828 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 0.8.0-SNAPSHOT + 0.8.0 arrow-tools Arrow Tools diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 46e06aa1e3f97..05c4efbb9b101 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 0.8.0-SNAPSHOT + 0.8.0 arrow-vector Arrow Vectors From 5afae14f40a471ef6f10af4e62ce32d3a9ca4455 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 12 Dec 2017 17:22:39 -0500 Subject: [PATCH 140/177] [maven-release-plugin] prepare for next development iteration Change-Id: I2a7909e2f0fa87780197270982ef941d89834cca --- java/format/pom.xml | 2 +- java/memory/pom.xml | 2 +- java/pom.xml | 4 ++-- java/tools/pom.xml | 2 +- java/vector/pom.xml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/format/pom.xml b/java/format/pom.xml index b05e47b95701f..3e6582d3fd639 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 0.8.0 + 0.9.0-SNAPSHOT arrow-format diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 5917634531f97..74ec45c1d2f97 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 0.8.0 + 0.9.0-SNAPSHOT arrow-memory Arrow Memory diff --git a/java/pom.xml b/java/pom.xml index 2c7078f8963ea..152deaa9c428e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-java-root - 0.8.0 + 0.9.0-SNAPSHOT pom Apache Arrow Java Root POM @@ -44,7 +44,7 @@ scm:git:https://github.com/apache/arrow.git scm:git:https://github.com/apache/arrow.git https://github.com/apache/arrow - apache-arrow-0.8.0 + apache-arrow-0.7.1 diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 9e9d80843d828..246f264a6def8 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 0.8.0 + 0.9.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 05c4efbb9b101..0f3e03e6902da 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 0.8.0 + 0.9.0-SNAPSHOT arrow-vector Arrow Vectors From 9e262f46a76fd02ee5226ee9e9250a917e93fe0f Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Mon, 18 Dec 2017 00:34:56 -0500 Subject: [PATCH 141/177] ARROW-1924: [Python] Bring back pickle=True option for serialization Author: Philipp Moritz Closes #1420 from pcmoritz/revert-to-pickle-arg and squashes the following commits: bfef3aeb [Philipp Moritz] fix windows test c1566538 [Philipp Moritz] fix remote serialization test on windows 3f58d0df [Philipp Moritz] fix windows 6a2a83dd [Philipp Moritz] add regression test 3eb93258 [Philipp Moritz] fix 518fb7d8 [Philipp Moritz] fix b4885862 [Philipp Moritz] revert to pickle=True argument for serialization --- python/pyarrow/serialization.pxi | 14 ++++++++-- python/pyarrow/serialization.py | 7 ++--- python/pyarrow/tests/test_serialization.py | 30 +++++++++++++++++++--- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index cbc5e3b8408e4..d95d582fe537e 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -47,6 +47,7 @@ cdef class SerializationContext: cdef: object type_to_type_id object whitelisted_types + object types_to_pickle object custom_serializers object custom_deserializers @@ -54,6 +55,7 @@ cdef class SerializationContext: # Types with special serialization handlers self.type_to_type_id = dict() self.whitelisted_types = dict() + self.types_to_pickle = set() self.custom_serializers = dict() self.custom_deserializers = dict() @@ -73,7 +75,7 @@ cdef class SerializationContext: return result - def register_type(self, type_, type_id, + def register_type(self, type_, type_id, pickle=False, custom_serializer=None, custom_deserializer=None): """EXPERIMENTAL: Add type to the list of types we can serialize. @@ -83,6 +85,9 @@ cdef class SerializationContext: The type that we can serialize. type_id : bytes A string of bytes used to identify the type. + pickle : bool + True if the serialization should be done with pickle. + False if it should be done efficiently with Arrow. custom_serializer : callable This argument is optional, but can be provided to serialize objects of the class in a particular way. @@ -92,6 +97,8 @@ cdef class SerializationContext: """ self.type_to_type_id[type_] = type_id self.whitelisted_types[type_id] = type_ + if pickle: + self.types_to_pickle.add(type_id) if custom_serializer is not None: self.custom_serializers[type_id] = custom_serializer self.custom_deserializers[type_id] = custom_deserializer @@ -111,7 +118,9 @@ cdef class SerializationContext: # use the closest match to type(obj) type_id = self.type_to_type_id[type_] - if type_id in self.custom_serializers: + if type_id in self.types_to_pickle: + serialized_obj = {"data": pickle.dumps(obj), "pickle": True} + elif type_id in self.custom_serializers: serialized_obj = {"data": self.custom_serializers[type_id](obj)} else: if is_named_tuple(type_): @@ -132,6 +141,7 @@ cdef class SerializationContext: # The object was pickled, so unpickle it. obj = pickle.loads(serialized_obj["data"]) else: + assert type_id not in self.types_to_pickle if type_id not in self.whitelisted_types: msg = "Type ID " + str(type_id) + " not registered in " \ "deserialization callback" diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py index 3059dfc1bfff2..689ec15d3292d 100644 --- a/python/pyarrow/serialization.py +++ b/python/pyarrow/serialization.py @@ -104,12 +104,9 @@ def _deserialize_default_dict(data): serialization_context.register_type( type(lambda: 0), "function", - custom_serializer=cloudpickle.dumps, - custom_deserializer=cloudpickle.loads) + pickle=True) - serialization_context.register_type(type, "type", - custom_serializer=cloudpickle.dumps, - custom_deserializer=cloudpickle.loads) + serialization_context.register_type(type, "type", pickle=True) serialization_context.register_type( np.ndarray, 'np.array', diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index 6d85621d411a7..f245dc299cb25 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -21,9 +21,9 @@ from collections import namedtuple, OrderedDict, defaultdict import datetime +import os import string import sys -import pickle import pyarrow as pa import numpy as np @@ -198,9 +198,7 @@ def make_serialization_context(): context.register_type(Baz, "Baz") context.register_type(Qux, "Quz") context.register_type(SubQux, "SubQux") - context.register_type(SubQuxPickle, "SubQuxPickle", - custom_serializer=pickle.dumps, - custom_deserializer=pickle.loads) + context.register_type(SubQuxPickle, "SubQuxPickle", pickle=True) context.register_type(Exception, "Exception") context.register_type(CustomError, "CustomError") context.register_type(Point, "Point") @@ -519,3 +517,27 @@ def test_serialize_to_components_invalid_cases(): with pytest.raises(pa.ArrowException): pa.deserialize_components(components) + + +@pytest.mark.skipif(os.name == 'nt', reason="deserialize_regex not pickleable") +def test_deserialize_in_different_process(): + from multiprocessing import Process, Queue + import re + + regex = re.compile(r"\d+\.\d*") + + serialization_context = pa.SerializationContext() + serialization_context.register_type(type(regex), "Regex", pickle=True) + + serialized = pa.serialize(regex, serialization_context) + serialized_bytes = serialized.to_buffer().to_pybytes() + + def deserialize_regex(serialized, q): + import pyarrow as pa + q.put(pa.deserialize(serialized)) + + q = Queue() + p = Process(target=deserialize_regex, args=(serialized_bytes, q)) + p.start() + assert q.get().pattern == regex.pattern + p.join() From 72e660a7c90f1846eac473c21fb718a2e23c476e Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Mon, 18 Dec 2017 16:52:22 +0100 Subject: [PATCH 142/177] ARROW-1933: [GLib] Fix build error with --with-arrow-cpp-build-dir The option is used in building deb package. `arrow/gpu/cuda_version.h` exists in build directory not source directory. Author: Kouhei Sutou Closes #1429 from kou/glib-fix-build-error-with-arrow-cpp-build-dir and squashes the following commits: 879d6bc [Kouhei Sutou] [GLib] Fix build error with --with-arrow-cpp-build-dir --- c_glib/arrow-glib/Makefile.am | 4 ++++ c_glib/arrow-gpu-glib/Makefile.am | 13 ++++++++++++- c_glib/configure.ac | 16 ++++++++++++++-- c_glib/example/Makefile.am | 4 ++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am index 39222a5480162..16d0703142798 100644 --- a/c_glib/arrow-glib/Makefile.am +++ b/c_glib/arrow-glib/Makefile.am @@ -212,6 +212,10 @@ if HAVE_INTROSPECTION INTROSPECTION_GIRS = INTROSPECTION_SCANNER_ARGS = INTROSPECTION_SCANNER_ENV = +if USE_ARROW_BUILD_DIR +INTROSPECTION_SCANNER_ENV += \ + LD_LIBRARY_PATH=$(ARROW_LIB_DIR):$${PKG_CONFIG_PATH} +endif if OS_MACOS INTROSPECTION_SCANNER_ENV += \ ARCHFLAGS= diff --git a/c_glib/arrow-gpu-glib/Makefile.am b/c_glib/arrow-gpu-glib/Makefile.am index 11ffa5ecbe367..1e1c02ac4a5c0 100644 --- a/c_glib/arrow-gpu-glib/Makefile.am +++ b/c_glib/arrow-gpu-glib/Makefile.am @@ -70,8 +70,14 @@ if HAVE_INTROSPECTION -include $(INTROSPECTION_MAKEFILE) INTROSPECTION_GIRS = INTROSPECTION_SCANNER_ARGS = -INTROSPECTION_SCANNER_ENV = \ +INTROSPECTION_SCANNER_ENV = +if USE_ARROW_BUILD_DIR +INTROSPECTION_SCANNER_ENV += \ + PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$(ARROW_BUILD_DIR)/src/arrow:$${PKG_CONFIG_PATH} +else +INTROSPECTION_SCANNER_ENV += \ PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH} +endif if OS_MACOS INTROSPECTION_SCANNER_ENV += \ ARCHFLAGS= @@ -88,6 +94,11 @@ ArrowGPU_1_0_gir_INCLUDES = \ Arrow-1.0 ArrowGPU_1_0_gir_CFLAGS = \ $(AM_CPPFLAGS) +ArrowGPU_1_0_gir_LDFLAGS = +if USE_ARROW_BUILD_DIR +ArrowGPU_1_0_gir_LDFLAGS += \ + -L$(ARROW_LIB_DIR) +endif ArrowGPU_1_0_gir_LIBS = \ $(abs_builddir)/../arrow-glib/libarrow-glib.la \ libarrow-gpu-glib.la diff --git a/c_glib/configure.ac b/c_glib/configure.ac index f06a017a10ba1..eabe7bad51227 100644 --- a/c_glib/configure.ac +++ b/c_glib/configure.ac @@ -89,16 +89,25 @@ AC_ARG_WITH(arrow-cpp-build-dir, [GARROW_ARROW_CPP_BUILD_DIR="$withval"], [GARROW_ARROW_CPP_BUILD_DIR=""]) if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then + USE_ARROW_BUILD_DIR=no + PKG_CHECK_MODULES([ARROW], [arrow arrow-compute]) PKG_CHECK_MODULES([ARROW_GPU], [arrow-gpu], [HAVE_ARROW_GPU=yes], [HAVE_ARROW_GPU=no]) else - ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src" + USE_ARROW_BUILD_DIR=yes + + ARROW_BUILD_DIR="${GARROW_ARROW_CPP_BUILD_DIR}" + AC_SUBST(ARROW_BUILD_DIR) + + ARROW_SOURCE_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src" + ARROW_BUILD_INCLUDE_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/src" ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}" + AC_SUBST(ARROW_LIB_DIR) - ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}" + ARROW_CFLAGS="-I${ARROW_BUILD_INCLUDE_DIR} -I${ARROW_SOURCE_INCLUDE_DIR}" ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow" AC_SUBST(ARROW_CFLAGS) AC_SUBST(ARROW_LIBS) @@ -115,6 +124,9 @@ else AC_SUBST(ARROW_GPU_LIBS) fi +AM_CONDITIONAL([USE_ARROW_BUILD_DIR], + [test "$USE_ARROW_BUILD_DIR" = "yes"]) + AM_CONDITIONAL([HAVE_ARROW_GPU], [test "$HAVE_ARROW_GPU" = "yes"]) if test "$HAVE_ARROW_GPU" = "yes"; then AC_DEFINE(HAVE_ARROW_GPU, [1], [Define to 1 if Apache Arrow supports GPU.]) diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am index 3eaf808bc5aa6..f4aca70c657f2 100644 --- a/c_glib/example/Makefile.am +++ b/c_glib/example/Makefile.am @@ -33,6 +33,10 @@ AM_CFLAGS = \ AM_LDFLAGS = \ $(GLIB_LIBS) \ $(builddir)/../arrow-glib/libarrow-glib.la +if USE_ARROW_BUILD_DIR +AM_LDFLAGS += \ + $(ARROW_LIBS) +endif noinst_PROGRAMS = \ build \ From db72bfe2b2141ac321251b29c1b63b3940623167 Mon Sep 17 00:00:00 2001 From: Jacques Nadeau Date: Sun, 17 Dec 2017 21:03:39 -0800 Subject: [PATCH 143/177] ARROW-1932: [Website] Update site for 0.8.0 release info --- dev/release/RELEASE_MANAGEMENT.md | 2 +- site/_release/0.8.0.md | 358 ++++++++++++++++++++++++++++++ site/_release/index.md | 2 + site/index.html | 2 +- site/install.md | 30 +-- 5 files changed, 377 insertions(+), 17 deletions(-) create mode 100644 site/_release/0.8.0.md diff --git a/dev/release/RELEASE_MANAGEMENT.md b/dev/release/RELEASE_MANAGEMENT.md index ce7835d5f5d85..73eaf5f95b3b0 100644 --- a/dev/release/RELEASE_MANAGEMENT.md +++ b/dev/release/RELEASE_MANAGEMENT.md @@ -63,7 +63,7 @@ in the `_release` subdirectory. The new contents of the new entry will go into a new Markdown file of the form `X.Y.Z.md`. You can start by copying one of the other release entries. -Generate a web-friendly changelog by running +Generate a web-friendly changelog by running (python3) ``` dev/release/changelog.py $VERSION 1 diff --git a/site/_release/0.8.0.md b/site/_release/0.8.0.md new file mode 100644 index 0000000000000..65b4210575055 --- /dev/null +++ b/site/_release/0.8.0.md @@ -0,0 +1,358 @@ +--- +layout: default +title: Apache Arrow 0.8.0 Release +permalink: /release/0.8.0.html +--- + + +# Apache Arrow 0.8.0 (18 December 2017) + +This is a major release. + +## Download + +* [**Source Artifacts**][2] +* [Git tag][1] + +## Contributors + +```shell +$ git shortlog -sn apache-arrow-0.7.1..apache-arrow-0.8.0 + 90 Wes McKinney + 23 Phillip Cloud + 21 Kouhei Sutou + 13 Licht-T + 12 Korn, Uwe + 12 Philipp Moritz + 12 Uwe L. Korn + 10 Bryan Cutler + 5 Li Jin + 5 Robert Nishihara + 4 Paul Taylor + 4 siddharth + 3 Max Risuhin + 3 Stephanie + 2 Rene Sugar + 2 Heimir Sverrisson + 2 Brian Hulette + 2 Yuliya Feldman + 2 dhirschf + 2 Matthias Vallentin + 1 vkorukanti + 1 Andrew Andrade + 1 Benjamin Goldberg + 1 Ivan Sadikov + 1 John Jenkins + 1 Joris Van den Bossche + 1 Lewis John McGibbney + 1 Lu Qi + 1 Manuel + 1 Nick White + 1 Ofek Lev + 1 Shixiong Zhu + 1 Siddharth Teotia + 1 Stephen G + 1 Victor Uriarte + 1 Wataru Shimizu + 1 ksdevlife + 1 lmeyerov + 1 rvernica + 1 Amir Malekpour +``` + +# Changelog + +## New Features and Improvements + +* [ARROW-1032](https://issues.apache.org/jira/browse/ARROW-1032) - [JS] Support custom_metadata +* [ARROW-1047](https://issues.apache.org/jira/browse/ARROW-1047) - [Java] Add generalized stream writer and reader interfaces that are decoupled from IO / message framing +* [ARROW-1087](https://issues.apache.org/jira/browse/ARROW-1087) - [Python] add get_include to expose directory containing header files +* [ARROW-1114](https://issues.apache.org/jira/browse/ARROW-1114) - [C++] Create Record Batch Builder class as a reusable and efficient way to transpose row-by-row data to columns +* [ARROW-1134](https://issues.apache.org/jira/browse/ARROW-1134) - [C++] Allow C++/CLI projects to build with Arrow +* [ARROW-1178](https://issues.apache.org/jira/browse/ARROW-1178) - [Python] Create alternative to Table.from_pandas that yields a list of RecordBatch objects with a given chunk size +* [ARROW-1226](https://issues.apache.org/jira/browse/ARROW-1226) - [C++] Improve / correct doxygen function documentation in arrow::ipc +* [ARROW-1250](https://issues.apache.org/jira/browse/ARROW-1250) - [Python] Define API for user type checking of array types +* [ARROW-1369](https://issues.apache.org/jira/browse/ARROW-1369) - Support boolean types in the javascript arrow reader library +* [ARROW-1371](https://issues.apache.org/jira/browse/ARROW-1371) - [Website] Add "Powered By" page to the website +* [ARROW-1455](https://issues.apache.org/jira/browse/ARROW-1455) - [Python] Add Dockerfile for validating Dask integration outside of usual CI +* [ARROW-1471](https://issues.apache.org/jira/browse/ARROW-1471) - [JAVA] Document requirements and non/requirements for ValueVector updates +* [ARROW-1472](https://issues.apache.org/jira/browse/ARROW-1472) - [JAVA] Design updated ValueVector Object Hierarchy +* [ARROW-1473](https://issues.apache.org/jira/browse/ARROW-1473) - [JAVA] Create Prototype Code Hierarchy (Implementation Phase 1) +* [ARROW-1474](https://issues.apache.org/jira/browse/ARROW-1474) - [JAVA] ValueVector hierarchy (Implementation Phase 2) +* [ARROW-1476](https://issues.apache.org/jira/browse/ARROW-1476) - [JAVA] Implement final ValueVector updates +* [ARROW-1482](https://issues.apache.org/jira/browse/ARROW-1482) - [C++] Implement casts between date32 and date64 +* [ARROW-1483](https://issues.apache.org/jira/browse/ARROW-1483) - [C++] Implement casts between time32 and time64 +* [ARROW-1484](https://issues.apache.org/jira/browse/ARROW-1484) - [C++] Implement (safe and unsafe) casts between timestamps and times of different units +* [ARROW-1486](https://issues.apache.org/jira/browse/ARROW-1486) - [C++] Decide if arrow::RecordBatch needs to be copyable +* [ARROW-1487](https://issues.apache.org/jira/browse/ARROW-1487) - [C++] Implement casts from List\ to List\, where a cast function is defined from any A to B +* [ARROW-1488](https://issues.apache.org/jira/browse/ARROW-1488) - [C++] Implement ArrayBuilder::Finish in terms of internal::ArrayData +* [ARROW-1498](https://issues.apache.org/jira/browse/ARROW-1498) - [GitHub] Add CONTRIBUTING.md and ISSUE_TEMPLATE.md +* [ARROW-1503](https://issues.apache.org/jira/browse/ARROW-1503) - [Python] Add serialization callbacks for pandas objects in pyarrow.serialize +* [ARROW-1522](https://issues.apache.org/jira/browse/ARROW-1522) - [C++] Support pyarrow.Buffer as built-in type in pyarrow.serialize +* [ARROW-1523](https://issues.apache.org/jira/browse/ARROW-1523) - [C++] Add helper data struct with methods for reading a validity bitmap possibly having a non-zero offset +* [ARROW-1524](https://issues.apache.org/jira/browse/ARROW-1524) - [C++] More graceful solution for handling non-zero offsets on inputs and outputs in compute library +* [ARROW-1525](https://issues.apache.org/jira/browse/ARROW-1525) - [C++] Change functions in arrow/compare.h to not return Status +* [ARROW-1526](https://issues.apache.org/jira/browse/ARROW-1526) - [Python] Unit tests to exercise code path in PARQUET-1100 +* [ARROW-1535](https://issues.apache.org/jira/browse/ARROW-1535) - [Python] Enable sdist source tarballs to build assuming that Arrow C++ libraries are available on the host system +* [ARROW-1538](https://issues.apache.org/jira/browse/ARROW-1538) - [C++] Support Ubuntu 14.04 in .deb packaging automation +* [ARROW-1539](https://issues.apache.org/jira/browse/ARROW-1539) - [C++] Remove functions deprecated as of 0.7.0 and prior releases +* [ARROW-1556](https://issues.apache.org/jira/browse/ARROW-1556) - [C++] Incorporate AssertArraysEqual function from PARQUET-1100 patch +* [ARROW-1559](https://issues.apache.org/jira/browse/ARROW-1559) - [C++] Kernel implementations for "unique" (compute distinct elements of array) +* [ARROW-1573](https://issues.apache.org/jira/browse/ARROW-1573) - [C++] Implement stateful kernel function that uses DictionaryBuilder to compute dictionary indices +* [ARROW-1575](https://issues.apache.org/jira/browse/ARROW-1575) - [Python] Add pyarrow.column factory function +* [ARROW-1577](https://issues.apache.org/jira/browse/ARROW-1577) - [JS] Package release script for NPM modules +* [ARROW-1588](https://issues.apache.org/jira/browse/ARROW-1588) - [C++/Format] Harden Decimal Format +* [ARROW-1593](https://issues.apache.org/jira/browse/ARROW-1593) - [PYTHON] serialize_pandas should pass through the preserve_index keyword +* [ARROW-1594](https://issues.apache.org/jira/browse/ARROW-1594) - [Python] Enable multi-threaded conversions in Table.from_pandas +* [ARROW-1600](https://issues.apache.org/jira/browse/ARROW-1600) - [C++] Zero-copy Buffer constructor from std::string +* [ARROW-1602](https://issues.apache.org/jira/browse/ARROW-1602) - [C++] Add IsValid/IsNotNull method to arrow::Array +* [ARROW-1603](https://issues.apache.org/jira/browse/ARROW-1603) - [C++] Add BinaryArray method to get a value as a std::string +* [ARROW-1604](https://issues.apache.org/jira/browse/ARROW-1604) - [Python] Support common type aliases in cast(...) and various type= arguments +* [ARROW-1605](https://issues.apache.org/jira/browse/ARROW-1605) - [Python] pyarrow.array should be able to yield smaller integer types without an explicit cast +* [ARROW-1607](https://issues.apache.org/jira/browse/ARROW-1607) - [C++] Implement DictionaryBuilder for Decimals +* [ARROW-1613](https://issues.apache.org/jira/browse/ARROW-1613) - [Java] ArrowReader should not close the input ReadChannel +* [ARROW-1616](https://issues.apache.org/jira/browse/ARROW-1616) - [Python] Add "write" method to RecordBatchStreamWriter that dispatches to write_table/write_back as appropriate +* [ARROW-1626](https://issues.apache.org/jira/browse/ARROW-1626) - Add make targets to run the inter-procedural static analysis tool called "infer". +* [ARROW-1627](https://issues.apache.org/jira/browse/ARROW-1627) - [JAVA] Reduce heap usage(Phase 2) - memory footprint in AllocationManager.BufferLedger +* [ARROW-1630](https://issues.apache.org/jira/browse/ARROW-1630) - [Serialization] Support Python datetime objects +* [ARROW-1631](https://issues.apache.org/jira/browse/ARROW-1631) - [C++] Add GRPC to ThirdpartyToolchain.cmake +* [ARROW-1635](https://issues.apache.org/jira/browse/ARROW-1635) - Add release management guide for PMCs +* [ARROW-1637](https://issues.apache.org/jira/browse/ARROW-1637) - [C++] IPC round-trip for null type +* [ARROW-1641](https://issues.apache.org/jira/browse/ARROW-1641) - [C++] Do not include in public headers +* [ARROW-1648](https://issues.apache.org/jira/browse/ARROW-1648) - C++: Add cast from Dictionary[NullType] to NullType +* [ARROW-1649](https://issues.apache.org/jira/browse/ARROW-1649) - C++: Print number of nulls in PrettyPrint for NullArray +* [ARROW-1651](https://issues.apache.org/jira/browse/ARROW-1651) - [JS] Lazy row accessor in Table +* [ARROW-1652](https://issues.apache.org/jira/browse/ARROW-1652) - [JS] Separate Vector into BatchVector and CompositeVector +* [ARROW-1654](https://issues.apache.org/jira/browse/ARROW-1654) - [Python] pa.DataType cannot be pickled +* [ARROW-1662](https://issues.apache.org/jira/browse/ARROW-1662) - Move OSX Dependency management into brew bundle Brewfiles +* [ARROW-1665](https://issues.apache.org/jira/browse/ARROW-1665) - [Serialization] Support more custom datatypes in the default serialization context +* [ARROW-1666](https://issues.apache.org/jira/browse/ARROW-1666) - [GLib] Enable gtk-doc on Travis CI Mac environment +* [ARROW-1667](https://issues.apache.org/jira/browse/ARROW-1667) - [GLib] Support Meson +* [ARROW-1671](https://issues.apache.org/jira/browse/ARROW-1671) - [C++] Change arrow::MakeArray to not return Status +* [ARROW-1675](https://issues.apache.org/jira/browse/ARROW-1675) - [Python] Use RecordBatch.from_pandas in FeatherWriter.write +* [ARROW-1677](https://issues.apache.org/jira/browse/ARROW-1677) - [Blog] Add blog post on Ray and Arrow Python serialization +* [ARROW-1679](https://issues.apache.org/jira/browse/ARROW-1679) - [GLib] Add garrow_record_batch_reader_read_next() +* [ARROW-1683](https://issues.apache.org/jira/browse/ARROW-1683) - [Python] Restore "TimestampType" to pyarrow namespace +* [ARROW-1684](https://issues.apache.org/jira/browse/ARROW-1684) - [Python] Simplify user API for reading nested Parquet columns +* [ARROW-1685](https://issues.apache.org/jira/browse/ARROW-1685) - [GLib] Add GArrowTableReader +* [ARROW-1689](https://issues.apache.org/jira/browse/ARROW-1689) - [Python] Categorical Indices Should Be Zero-Copy +* [ARROW-1690](https://issues.apache.org/jira/browse/ARROW-1690) - [GLib] Add garrow_array_is_valid() +* [ARROW-1691](https://issues.apache.org/jira/browse/ARROW-1691) - [Java] Conform Java Decimal type implementation to format decisions in ARROW-1588 +* [ARROW-1697](https://issues.apache.org/jira/browse/ARROW-1697) - [GitHub] Add ISSUE_TEMPLATE.md +* [ARROW-1701](https://issues.apache.org/jira/browse/ARROW-1701) - [Serialization] Support zero copy PyTorch Tensor serialization +* [ARROW-1702](https://issues.apache.org/jira/browse/ARROW-1702) - Update jemalloc in manylinux1 build +* [ARROW-1703](https://issues.apache.org/jira/browse/ARROW-1703) - [C++] Vendor exact version of jemalloc we depend on +* [ARROW-1707](https://issues.apache.org/jira/browse/ARROW-1707) - Update dev README after movement to GitBox +* [ARROW-1710](https://issues.apache.org/jira/browse/ARROW-1710) - [Java] Remove non-nullable vectors in new vector class hierarchy +* [ARROW-1716](https://issues.apache.org/jira/browse/ARROW-1716) - [Format/JSON] Use string integer value for Decimals in JSON +* [ARROW-1717](https://issues.apache.org/jira/browse/ARROW-1717) - [Java] Remove public static helper method in vector classes for JSONReader/Writer +* [ARROW-1718](https://issues.apache.org/jira/browse/ARROW-1718) - [Python] Implement casts from timestamp to date32/date64 and support in Array.from_pandas +* [ARROW-1719](https://issues.apache.org/jira/browse/ARROW-1719) - [Java] Remove accessor/mutator +* [ARROW-1721](https://issues.apache.org/jira/browse/ARROW-1721) - [Python] Support null mask in places where it isn't supported in numpy_to_arrow.cc +* [ARROW-1724](https://issues.apache.org/jira/browse/ARROW-1724) - [Packaging] Support Ubuntu 17.10 +* [ARROW-1725](https://issues.apache.org/jira/browse/ARROW-1725) - [Packaging] Upload .deb for Ubuntu 17.10 +* [ARROW-1726](https://issues.apache.org/jira/browse/ARROW-1726) - [GLib] Add setup description to verify C GLib build +* [ARROW-1727](https://issues.apache.org/jira/browse/ARROW-1727) - [Format] Expand Arrow streaming format to permit new dictionaries and deltas / additions to existing dictionaries +* [ARROW-1728](https://issues.apache.org/jira/browse/ARROW-1728) - [C++] Run clang-format checks in Travis CI +* [ARROW-1734](https://issues.apache.org/jira/browse/ARROW-1734) - C++/Python: Add cast function on Column-level +* [ARROW-1736](https://issues.apache.org/jira/browse/ARROW-1736) - [GLib] Add GArrowCastOptions:allow-time-truncate +* [ARROW-1737](https://issues.apache.org/jira/browse/ARROW-1737) - [GLib] Use G_DECLARE_DERIVABLE_TYPE +* [ARROW-1746](https://issues.apache.org/jira/browse/ARROW-1746) - [Python] Add build dependencies for Arch Linux +* [ARROW-1747](https://issues.apache.org/jira/browse/ARROW-1747) - [C++] Don't export symbols of statically linked libraries +* [ARROW-1748](https://issues.apache.org/jira/browse/ARROW-1748) - [GLib] Add GArrowRecordBatchBuilder +* [ARROW-1750](https://issues.apache.org/jira/browse/ARROW-1750) - [C++] Remove the need for arrow/util/random.h +* [ARROW-1752](https://issues.apache.org/jira/browse/ARROW-1752) - [Packaging] Add GPU packages for Debian and Ubuntu +* [ARROW-1753](https://issues.apache.org/jira/browse/ARROW-1753) - [Python] Provide for matching subclasses with register_type in serialization context +* [ARROW-1755](https://issues.apache.org/jira/browse/ARROW-1755) - [C++] Add build options for MSVC to use static runtime libraries +* [ARROW-1758](https://issues.apache.org/jira/browse/ARROW-1758) - [Python] Remove pickle=True option for object serialization +* [ARROW-1763](https://issues.apache.org/jira/browse/ARROW-1763) - [Python] DataType should be hashable +* [ARROW-1765](https://issues.apache.org/jira/browse/ARROW-1765) - [Doc] Use dependencies from conda in C++ docker build +* [ARROW-1767](https://issues.apache.org/jira/browse/ARROW-1767) - [C++] Support file reads and writes over 2GB on Windows +* [ARROW-1772](https://issues.apache.org/jira/browse/ARROW-1772) - [C++] Add public-api-test module in style of parquet-cpp +* [ARROW-1773](https://issues.apache.org/jira/browse/ARROW-1773) - [C++] Add casts from date/time types to compatible signed integers +* [ARROW-1775](https://issues.apache.org/jira/browse/ARROW-1775) - Ability to abort created but unsealed Plasma objects +* [ARROW-1777](https://issues.apache.org/jira/browse/ARROW-1777) - [C++] Add static ctor ArrayData::Make for nicer syntax in places +* [ARROW-1779](https://issues.apache.org/jira/browse/ARROW-1779) - [Java] Integration test breaks without zeroing out validity vectors +* [ARROW-1782](https://issues.apache.org/jira/browse/ARROW-1782) - [Python] Expose compressors as pyarrow.compress, pyarrow.decompress +* [ARROW-1783](https://issues.apache.org/jira/browse/ARROW-1783) - [Python] Convert SerializedPyObject to/from sequence of component buffers with minimal memory allocation / copying +* [ARROW-1784](https://issues.apache.org/jira/browse/ARROW-1784) - [Python] Read and write pandas.DataFrame in pyarrow.serialize by decomposing the BlockManager rather than coercing to Arrow format +* [ARROW-1785](https://issues.apache.org/jira/browse/ARROW-1785) - [Format/C++/Java] Remove VectorLayout metadata from Flatbuffers metadata +* [ARROW-1787](https://issues.apache.org/jira/browse/ARROW-1787) - [Python] Support reading parquet files into DataFrames in a backward compatible way +* [ARROW-1794](https://issues.apache.org/jira/browse/ARROW-1794) - [C++/Python] Rename DecimalArray to Decimal128Array +* [ARROW-1801](https://issues.apache.org/jira/browse/ARROW-1801) - [Docs] Update install instructions to use red-data-tools repos +* [ARROW-1802](https://issues.apache.org/jira/browse/ARROW-1802) - [GLib] Add Arrow GPU support +* [ARROW-1806](https://issues.apache.org/jira/browse/ARROW-1806) - [GLib] Add garrow_record_batch_writer_write_table() +* [ARROW-1808](https://issues.apache.org/jira/browse/ARROW-1808) - [C++] Make RecordBatch interface virtual to permit record batches that lazy-materialize columns +* [ARROW-1809](https://issues.apache.org/jira/browse/ARROW-1809) - [GLib] Use .xml instead of .sgml for GTK-Doc main file +* [ARROW-1810](https://issues.apache.org/jira/browse/ARROW-1810) - [Plasma] Remove test shell scripts +* [ARROW-1817](https://issues.apache.org/jira/browse/ARROW-1817) - Configure JsonFileReader to read NaN for floats +* [ARROW-1818](https://issues.apache.org/jira/browse/ARROW-1818) - Examine Java Dependencies +* [ARROW-1819](https://issues.apache.org/jira/browse/ARROW-1819) - [Java] Remove legacy vector classes +* [ARROW-1826](https://issues.apache.org/jira/browse/ARROW-1826) - [JAVA] Avoid branching at cell level (copyFrom) +* [ARROW-1827](https://issues.apache.org/jira/browse/ARROW-1827) - [Java] Add checkstyle config file and header file +* [ARROW-1828](https://issues.apache.org/jira/browse/ARROW-1828) - [C++] Implement hash kernel specialization for BooleanType +* [ARROW-1834](https://issues.apache.org/jira/browse/ARROW-1834) - [Doc] Build documentation in separate build folders +* [ARROW-1838](https://issues.apache.org/jira/browse/ARROW-1838) - [C++] Use compute::Datum uniformly for input argument to kernels +* [ARROW-1841](https://issues.apache.org/jira/browse/ARROW-1841) - [JS] Update text-encoding-utf-8 and tslib for node ESModules support +* [ARROW-1844](https://issues.apache.org/jira/browse/ARROW-1844) - [C++] Basic benchmark suite for hash kernels +* [ARROW-1849](https://issues.apache.org/jira/browse/ARROW-1849) - [GLib] Add input checks to GArrowRecordBatch +* [ARROW-1850](https://issues.apache.org/jira/browse/ARROW-1850) - [C++] Use const void* in Writable::Write instead of const uint8_t* +* [ARROW-1854](https://issues.apache.org/jira/browse/ARROW-1854) - [Python] Improve performance of serializing object dtype ndarrays +* [ARROW-1855](https://issues.apache.org/jira/browse/ARROW-1855) - [GLib] Add workaround for build failure on macOS +* [ARROW-1857](https://issues.apache.org/jira/browse/ARROW-1857) - [Python] Add switch for boost linkage with static parquet in wheels +* [ARROW-1859](https://issues.apache.org/jira/browse/ARROW-1859) - [GLib] Add GArrowDictionaryDataType +* [ARROW-1862](https://issues.apache.org/jira/browse/ARROW-1862) - [GLib] Add GArrowDictionaryArray +* [ARROW-1864](https://issues.apache.org/jira/browse/ARROW-1864) - [Java] Upgrade Netty to 4.1.x +* [ARROW-1867](https://issues.apache.org/jira/browse/ARROW-1867) - [Java] Add BitVector APIs from old vector class +* [ARROW-1874](https://issues.apache.org/jira/browse/ARROW-1874) - [GLib] Add garrow_array_unique() +* [ARROW-1878](https://issues.apache.org/jira/browse/ARROW-1878) - [GLib] Add garrow_array_dictionary_encode() +* [ARROW-1884](https://issues.apache.org/jira/browse/ARROW-1884) - [C++] Make JsonReader/JsonWriter classes internal APIs +* [ARROW-1885](https://issues.apache.org/jira/browse/ARROW-1885) - [Java] Restore previous MapVector class names +* [ARROW-1901](https://issues.apache.org/jira/browse/ARROW-1901) - [Python] Support recursive mkdir for DaskFilesystem +* [ARROW-1902](https://issues.apache.org/jira/browse/ARROW-1902) - [Python] Remove mkdir race condition from write_to_dataset +* [ARROW-1905](https://issues.apache.org/jira/browse/ARROW-1905) - [Python] Add more functions for checking exact types in pyarrow.types +* [ARROW-1911](https://issues.apache.org/jira/browse/ARROW-1911) - Add Graphistry to Arrow JS proof points +* [ARROW-480](https://issues.apache.org/jira/browse/ARROW-480) - [Python] Add accessors for Parquet column statistics +* [ARROW-504](https://issues.apache.org/jira/browse/ARROW-504) - [Python] Add adapter to write pandas.DataFrame in user-selected chunk size to streaming format +* [ARROW-507](https://issues.apache.org/jira/browse/ARROW-507) - [C++/Python] Construct List container from offsets and values subarrays +* [ARROW-541](https://issues.apache.org/jira/browse/ARROW-541) - [JS] Implement JavaScript-compatible implementation +* [ARROW-571](https://issues.apache.org/jira/browse/ARROW-571) - [Python] Add APIs to build Parquet files incrementally from Arrow tables +* [ARROW-587](https://issues.apache.org/jira/browse/ARROW-587) - Add JIRA fix version to merge tool +* [ARROW-609](https://issues.apache.org/jira/browse/ARROW-609) - [C++] Function for casting from days since UNIX epoch to int64 date +* [ARROW-838](https://issues.apache.org/jira/browse/ARROW-838) - [Python] Efficient construction of arrays from non-pandas 1D NumPy arrays +* [ARROW-905](https://issues.apache.org/jira/browse/ARROW-905) - [Docs] Add Dockerfile for reproducible documentation generation +* [ARROW-942](https://issues.apache.org/jira/browse/ARROW-942) - Support integration testing on Python 2.7 +* [ARROW-950](https://issues.apache.org/jira/browse/ARROW-950) - [Site] Add Google Analytics tag +* [ARROW-972](https://issues.apache.org/jira/browse/ARROW-972) - [Python] Add test cases and basic APIs for UnionArray + +## Bug Fixes + +* [ARROW-1282](https://issues.apache.org/jira/browse/ARROW-1282) - Large memory reallocation by Arrow causes hang in jemalloc +* [ARROW-1341](https://issues.apache.org/jira/browse/ARROW-1341) - [C++] Deprecate arrow::MakeTable in favor of new ctor from ARROW-1334 +* [ARROW-1347](https://issues.apache.org/jira/browse/ARROW-1347) - [JAVA] List null type should use consistent name for inner field +* [ARROW-1398](https://issues.apache.org/jira/browse/ARROW-1398) - [Python] No support reading columns of type decimal(19,4) +* [ARROW-1409](https://issues.apache.org/jira/browse/ARROW-1409) - [Format] Use for "page" attribute in Buffer in metadata +* [ARROW-1540](https://issues.apache.org/jira/browse/ARROW-1540) - [C++] Fix valgrind warnings in cuda-test if possible +* [ARROW-1541](https://issues.apache.org/jira/browse/ARROW-1541) - [C++] Race condition with arrow_gpu +* [ARROW-1543](https://issues.apache.org/jira/browse/ARROW-1543) - [C++] row_wise_conversion example doesn't correspond to ListBuilder constructor arguments +* [ARROW-1549](https://issues.apache.org/jira/browse/ARROW-1549) - [JS] Integrate auto-generated Arrow test files +* [ARROW-1555](https://issues.apache.org/jira/browse/ARROW-1555) - [Python] write_to_dataset on s3 +* [ARROW-1584](https://issues.apache.org/jira/browse/ARROW-1584) - [PYTHON] serialize_pandas on empty dataframe +* [ARROW-1585](https://issues.apache.org/jira/browse/ARROW-1585) - serialize_pandas round trip fails on integer columns +* [ARROW-1586](https://issues.apache.org/jira/browse/ARROW-1586) - [PYTHON] serialize_pandas roundtrip loses columns name +* [ARROW-1609](https://issues.apache.org/jira/browse/ARROW-1609) - Plasma: Build fails with Xcode 9.0 +* [ARROW-1615](https://issues.apache.org/jira/browse/ARROW-1615) - CXX flags for development more permissive than Travis CI builds +* [ARROW-1617](https://issues.apache.org/jira/browse/ARROW-1617) - [Python] Do not use symlinks in python/cmake_modules +* [ARROW-1620](https://issues.apache.org/jira/browse/ARROW-1620) - Python: Download Boost in manylinux1 build from bintray +* [ARROW-1624](https://issues.apache.org/jira/browse/ARROW-1624) - [C++] Follow up fixes / tweaks to compiler warnings for Plasma / LLVM 4.0, add to readme +* [ARROW-1625](https://issues.apache.org/jira/browse/ARROW-1625) - [Serialization] Support OrderedDict properly +* [ARROW-1629](https://issues.apache.org/jira/browse/ARROW-1629) - [C++] Fix problematic code paths identified by infer tool +* [ARROW-1633](https://issues.apache.org/jira/browse/ARROW-1633) - [Python] numpy "unicode" arrays not understood +* [ARROW-1640](https://issues.apache.org/jira/browse/ARROW-1640) - Resolve OpenSSL issues in Travis CI +* [ARROW-1647](https://issues.apache.org/jira/browse/ARROW-1647) - [Plasma] Potential bug when reading/writing messages. +* [ARROW-1653](https://issues.apache.org/jira/browse/ARROW-1653) - [Plasma] Use static cast to avoid compiler warning. +* [ARROW-1656](https://issues.apache.org/jira/browse/ARROW-1656) - [C++] Endianness Macro is Incorrect on Windows And Mac +* [ARROW-1657](https://issues.apache.org/jira/browse/ARROW-1657) - [C++] Multithreaded Read Test Failing on Arch Linux +* [ARROW-1658](https://issues.apache.org/jira/browse/ARROW-1658) - [Python] Out of bounds dictionary indices causes segfault after converting to pandas +* [ARROW-1663](https://issues.apache.org/jira/browse/ARROW-1663) - [Java] Follow up on ARROW-1347 and make schema backward compatible +* [ARROW-1670](https://issues.apache.org/jira/browse/ARROW-1670) - [Python] Speed up deserialization code path +* [ARROW-1672](https://issues.apache.org/jira/browse/ARROW-1672) - [Python] Failure to write Feather bytes column +* [ARROW-1673](https://issues.apache.org/jira/browse/ARROW-1673) - [Python] NumPy boolean arrays get converted to uint8 arrays on NdarrayToTensor roundtrip +* [ARROW-1676](https://issues.apache.org/jira/browse/ARROW-1676) - [C++] Correctly truncate oversized validity bitmaps when writing Feather format +* [ARROW-1678](https://issues.apache.org/jira/browse/ARROW-1678) - [Python] Incorrect serialization of numpy.float16 +* [ARROW-1680](https://issues.apache.org/jira/browse/ARROW-1680) - [Python] Timestamp unit change not done in from_pandas() conversion +* [ARROW-1686](https://issues.apache.org/jira/browse/ARROW-1686) - Documentation generation script creates "apidocs" directory under site/java +* [ARROW-1693](https://issues.apache.org/jira/browse/ARROW-1693) - [JS] Error reading dictionary-encoded integration test files +* [ARROW-1695](https://issues.apache.org/jira/browse/ARROW-1695) - [Serialization] Fix reference counting of numpy arrays created in custom serialializer +* [ARROW-1698](https://issues.apache.org/jira/browse/ARROW-1698) - [JS] File reader attempts to load the same dictionary batch more than once +* [ARROW-1704](https://issues.apache.org/jira/browse/ARROW-1704) - [GLib] Go example in test suite is broken +* [ARROW-1708](https://issues.apache.org/jira/browse/ARROW-1708) - [JS] Linter problem breaks master build +* [ARROW-1709](https://issues.apache.org/jira/browse/ARROW-1709) - [C++] Decimal.ToString is incorrect for negative scale +* [ARROW-1711](https://issues.apache.org/jira/browse/ARROW-1711) - [Python] flake8 checks still not failing builds +* [ARROW-1714](https://issues.apache.org/jira/browse/ARROW-1714) - [Python] No named pd.Series name serialized as u'None' +* [ARROW-1720](https://issues.apache.org/jira/browse/ARROW-1720) - [Python] Segmentation fault while trying to access an out-of-bound chunk +* [ARROW-1723](https://issues.apache.org/jira/browse/ARROW-1723) - Windows: __declspec(dllexport) specified when building arrow static library +* [ARROW-1730](https://issues.apache.org/jira/browse/ARROW-1730) - [Python] Incorrect result from pyarrow.array when passing timestamp type +* [ARROW-1732](https://issues.apache.org/jira/browse/ARROW-1732) - [Python] RecordBatch.from_pandas fails on DataFrame with no columns when preserve_index=False +* [ARROW-1735](https://issues.apache.org/jira/browse/ARROW-1735) - [C++] Cast kernels cannot write into sliced output array +* [ARROW-1738](https://issues.apache.org/jira/browse/ARROW-1738) - [Python] Wrong datetime conversion when pa.array with unit +* [ARROW-1739](https://issues.apache.org/jira/browse/ARROW-1739) - [Python] Fix usages of assertRaises causing broken build +* [ARROW-1742](https://issues.apache.org/jira/browse/ARROW-1742) - C++: clang-format is not detected correct on OSX anymore +* [ARROW-1743](https://issues.apache.org/jira/browse/ARROW-1743) - [Python] Table to_pandas fails when index contains categorical column +* [ARROW-1745](https://issues.apache.org/jira/browse/ARROW-1745) - Compilation failure on Mac OS in plasma tests +* [ARROW-1749](https://issues.apache.org/jira/browse/ARROW-1749) - [C++] Handle range of Decimal128 values that require 39 digits to be displayed +* [ARROW-1751](https://issues.apache.org/jira/browse/ARROW-1751) - [Python] Pandas 0.21.0 introduces a breaking API change for MultiIndex construction +* [ARROW-1754](https://issues.apache.org/jira/browse/ARROW-1754) - [Python] Fix buggy Parquet roundtrip when an index name is the same as a column name +* [ARROW-1756](https://issues.apache.org/jira/browse/ARROW-1756) - [Python] Observed int32 overflow in Feather write/read path +* [ARROW-1762](https://issues.apache.org/jira/browse/ARROW-1762) - [C++] unittest failure for language environment +* [ARROW-1764](https://issues.apache.org/jira/browse/ARROW-1764) - [Python] Add -c conda-forge for Windows dev installation instructions +* [ARROW-1766](https://issues.apache.org/jira/browse/ARROW-1766) - [GLib] Fix failing builds on OSX +* [ARROW-1768](https://issues.apache.org/jira/browse/ARROW-1768) - [Python] Fix suppressed exception in ParquetWriter.__del__ +* [ARROW-1770](https://issues.apache.org/jira/browse/ARROW-1770) - [GLib] Fix GLib compiler warning +* [ARROW-1771](https://issues.apache.org/jira/browse/ARROW-1771) - [C++] ARROW-1749 Breaks Public API test in parquet-cpp +* [ARROW-1776](https://issues.apache.org/jira/browse/ARROW-1776) - [C++[ arrow::gpu::CudaContext::bytes_allocated() isn't defined +* [ARROW-1778](https://issues.apache.org/jira/browse/ARROW-1778) - [Python] Link parquet-cpp statically, privately in manylinux1 wheels +* [ARROW-1781](https://issues.apache.org/jira/browse/ARROW-1781) - [CI] OSX Builds on Travis-CI time out often +* [ARROW-1788](https://issues.apache.org/jira/browse/ARROW-1788) - Plasma store crashes when trying to abort objects for disconnected client +* [ARROW-1791](https://issues.apache.org/jira/browse/ARROW-1791) - Integration tests generate date[DAY] values outside of reasonable range +* [ARROW-1793](https://issues.apache.org/jira/browse/ARROW-1793) - [Integration] fix a typo for README.md +* [ARROW-1800](https://issues.apache.org/jira/browse/ARROW-1800) - [C++] Fix and simplify random_decimals +* [ARROW-1805](https://issues.apache.org/jira/browse/ARROW-1805) - [Python] ignore non-parquet files when exploring dataset +* [ARROW-1811](https://issues.apache.org/jira/browse/ARROW-1811) - [C++/Python] Rename all Decimal based APIs to Decimal128 +* [ARROW-1812](https://issues.apache.org/jira/browse/ARROW-1812) - Plasma store modifies hash table while iterating during client disconnect +* [ARROW-1821](https://issues.apache.org/jira/browse/ARROW-1821) - Add integration test case to explicitly check for optional validity buffer +* [ARROW-1829](https://issues.apache.org/jira/browse/ARROW-1829) - [Plasma] Clean up eviction policy bookkeeping +* [ARROW-1830](https://issues.apache.org/jira/browse/ARROW-1830) - [Python] Error when loading all the files in a dictionary +* [ARROW-1836](https://issues.apache.org/jira/browse/ARROW-1836) - [C++] Fix C4996 warning from arrow/util/variant.h on MSVC builds +* [ARROW-1839](https://issues.apache.org/jira/browse/ARROW-1839) - [C++/Python] Add Decimal Parquet Read/Write Tests +* [ARROW-1840](https://issues.apache.org/jira/browse/ARROW-1840) - [Website] The installation command failed on Windows10 anaconda environment. +* [ARROW-1845](https://issues.apache.org/jira/browse/ARROW-1845) - [Python] Expose Decimal128Type +* [ARROW-1852](https://issues.apache.org/jira/browse/ARROW-1852) - [Plasma] Make retrieving manager file descriptor const +* [ARROW-1853](https://issues.apache.org/jira/browse/ARROW-1853) - [Plasma] Fix off-by-one error in retry processing +* [ARROW-1863](https://issues.apache.org/jira/browse/ARROW-1863) - [Python] PyObjectStringify could render bytes-like output for more types of objects +* [ARROW-1865](https://issues.apache.org/jira/browse/ARROW-1865) - [C++] Adding a column to an empty Table fails +* [ARROW-1869](https://issues.apache.org/jira/browse/ARROW-1869) - Fix typo in LowCostIdentityHashMap +* [ARROW-1871](https://issues.apache.org/jira/browse/ARROW-1871) - [Python/C++] Appending Python Decimals with different scales requires rescaling +* [ARROW-1873](https://issues.apache.org/jira/browse/ARROW-1873) - [Python] Segmentation fault when loading total 2GB of parquet files +* [ARROW-1877](https://issues.apache.org/jira/browse/ARROW-1877) - Incorrect comparison in JsonStringArrayList.equals +* [ARROW-1879](https://issues.apache.org/jira/browse/ARROW-1879) - [Python] Dask integration tests are not skipped if dask is not installed +* [ARROW-1881](https://issues.apache.org/jira/browse/ARROW-1881) - [Python] setuptools_scm picks up JS version tags +* [ARROW-1882](https://issues.apache.org/jira/browse/ARROW-1882) - [C++] Reintroduce DictionaryBuilder +* [ARROW-1883](https://issues.apache.org/jira/browse/ARROW-1883) - [Python] BUG: Table.to_pandas metadata checking fails if columns are not present +* [ARROW-1889](https://issues.apache.org/jira/browse/ARROW-1889) - [Python] --exclude is not available in older git versions +* [ARROW-1890](https://issues.apache.org/jira/browse/ARROW-1890) - [Python] Masking for date32 arrays not working +* [ARROW-1891](https://issues.apache.org/jira/browse/ARROW-1891) - [Python] NaT date32 values are only converted to nulls if from_pandas is used +* [ARROW-1892](https://issues.apache.org/jira/browse/ARROW-1892) - [Python] Unknown list item type: binary +* [ARROW-1893](https://issues.apache.org/jira/browse/ARROW-1893) - [Python] test_primitive_serialization fails on Python 2.7.3 +* [ARROW-1895](https://issues.apache.org/jira/browse/ARROW-1895) - [Python] Add field_name to pandas index metadata +* [ARROW-1897](https://issues.apache.org/jira/browse/ARROW-1897) - [Python] Incorrect numpy_type for pandas metadata of Categoricals +* [ARROW-1904](https://issues.apache.org/jira/browse/ARROW-1904) - [C++] Deprecate PrimitiveArray::raw_values +* [ARROW-1906](https://issues.apache.org/jira/browse/ARROW-1906) - [Python] Creating a pyarrow.Array with timestamp of different unit is not casted +* [ARROW-1908](https://issues.apache.org/jira/browse/ARROW-1908) - [Python] Construction of arrow table from pandas DataFrame with duplicate column names crashes +* [ARROW-1910](https://issues.apache.org/jira/browse/ARROW-1910) - CPP README Brewfile link incorrect +* [ARROW-1914](https://issues.apache.org/jira/browse/ARROW-1914) - [C++] make -j may fail to build with -DARROW_GPU=on +* [ARROW-1915](https://issues.apache.org/jira/browse/ARROW-1915) - [Python] Parquet tests should be optional +* [ARROW-1916](https://issues.apache.org/jira/browse/ARROW-1916) - [Java] Do not exclude java/dev/checkstyle from source releases +* [ARROW-1917](https://issues.apache.org/jira/browse/ARROW-1917) - [GLib] Must set GI_TYPELIB_PATH in verify-release-candidate.sh +* [ARROW-226](https://issues.apache.org/jira/browse/ARROW-226) - [C++] libhdfs: feedback to help determining cause of failure in opening file path +* [ARROW-641](https://issues.apache.org/jira/browse/ARROW-641) - [C++] Do not build/run io-hdfs-test if ARROW_HDFS=off + +[1]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0 +[2]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/ diff --git a/site/_release/index.md b/site/_release/index.md index 4752e980b1916..e22df92b6cb41 100644 --- a/site/_release/index.md +++ b/site/_release/index.md @@ -26,6 +26,7 @@ limitations under the License. Navigate to the release page for downloads and the changelog. +* [0.8.0 (18 December 2017)][10] * [0.7.1 (1 October 2017)][9] * [0.7.0 (17 September 2017)][8] * [0.6.0 (14 August 2017)][7] @@ -45,3 +46,4 @@ Navigate to the release page for downloads and the changelog. [7]: {{ site.baseurl }}/release/0.6.0.html [8]: {{ site.baseurl }}/release/0.7.0.html [9]: {{ site.baseurl }}/release/0.7.1.html +[10]: {{ site.baseurl }}/release/0.8.0.html diff --git a/site/index.html b/site/index.html index 54f67c1a245d0..ffa8d54af2f4f 100644 --- a/site/index.html +++ b/site/index.html @@ -7,7 +7,7 @@

    Apache Arrow

    Powering Columnar In-Memory Analytics

    Join Mailing List - Install (0.7.1 Release - October 1, 2017) + Install (0.8.0 Release - December 18, 2017)

    See Latest News

    diff --git a/site/install.md b/site/install.md index 67b26983b46c4..6e67d790a9d31 100644 --- a/site/install.md +++ b/site/install.md @@ -20,17 +20,17 @@ limitations under the License. {% endcomment %} --> -## Current Version: 0.7.1 +## Current Version: 0.8.0 -### Released: 1 October 2017 +### Released: 18 December 2017 See the [release notes][10] for more about what's new. ### Source release -* **Source Release**: [apache-arrow-0.7.1.tar.gz][6] +* **Source Release**: [apache-arrow-0.8.0.tar.gz][6] * **Verification**: [sha512][3], [asc][7] -* [Git tag 0e21f84][2] +* [Git tag 1d689e5][2] ### Java Packages @@ -52,8 +52,8 @@ Install them with: ```shell -conda install arrow-cpp=0.7.* -c conda-forge -conda install pyarrow=0.7.* -c conda-forge +conda install arrow-cpp=0.8.* -c conda-forge +conda install pyarrow=0.8.* -c conda-forge ``` ### Python Wheels on PyPI (Unofficial) @@ -61,10 +61,10 @@ conda install pyarrow=0.7.* -c conda-forge We have provided binary wheels on PyPI for Linux, macOS, and Windows: ```shell -pip install pyarrow==0.7.* +pip install pyarrow==0.8.* ``` -We recommend pinning `0.7.*` in `requirements.txt` to install the latest patch +We recommend pinning `0.8.*` in `requirements.txt` to install the latest patch release. These include the Apache Arrow and Apache Parquet C++ binary libraries bundled @@ -157,13 +157,13 @@ conda install arrow-cpp -c twosigma conda install pyarrow -c twosigma ``` -[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/ -[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.7.1 -[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz.sha512 -[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.7.1%22 +[1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/ +[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0 +[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512 +[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.8.0%22 [5]: http://conda-forge.github.io -[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz -[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.7.1/apache-arrow-0.7.1.tar.gz.asc +[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz +[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc [8]: https://github.com/red-data-tools/parquet-glib [9]: https://github.com/red-data-tools/arrow-packages -[10]: http://arrow.apache.org/release/0.7.1.html +[10]: http://arrow.apache.org/release/0.8.0.html From 44c5a7c77a8bbc33916aeb718d31027a518a88b7 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Mon, 18 Dec 2017 22:46:19 -0500 Subject: [PATCH 144/177] ARROW-1926: [GLib] Add garrow_timestamp_data_type_get_unit() This is not a 0.8.0 broker. If 0.8.0 RC2 is dropped, I hope that 0.8.0 includes this. If RC2 has no problem, I hope that 0.9.0 includes this. Author: Kouhei Sutou Closes #1424 from kou/glib-add-timestamp-data-type-get-unit and squashes the following commits: a88771b2 [Kouhei Sutou] [GLib] Add garrow_timestamp_data_type_get_unit() --- c_glib/arrow-glib/basic-data-type.cpp | 18 ++++++++++++++++++ c_glib/arrow-glib/basic-data-type.h | 2 ++ c_glib/test/test-timestamp-data-type.rb | 16 ++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/c_glib/arrow-glib/basic-data-type.cpp b/c_glib/arrow-glib/basic-data-type.cpp index dcafe06d67581..a5f7aed1b8f76 100644 --- a/c_glib/arrow-glib/basic-data-type.cpp +++ b/c_glib/arrow-glib/basic-data-type.cpp @@ -805,6 +805,24 @@ garrow_timestamp_data_type_new(GArrowTimeUnit unit) return data_type; } +/** + * garrow_timestamp_data_type_get_unit: + * @timestamp_data_type: The #GArrowTimestampDataType. + * + * Returns: The unit of the timestamp data type. + * + * Since: 0.8.0 + */ +GArrowTimeUnit +garrow_timestamp_data_type_get_unit(GArrowTimestampDataType *timestamp_data_type) +{ + const auto arrow_data_type = + garrow_data_type_get_raw(GARROW_DATA_TYPE(timestamp_data_type)); + const auto arrow_timestamp_data_type = + std::static_pointer_cast(arrow_data_type); + return garrow_time_unit_from_raw(arrow_timestamp_data_type->unit()); +} + G_DEFINE_TYPE(GArrowTimeDataType, \ garrow_time_data_type, \ diff --git a/c_glib/arrow-glib/basic-data-type.h b/c_glib/arrow-glib/basic-data-type.h index bc0192f1b0062..469590191d4e1 100644 --- a/c_glib/arrow-glib/basic-data-type.h +++ b/c_glib/arrow-glib/basic-data-type.h @@ -774,6 +774,8 @@ struct _GArrowTimestampDataTypeClass GType garrow_timestamp_data_type_get_type (void) G_GNUC_CONST; GArrowTimestampDataType *garrow_timestamp_data_type_new (GArrowTimeUnit unit); +GArrowTimeUnit +garrow_timestamp_data_type_get_unit (GArrowTimestampDataType *timestamp_data_type); #define GARROW_TYPE_TIME_DATA_TYPE \ diff --git a/c_glib/test/test-timestamp-data-type.rb b/c_glib/test/test-timestamp-data-type.rb index 83038876926f0..bbc597c160ebc 100644 --- a/c_glib/test/test-timestamp-data-type.rb +++ b/c_glib/test/test-timestamp-data-type.rb @@ -29,6 +29,10 @@ def setup def test_to_s assert_equal("timestamp[s]", @data_type.to_s) end + + def test_unit + assert_equal(Arrow::TimeUnit::SECOND, @data_type.unit) + end end sub_test_case("millisecond") do @@ -39,6 +43,10 @@ def setup def test_to_s assert_equal("timestamp[ms]", @data_type.to_s) end + + def test_unit + assert_equal(Arrow::TimeUnit::MILLI, @data_type.unit) + end end sub_test_case("micro") do @@ -49,6 +57,10 @@ def setup def test_to_s assert_equal("timestamp[us]", @data_type.to_s) end + + def test_unit + assert_equal(Arrow::TimeUnit::MICRO, @data_type.unit) + end end sub_test_case("nano") do @@ -59,5 +71,9 @@ def setup def test_to_s assert_equal("timestamp[ns]", @data_type.to_s) end + + def test_unit + assert_equal(Arrow::TimeUnit::NANO, @data_type.unit) + end end end From 9335e28f2be9da7003638b690cefbe9dc48dc469 Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Mon, 18 Dec 2017 22:53:28 -0500 Subject: [PATCH 145/177] ARROW-1832: [JS] Implement JSON reader for integration tests Add JSON reader, as well as `js/bin/integration.js` script for running integration test validation Author: Paul Taylor Author: Brian Hulette Author: Brian Hulette Closes #1343 from TheNeuralBit/json-reader and squashes the following commits: bd6e80cf [Paul Taylor] print correct error messages f1a51bdd [Paul Taylor] update example html file for new API bb0059b8 [Paul Taylor] fix off by one error reading buffers from metadata v3 arrows 6da297ca [Paul Taylor] update CI and JS integration scripts to invoke jest integration tests 03d82bd4 [Paul Taylor] add integration tests to gulp test task 937dbf84 [Paul Taylor] split out integration and unit tests e0354a76 [Paul Taylor] quote enum keys to save from mangling, add JSON customMetadata map c49f43a1 [Paul Taylor] use string indexers to protect JSON fields from closure compiler's mangler, fix es5 umd build 97f8e5e4 [Paul Taylor] really flatten buffers from json 7b6ea0a3 [Paul Taylor] fix a few json reader typos fa352ead [Paul Taylor] update tests f81dcb9c [Paul Taylor] move arrow2csv into src so it's distributed in the npm packages b5f14702 [Paul Taylor] add Arrow types AST, refactor buffer + json reader to emit type AST nodes 85bf03c9 [Brian Hulette] linter fixes 793f9e56 [Brian Hulette] only use json-bignum in bin/integration eaa5de4a [Brian Hulette] add dictionary-encoded vectors 12f99de0 [Brian Hulette] Add JSON support for Date/Time/Timestamp vectors 53490806 [Brian Hulette] move test data creation after integration_test.py 68c23494 [Paul Taylor] update npm script name in integration runner f50356e0 [Paul Taylor] run the js build before integration.py dcc85f92 [Brian Hulette] linter fixes 313cd587 [Brian Hulette] Add int-test to test-task 86b53b47 [Brian Hulette] Use Int128.fromString in JSON reader 148e997f [Brian Hulette] cleanup a2befbb4 [Brian Hulette] Switch endianness of Int64/128 a1ea88f8 [Brian Hulette] Now uses Uint32 for all internal buffers 02a7838d [Brian Hulette] WIP Int64, Int128 645b844b [Brian Hulette] JS integration script uses new Table.from for JSON c1f3f6ad [Paul Taylor] move createTypedArray and createValidityArray to VectorReaderContext 1191a277 [Paul Taylor] refactor `Table.from()` to accept a JSON object or string 02ea8a67 [Paul Taylor] refactor traits to be compatible with closure compiler's full ES6 -> ES5 01de162b [Paul Taylor] move generated format to format/fb folder, fix closure compiler es5 build ad417411 [Brian Hulette] Fix bug with zero-length vectors b17367cc [Brian Hulette] Add list,struct to JSON reader e3d6d622 [Brian Hulette] linter fixes 1e64707c [Brian Hulette] Add JS integration script and integration_test.py JS runner 7e33b1c4 [Brian Hulette] Add JSON reader --- ci/travis_script_integration.sh | 26 +- integration/integration_test.py | 47 +- js/bin/integration.js | 86 ++++ js/examples/read_file.html | 8 +- js/gulp/argv.js | 6 +- js/gulp/closure-task.js | 29 +- js/gulp/package-task.js | 7 +- js/gulp/test-task.js | 15 +- js/gulp/typescript-task.js | 4 +- js/gulp/util.js | 5 +- js/gulpfile.js | 4 +- js/package.json | 46 +- js/src/Arrow.ts | 17 +- js/{bin/arrow2csv.js => src/bin/arrow2csv.ts} | 78 +++- js/src/format/arrow.ts | 32 ++ js/src/format/fb.ts | 234 ++++++++++ js/src/format/{ => fb}/File.ts | 0 js/src/format/{ => fb}/File_generated.js | 0 js/src/format/{ => fb}/Message.ts | 0 js/src/format/{ => fb}/Message_generated.js | 0 js/src/format/{ => fb}/Schema.ts | 0 js/src/format/{ => fb}/Schema_generated.js | 82 ++-- js/src/format/json.ts | 173 ++++++++ js/src/format/types.ts | 393 +++++++++++++++++ js/src/reader/arrow.ts | 189 +------- js/src/reader/buffer.ts | 229 ++++++++++ js/src/reader/format.ts | 112 ----- js/src/reader/json.ts | 176 ++++++++ js/src/reader/vector.ts | 412 +++++++++--------- js/src/util/int.ts | 320 ++++++++++++++ js/src/util/layout.ts | 200 +++++++++ js/src/vector/arrow.ts | 267 +++--------- js/src/vector/table.ts | 28 +- js/src/vector/traits/field.ts | 71 +++ js/src/vector/{traits.ts => traits/mixins.ts} | 46 +- js/src/vector/traits/nullable.ts | 70 +++ js/src/vector/traits/nullablefield.ts | 71 +++ js/src/vector/traits/vectors.ts | 75 ++++ js/src/vector/vector.ts | 3 +- js/test/Arrow.ts | 3 +- js/test/{ => integration}/test-config.ts | 0 .../validate-tests.ts} | 74 ++-- js/test/reader-tests.ts | 69 --- js/test/table-tests.ts | 136 ------ js/test/unit/int-tests.ts | 227 ++++++++++ js/test/{ => unit}/vector-tests.ts | 5 +- js/tsconfig/tsconfig.es5.cls.json | 6 +- 47 files changed, 2978 insertions(+), 1103 deletions(-) create mode 100755 js/bin/integration.js rename js/{bin/arrow2csv.js => src/bin/arrow2csv.ts} (53%) mode change 100755 => 100644 create mode 100644 js/src/format/arrow.ts create mode 100644 js/src/format/fb.ts rename js/src/format/{ => fb}/File.ts (100%) rename js/src/format/{ => fb}/File_generated.js (100%) rename js/src/format/{ => fb}/Message.ts (100%) rename js/src/format/{ => fb}/Message_generated.js (100%) rename js/src/format/{ => fb}/Schema.ts (100%) rename js/src/format/{ => fb}/Schema_generated.js (97%) create mode 100644 js/src/format/json.ts create mode 100644 js/src/format/types.ts create mode 100644 js/src/reader/buffer.ts delete mode 100644 js/src/reader/format.ts create mode 100644 js/src/reader/json.ts create mode 100644 js/src/util/int.ts create mode 100644 js/src/util/layout.ts create mode 100644 js/src/vector/traits/field.ts rename js/src/vector/{traits.ts => traits/mixins.ts} (67%) create mode 100644 js/src/vector/traits/nullable.ts create mode 100644 js/src/vector/traits/nullablefield.ts create mode 100644 js/src/vector/traits/vectors.ts rename js/test/{ => integration}/test-config.ts (100%) rename js/test/{integration-tests.ts => integration/validate-tests.ts} (58%) delete mode 100644 js/test/reader-tests.ts delete mode 100644 js/test/table-tests.ts create mode 100644 js/test/unit/int-tests.ts rename js/test/{ => unit}/vector-tests.ts (99%) diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh index 9514a30a88b08..0c415dc4865cd 100755 --- a/ci/travis_script_integration.sh +++ b/ci/travis_script_integration.sh @@ -20,6 +20,7 @@ set -e source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug pushd $ARROW_JAVA_DIR @@ -28,8 +29,15 @@ mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false) popd +pushd $ARROW_JS_DIR + +# lint and compile JS source +npm run lint +npm run build + +popd + pushd $ARROW_INTEGRATION_DIR -export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug CONDA_ENV_NAME=arrow-integration-test conda create -y -q -n $CONDA_ENV_NAME python=3.5 @@ -45,21 +53,11 @@ python integration_test.py --debug popd -pushd $ARROW_JS_DIR +# pushd $ARROW_JS_DIR -# lint and compile JS source -npm run lint -npm run build -# create initial test data -npm run create:testdata -# run once to write the snapshots -npm test -- -t ts -u --integration -# run again to test all builds against the snapshots -npm test -- --integration # run tests against source to generate coverage data -npm run test:coverage -- --integration +# npm run test:coverage # Uncomment to upload to coveralls # cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js; - -popd \ No newline at end of file +# popd \ No newline at end of file diff --git a/integration/integration_test.py b/integration/integration_test.py index 5468ab19de8dc..79b098a614a6c 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -868,8 +868,8 @@ def __init__(self, json_files, testers, debug=False): self.debug = debug def run(self): - for producer, consumer in itertools.product(self.testers, - self.testers): + for producer, consumer in itertools.product(filter(lambda t: t.PRODUCER, self.testers), + filter(lambda t: t.CONSUMER, self.testers)): self._compare_implementations(producer, consumer) def _compare_implementations(self, producer, consumer): @@ -909,6 +909,8 @@ def _compare_implementations(self, producer, consumer): class Tester(object): + PRODUCER = False + CONSUMER = False def __init__(self, debug=False): self.debug = debug @@ -927,6 +929,8 @@ def validate(self, json_path, arrow_path): class JavaTester(Tester): + PRODUCER = True + CONSUMER = True _arrow_version = load_version_from_pom() ARROW_TOOLS_JAR = os.environ.get( @@ -978,6 +982,8 @@ def file_to_stream(self, file_path, stream_path): class CPPTester(Tester): + PRODUCER = True + CONSUMER = True EXE_PATH = os.environ.get( 'ARROW_CPP_EXE_PATH', @@ -1025,6 +1031,41 @@ def file_to_stream(self, file_path, stream_path): print(cmd) os.system(cmd) +class JSTester(Tester): + PRODUCER = False + CONSUMER = True + + INTEGRATION_EXE = os.path.join(ARROW_HOME, 'js/bin/integration.js') + + name = 'JS' + + def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): + cmd = [self.INTEGRATION_EXE] + + if arrow_path is not None: + cmd.extend(['-a', arrow_path]) + + if json_path is not None: + cmd.extend(['-j', json_path]) + + cmd.extend(['--mode', command]) + + if self.debug: + print(' '.join(cmd)) + + run_cmd(cmd) + + def validate(self, json_path, arrow_path): + return self._run(arrow_path, json_path, 'VALIDATE') + + def stream_to_file(self, stream_path, file_path): + # Just copy stream to file, we can read the stream directly + cmd = ['cp', stream_path, file_path] + cmd = ' '.join(cmd) + if self.debug: + print(cmd) + os.system(cmd) + def get_static_json_files(): glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json') @@ -1032,7 +1073,7 @@ def get_static_json_files(): def run_all_tests(debug=False): - testers = [CPPTester(debug=debug), JavaTester(debug=debug)] + testers = [CPPTester(debug=debug), JavaTester(debug=debug), JSTester(debug=debug)] static_json_files = get_static_json_files() generated_json_files = get_generated_json_files() json_files = static_json_files + generated_json_files diff --git a/js/bin/integration.js b/js/bin/integration.js new file mode 100755 index 0000000000000..fe32433d3845a --- /dev/null +++ b/js/bin/integration.js @@ -0,0 +1,86 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +var path = require('path'); +var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`)); +var child_process = require(`child_process`); +var optionList = [ + { + type: String, + name: 'mode', + description: 'The integration test to run' + }, + { + type: String, + name: 'arrow', alias: 'a', + description: 'The Arrow file to read/write' + }, + { + type: String, + name: 'json', alias: 'j', + description: 'The JSON file to read/write' + } +]; + +var argv = require(`command-line-args`)(optionList, { partial: true }); + +function print_usage() { + console.log(require('command-line-usage')([ + { + header: 'integration', + content: 'Script for running Arrow integration tests' + }, + { + header: 'Synopsis', + content: [ + '$ integration.js -j file.json -a file.arrow --mode validate' + ] + }, + { + header: 'Options', + optionList: [ + ...optionList, + { + name: 'help', + description: 'Print this usage guide.' + } + ] + }, + ])); + process.exit(1); +} + +if (!argv.arrow || !argv.json || !argv.mode) { + return print_usage(); +} + +switch (argv.mode.toUpperCase()) { + case 'VALIDATE': + child_process.spawnSync( + gulp, + [`test`, `-i`].concat(process.argv.slice(2)), + { + cwd: path.resolve(__dirname, '..'), + stdio: ['ignore', 'inherit', 'inherit'] + } + ); + break; + default: + print_usage(); +} diff --git a/js/examples/read_file.html b/js/examples/read_file.html index 2a1ebaba82c50..3093622fcc1d6 100644 --- a/js/examples/read_file.html +++ b/js/examples/read_file.html @@ -40,7 +40,7 @@ } reader.onload = function (evt) { - var arrowTable = Arrow.Table.from(new Uint8Array(evt.target.result)); + var arrowTable = Arrow.Table.from([new Uint8Array(evt.target.result)]); var thead = document.getElementById("thead"); var tbody = document.getElementById("tbody"); @@ -53,13 +53,13 @@ } var header_row = document.createElement("tr"); - for (let column of arrowTable.cols()) { + for (let column of arrowTable.columns) { addCell(header_row, "th", column.name); } thead.appendChild(header_row); - for (let row of arrowTable.rows(true)) { + for (let row of arrowTable) { var tr = document.createElement("tr"); for (let cell of row) { addCell(tr, "td", @@ -85,6 +85,6 @@
    - + diff --git a/js/gulp/argv.js b/js/gulp/argv.js index 253553c733c8a..6f80912e97e52 100644 --- a/js/gulp/argv.js +++ b/js/gulp/argv.js @@ -16,12 +16,14 @@ // under the License. const argv = require(`command-line-args`)([ - { name: `all`, alias: `a`, type: Boolean }, + { name: `all`, type: Boolean }, { name: 'update', alias: 'u', type: Boolean }, { name: 'verbose', alias: 'v', type: Boolean }, { name: `target`, type: String, defaultValue: `` }, { name: `module`, type: String, defaultValue: `` }, { name: `coverage`, type: Boolean, defaultValue: false }, + { name: `json_file`, alias: `j`, type: String, defaultValue: null }, + { name: `arrow_file`, alias: `a`, type: String, defaultValue: null }, { name: `integration`, alias: `i`, type: Boolean, defaultValue: false }, { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] }, { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }, @@ -36,4 +38,4 @@ argv.module && !modules.length && modules.push(argv.module); (argv.all || !targets.length) && targets.push(`all`); (argv.all || !modules.length) && modules.push(`all`); -module.exports = { argv, targets, modules }; \ No newline at end of file +module.exports = { argv, targets, modules }; diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js index a1f0a9a698843..1bd872fd3044a 100644 --- a/js/gulp/closure-task.js +++ b/js/gulp/closure-task.js @@ -27,6 +27,8 @@ const gulp = require('gulp'); const path = require('path'); const sourcemaps = require('gulp-sourcemaps'); const { memoizeTask } = require('./memoize-task'); +const ASTBuilders = require('ast-types').builders; +const transformAST = require('gulp-transform-js-ast'); const { Observable, ReplaySubject } = require('rxjs'); const closureCompiler = require('google-closure-compiler').gulp(); @@ -48,6 +50,9 @@ const closureTask = ((cache) => memoizeTask(cache, function closure(target, form ], { base: `./` }), sourcemaps.init(), closureCompiler(createClosureArgs(entry, externs)), + // Strip out closure compiler's error-throwing iterator-return methods + // see this issue: https://github.com/google/closure-compiler/issues/2728 + transformAST(iteratorReturnVisitor), // rename the sourcemaps from *.js.map files to *.min.js.map sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }), gulp.dest(out) @@ -62,8 +67,9 @@ const createClosureArgs = (entry, externs) => ({ externs: `${externs}.js`, entry_point: `${entry}.js`, module_resolution: `NODE`, - // formatting: `PRETTY_PRINT`, + // formatting: `PRETTY_PRINT`, debug: true, compilation_level: `ADVANCED`, + allow_method_call_decomposing: true, package_json_entry_names: `module,jsnext:main,main`, assume_function_wrapper: true, js_output_file: `${mainExport}.js`, @@ -95,3 +101,24 @@ const createClosureArgs = (entry, externs) => ({ module.exports = closureTask; module.exports.closureTask = closureTask; + +const iteratorReturnVisitor = { + visitObjectExpression(p) { + const node = p.node, value = p.value; + if (!node.properties || !(node.properties.length === 3)) { return value; } + if (!propertyIsThrowingIteratorReturn(node.properties[2])) { return value; } + value.properties = value.properties.slice(0, 2); + return value; + } +}; + +function propertyIsThrowingIteratorReturn(p) { + if (!p || !(p.kind === 'init')) { return false; } + if (!p.key || !(p.key.type === 'Identifier') || !(p.key.name === 'return')) { return false; } + if (!p.value || !(p.value.type === 'FunctionExpression') || !p.value.params || !(p.value.params.length === 0)) { return false; } + if (!p.value.body || !p.value.body.body || !(p.value.body.body.length === 1) || !(p.value.body.body[0].type === 'ThrowStatement')) { return false; } + if (!p.value.body.body[0].argument || !(p.value.body.body[0].argument.type === 'CallExpression')) { return false; } + if (!p.value.body.body[0].argument.arguments || !(p.value.body.body[0].argument.arguments.length === 1)) { return false; } + if (!p.value.body.body[0].argument.arguments[0] || !(p.value.body.body[0].argument.arguments[0].type === 'Literal')) { return false; } + return p.value.body.body[0].argument.arguments[0].value === 'Not yet implemented'; +} \ No newline at end of file diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js index 824284e6a158a..fc959643503bd 100644 --- a/js/gulp/package-task.js +++ b/js/gulp/package-task.js @@ -53,7 +53,12 @@ const createMainPackageJson = (target, format) => (orig) => ({ const createTypeScriptPackageJson = (target, format) => (orig) => ({ ...createScopedPackageJSON(target, format)(orig), - main: `${mainExport}.ts`, types: `${mainExport}.ts` + main: `${mainExport}.ts`, types: `${mainExport}.ts`, + dependencies: { + '@types/flatbuffers': '*', + '@types/node': '*', + ...orig.dependencies + } }); const createScopedPackageJSON = (target, format) => (({ name, ...orig }) => diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js index ac9755e11ded3..f21aaf2364d03 100644 --- a/js/gulp/test-task.js +++ b/js/gulp/test-task.js @@ -36,16 +36,20 @@ argv.coverage && jestArgv.push(`--coverage`); const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`]; const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`)); +const testOptions = { + env: { ...process.env }, + stdio: [`ignore`, `inherit`, `inherit`], +}; const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) { const opts = { ...testOptions }; const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv]; - if (!argv.integration) { - args.push('test/vector-tests.ts'); - } + args.push(`test/${argv.integration ? `integration/*` : `unit/*`}`); opts.env = { ...opts.env, TEST_TARGET: target, TEST_MODULE: format, + JSON_PATH: argv.json_file, + ARROW_PATH: argv.arrow_file, TEST_TS_SOURCE: !!argv.coverage, TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]), TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]), @@ -53,10 +57,7 @@ const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function return !debug ? child_process.spawn(jest, args, opts) : child_process.exec(`node --inspect-brk ${jest} ${args.join(` `)}`, opts); -}))({}, jestArgv, { - env: { ...process.env }, - stdio: [`ignore`, `inherit`, `inherit`], -}); +}))({}, jestArgv, testOptions); module.exports = testTask; module.exports.testTask = testTask; diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js index 2fd9f1350a6f1..8b755cf7f1624 100644 --- a/js/gulp/typescript-task.js +++ b/js/gulp/typescript-task.js @@ -52,10 +52,10 @@ function maybeCopyRawJSArrowFormatFiles(target, format) { return Observable.empty(); } return Observable.defer(async () => { - const outFormatDir = path.join(targetDir(target, format), `format`); + const outFormatDir = path.join(targetDir(target, format), `format`, `fb`); await del(path.join(outFormatDir, '*.js')); await observableFromStreams( - gulp.src(path.join(`src`, `format`, `*_generated.js`)), + gulp.src(path.join(`src`, `format`, `fb`, `*_generated.js`)), gulpRename((p) => { p.basename = p.basename.replace(`_generated`, ``); }), gulp.dest(outFormatDir) ).toPromise(); diff --git a/js/gulp/util.js b/js/gulp/util.js index 23476da2df53b..ba6ebece51bba 100644 --- a/js/gulp/util.js +++ b/js/gulp/util.js @@ -27,11 +27,12 @@ const npmOrgName = `@${npmPkgName}`; const releasesRootDir = `targets`; const knownTargets = [`es5`, `es2015`, `esnext`]; const knownModules = [`cjs`, `esm`, `cls`, `umd`]; -const moduleFormatsToSkipCombosOf = { cls: true }; +const moduleFormatsToSkipCombosOf = { cls: { test: true, integration: true } }; const packageJSONFields = [ `version`, `license`, `description`, `author`, `homepage`, `repository`, - `bugs`, `keywords`, `dependencies` + `bugs`, `keywords`, `dependencies`, + `bin` ]; const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => { diff --git a/js/gulpfile.js b/js/gulpfile.js index a60363214bb94..7b82962035e46 100644 --- a/js/gulpfile.js +++ b/js/gulpfile.js @@ -93,13 +93,13 @@ gulp.task(`debug`, gulp.series(getTasks(`debug`))); gulp.task(`clean`, gulp.parallel(getTasks(`clean`))); gulp.task(`build`, buildConcurrent(getTasks(`build`))); gulp.task(`default`, gulp.series(`build`, `test`)); - + function getTasks(name) { const tasks = []; if (targets.indexOf(`ts`) !== -1) tasks.push(`${name}:ts`); if (targets.indexOf(npmPkgName) !== -1) tasks.push(`${name}:${npmPkgName}`); for (const [target, format] of combinations(targets, modules)) { - if (moduleFormatsToSkipCombosOf[format] && name === `test`) { + if (moduleFormatsToSkipCombosOf[format] && moduleFormatsToSkipCombosOf[format][name]) { continue; } tasks.push(`${name}:${taskName(target, format)}`); diff --git a/js/package.json b/js/package.json index 8e5c42ef7d1e3..179107b63db11 100644 --- a/js/package.json +++ b/js/package.json @@ -1,14 +1,10 @@ { - "name": "apache-arrow", "version": "0.2.0", - "types": "./types/Arrow.d.ts", - "typings": "./types/Arrow.d.ts", - "main": "./targets/es5/cjs/Arrow.js", - "module": "./targets/es5/esm/Arrow.js", - "browser": "./targets/es5/umd/Arrow.js", - "jsnext:main": "./targets/es2015/esm/Arrow.js", - "esnext:main": "./targets/esnext/esm/Arrow.js", + "name": "apache-arrow", "description": "Apache Arrow columnar in-memory format", + "bin": { + "arrow2csv": "bin/arrow2csv" + }, "scripts": { "lerna": "lerna", "test": "gulp test", @@ -57,50 +53,54 @@ "command-line-usage": "4.0.1" }, "dependencies": { + "command-line-args": "4.0.7", + "command-line-usage": "4.0.2", "flatbuffers": "trxcllnt/flatbuffers-esm", + "json-bignum": "0.0.3", "text-encoding-utf-8": "^1.0.2", - "tslib": "^1.8.0" + "tslib": "1.8.1" }, "devDependencies": { - "@std/esm": "0.13.0", + "@std/esm": "0.18.0", "@types/flatbuffers": "1.6.5", - "@types/glob": "5.0.33", - "@types/jest": "21.1.6", - "@types/node": "8.0.51", + "@types/glob": "5.0.34", + "@types/jest": "21.1.8", + "@types/node": "8.5.0", "@types/text-encoding": "0.0.32", + "ast-types": "0.10.1", "benchmark": "2.1.4", - "command-line-args": "4.0.7", "coveralls": "3.0.0", "del": "3.0.0", "esdoc": "1.0.3", "esdoc-standard-plugin": "1.0.0", "glob": "7.1.2", - "google-closure-compiler": "20171112.0.0", + "google-closure-compiler": "20171203.0.0", "gulp": "github:gulpjs/gulp#4.0", "gulp-json-transform": "0.4.5", "gulp-rename": "1.2.2", "gulp-sourcemaps": "2.6.1", + "gulp-transform-js-ast": "1.0.2", "gulp-typescript": "3.2.3", - "ix": "2.3.1", + "ix": "2.3.4", "jest": "21.2.1", "jest-environment-node-debug": "2.0.0", "json": "9.0.6", "lerna": "2.5.1", - "lint-staged": "5.0.0", + "lint-staged": "6.0.0", "merge2": "1.2.0", "mkdirp": "0.5.1", "npm-run-all": "4.1.2", "pump": "1.0.2", "rimraf": "2.6.2", - "rxjs": "5.5.2", + "rxjs": "5.5.5", "shx": "0.2.2", "source-map-loader": "0.2.3", - "trash": "4.1.0", - "ts-jest": "21.2.1", + "trash": "4.2.1", + "ts-jest": "21.2.4", "tslint": "5.8.0", - "typescript": "2.6.1", - "uglifyjs-webpack-plugin": "1.0.1", - "webpack": "3.8.1", + "typescript": "2.6.2", + "uglifyjs-webpack-plugin": "1.1.2", + "webpack": "3.10.0", "xml2js": "0.4.19" }, "lint-staged": { diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts index e6527b34e10dd..3a8943434eece 100644 --- a/js/src/Arrow.ts +++ b/js/src/Arrow.ts @@ -20,7 +20,8 @@ import { Vector } from './vector/vector'; import { Utf8Vector } from './vector/utf8'; import { DictionaryVector } from './vector/dictionary'; import { StructVector, StructRow } from './vector/struct'; -import { readVectors, readVectorsAsync } from './reader/arrow'; +import { read, readAsync } from './reader/arrow'; +import { Uint64, Int64, Int128 } from './util/int'; import { ListVector, BinaryVector, FixedSizeListVector } from './vector/list'; import { @@ -44,8 +45,16 @@ import { TimestampVector, } from './vector/numeric'; +// closure compiler always erases static method names: +// https://github.com/google/closure-compiler/issues/1776 +// set them via string indexers to save them from the mangler +Table['from'] = Table.from; +Table['fromAsync'] = Table.fromAsync; +BoolVector['pack'] = BoolVector.pack; + +export { read, readAsync }; export { Table, Vector, StructRow }; -export { readVectors, readVectorsAsync }; +export { Uint64, Int64, Int128 }; export { NumericVectorConstructor } from './vector/numeric'; export { List, TypedArray, TypedArrayConstructor } from './vector/types'; export { @@ -80,8 +89,8 @@ try { const Arrow = eval('exports'); if (typeof Arrow === 'object') { // string indexers tell closure compiler not to rename these properties - Arrow['readVectors'] = readVectors; - Arrow['readVectorsAsync'] = readVectorsAsync; + Arrow['read'] = read; + Arrow['readAsync'] = readAsync; Arrow['Table'] = Table; Arrow['Vector'] = Vector; Arrow['StructRow'] = StructRow; diff --git a/js/bin/arrow2csv.js b/js/src/bin/arrow2csv.ts old mode 100755 new mode 100644 similarity index 53% rename from js/bin/arrow2csv.js rename to js/src/bin/arrow2csv.ts index f316b84b69e58..01ef0b848ce75 --- a/js/bin/arrow2csv.js +++ b/js/src/bin/arrow2csv.ts @@ -1,4 +1,4 @@ -#! /usr/bin/env node +// #! /usr/bin/env node // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file @@ -17,13 +17,19 @@ // specific language governing permissions and limitations // under the License. -var fs = require('fs'); -var Table = require('../dist/Arrow.js').Table; -var optionList = [ +/* tslint:disable */ + +import * as Arrow from '../Arrow'; + +(function() { + +const fs = require('fs'); +const { parse } = require('json-bignum'); +const optionList = [ { type: String, - name: 'schema', - alias: 's', multiple: true, + name: 'schema', alias: 's', + optional: true, multiple: true, typeLabel: '[underline]{columns}', description: 'A space-delimited list of column names' }, @@ -34,12 +40,10 @@ var optionList = [ } ]; -var argv = require(`command-line-args`)(optionList, { partial: true }); -var files = [argv.file, ...(argv._unknown || [])].filter(Boolean); - -// console.log(JSON.stringify(argv)); +const argv = require(`command-line-args`)(optionList, { partial: true }); +const files = [argv.file, ...(argv._unknown || [])].filter(Boolean); -if (!argv.schema || !files.length) { +if (!files.length) { console.log(require('command-line-usage')([ { header: 'arrow2csv', @@ -81,9 +85,51 @@ if (!argv.schema || !files.length) { } files.forEach((source) => { - var allColumns = Table.from(fs.readFileSync(source)); - var selectedColumns = new Table(argv.schema.map((columnName) => { - return allColumns.getColumn(columnName); - })); - console.log(selectedColumns.toString()); + let table: any, input = fs.readFileSync(source); + try { + table = Arrow.Table.from([input]); + } catch (e) { + table = Arrow.Table.from(parse(input + '')); + } + if (argv.schema && argv.schema.length) { + table = table.select(...argv.schema); + } + printTable(table); }); + +function printTable(table: Arrow.Table) { + let header = [...table.columns.map((_, i) => table.key(i))].map(stringify); + let maxColumnWidths = header.map(x => x.length); + // Pass one to convert to strings and count max column widths + for (let i = -1, n = table.length - 1; ++i < n;) { + let val, + row = [i, ...table.get(i)]; + for (let j = -1, k = row.length; ++j < k; ) { + val = stringify(row[j]); + maxColumnWidths[j] = Math.max(maxColumnWidths[j], val.length); + } + } + console.log(header.map((x, j) => leftPad(x, ' ', maxColumnWidths[j])).join(' | ')); + // Pass two to pad each one to max column width + for (let i = -1, n = table.length; ++i < n; ) { + console.log( + [...table.get(i)] + .map(stringify) + .map((x, j) => leftPad(x, ' ', maxColumnWidths[j])) + .join(' | ') + ); + } +} + +function leftPad(str: string, fill: string, n: number) { + return (new Array(n + 1).join(fill) + str).slice(-1 * n); +} + +function stringify(x: any) { + return typeof x === 'string' ? `"${x}"` + : Array.isArray(x) ? JSON.stringify(x) + : ArrayBuffer.isView(x) ? `[${x}]` + : `${x}`; +} + +})(); \ No newline at end of file diff --git a/js/src/format/arrow.ts b/js/src/format/arrow.ts new file mode 100644 index 0000000000000..14adf9040a47f --- /dev/null +++ b/js/src/format/arrow.ts @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { footerFromByteBuffer, messageFromByteBuffer } from './fb'; +import { schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON } from './json'; +import { + IntBitWidth, TimeBitWidth, + VisitorNode, Visitor, Footer, Block, Message, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode, + Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_, +} from './types'; + +export { + IntBitWidth, TimeBitWidth, + footerFromByteBuffer, messageFromByteBuffer, + schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON, + VisitorNode, Visitor, Footer, Block, Message, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode, + Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_ as Map, +}; diff --git a/js/src/format/fb.ts b/js/src/format/fb.ts new file mode 100644 index 0000000000000..fdf7f7b0ed99a --- /dev/null +++ b/js/src/format/fb.ts @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as File_ from './fb/File'; +import * as Schema_ from './fb/Schema'; +import * as Message_ from './fb/Message'; +import { flatbuffers } from 'flatbuffers'; +import ByteBuffer = flatbuffers.ByteBuffer; +import Type = Schema_.org.apache.arrow.flatbuf.Type; +import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; +import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion; +import _Footer = File_.org.apache.arrow.flatbuf.Footer; +import _Block = File_.org.apache.arrow.flatbuf.Block; +import _Message = Message_.org.apache.arrow.flatbuf.Message; +import _Schema = Schema_.org.apache.arrow.flatbuf.Schema; +import _Field = Schema_.org.apache.arrow.flatbuf.Field; +import _RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; +import _DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch; +import _FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; +import _Buffer = Schema_.org.apache.arrow.flatbuf.Buffer; +import _DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding; +import _Null = Schema_.org.apache.arrow.flatbuf.Null; +import _Int = Schema_.org.apache.arrow.flatbuf.Int; +import _FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint; +import _Binary = Schema_.org.apache.arrow.flatbuf.Binary; +import _Bool = Schema_.org.apache.arrow.flatbuf.Bool; +import _Utf8 = Schema_.org.apache.arrow.flatbuf.Utf8; +import _Decimal = Schema_.org.apache.arrow.flatbuf.Decimal; +import _Date = Schema_.org.apache.arrow.flatbuf.Date; +import _Time = Schema_.org.apache.arrow.flatbuf.Time; +import _Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp; +import _Interval = Schema_.org.apache.arrow.flatbuf.Interval; +import _List = Schema_.org.apache.arrow.flatbuf.List; +import _Struct = Schema_.org.apache.arrow.flatbuf.Struct_; +import _Union = Schema_.org.apache.arrow.flatbuf.Union; +import _FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary; +import _FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList; +import _Map = Schema_.org.apache.arrow.flatbuf.Map; + +import { + IntBitWidth, TimeBitWidth, + Footer, Block, Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode, + Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_, +} from './types'; + +export function footerFromByteBuffer(bb: ByteBuffer) { + const f = _Footer.getRootAsFooter(bb), s = f.schema()!; + return new Footer( + dictionaryBatchesFromFooter(f), recordBatchesFromFooter(f), + new Schema(f.version(), fieldsFromSchema(s), customMetadata(s), s.endianness()) + ); +} + +export function messageFromByteBuffer(bb: ByteBuffer) { + const m = _Message.getRootAsMessage(bb)!, type = m.headerType(), version = m.version(); + switch (type) { + case MessageHeader.Schema: return schemaFromMessage(version, m.header(new _Schema())!); + case MessageHeader.RecordBatch: return recordBatchFromMessage(version, m.header(new _RecordBatch())!); + case MessageHeader.DictionaryBatch: return dictionaryBatchFromMessage(version, m.header(new _DictionaryBatch())!); + } + return null; + // throw new Error(`Unrecognized Message type '${type}'`); +} + +function schemaFromMessage(version: MetadataVersion, s: _Schema) { + return new Schema(version, fieldsFromSchema(s), customMetadata(s), s.endianness()); +} + +function recordBatchFromMessage(version: MetadataVersion, b: _RecordBatch) { + return new RecordBatch(version, b.length(), fieldNodesFromRecordBatch(b), buffersFromRecordBatch(b, version)); +} + +function dictionaryBatchFromMessage(version: MetadataVersion, d: _DictionaryBatch) { + return new DictionaryBatch(version, recordBatchFromMessage(version, d.data()!), d.id(), d.isDelta()); +} + +function dictionaryBatchesFromFooter(f: _Footer) { + const blocks = [] as Block[]; + for (let b: _Block, i = -1, n = f && f.dictionariesLength(); ++i < n;) { + if (b = f.dictionaries(i)!) { + blocks.push(new Block(b.metaDataLength(), b.bodyLength(), b.offset())); + } + } + return blocks; +} + +function recordBatchesFromFooter(f: _Footer) { + const blocks = [] as Block[]; + for (let b: _Block, i = -1, n = f && f.recordBatchesLength(); ++i < n;) { + if (b = f.recordBatches(i)!) { + blocks.push(new Block(b.metaDataLength(), b.bodyLength(), b.offset())); + } + } + return blocks; +} + +function fieldsFromSchema(s: _Schema) { + const fields = [] as Field[]; + for (let i = -1, n = s && s.fieldsLength(); ++i < n;) { + fields.push(field(s.fields(i)!)); + } + return fields; +} + +function fieldsFromField(f: _Field) { + const fields = [] as Field[]; + for (let i = -1, n = f && f.childrenLength(); ++i < n;) { + fields.push(field(f.children(i)!)); + } + return fields; +} + +function fieldNodesFromRecordBatch(b: _RecordBatch) { + const fieldNodes = [] as FieldNode[]; + for (let i = -1, n = b.nodesLength(); ++i < n;) { + fieldNodes.push(fieldNodeFromRecordBatch(b.nodes(i)!)); + } + return fieldNodes; +} + +function buffersFromRecordBatch(b: _RecordBatch, version: MetadataVersion) { + const buffers = [] as Buffer[]; + for (let i = -1, n = b.buffersLength(); ++i < n;) { + let buffer = b.buffers(i)!; + // If this Arrow buffer was written before version 4, + // advance the buffer's bb_pos 8 bytes to skip past + // the now-removed page id field. + if (version < MetadataVersion.V4) { + buffer.bb_pos += (8 * (i + 1)); + } + buffers.push(bufferFromRecordBatch(buffer)); + } + return buffers; +} + +function field(f: _Field) { + return new Field( + f.name()!, + typeFromField(f), + f.typeType(), + f.nullable(), + fieldsFromField(f), + customMetadata(f), + dictionaryEncodingFromField(f) + ); +} + +function dictionaryEncodingFromField(f: _Field) { + let t: _Int | null; + let e: _DictionaryEncoding | null; + if (e = f.dictionary()) { + if (t = e.indexType()) { + return new DictionaryEncoding(new Int(t.isSigned(), t.bitWidth() as IntBitWidth), e.id(), e.isOrdered()); + } + return new DictionaryEncoding(null, e.id(), e.isOrdered()); + } + return undefined; +} + +function customMetadata(parent?: _Schema | _Field | null) { + const data = new Map(); + if (parent) { + for (let entry, key, i = -1, n = parent.customMetadataLength() | 0; ++i < n;) { + if ((entry = parent.customMetadata(i)) && (key = entry.key()) != null) { + data.set(key, entry.value()!); + } + } + } + return data; +} + +function fieldNodeFromRecordBatch(f: _FieldNode) { + return new FieldNode(f.length(), f.nullCount()); +} + +function bufferFromRecordBatch(b: _Buffer) { + return new Buffer(b.offset(), b.length()); +} + +function typeFromField(f: _Field) { + switch (f.typeType()) { + case Type.NONE: return nullFromField(f.type(new _Null())!); + case Type.Null: return nullFromField(f.type(new _Null())!); + case Type.Int: return intFromField(f.type(new _Int())!); + case Type.FloatingPoint: return floatingPointFromField(f.type(new _FloatingPoint())!); + case Type.Binary: return binaryFromField(f.type(new _Binary())!); + case Type.Utf8: return utf8FromField(f.type(new _Utf8())!); + case Type.Bool: return boolFromField(f.type(new _Bool())!); + case Type.Decimal: return decimalFromField(f.type(new _Decimal())!); + case Type.Date: return dateFromField(f.type(new _Date())!); + case Type.Time: return timeFromField(f.type(new _Time())!); + case Type.Timestamp: return timestampFromField(f.type(new _Timestamp())!); + case Type.Interval: return intervalFromField(f.type(new _Interval())!); + case Type.List: return listFromField(f.type(new _List())!); + case Type.Struct_: return structFromField(f.type(new _Struct())!); + case Type.Union: return unionFromField(f.type(new _Union())!); + case Type.FixedSizeBinary: return fixedSizeBinaryFromField(f.type(new _FixedSizeBinary())!); + case Type.FixedSizeList: return fixedSizeListFromField(f.type(new _FixedSizeList())!); + case Type.Map: return mapFromField(f.type(new _Map())!); + } + throw new Error(`Unrecognized type ${f.typeType()}`); +} + +function nullFromField(_type: _Null) { return new Null(); } +function intFromField(_type: _Int) { return new Int(_type.isSigned(), _type.bitWidth() as IntBitWidth); } +function floatingPointFromField(_type: _FloatingPoint) { return new FloatingPoint(_type.precision()); } +function binaryFromField(_type: _Binary) { return new Binary(); } +function utf8FromField(_type: _Utf8) { return new Utf8(); } +function boolFromField(_type: _Bool) { return new Bool(); } +function decimalFromField(_type: _Decimal) { return new Decimal(_type.scale(), _type.precision()); } +function dateFromField(_type: _Date) { return new Date(_type.unit()); } +function timeFromField(_type: _Time) { return new Time(_type.unit(), _type.bitWidth() as TimeBitWidth); } +function timestampFromField(_type: _Timestamp) { return new Timestamp(_type.unit(), _type.timezone()); } +function intervalFromField(_type: _Interval) { return new Interval(_type.unit()); } +function listFromField(_type: _List) { return new List(); } +function structFromField(_type: _Struct) { return new Struct(); } +function unionFromField(_type: _Union) { return new Union(_type.mode(), (_type.typeIdsArray() || []) as Type[]); } +function fixedSizeBinaryFromField(_type: _FixedSizeBinary) { return new FixedSizeBinary(_type.byteWidth()); } +function fixedSizeListFromField(_type: _FixedSizeList) { return new FixedSizeList(_type.listSize()); } +function mapFromField(_type: _Map) { return new Map_(_type.keysSorted()); } diff --git a/js/src/format/File.ts b/js/src/format/fb/File.ts similarity index 100% rename from js/src/format/File.ts rename to js/src/format/fb/File.ts diff --git a/js/src/format/File_generated.js b/js/src/format/fb/File_generated.js similarity index 100% rename from js/src/format/File_generated.js rename to js/src/format/fb/File_generated.js diff --git a/js/src/format/Message.ts b/js/src/format/fb/Message.ts similarity index 100% rename from js/src/format/Message.ts rename to js/src/format/fb/Message.ts diff --git a/js/src/format/Message_generated.js b/js/src/format/fb/Message_generated.js similarity index 100% rename from js/src/format/Message_generated.js rename to js/src/format/fb/Message_generated.js diff --git a/js/src/format/Schema.ts b/js/src/format/fb/Schema.ts similarity index 100% rename from js/src/format/Schema.ts rename to js/src/format/fb/Schema.ts diff --git a/js/src/format/Schema_generated.js b/js/src/format/fb/Schema_generated.js similarity index 97% rename from js/src/format/Schema_generated.js rename to js/src/format/fb/Schema_generated.js index f89cf2fa8fa0a..ebed8a90645c8 100644 --- a/js/src/format/Schema_generated.js +++ b/js/src/format/fb/Schema_generated.js @@ -31,65 +31,65 @@ org.apache.arrow.flatbuf.MetadataVersion = { /** * 0.1.0 */ - V1: 0, 0: 'V1', + 'V1': 0, 0: 'V1', /** * 0.2.0 */ - V2: 1, 1: 'V2', + 'V2': 1, 1: 'V2', /** * 0.3.0 -> 0.7.1 */ - V3: 2, 2: 'V3', + 'V3': 2, 2: 'V3', /** * >= 0.8.0 */ - V4: 3, 3: 'V4' + 'V4': 3, 3: 'V4' }; /** * @enum */ org.apache.arrow.flatbuf.UnionMode = { - Sparse: 0, 0: 'Sparse', - Dense: 1, 1: 'Dense', + 'Sparse': 0, 0: 'Sparse', + 'Dense': 1, 1: 'Dense', }; /** * @enum */ org.apache.arrow.flatbuf.Precision = { - HALF: 0, 0: 'HALF', - SINGLE: 1, 1: 'SINGLE', - DOUBLE: 2, 2: 'DOUBLE', + 'HALF': 0, 0: 'HALF', + 'SINGLE': 1, 1: 'SINGLE', + 'DOUBLE': 2, 2: 'DOUBLE', }; /** * @enum */ org.apache.arrow.flatbuf.DateUnit = { - DAY: 0, 0: 'DAY', - MILLISECOND: 1, 1: 'MILLISECOND', + 'DAY': 0, 0: 'DAY', + 'MILLISECOND': 1, 1: 'MILLISECOND', }; /** * @enum */ org.apache.arrow.flatbuf.TimeUnit = { - SECOND: 0, 0: 'SECOND', - MILLISECOND: 1, 1: 'MILLISECOND', - MICROSECOND: 2, 2: 'MICROSECOND', - NANOSECOND: 3, 3: 'NANOSECOND', + 'SECOND': 0, 0: 'SECOND', + 'MILLISECOND': 1, 1: 'MILLISECOND', + 'MICROSECOND': 2, 2: 'MICROSECOND', + 'NANOSECOND': 3, 3: 'NANOSECOND', }; /** * @enum */ org.apache.arrow.flatbuf.IntervalUnit = { - YEAR_MONTH: 0, 0: 'YEAR_MONTH', - DAY_TIME: 1, 1: 'DAY_TIME', + 'YEAR_MONTH': 0, 0: 'YEAR_MONTH', + 'DAY_TIME': 1, 1: 'DAY_TIME', }; /** @@ -100,24 +100,24 @@ org.apache.arrow.flatbuf.IntervalUnit = { * @enum */ org.apache.arrow.flatbuf.Type = { - NONE: 0, 0: 'NONE', - Null: 1, 1: 'Null', - Int: 2, 2: 'Int', - FloatingPoint: 3, 3: 'FloatingPoint', - Binary: 4, 4: 'Binary', - Utf8: 5, 5: 'Utf8', - Bool: 6, 6: 'Bool', - Decimal: 7, 7: 'Decimal', - Date: 8, 8: 'Date', - Time: 9, 9: 'Time', - Timestamp: 10, 10: 'Timestamp', - Interval: 11, 11: 'Interval', - List: 12, 12: 'List', - Struct_: 13, 13: 'Struct_', - Union: 14, 14: 'Union', - FixedSizeBinary: 15, 15: 'FixedSizeBinary', - FixedSizeList: 16, 16: 'FixedSizeList', - Map: 17, 17: 'Map' + 'NONE': 0, 0: 'NONE', + 'Null': 1, 1: 'Null', + 'Int': 2, 2: 'Int', + 'FloatingPoint': 3, 3: 'FloatingPoint', + 'Binary': 4, 4: 'Binary', + 'Utf8': 5, 5: 'Utf8', + 'Bool': 6, 6: 'Bool', + 'Decimal': 7, 7: 'Decimal', + 'Date': 8, 8: 'Date', + 'Time': 9, 9: 'Time', + 'Timestamp': 10, 10: 'Timestamp', + 'Interval': 11, 11: 'Interval', + 'List': 12, 12: 'List', + 'Struct_': 13, 13: 'Struct_', + 'Union': 14, 14: 'Union', + 'FixedSizeBinary': 15, 15: 'FixedSizeBinary', + 'FixedSizeList': 16, 16: 'FixedSizeList', + 'Map': 17, 17: 'Map' }; /** @@ -130,22 +130,22 @@ org.apache.arrow.flatbuf.VectorType = { /** * used in List type, Dense Union and variable length primitive types (String, Binary) */ - OFFSET: 0, 0: 'OFFSET', + 'OFFSET': 0, 0: 'OFFSET', /** * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector */ - DATA: 1, 1: 'DATA', + 'DATA': 1, 1: 'DATA', /** * Bit vector indicating if each value is null */ - VALIDITY: 2, 2: 'VALIDITY', + 'VALIDITY': 2, 2: 'VALIDITY', /** * Type vector used in Union type */ - TYPE: 3, 3: 'TYPE' + 'TYPE': 3, 3: 'TYPE' }; /** @@ -155,8 +155,8 @@ org.apache.arrow.flatbuf.VectorType = { * @enum */ org.apache.arrow.flatbuf.Endianness = { - Little: 0, 0: 'Little', - Big: 1, 1: 'Big', + 'Little': 0, 0: 'Little', + 'Big': 1, 1: 'Big', }; /** diff --git a/js/src/format/json.ts b/js/src/format/json.ts new file mode 100644 index 0000000000000..3da3db6d5fea3 --- /dev/null +++ b/js/src/format/json.ts @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Schema_ from './fb/Schema'; +import { flatbuffers } from 'flatbuffers'; +import Long = flatbuffers.Long; +import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion; +import Type = Schema_.org.apache.arrow.flatbuf.Type; +import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit; +import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit; +import Precision = Schema_.org.apache.arrow.flatbuf.Precision; +import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit; +import { + IntBitWidth, TimeBitWidth, + Schema, RecordBatch, DictionaryBatch, Field, DictionaryEncoding, Buffer, FieldNode, + Null, Int, FloatingPoint, Binary, Bool, Utf8, Decimal, Date, Time, Timestamp, Interval, List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_, +} from './types'; + +export function schemaFromJSON(s: any): Schema { + // todo: metadataFromJSON + return new Schema( + MetadataVersion.V4, + fieldsFromJSON(s['fields']), + customMetadata(s['customMetadata']) + ); +} + +export function recordBatchFromJSON(b: any): RecordBatch { + return new RecordBatch( + MetadataVersion.V4, + new Long(b['count'], 0), + fieldNodesFromJSON(b['columns']), + buffersFromJSON(b['columns']) + ); +} + +export function dictionaryBatchFromJSON(b: any): DictionaryBatch { + return new DictionaryBatch( + MetadataVersion.V4, + recordBatchFromJSON(b['data']), + new Long(b['id'], 0), b['isDelta'] + ); +} + +function fieldsFromJSON(fs: any[]): Field[] { + return (fs || []).map(fieldFromJSON); +} + +function fieldNodesFromJSON(xs: any[]): FieldNode[] { + return (xs || []).reduce((fieldNodes, column: any) => [ + ...fieldNodes, + new FieldNode( + new Long(column['count'], 0), + new Long(nullCountFromJSON(column['VALIDITY']), 0) + ), + ...fieldNodesFromJSON(column['children']) + ], [] as FieldNode[]); +} + +function buffersFromJSON(xs: any[], buffers: Buffer[] = []): Buffer[] { + for (let i = -1, n = (xs || []).length; ++i < n;) { + const column = xs[i]; + column['VALIDITY'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['VALIDITY'].length, 0))); + column['OFFSET'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['OFFSET'].length, 0))); + column['DATA'] && buffers.push(new Buffer(new Long(buffers.length, 0), new Long(column['DATA'].length, 0))); + buffers = buffersFromJSON(column['children'], buffers); + } + return buffers; +} + +function nullCountFromJSON(validity: number[]) { + return (validity || []).reduce((sum, val) => sum + +(val === 0), 0); +} + +function fieldFromJSON(f: any) { + return new Field( + f['name'], + typeFromJSON(f['type']), + namesToTypeMap[f['type']['name']], + f.nullable, + fieldsFromJSON(f['children']), + customMetadata(f['customMetadata']), + dictionaryEncodingFromJSON(f['dictionary']) + ); +} + +function dictionaryEncodingFromJSON(d: any) { + return !d ? null : new DictionaryEncoding( + d.indexType ? intFromJSON(d.indexType) : null, + new Long(d.id, 0), d.isOrdered + ); +} + +function customMetadata(metadata?: any) { + return new Map(Object.entries(metadata || {})); +} + +const namesToTypeMap: { [n: string]: Type } = { + 'NONE': Type.NONE, + 'null': Type.Null, + 'int': Type.Int, + 'floatingpoint': Type.FloatingPoint, + 'binary': Type.Binary, + 'bool': Type.Bool, + 'utf8': Type.Utf8, + 'decimal': Type.Decimal, + 'date': Type.Date, + 'time': Type.Time, + 'timestamp': Type.Timestamp, + 'interval': Type.Interval, + 'list': Type.List, + 'struct': Type.Struct_, + 'union': Type.Union, + 'fixedsizebinary': Type.FixedSizeBinary, + 'fixedsizelist': Type.FixedSizeList, + 'map': Type.Map, +}; + +function typeFromJSON(t: any) { + switch (namesToTypeMap[t['name']]) { + case Type.NONE: return nullFromJSON(t); + case Type.Null: return nullFromJSON(t); + case Type.Int: return intFromJSON(t); + case Type.FloatingPoint: return floatingPointFromJSON(t); + case Type.Binary: return binaryFromJSON(t); + case Type.Utf8: return utf8FromJSON(t); + case Type.Bool: return boolFromJSON(t); + case Type.Decimal: return decimalFromJSON(t); + case Type.Date: return dateFromJSON(t); + case Type.Time: return timeFromJSON(t); + case Type.Timestamp: return timestampFromJSON(t); + case Type.Interval: return intervalFromJSON(t); + case Type.List: return listFromJSON(t); + case Type.Struct_: return structFromJSON(t); + case Type.Union: return unionFromJSON(t); + case Type.FixedSizeBinary: return fixedSizeBinaryFromJSON(t); + case Type.FixedSizeList: return fixedSizeListFromJSON(t); + case Type.Map: return mapFromJSON(t); + } + throw new Error(`Unrecognized type ${t['name']}`); +} + +function nullFromJSON(_type: any) { return new Null(); } +function intFromJSON(_type: any) { return new Int(_type['isSigned'], _type['bitWidth'] as IntBitWidth); } +function floatingPointFromJSON(_type: any) { return new FloatingPoint(Precision[_type['precision']] as any); } +function binaryFromJSON(_type: any) { return new Binary(); } +function utf8FromJSON(_type: any) { return new Utf8(); } +function boolFromJSON(_type: any) { return new Bool(); } +function decimalFromJSON(_type: any) { return new Decimal(_type['scale'], _type['precision']); } +function dateFromJSON(_type: any) { return new Date(DateUnit[_type['unit']] as any); } +function timeFromJSON(_type: any) { return new Time(TimeUnit[_type['unit']] as any, _type['bitWidth'] as TimeBitWidth); } +function timestampFromJSON(_type: any) { return new Timestamp(TimeUnit[_type['unit']] as any, _type['timezone']); } +function intervalFromJSON(_type: any) { return new Interval(IntervalUnit[_type['unit']] as any); } +function listFromJSON(_type: any) { return new List(); } +function structFromJSON(_type: any) { return new Struct(); } +function unionFromJSON(_type: any) { return new Union(_type['mode'], (_type['typeIdsArray'] || []) as Type[]); } +function fixedSizeBinaryFromJSON(_type: any) { return new FixedSizeBinary(_type['byteWidth']); } +function fixedSizeListFromJSON(_type: any) { return new FixedSizeList(_type['listSize']); } +function mapFromJSON(_type: any) { return new Map_(_type['keysSorted']); } diff --git a/js/src/format/types.ts b/js/src/format/types.ts new file mode 100644 index 0000000000000..09df8ccbbdf7c --- /dev/null +++ b/js/src/format/types.ts @@ -0,0 +1,393 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* tslint:disable:class-name */ + +import { align } from '../util/layout'; +import * as Schema_ from './fb/Schema'; +import * as Message_ from './fb/Message'; +import { flatbuffers } from 'flatbuffers'; +import Long = flatbuffers.Long; +import Type = Schema_.org.apache.arrow.flatbuf.Type; +import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit; +import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit; +import Precision = Schema_.org.apache.arrow.flatbuf.Precision; +import UnionMode = Schema_.org.apache.arrow.flatbuf.UnionMode; +import Endianness = Schema_.org.apache.arrow.flatbuf.Endianness; +import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit; +import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; +import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion; + +export type IntBitWidth = 8 | 16 | 32 | 64; +export type TimeBitWidth = IntBitWidth | 128; + +export interface VisitorNode { + accept(visitor: Visitor): any; +} + +export abstract class Visitor { + visit(node: VisitorNode): T { + return node.accept(this); + } + visitMany(nodes: VisitorNode[]): T[] { + return nodes.map((node) => this.visit(node)); + } + abstract visitFooter(node: Footer): any; + abstract visitBlock(node: Block): any; + abstract visitMessage(node: Message): any; + abstract visitSchema(node: Schema): any; + abstract visitField(node: Field): any; + abstract visitBuffer(node: Buffer): any; + abstract visitFieldNode(node: FieldNode): any; + abstract visitRecordBatch(node: RecordBatch): any; + abstract visitDictionaryBatch(node: DictionaryBatch): any; + abstract visitDictionaryEncoding(node: DictionaryEncoding): any; + abstract visitNullFieldType(node: Null): any; + abstract visitIntFieldType(node: Int): any; + abstract visitFloatingPointFieldType(node: FloatingPoint): any; + abstract visitBinaryFieldType(node: Binary): any; + abstract visitBoolFieldType(node: Bool): any; + abstract visitUtf8FieldType(node: Utf8): any; + abstract visitDecimalFieldType(node: Decimal): any; + abstract visitDateFieldType(node: Date): any; + abstract visitTimeFieldType(node: Time): any; + abstract visitTimestampFieldType(node: Timestamp): any; + abstract visitIntervalFieldType(node: Interval): any; + abstract visitListFieldType(node: List): any; + abstract visitStructFieldType(node: Struct): any; + abstract visitUnionFieldType(node: Union): any; + abstract visitFixedSizeBinaryFieldType(node: FixedSizeBinary): any; + abstract visitFixedSizeListFieldType(node: FixedSizeList): any; + abstract visitMapFieldType(node: Map_): any; +} + +export class Footer implements VisitorNode { + constructor(public dictionaryBatches: Block[], public recordBatches: Block[], public schema: Schema) {} + accept(visitor: Visitor): any { + return visitor.visitFooter(this); + } +} + +export class Block implements VisitorNode { + constructor(public metaDataLength: number, public bodyLength: Long, public offset: Long) {} + accept(visitor: Visitor): any { + return visitor.visitBlock(this); + } +} + +export class Message implements VisitorNode { + constructor(public version: MetadataVersion, public bodyLength: Long, public headerType: MessageHeader) {} + isSchema(): this is Schema { return this.headerType === MessageHeader.Schema; } + isRecordBatch(): this is RecordBatch { return this.headerType === MessageHeader.RecordBatch; } + isDictionaryBatch(): this is DictionaryBatch { return this.headerType === MessageHeader.DictionaryBatch; } + accept(visitor: Visitor): any { + visitor.visitMessage(this); + } +} + +export class Schema extends Message { + public dictionaries: Map; + constructor(version: MetadataVersion, public fields: Field[], public customMetadata?: Map, public endianness = Endianness.Little) { + super(version, Long.ZERO, MessageHeader.Schema); + const dictionaries = [] as Field[]; + for (let f: Field, i = -1, n = fields.length; ++i < n;) { + if ((f = fields[i])) { + f.dictionary && dictionaries.push(f); + dictionaries.push(...f.dictionaries); + } + } + this.dictionaries = new Map(dictionaries.map<[string, Field]>((f) => [ + f.dictionary!.dictionaryId.toFloat64().toString(), f + ])); + } + accept(visitor: Visitor): any { + return visitor.visitSchema(this); + } +} + +export class RecordBatch extends Message { + constructor(version: MetadataVersion, public length: Long, public fieldNodes: FieldNode[], public buffers: Buffer[]) { + super(version, new Long(buffers.reduce((s, b) => align(s + b.length.low + (b.offset.low - s), 8), 0), 0), MessageHeader.RecordBatch); + } + accept(visitor: Visitor) { + return visitor.visitRecordBatch(this); + } +} + +export class DictionaryBatch extends Message { + constructor(version: MetadataVersion, public dictionary: RecordBatch, public dictionaryId: Long, public isDelta: boolean) { + super(version, dictionary.bodyLength, MessageHeader.DictionaryBatch); + } + get fieldNodes(): FieldNode[] { return this.dictionary.fieldNodes; } + get buffers(): Buffer[] { return this.dictionary.buffers; } + accept(visitor: Visitor) { + return visitor.visitDictionaryBatch(this); + } + static atomicDictionaryId = 0; +} + +export class Field implements VisitorNode { + public dictionaries: Field[]; + constructor(public name: string, + public type: FieldType, + public typeType: Type, + public nullable = false, + public children: Field[] = [], + public metadata?: Map | null, + public dictionary?: DictionaryEncoding | null) { + const dictionaries = [] as Field[]; + for (let f: Field, i = -1, n = children.length; ++i < n;) { + if ((f = children[i])) { + f.dictionary && dictionaries.push(f); + dictionaries.push(...f.dictionaries); + } + } + this.dictionaries = dictionaries; + } + accept(visitor: Visitor): any { + return visitor.visitField(this); + } + indexField() { + return !this.dictionary ? this : new Field( + this.name, + this.dictionary.indexType, this.dictionary.indexType.type, + this.nullable, this.children, this.metadata, this.dictionary + ); + } + toString() { return `Field name[${this.name}], nullable[${this.nullable}], type[${this.type.toString()}]`; } +} + +export class Buffer implements VisitorNode { + constructor(public offset: Long, public length: Long) {} + accept(visitor: Visitor) { + return visitor.visitBuffer(this); + } +} + +export class FieldNode implements VisitorNode { + constructor(public length: Long, public nullCount: Long) {} + accept(visitor: Visitor) { + return visitor.visitFieldNode(this); + } +} + +export class DictionaryEncoding implements VisitorNode { + public isOrdered: boolean; + public dictionaryId: Long; + public indexType: Int; + constructor(indexType?: Int | null, dictionaryId?: Long | null, isOrdered?: boolean | null) { + this.isOrdered = isOrdered || false; + /* a dictionary index defaults to signed 32 bit int if unspecified */ + this.indexType = indexType || new Int(true, 32); + this.dictionaryId = dictionaryId || new Long(DictionaryBatch.atomicDictionaryId++, 0); + } + accept(visitor: Visitor): any { + return visitor.visitDictionaryEncoding(this); + } +} + +export abstract class FieldType implements VisitorNode { + constructor(public type: Type) {} + abstract accept(visitor: Visitor): any; + isNull(): this is Null { return this.type === Type.Null; } + isInt(): this is Int { return this.type === Type.Int; } + isFloatingPoint(): this is FloatingPoint { return this.type === Type.FloatingPoint; } + isBinary(): this is Binary { return this.type === Type.Binary; } + isUtf8(): this is Utf8 { return this.type === Type.Utf8; } + isBool(): this is Bool { return this.type === Type.Bool; } + isDecimal(): this is Decimal { return this.type === Type.Decimal; } + isDate(): this is Date { return this.type === Type.Date; } + isTime(): this is Time { return this.type === Type.Time; } + isTimestamp(): this is Timestamp { return this.type === Type.Timestamp; } + isInterval(): this is Interval { return this.type === Type.Interval; } + isList(): this is List { return this.type === Type.List; } + isStruct(): this is Struct { return this.type === Type.Struct_; } + isUnion(): this is Union { return this.type === Type.Union; } + isFixedSizeBinary(): this is FixedSizeBinary { return this.type === Type.FixedSizeBinary; } + isFixedSizeList(): this is FixedSizeList { return this.type === Type.FixedSizeList; } + isMap(): this is Map_ { return this.type === Type.Map; } +} + +export class Null extends FieldType { + toString() { return `Null`; } + constructor() { + super(Type.Null); + } + accept(visitor: Visitor) { + return visitor.visitNullFieldType(this); + } +} + +export class Int extends FieldType { + toString() { return `Int isSigned[${this.isSigned}], bitWidth[${this.bitWidth}]`; } + constructor(public isSigned: boolean, public bitWidth: IntBitWidth) { + super(Type.Int); + } + accept(visitor: Visitor) { + return visitor.visitIntFieldType(this); + } +} + +export class FloatingPoint extends FieldType { + toString() { return `FloatingPoint precision`; } + constructor(public precision: Precision) { + super(Type.FloatingPoint); + } + accept(visitor: Visitor) { + return visitor.visitFloatingPointFieldType(this); + } +} + +export class Binary extends FieldType { + toString() { return `Binary`; } + constructor() { + super(Type.Binary); + } + accept(visitor: Visitor) { + return visitor.visitBinaryFieldType(this); + } +} + +export class Utf8 extends FieldType { + toString() { return `Utf8`; } + constructor() { + super(Type.Utf8); + } + accept(visitor: Visitor) { + return visitor.visitUtf8FieldType(this); + } +} + +export class Bool extends FieldType { + toString() { return `Bool`; } + constructor() { + super(Type.Bool); + } + accept(visitor: Visitor) { + return visitor.visitBoolFieldType(this); + } +} + +export class Decimal extends FieldType { + toString() { return `Decimal scale[${this.scale}], precision[${this.precision}]`; } + constructor(public scale: number, public precision: number) { + super(Type.Decimal); + } + accept(visitor: Visitor) { + return visitor.visitDecimalFieldType(this); + } +} + +export class Date extends FieldType { + toString() { return `Date unit[${this.unit}]`; } + constructor(public unit: DateUnit) { + super(Type.Date); + } + accept(visitor: Visitor) { + return visitor.visitDateFieldType(this); + } +} + +export class Time extends FieldType { + toString() { return `Time unit[${this.unit}], bitWidth[${this.bitWidth}]`; } + constructor(public unit: TimeUnit, public bitWidth: TimeBitWidth) { + super(Type.Time); + } + accept(visitor: Visitor) { + return visitor.visitTimeFieldType(this); + } +} + +export class Timestamp extends FieldType { + toString() { return `Timestamp unit[${this.unit}], timezone[${this.timezone}]`; } + constructor(public unit: TimeUnit, public timezone?: string | null) { + super(Type.Timestamp); + } + accept(visitor: Visitor) { + return visitor.visitTimestampFieldType(this); + } +} + +export class Interval extends FieldType { + toString() { return `Interval unit[${this.unit}]`; } + constructor(public unit: IntervalUnit) { + super(Type.Interval); + } + accept(visitor: Visitor) { + return visitor.visitIntervalFieldType(this); + } +} + +export class List extends FieldType { + toString() { return `List`; } + constructor() { + super(Type.List); + } + accept(visitor: Visitor) { + return visitor.visitListFieldType(this); + } +} + +export class Struct extends FieldType { + toString() { return `Struct`; } + constructor() { + super(Type.Struct_); + } + accept(visitor: Visitor) { + return visitor.visitStructFieldType(this); + } +} + +export class Union extends FieldType { + toString() { return `Union mode[${this.mode}], typeIds[${this.typeIds}]`; } + constructor(public mode: UnionMode, public typeIds: Type[]) { + super(Type.Union); + } + accept(visitor: Visitor) { + return visitor.visitUnionFieldType(this); + } +} + +export class FixedSizeBinary extends FieldType { + toString() { return `FixedSizeBinary byteWidth[${this.byteWidth}]`; } + constructor(public byteWidth: number) { + super(Type.FixedSizeBinary); + } + accept(visitor: Visitor) { + return visitor.visitFixedSizeBinaryFieldType(this); + } +} + +export class FixedSizeList extends FieldType { + toString() { return `FixedSizeList listSize[${this.listSize}]`; } + constructor(public listSize: number) { + super(Type.FixedSizeList); + } + accept(visitor: Visitor) { + return visitor.visitFixedSizeListFieldType(this); + } +} + +export class Map_ extends FieldType { + toString() { return `Map keysSorted[${this.keysSorted}]`; } + constructor(public keysSorted: boolean) { + super(Type.Map); + } + accept(visitor: Visitor) { + return visitor.visitMapFieldType(this); + } +} diff --git a/js/src/reader/arrow.ts b/js/src/reader/arrow.ts index 8d3aafc722160..cf8a3d6a281a2 100644 --- a/js/src/reader/arrow.ts +++ b/js/src/reader/arrow.ts @@ -15,181 +15,34 @@ // specific language governing permissions and limitations // under the License. +import { readJSON } from './json'; +import { readBuffers, readBuffersAsync } from './buffer'; +import { readVectors, readVectorsAsync } from './vector'; import { Vector } from '../vector/vector'; -import { flatbuffers } from 'flatbuffers'; -import { readVector, readValueVector } from './vector'; -import { - readFileFooter, readFileMessages, - readStreamSchema, readStreamMessages -} from './format'; -import * as File_ from '../format/File'; -import * as Schema_ from '../format/Schema'; -import * as Message_ from '../format/Message'; +export { readJSON }; +export { readBuffers, readBuffersAsync }; +export { readVectors, readVectorsAsync }; -import ByteBuffer = flatbuffers.ByteBuffer; -import Footer = File_.org.apache.arrow.flatbuf.Footer; -import Field = Schema_.org.apache.arrow.flatbuf.Field; -import Schema = Schema_.org.apache.arrow.flatbuf.Schema; -import Message = Message_.org.apache.arrow.flatbuf.Message; -import ArrowBuffer = Schema_.org.apache.arrow.flatbuf.Buffer; -import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; -import RecordBatch = Message_.org.apache.arrow.flatbuf.RecordBatch; -import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; -import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion; -import DictionaryBatch = Message_.org.apache.arrow.flatbuf.DictionaryBatch; -import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding; - -export type ArrowReaderContext = { - schema?: Schema; - footer?: Footer | null; - dictionaries: Map; - dictionaryEncodedFields: Map; - readMessages: (bb: ByteBuffer, footer: Footer) => Iterable; -}; - -export interface VectorReaderContext { - offset: number; - bytes: Uint8Array; - batch: RecordBatch; - dictionaries: Map; - readNextNode(): FieldNode; - readNextBuffer(): ArrowBuffer; -} - -export function* readVectors(buffers: Iterable, context?: ArrowReaderContext) { - const context_ = context || {} as ArrowReaderContext; - for (const buffer of buffers) { - yield* readBuffer(toByteBuffer(buffer), context_); +export function* read(sources: Iterable | object | string) { + let input: any = sources; + let batches: Iterable; + if (typeof input === 'string') { + try { input = JSON.parse(input); } + catch (e) { input = sources; } } -} - -export async function* readVectorsAsync(buffers: AsyncIterable, context?: ArrowReaderContext) { - const context_ = context || {} as ArrowReaderContext; - for await (const buffer of buffers) { - yield* readBuffer(toByteBuffer(buffer), context_); - } -} - -function* readBuffer(bb: ByteBuffer, readerContext: ArrowReaderContext) { - - let { schema, footer, readMessages, dictionaryEncodedFields, dictionaries } = readerContext; - - if (!schema) { - ({ schema, footer, readMessages, dictionaryEncodedFields } = readSchema(bb)); - readerContext.schema = schema; - readerContext.readMessages = readMessages; - readerContext.dictionaryEncodedFields = dictionaryEncodedFields; - readerContext.dictionaries = dictionaries = new Map(); - } - - const fieldsLength = schema.fieldsLength(); - const context = new BufferReaderContext(bb.bytes(), dictionaries); - - for (const message of readMessages(bb, footer!)) { - - let id: string; - let field: Field; - let vector: Vector; - let vectors: Array; - - context.message = message; - - if (message.headerType() === MessageHeader.DictionaryBatch) { - let batch: DictionaryBatch; - if (batch = message.header(new DictionaryBatch())!) { - context.batch = batch.data()!; - id = batch.id().toFloat64().toString(); - field = dictionaryEncodedFields.get(id)!; - vector = readValueVector(field, context); - if (batch.isDelta() && dictionaries.has(id)) { - vector = dictionaries.get(id)!.concat(vector); - } - dictionaries.set(id, vector); - } - continue; - } - - vectors = new Array(fieldsLength); - context.batch = message.header(new RecordBatch())!; - - for (let i = -1; ++i < fieldsLength;) { - if ((field = schema.fields(i)!) || (vectors[i] = null as any)) { - vectors[i] = readVector(field, context); - } - } - - yield vectors; - } -} - -function readSchema(bb: ByteBuffer) { - let schema: Schema, readMessages, footer = readFileFooter(bb); - if (footer) { - schema = footer.schema()!; - readMessages = readFileMessages; - } else if (schema = readStreamSchema(bb)!) { - readMessages = readStreamMessages; + if (!input || typeof input !== 'object') { + batches = (typeof input === 'string') ? readVectors(readBuffers([input])) : []; } else { - throw new Error('Invalid Arrow buffer'); - } - return { schema, footer, readMessages, dictionaryEncodedFields: readDictionaryEncodedFields(schema, new Map()) }; -} - -function readDictionaryEncodedFields(parent: Schema | Field, fields: Map) { - let field: Field, encoding: DictionaryEncoding, id: string; - let getField = parent instanceof Field ? parent.children : parent.fields; - let getFieldCount = parent instanceof Field ? parent.childrenLength : parent.fieldsLength; - for (let i = -1, n = getFieldCount.call(parent); ++i < n;) { - if (field = getField.call(parent, i)!) { - if ((encoding = field.dictionary()!) && - (id = encoding.id().toFloat64().toString())) { - !fields.has(id) && fields.set(id, field); - } - readDictionaryEncodedFields(field, fields); - } + batches = (typeof input[Symbol.iterator] === 'function') + ? readVectors(readBuffers(input)) + : readVectors(readJSON(input)); } - return fields; + yield* batches; } -function toByteBuffer(bytes?: Uint8Array | Buffer | string) { - let arr: Uint8Array = bytes as any || new Uint8Array(0); - if (typeof bytes === 'string') { - arr = new Uint8Array(bytes.length); - for (let i = -1, n = bytes.length; ++i < n;) { - arr[i] = bytes.charCodeAt(i); - } - return new ByteBuffer(arr); +export async function* readAsync(sources: AsyncIterable) { + for await (let vectors of readVectorsAsync(readBuffersAsync(sources))) { + yield vectors; } - return new ByteBuffer(arr); } - -class BufferReaderContext implements VectorReaderContext { - public offset: number; - public batch: RecordBatch; - private nodeIndex: number; - private bufferIndex: number; - private metadataVersion: MetadataVersion; - constructor(public bytes: Uint8Array, - public dictionaries: Map) { - } - set message(m: Message) { - this.nodeIndex = 0; - this.bufferIndex = 0; - this.offset = m.bb.position(); - this.metadataVersion = m.version(); - } - public readNextNode() { - return this.batch.nodes(this.nodeIndex++)!; - } - public readNextBuffer() { - const buffer = this.batch.buffers(this.bufferIndex++)!; - // If this Arrow buffer was written before version 4, - // advance the buffer's bb_pos 8 bytes to skip past - // the now-removed page id field. - if (this.metadataVersion < MetadataVersion[`V4`]) { - buffer.bb_pos += (8 * this.bufferIndex); - } - return buffer; - } -} \ No newline at end of file diff --git a/js/src/reader/buffer.ts b/js/src/reader/buffer.ts new file mode 100644 index 0000000000000..c7b90507e396f --- /dev/null +++ b/js/src/reader/buffer.ts @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { flatbuffers } from 'flatbuffers'; +import { VectorLayoutReader } from './vector'; +import { TypedArray, TypedArrayConstructor } from '../vector/types'; +import { footerFromByteBuffer, messageFromByteBuffer } from '../format/fb'; +import { Footer, Schema, RecordBatch, DictionaryBatch, Field, Buffer, FieldNode } from '../format/arrow'; +import ByteBuffer = flatbuffers.ByteBuffer; + +export function* readBuffers(sources: Iterable) { + let schema: Schema | null = null; + let readMessages: ((bb: ByteBuffer) => IterableIterator) | null = null; + for (const source of sources) { + const bb = toByteBuffer(source); + if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) { + for (const message of readMessages(bb)) { + yield { + schema, message, reader: new BufferVectorLayoutReader( + bb, + (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes), + (function* (buffers) { yield* buffers; })(message.buffers) + ) as VectorLayoutReader + }; + } + } + } +} + +export async function* readBuffersAsync(sources: AsyncIterable) { + let schema: Schema | null = null; + let readMessages: ((bb: ByteBuffer) => IterableIterator) | null = null; + for await (const source of sources) { + const bb = toByteBuffer(source); + if ((!schema && ({ schema, readMessages } = readSchema(bb))) && schema && readMessages) { + for (const message of readMessages(bb)) { + yield { + schema, message, reader: new BufferVectorLayoutReader( + bb, + (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes), + (function* (buffers) { yield* buffers; })(message.buffers) + ) as VectorLayoutReader + }; + } + } + } +} + +function toByteBuffer(bytes?: Uint8Array | NodeBuffer | string) { + let arr: Uint8Array = bytes as any || new Uint8Array(0); + if (typeof bytes === 'string') { + arr = new Uint8Array(bytes.length); + for (let i = -1, n = bytes.length; ++i < n;) { + arr[i] = bytes.charCodeAt(i); + } + return new ByteBuffer(arr); + } + return new ByteBuffer(arr); +} + +function readSchema(bb: ByteBuffer) { + let schema: Schema, readMessages, footer: Footer | null; + if (footer = readFileSchema(bb)) { + schema = footer.schema!; + readMessages = readFileMessages(footer); + } else if (schema = readStreamSchema(bb)!) { + readMessages = readStreamMessages; + } else { + throw new Error('Invalid Arrow buffer'); + } + return { schema, readMessages }; +} + +const PADDING = 4; +const MAGIC_STR = 'ARROW1'; +const MAGIC = new Uint8Array(MAGIC_STR.length); +for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) { + MAGIC[i] = MAGIC_STR.charCodeAt(i); +} + +function checkForMagicArrowString(buffer: Uint8Array, index = 0) { + for (let i = -1, n = MAGIC.length; ++i < n;) { + if (MAGIC[i] !== buffer[index + i]) { + return false; + } + } + return true; +} + +const magicLength = MAGIC.length; +const magicAndPadding = magicLength + PADDING; +const magicX2AndPadding = magicLength * 2 + PADDING; + +function readStreamSchema(bb: ByteBuffer) { + if (!checkForMagicArrowString(bb.bytes(), 0)) { + for (const message of readMessages(bb)) { + if (message.isSchema()) { + return message as Schema; + } + } + } + return null; +} + +function* readStreamMessages(bb: ByteBuffer) { + for (const message of readMessages(bb)) { + if (message.isRecordBatch()) { + yield message; + } else if (message.isDictionaryBatch()) { + yield message; + } else { + continue; + } + // position the buffer after the body to read the next message + bb.setPosition(bb.position() + message.bodyLength.low); + } +} + +function readFileSchema(bb: ByteBuffer) { + let fileLength = bb.capacity(), footerLength: number, footerOffset: number; + if ((fileLength < magicX2AndPadding /* Arrow buffer too small */) || + (!checkForMagicArrowString(bb.bytes(), 0) /* Missing magic start */) || + (!checkForMagicArrowString(bb.bytes(), fileLength - magicLength) /* Missing magic end */) || + (/* Invalid footer length */ + (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 && + (footerLength + magicX2AndPadding > fileLength))) { + return null; + } + bb.setPosition(footerOffset - footerLength); + return footerFromByteBuffer(bb); +} + +function readFileMessages(footer: Footer) { + return function* (bb: ByteBuffer) { + for (let i = -1, batches = footer.dictionaryBatches, n = batches.length; ++i < n;) { + bb.setPosition(batches[i].offset.low); + yield readMessage(bb, bb.readInt32(bb.position())) as DictionaryBatch; + } + for (let i = -1, batches = footer.recordBatches, n = batches.length; ++i < n;) { + bb.setPosition(batches[i].offset.low); + yield readMessage(bb, bb.readInt32(bb.position())) as RecordBatch; + } + }; +} + +function* readMessages(bb: ByteBuffer) { + let length: number, message: Schema | RecordBatch | DictionaryBatch; + while (bb.position() < bb.capacity() && + (length = bb.readInt32(bb.position())) > 0) { + if (message = readMessage(bb, length)!) { + yield message; + } + } +} + +function readMessage(bb: ByteBuffer, length: number) { + bb.setPosition(bb.position() + PADDING); + const message = messageFromByteBuffer(bb); + bb.setPosition(bb.position() + length); + return message; +} + +class BufferVectorLayoutReader implements VectorLayoutReader { + private offset: number; + private bytes: Uint8Array; + constructor(bb: ByteBuffer, private fieldNodes: Iterator, private buffers: Iterator) { + this.bytes = bb.bytes(); + this.offset = bb.position(); + } + readContainerLayout(field: Field) { + const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value) + }; + } + readFixedWidthLayout(field: Field, dataType: TypedArrayConstructor) { + const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value), + data: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, dataType) + }; + } + readBinaryLayout(field: Field) { + const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value), + offsets: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Int32Array), + data: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Uint8Array) + }; + } + readVariableWidthLayout(field: Field) { + const { bytes, offset, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(bytes, field, fieldNode, offset, buffers.next().value), + offsets: createTypedArray(bytes, field, fieldNode, offset, buffers.next().value, Int32Array) + }; + } +} + +function createValidityArray(bytes: Uint8Array, field: Field, fieldNode: FieldNode, offset: number, buffer: Buffer) { + return field.nullable && fieldNode.nullCount.low > 0 && + createTypedArray(bytes, field, fieldNode, offset, buffer, Uint8Array) || null; +} + +function createTypedArray(bytes: Uint8Array, _field: Field, _fieldNode: FieldNode, offset: number, buffer: Buffer, ArrayConstructor: TypedArrayConstructor): T { + return new ArrayConstructor( + bytes.buffer, + bytes.byteOffset + offset + buffer.offset.low, + buffer.length.low / ArrayConstructor.BYTES_PER_ELEMENT + ); +} diff --git a/js/src/reader/format.ts b/js/src/reader/format.ts deleted file mode 100644 index fd8f1b40d918a..0000000000000 --- a/js/src/reader/format.ts +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { flatbuffers } from 'flatbuffers'; -import * as File_ from '../format/File'; -import * as Schema_ from '../format/Schema'; -import * as Message_ from '../format/Message'; -import ByteBuffer = flatbuffers.ByteBuffer; -import Footer = File_.org.apache.arrow.flatbuf.Footer; -import Schema = Schema_.org.apache.arrow.flatbuf.Schema; -import Message = Message_.org.apache.arrow.flatbuf.Message; -import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader; - -const PADDING = 4; -const MAGIC_STR = 'ARROW1'; -const MAGIC = new Uint8Array(MAGIC_STR.length); -for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) { - MAGIC[i] = MAGIC_STR.charCodeAt(i); -} - -export function _checkMagic(buffer: Uint8Array, index = 0) { - for (let i = -1, n = MAGIC.length; ++i < n;) { - if (MAGIC[i] !== buffer[index + i]) { - return false; - } - } - return true; -} - -const magicLength = MAGIC.length; -const magicAndPadding = magicLength + PADDING; -const magicX2AndPadding = magicLength * 2 + PADDING; - -export function readStreamSchema(bb: ByteBuffer) { - if (!_checkMagic(bb.bytes(), 0)) { - for (const message of readMessages(bb)) { - if (message.headerType() === MessageHeader.Schema) { - return message.header(new Schema()); - } - } - } - return null; -} - -export function readFileFooter(bb: ByteBuffer) { - let fileLength = bb.capacity(); - let footerLength: number, footerOffset: number; - if ((fileLength < magicX2AndPadding /* Arrow buffer too small */) || - (!_checkMagic(bb.bytes(), 0) /* Missing magic start */) || - (!_checkMagic(bb.bytes(), fileLength - magicLength) /* Missing magic end */) || - (/* Invalid footer length */ - (footerLength = bb.readInt32(footerOffset = fileLength - magicAndPadding)) < 1 && - (footerLength + magicX2AndPadding > fileLength))) { - return null; - } - bb.setPosition(footerOffset - footerLength); - return Footer.getRootAsFooter(bb); -} - -export function* readFileMessages(bb: ByteBuffer, footer: Footer) { - for (let i = -1, n = footer.dictionariesLength(); ++i < n;) { - bb.setPosition(footer.dictionaries(i)!.offset().low); - yield readMessage(bb, bb.readInt32(bb.position())); - } - for (let i = -1, n = footer.recordBatchesLength(); ++i < n;) { - bb.setPosition(footer.recordBatches(i)!.offset().low); - yield readMessage(bb, bb.readInt32(bb.position())); - } -} - -export function readMessage(bb: ByteBuffer, length: number) { - bb.setPosition(bb.position() + PADDING); - const message = Message.getRootAsMessage(bb); - bb.setPosition(bb.position() + length); - return message; -} - -export function* readMessages(bb: ByteBuffer) { - let length; - while (bb.position() < bb.capacity() && - (length = bb.readInt32(bb.position())) > 0) { - yield readMessage(bb, length); - } -} - -export function* readStreamMessages(bb: ByteBuffer) { - for (const message of readMessages(bb)) { - switch (message.headerType()) { - case MessageHeader.RecordBatch: - case MessageHeader.DictionaryBatch: - yield message; - break; - default: continue; - } - // position the buffer after the body to read the next message - bb.setPosition(bb.position() + message.bodyLength().low); - } -} diff --git a/js/src/reader/json.ts b/js/src/reader/json.ts new file mode 100644 index 0000000000000..49431496354e8 --- /dev/null +++ b/js/src/reader/json.ts @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Schema_ from '../format/fb/Schema'; +import { Int64, Int128 } from '../util/int'; +import { VectorLayoutReader } from './vector'; +import { TextEncoder } from 'text-encoding-utf-8'; +import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit; +import { TypedArray, TypedArrayConstructor } from '../vector/types'; +import { schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON } from '../format/json'; +import { Schema, RecordBatch, DictionaryBatch, Field, Buffer, FieldNode } from '../format/arrow'; +export { Schema, RecordBatch, DictionaryBatch }; + +export function* readJSON(json: any) { + const schema = schemaFromJSON(json['schema']); + for (const batch of (json['dictionaries'] || [])) { + const message = dictionaryBatchFromJSON(batch); + yield { + schema, message, reader: new JSONVectorLayoutReader( + flattenDataSources(batch['data']['columns']), + (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes), + (function* (buffers) { yield* buffers; })(message.buffers) + ) as VectorLayoutReader + }; + } + for (const batch of (json['batches'] || [])) { + const message = recordBatchFromJSON(batch); + yield { + schema, message, reader: new JSONVectorLayoutReader( + flattenDataSources(batch['columns']), + (function* (fieldNodes) { yield* fieldNodes; })(message.fieldNodes), + (function* (buffers) { yield* buffers; })(message.buffers) + ) as VectorLayoutReader + }; + } +} + +function flattenDataSources(xs: any[]): any[][] { + return (xs || []).reduce((buffers, column: any) => [ + ...buffers, + ...(column['VALIDITY'] && [column['VALIDITY']] || []), + ...(column['OFFSET'] && [column['OFFSET']] || []), + ...(column['DATA'] && [column['DATA']] || []), + ...flattenDataSources(column['children']) + ], [] as any[][]); +} + +class JSONVectorLayoutReader implements VectorLayoutReader { + constructor(private sources: any[][], private fieldNodes: Iterator, private buffers: Iterator) {} + readContainerLayout(field: Field) { + const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(sources, field, fieldNode, buffers.next().value) + }; + } + readFixedWidthLayout(field: Field, dataType: TypedArrayConstructor) { + const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(sources, field, fieldNode, buffers.next().value), + data: createDataArray(sources, field, fieldNode, buffers.next().value, dataType) + }; + } + readBinaryLayout(field: Field) { + const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(sources, field, fieldNode, buffers.next().value), + offsets: new Int32Array(sources[buffers.next().value.offset.low]), + data: createDataArray(sources, field, fieldNode, buffers.next().value, Uint8Array) + }; + } + readVariableWidthLayout(field: Field) { + const { sources, buffers } = this, fieldNode = this.fieldNodes.next().value; + return { + field, fieldNode, + validity: createValidityArray(sources, field, fieldNode, buffers.next().value), + offsets: new Int32Array(sources[buffers.next().value.offset.low]), + }; + } +} + +function createValidityArray(sources: any[][], field: Field, fieldNode: FieldNode, buffer: Buffer) { + return field.nullable && fieldNode.nullCount.low > 0 && + booleanFromJSON(sources[buffer.offset.low]) || null; +} + +const encoder = new TextEncoder('utf-8'); + +function createDataArray(sources: any[][], field: Field, _fieldNode: FieldNode, buffer: Buffer, ArrayConstructor: TypedArrayConstructor): T { + let type = field.type, data: ArrayLike | ArrayBufferLike; + if (type.isTimestamp() === true) { + data = int64sFromJSON(sources[buffer.offset.low] as string[]); + } else if ((type.isInt() || type.isTime()) && type.bitWidth === 64) { + data = int64sFromJSON(sources[buffer.offset.low] as string[]); + } else if (type.isDate() && type.unit === DateUnit.MILLISECOND) { + data = int64sFromJSON(sources[buffer.offset.low] as string[]); + } else if (type.isDecimal() === true) { + data = decimalFromJSON(sources[buffer.offset.low] as string[]); + } else if (type.isBinary() === true) { + data = binaryFromJSON(sources[buffer.offset.low] as string[]); + } else if (type.isBool() === true) { + data = booleanFromJSON(sources[buffer.offset.low] as number[]).buffer; + } else if (type.isUtf8() === true) { + data = encoder.encode((sources[buffer.offset.low] as string[]).join('')); + } else { + data = (sources[buffer.offset.low]).map((x) => +x); + } + return new ArrayConstructor(data); +} + +function int64sFromJSON(values: string[]) { + const data = new Uint32Array(values.length * 2); + for (let i = -1, n = values.length; ++i < n;) { + // Force all values (even numbers) to be parsed as strings since + // pulling out high and low bits seems to lose precision sometimes + // For example: + // > -4613034156400212000 >>> 0 + // 721782784 + // The correct lower 32-bits are 721782752 + Int64.fromString(values[i].toString(), new Uint32Array(data.buffer, data.byteOffset + 2 * i * 4, 2)); + } + return data.buffer; +} + +function decimalFromJSON(values: string[]) { + const data = new Uint32Array(values.length * 4); + for (let i = -1, n = values.length; ++i < n;) { + Int128.fromString(values[i], new Uint32Array(data.buffer, data.byteOffset + 4 * 4 * i, 4)); + } + return data.buffer; +} + +function binaryFromJSON(values: string[]) { + // "DATA": ["49BC7D5B6C47D2","3F5FB6D9322026"] + // There are definitely more efficient ways to do this... but it gets the + // job done. + const joined = values.join(''); + const data = new Uint8Array(joined.length / 2); + for (let i = 0; i < joined.length; i += 2) { + data[i >> 1] = parseInt(joined.substr(i, 2), 16); + } + return data.buffer; +} + +function booleanFromJSON(arr: number[]) { + let xs = [], n, i = 0; + let bit = 0, byte = 0; + for (const value of arr) { + value && (byte |= 1 << bit); + if (++bit === 8) { + xs[i++] = byte; + byte = bit = 0; + } + } + if (i === 0 || bit > 0) { xs[i++] = byte; } + if (i % 8 && (n = i + 8 - i % 8)) { + do { xs[i] = 0; } while (++i < n); + } + return new Uint8Array(xs); +} diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts index 0f95b769e141f..3bd6d2bb67650 100644 --- a/js/src/reader/vector.ts +++ b/js/src/reader/vector.ts @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -import { VectorReaderContext } from './arrow'; -import * as Schema_ from '../format/Schema'; -import * as Message_ from '../format/Message'; +import * as Schema_ from '../format/fb/Schema'; import { TypedArray, TypedArrayConstructor } from '../vector/types'; +import { Schema, RecordBatch, DictionaryBatch, Field, FieldNode } from '../format/arrow'; +import { Int, Date, Time, Timestamp, Decimal, FixedSizeList, FixedSizeBinary, FloatingPoint } from '../format/arrow'; import { Vector, BoolVector, BinaryVector, DictionaryVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, @@ -28,244 +28,228 @@ import { Date32Vector, Date64Vector, Time32Vector, Time64Vector, TimestampVector, } from '../vector/arrow'; -import Int = Schema_.org.apache.arrow.flatbuf.Int; -import Date = Schema_.org.apache.arrow.flatbuf.Date; -import Time = Schema_.org.apache.arrow.flatbuf.Time; import Type = Schema_.org.apache.arrow.flatbuf.Type; -import Field = Schema_.org.apache.arrow.flatbuf.Field; -import Buffer = Schema_.org.apache.arrow.flatbuf.Buffer; -import Decimal = Schema_.org.apache.arrow.flatbuf.Decimal; import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit; import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit; -// import Interval = Schema_.org.apache.arrow.flatbuf.Interval; -import Timestamp = Schema_.org.apache.arrow.flatbuf.Timestamp; -// import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit; import Precision = Schema_.org.apache.arrow.flatbuf.Precision; -import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; -import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList; -import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint; -import FixedSizeBinary = Schema_.org.apache.arrow.flatbuf.FixedSizeBinary; +// import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit; -export function readVector(field: Field, state: VectorReaderContext) { - return readDictionaryVector(field, state) || readValueVector(field, state); +export interface ContainerLayout { + fieldNode: FieldNode; + validity: Uint8Array | null | void; } -/* a dictionary index defaults to signed 32 bit int if unspecified */ -const defaultDictionaryIndexType = { bitWidth: () => 32, isSigned: () => true } as Int; -const intVectors = [ - [/* unsigned */ [Uint8Vector, Uint8Array ], /* signed */ [Int8Vector , Int8Array ]], - [/* unsigned */ [Uint16Vector, Uint16Array], /* signed */ [Int16Vector, Int16Array]], - [/* unsigned */ [Uint32Vector, Uint32Array], /* signed */ [Int32Vector, Int32Array]],, - [/* unsigned */ [Uint64Vector, Uint32Array], /* signed */ [Int64Vector, Int32Array]] -] as [any, TypedArrayConstructor][][]; - -export function readDictionaryVector(field: Field, state: VectorReaderContext) { - const encoding = field.dictionary()!; - if (encoding) { - const type = encoding.indexType() || defaultDictionaryIndexType; - const data = state.dictionaries.get(encoding.id().toFloat64().toString())!; - const [IntVector, IntArray] = intVectors[type.bitWidth() >>> 4]![+type.isSigned()]; - const { fieldNode, validity, data: keys } = readNumericBuffers(field, state, IntArray); - return new DictionaryVector({ - validity, data, field, fieldNode, - keys: new IntVector({ field, fieldNode, data: keys }) - }); - } - return null; +export interface VariableWidthLayout { + fieldNode: FieldNode; + offsets: Int32Array; + validity: Uint8Array | null | void; } -export function readValueVector(field: Field, state: VectorReaderContext): Vector { - switch (field.typeType()) { - case Type.NONE: return readNullVector(); - case Type.Null: return readNullVector(); - // case Type.Map: return readMapVector(field, state); - case Type.Int: return readIntVector(field, state); - case Type.Bool: return readBoolVector(field, state); - case Type.Date: return readDateVector(field, state); - case Type.List: return readListVector(field, state); - case Type.Utf8: return readUtf8Vector(field, state); - case Type.Time: return readTimeVector(field, state); - // case Type.Union: return readUnionVector(field, state); - case Type.Binary: return readBinaryVector(field, state); - case Type.Decimal: return readDecimalVector(field, state); - case Type.Struct_: return readStructVector(field, state); - case Type.FloatingPoint: return readFloatVector(field, state); - case Type.Timestamp: return readTimestampVector(field, state); - case Type.FixedSizeList: return readFixedSizeListVector(field, state); - case Type.FixedSizeBinary: return readFixedSizeBinaryVector(field, state); - } - throw new Error(`Unrecognized Vector { name: ${Type[field.typeType()]}, type: ${field.typeType()} }`); -} - -export function readNullVector() { - return new Vector(); +export interface BinaryLayout extends FixedWidthLayout { + offsets: Int32Array; } -export function readBoolVector(field: Field, state: VectorReaderContext) { - return new BoolVector(readNumericBuffers(field, state, Uint8Array)); +export interface FixedWidthLayout { + fieldNode: FieldNode; + data: T; + validity: Uint8Array | null | void; } -export function readDateVector(field: Field, state: VectorReaderContext) { - const type = field.type(new Date())!; - switch (type.unit()) { - case DateUnit.DAY: return new Date32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] }); - case DateUnit.MILLISECOND: return new Date64Vector({ ...readNumericBuffers(field, state, Int32Array), unit: DateUnit[type.unit()] }); +export function* readVectors(messages: Iterable<{ schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorLayoutReader }>) { + const dictionaries = new Map(); + for (const { schema, message, reader } of messages) { + yield* readMessageVectors(schema, message, new VectorReader(dictionaries, reader)); } - throw new Error(`Unrecognized Date { unit: ${type.unit()} }`); } -export function readTimeVector(field: Field, state: VectorReaderContext) { - const type = field.type(new Time())!; - switch (type.bitWidth()) { - case 32: return new Time32Vector({ ...readNumericBuffers(field, state, Int32Array), unit: TimeUnit[type.unit()] }); - case 64: return new Time64Vector({ ...readNumericBuffers(field, state, Uint32Array), unit: TimeUnit[type.unit()] }); +export async function* readVectorsAsync(messages: AsyncIterable<{ schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorLayoutReader }>) { + const dictionaries = new Map(); + for await (const { schema, message, reader } of messages) { + yield* readMessageVectors(schema, message, new VectorReader(dictionaries, reader)); } - throw new Error(`Unrecognized Time { unit: ${type.unit()}, bitWidth: ${type.bitWidth()} }`); } -export function readTimestampVector(field: Field, state: VectorReaderContext) { - const type = field.type(new Timestamp())!; - const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array); - return new TimestampVector({ - field, fieldNode, validity, data, - timezone: type.timezone()!, - unit: TimeUnit[type.unit()], - }); -} - -export function readListVector(field: Field, state: VectorReaderContext) { - const { fieldNode, validity, offsets } = readListBuffers(field, state); - return new ListVector({ - field, fieldNode, validity, offsets, - values: readVector(field.children(0)!, state) - }); -} - -export function readStructVector(field: Field, state: VectorReaderContext) { - const n = field.childrenLength(); - const columns = new Array(n); - const fieldNode = state.readNextNode(); - const validity = readValidityBuffer(field, fieldNode, state); - for (let i = -1, child: Field; ++i < n;) { - if (child = field.children(i)!) { - columns[i] = readVector(child, state); +function* readMessageVectors(schema: Schema, message: RecordBatch | DictionaryBatch, reader: VectorReader) { + if (message.isRecordBatch() === true) { + yield schema.fields.map((field) => reader.readVector(field)); + } else if (message.isDictionaryBatch()) { + let id = message.dictionaryId.toFloat64().toString(); + let vector = reader.readValueVector(schema.dictionaries.get(id)!); + if (message.isDelta) { + vector = reader.dictionaries.get(id)!.concat(vector); } + reader.dictionaries.set(id, vector); } - return new StructVector({ field, fieldNode, validity, columns }); -} - -export function readBinaryVector(field: Field, state: VectorReaderContext) { - return new BinaryVector(readBinaryBuffers(field, state)); -} - -export function readDecimalVector(field: Field, state: VectorReaderContext) { - const type = field.type(new Decimal())!; - const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint32Array); - return new DecimalVector({ - scale: type.scale(), - precision: type.precision(), - field, fieldNode, validity, data - }); -} - -export function readUtf8Vector(field: Field, state: VectorReaderContext) { - const { fieldNode, validity, offsets, data } = readBinaryBuffers(field, state); - return new Utf8Vector({ - field, fieldNode, - values: new BinaryVector({ - validity, offsets, data - }) - }); -} - -export function readFixedSizeListVector(field: Field, state: VectorReaderContext) { - const type = field.type(new FixedSizeList())!; - const fieldNode = state.readNextNode(); - const validity = readValidityBuffer(field, fieldNode, state); - return new FixedSizeListVector({ - field, fieldNode, validity, - size: type.listSize(), - values: readVector(field.children(0)!, state) - }); } -export function readFixedSizeBinaryVector(field: Field, state: VectorReaderContext) { - const type = field.type(new FixedSizeBinary())!; - const { fieldNode, validity, data } = readNumericBuffers(field, state, Uint8Array); - return new FixedSizeListVector({ - size: type.byteWidth(), - field, fieldNode, validity, - values: new Uint8Vector({ data }) - }); +export interface VectorLayoutReader { + readBinaryLayout(field: Field): BinaryLayout; + readContainerLayout(field: Field): ContainerLayout; + readVariableWidthLayout(field: Field): VariableWidthLayout; + readFixedWidthLayout(field: Field, TypedArrayConstructor: TypedArrayConstructor): FixedWidthLayout; } -export function readFloatVector(field: Field, state: VectorReaderContext) { - const type = field.type(new FloatingPoint())!; - switch (type.precision()) { - case Precision.HALF: return new Float16Vector(readNumericBuffers(field, state, Uint16Array)); - case Precision.SINGLE: return new Float32Vector(readNumericBuffers(field, state, Float32Array)); - case Precision.DOUBLE: return new Float64Vector(readNumericBuffers(field, state, Float64Array)); +export class VectorReader implements VectorLayoutReader { + constructor(public dictionaries: Map, protected layout: VectorLayoutReader) {} + readVector(field: Field): Vector { + return this.readDictionaryVector(field) || this.readValueVector(field); } - throw new Error(`Unrecognized FloatingPoint { precision: ${type.precision()} }`); -} - -export function readIntVector(field: Field, state: VectorReaderContext) { - const type = field.type(new Int())!; - if (type.isSigned()) { - switch (type.bitWidth()) { - case 8: return new Int8Vector(readNumericBuffers(field, state, Int8Array)); - case 16: return new Int16Vector(readNumericBuffers(field, state, Int16Array)); - case 32: return new Int32Vector(readNumericBuffers(field, state, Int32Array)); - case 64: return new Int64Vector(readNumericBuffers(field, state, Int32Array)); + readDictionaryVector(field: Field) { + const encoding = field.dictionary; + if (encoding) { + const keys = this.readIntVector(field.indexField()); + const data = this.dictionaries.get(encoding.dictionaryId.toFloat64().toString())!; + return new DictionaryVector({ + field, data, keys, + validity: (keys as any).validity, + fieldNode: (keys as any).fieldNode, + }); } + return null; } - switch (type.bitWidth()) { - case 8: return new Uint8Vector(readNumericBuffers(field, state, Uint8Array)); - case 16: return new Uint16Vector(readNumericBuffers(field, state, Uint16Array)); - case 32: return new Uint32Vector(readNumericBuffers(field, state, Uint32Array)); - case 64: return new Uint64Vector(readNumericBuffers(field, state, Uint32Array)); + readValueVector(field: Field) { + switch (field.typeType) { + case Type.NONE: return this.readNullVector(); + case Type.Null: return this.readNullVector(); + // case Type.Map: return this.readMapVector(field); + case Type.Int: return this.readIntVector(field); + case Type.Bool: return this.readBoolVector(field); + case Type.Date: return this.readDateVector(field); + case Type.List: return this.readListVector(field); + case Type.Utf8: return this.readUtf8Vector(field); + case Type.Time: return this.readTimeVector(field); + // case Type.Union: return this.readUnionVector(field); + case Type.Binary: return this.readBinaryVector(field); + case Type.Decimal: return this.readDecimalVector(field); + case Type.Struct_: return this.readStructVector(field); + case Type.FloatingPoint: return this.readFloatVector(field); + case Type.Timestamp: return this.readTimestampVector(field); + case Type.FixedSizeList: return this.readFixedSizeListVector(field); + case Type.FixedSizeBinary: return this.readFixedSizeBinaryVector(field); + } + throw new Error(`Unrecognized ${field.toString()}`); + } + readNullVector() { + return new Vector(); + } + readBoolVector(field: Field) { + return new BoolVector(this.readFixedWidthLayout(field, Uint8Array)); + } + readDateVector(field: Field) { + const type = field.type as Date; + switch (type.unit) { + case DateUnit.DAY: return new Date32Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: DateUnit[type.unit] }); + case DateUnit.MILLISECOND: return new Date64Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: DateUnit[type.unit] }); + } + throw new Error(`Unrecognized ${type.toString()}`); + } + readTimeVector(field: Field) { + const type = field.type as Time; + switch (type.bitWidth) { + case 32: return new Time32Vector({ ...this.readFixedWidthLayout(field, Int32Array), unit: TimeUnit[type.unit] }); + case 64: return new Time64Vector({ ...this.readFixedWidthLayout(field, Uint32Array), unit: TimeUnit[type.unit] }); + } + throw new Error(`Unrecognized ${type.toString()}`); + } + readTimestampVector(field: Field) { + const type = field.type as Timestamp; + const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint32Array); + return new TimestampVector({ + field, fieldNode, validity, data, + timezone: type.timezone!, + unit: TimeUnit[type.unit], + }); + } + readListVector(field: Field) { + const { fieldNode, validity, offsets } = this.readVariableWidthLayout(field); + return new ListVector({ + field, fieldNode, validity, offsets, + values: this.readVector(field.children[0]) + }); + } + readStructVector(field: Field) { + const { fieldNode, validity } = this.readContainerLayout(field); + return new StructVector({ + field, fieldNode, validity, + columns: field.children.map((field) => this.readVector(field)) + }); + } + readBinaryVector(field: Field) { + return new BinaryVector(this.readBinaryLayout(field)); + } + readDecimalVector(field: Field) { + const type = field.type as Decimal; + const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint32Array); + return new DecimalVector({ + scale: type.scale, + precision: type.precision, + field, fieldNode, validity, data + }); + } + readUtf8Vector(field: Field) { + const { fieldNode, validity, offsets, data } = this.readBinaryLayout(field); + return new Utf8Vector({ + field, fieldNode, + values: new BinaryVector({ + validity, offsets, data + }) + }); + } + readFixedSizeListVector(field: Field) { + const type = field.type as FixedSizeList; + const { fieldNode, validity } = this.readContainerLayout(field); + return new FixedSizeListVector({ + field, fieldNode, validity, + size: type.listSize, + values: this.readVector(field.children[0]) + }); + } + readFixedSizeBinaryVector(field: Field) { + const type = field.type as FixedSizeBinary; + const { fieldNode, validity, data } = this.readFixedWidthLayout(field, Uint8Array); + return new FixedSizeListVector({ + size: type.byteWidth, + field, fieldNode, validity, + values: new Uint8Vector({ data }) + }); + } + readFloatVector(field: Field) { + const type = field.type as FloatingPoint; + switch (type.precision) { + case Precision.HALF: return new Float16Vector(this.readFixedWidthLayout(field, Uint16Array)); + case Precision.SINGLE: return new Float32Vector(this.readFixedWidthLayout(field, Float32Array)); + case Precision.DOUBLE: return new Float64Vector(this.readFixedWidthLayout(field, Float64Array)); + } + throw new Error(`Unrecognized FloatingPoint { precision: ${type.precision} }`); + } + readIntVector(field: Field) { + const type = field.type as Int; + if (type.isSigned) { + switch (type.bitWidth) { + case 8: return new Int8Vector(this.readFixedWidthLayout(field, Int8Array)); + case 16: return new Int16Vector(this.readFixedWidthLayout(field, Int16Array)); + case 32: return new Int32Vector(this.readFixedWidthLayout(field, Int32Array)); + case 64: return new Int64Vector(this.readFixedWidthLayout(field, Int32Array)); + } + } + switch (type.bitWidth) { + case 8: return new Uint8Vector(this.readFixedWidthLayout(field, Uint8Array)); + case 16: return new Uint16Vector(this.readFixedWidthLayout(field, Uint16Array)); + case 32: return new Uint32Vector(this.readFixedWidthLayout(field, Uint32Array)); + case 64: return new Uint64Vector(this.readFixedWidthLayout(field, Uint32Array)); + } + throw new Error(`Unrecognized Int { isSigned: ${type.isSigned}, bitWidth: ${type.bitWidth} }`); + } + readContainerLayout(field: Field) { + return this.layout.readContainerLayout(field); + } + readBinaryLayout(field: Field) { + return this.layout.readBinaryLayout(field); + } + readVariableWidthLayout(field: Field) { + return this.layout.readVariableWidthLayout(field); + } + readFixedWidthLayout(field: Field, TypedArrayConstructor: TypedArrayConstructor) { + return this.layout.readFixedWidthLayout(field, TypedArrayConstructor); } - throw new Error(`Unrecognized Int { isSigned: ${type.isSigned()}, bitWidth: ${type.bitWidth()} }`); -} - -function readListBuffers(field: Field, state: VectorReaderContext) { - const fieldNode = state.readNextNode(); - const validity = readValidityBuffer(field, fieldNode, state); - const offsets = readDataBuffer(Int32Array, state); - return { field, fieldNode, validity, offsets }; -} - -function readBinaryBuffers(field: Field, state: VectorReaderContext) { - const fieldNode = state.readNextNode(); - const validity = readValidityBuffer(field, fieldNode, state); - const offsets = readDataBuffer(Int32Array, state); - const data = readDataBuffer(Uint8Array, state); - return { field, fieldNode, validity, offsets, data }; -} - -function readNumericBuffers(field: Field, state: VectorReaderContext, ArrayConstructor: TypedArrayConstructor) { - const fieldNode = state.readNextNode(); - const validity = readValidityBuffer(field, fieldNode, state); - const data = readDataBuffer(ArrayConstructor, state); - return { field, fieldNode, validity, data }; -} - -function readDataBuffer(ArrayConstructor: TypedArrayConstructor, state: VectorReaderContext) { - return createTypedArray(ArrayConstructor, state.bytes, state.offset, state.readNextBuffer()); -} - -function readValidityBuffer(field: Field, fieldNode: FieldNode, state: VectorReaderContext) { - return createValidityArray(field, fieldNode, state.bytes, state.offset, state.readNextBuffer()); -} - -function createValidityArray(field: Field, fieldNode: FieldNode, bytes: Uint8Array, offset: number, buffer: Buffer) { - return field.nullable() && fieldNode.nullCount().low > 0 && createTypedArray(Uint8Array, bytes, offset, buffer) || null; -} - -function createTypedArray(ArrayConstructor: TypedArrayConstructor, bytes: Uint8Array, offset: number, buffer: Buffer) { - return new ArrayConstructor( - bytes.buffer, - bytes.byteOffset + offset + buffer.offset().low, - buffer.length().low / ArrayConstructor.BYTES_PER_ELEMENT - ); } diff --git a/js/src/util/int.ts b/js/src/util/int.ts new file mode 100644 index 0000000000000..9088e7b995573 --- /dev/null +++ b/js/src/util/int.ts @@ -0,0 +1,320 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const carryBit16 = 1 << 16; + +function intAsHex(value: number): string { + if (value < 0) { + value = 0xFFFFFFFF + value + 1; + } + return `0x${value.toString(16)}`; +} + +const kInt32DecimalDigits = 8; +const kPowersOfTen = [1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000]; + +export class BaseInt64 { + constructor (protected buffer: Uint32Array) {} + + high(): number { return this.buffer[1]; } + low (): number { return this.buffer[0]; } + + protected _times(other: BaseInt64) { + // Break the left and right numbers into 16 bit chunks + // so that we can multiply them without overflow. + const L = new Uint32Array([ + this.buffer[1] >>> 16, + this.buffer[1] & 0xFFFF, + this.buffer[0] >>> 16, + this.buffer[0] & 0xFFFF + ]); + + const R = new Uint32Array([ + other.buffer[1] >>> 16, + other.buffer[1] & 0xFFFF, + other.buffer[0] >>> 16, + other.buffer[0] & 0xFFFF + ]); + + let product = L[3] * R[3]; + this.buffer[0] = product & 0xFFFF; + + let sum = product >>> 16; + + product = L[2] * R[3]; + sum += product; + + product = (L[3] * R[2]) >>> 0; + sum += product; + + this.buffer[0] += sum << 16; + + this.buffer[1] = (sum >>> 0 < product ? carryBit16 : 0); + + this.buffer[1] += sum >>> 16; + this.buffer[1] += L[1] * R[3] + L[2] * R[2] + L[3] * R[1]; + this.buffer[1] += (L[0] * R[3] + L[1] * R[2] + L[2] * R[1] + L[3] * R[0]) << 16; + + return this; + } + + protected _plus(other: BaseInt64) { + const sum = (this.buffer[0] + other.buffer[0]) >>> 0; + this.buffer[1] += other.buffer[1]; + if (sum < (this.buffer[0] >>> 0)) { + ++this.buffer[1]; + } + this.buffer[0] = sum; + } + + lessThan(other: BaseInt64): boolean { + return this.buffer[1] < other.buffer[1] || + (this.buffer[1] === other.buffer[1] && this.buffer[0] < other.buffer[0]); + } + + equals(other: BaseInt64): boolean { + return this.buffer[1] === other.buffer[1] && this.buffer[0] == other.buffer[0]; + } + + greaterThan(other: BaseInt64): boolean { + return other.lessThan(this); + } + + hex(): string { + return `${intAsHex(this.buffer[1])} ${intAsHex(this.buffer[0])}`; + } +} + +export class Uint64 extends BaseInt64 { + times(other: Uint64): Uint64 { + this._times(other); + return this; + } + + plus(other: Uint64): Uint64 { + this._plus(other); + return this; + } + + static multiply(left: Uint64, right: Uint64): Uint64 { + let rtrn = new Uint64(new Uint32Array(left.buffer)); + return rtrn.times(right); + } + + static add(left: Uint64, right: Uint64): Uint64 { + let rtrn = new Uint64(new Uint32Array(left.buffer)); + return rtrn.plus(right); + } +} + +export class Int64 extends BaseInt64 { + negate(): Int64 { + this.buffer[0] = ~this.buffer[0] + 1; + this.buffer[1] = ~this.buffer[1]; + + if (this.buffer[0] == 0) { ++this.buffer[1]; } + return this; + } + + times(other: Int64): Int64 { + this._times(other); + return this; + } + + plus(other: Int64): Int64 { + this._plus(other); + return this; + } + + lessThan(other: Int64): boolean { + // force high bytes to be signed + const this_high = this.buffer[1] << 0; + const other_high = other.buffer[1] << 0; + return this_high < other_high || + (this_high === other_high && this.buffer[0] < other.buffer[0]); + } + + static fromString(str: string, out_buffer = new Uint32Array(2)): Int64 { + // TODO: Assert that out_buffer is 0 and length = 2 + const negate = str.startsWith('-'); + const length = str.length; + + let out = new Int64(out_buffer); + for (let posn = negate ? 1 : 0; posn < length;) { + const group = kInt32DecimalDigits < length - posn ? + kInt32DecimalDigits : length - posn; + const chunk = new Int64(new Uint32Array([parseInt(str.substr(posn, group), 10), 0])); + const multiple = new Int64(new Uint32Array([kPowersOfTen[group], 0])); + + out.times(multiple); + out.plus(chunk); + + posn += group; + } + + return negate ? out.negate() : out; + } + + static multiply(left: Int64, right: Int64): Int64 { + let rtrn = new Int64(new Uint32Array(left.buffer)); + return rtrn.times(right); + } + + static add(left: Int64, right: Int64): Int64 { + let rtrn = new Int64(new Uint32Array(left.buffer)); + return rtrn.plus(right); + } +} + +export class Int128 { + constructor (private buffer: Uint32Array) { + // buffer[3] MSB (high) + // buffer[2] + // buffer[1] + // buffer[0] LSB (low) + } + + high(): Int64 { + return new Int64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset + 8, 2)); + } + + low(): Int64 { + return new Int64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset, 2)); + } + + negate(): Int128 { + this.buffer[0] = ~this.buffer[0] + 1; + this.buffer[1] = ~this.buffer[1]; + this.buffer[2] = ~this.buffer[2]; + this.buffer[3] = ~this.buffer[3]; + + if (this.buffer[0] == 0) { ++this.buffer[1]; } + if (this.buffer[1] == 0) { ++this.buffer[2]; } + if (this.buffer[2] == 0) { ++this.buffer[3]; } + return this; + } + + times(other: Int128): Int128 { + // Break the left and right numbers into 32 bit chunks + // so that we can multiply them without overflow. + const L0 = new Uint64(new Uint32Array([this.buffer[3], 0])); + const L1 = new Uint64(new Uint32Array([this.buffer[2], 0])); + const L2 = new Uint64(new Uint32Array([this.buffer[1], 0])); + const L3 = new Uint64(new Uint32Array([this.buffer[0], 0])); + + const R0 = new Uint64(new Uint32Array([other.buffer[3], 0])); + const R1 = new Uint64(new Uint32Array([other.buffer[2], 0])); + const R2 = new Uint64(new Uint32Array([other.buffer[1], 0])); + const R3 = new Uint64(new Uint32Array([other.buffer[0], 0])); + + let product = Uint64.multiply(L3, R3); + this.buffer[0] = product.low(); + + let sum = new Uint64(new Uint32Array([product.high(), 0])); + + product = Uint64.multiply(L2, R3); + sum.plus(product); + + product = Uint64.multiply(L3, R2); + sum.plus(product); + + this.buffer[1] = sum.low(); + + this.buffer[3] = (sum.lessThan(product) ? 1 : 0); + + this.buffer[2] = sum.high(); + let high = new Uint64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset + 8, 2)); + + high.plus(Uint64.multiply(L1, R3)) + .plus(Uint64.multiply(L2, R2)) + .plus(Uint64.multiply(L3, R1)); + this.buffer[3] += Uint64.multiply(L0, R3) + .plus(Uint64.multiply(L1, R2)) + .plus(Uint64.multiply(L2, R1)) + .plus(Uint64.multiply(L3, R0)).low(); + + return this; + } + + plus(other: Int128): Int128 { + let sums = new Uint32Array(4); + sums[3] = (this.buffer[3] + other.buffer[3]) >>> 0; + sums[2] = (this.buffer[2] + other.buffer[2]) >>> 0; + sums[1] = (this.buffer[1] + other.buffer[1]) >>> 0; + sums[0] = (this.buffer[0] + other.buffer[0]) >>> 0; + + if (sums[0] < (this.buffer[0] >>> 0)) { + ++sums[1]; + } + if (sums[1] < (this.buffer[1] >>> 0)) { + ++sums[2]; + } + if (sums[2] < (this.buffer[2] >>> 0)) { + ++sums[3]; + } + + this.buffer[3] = sums[3]; + this.buffer[2] = sums[2]; + this.buffer[1] = sums[1]; + this.buffer[0] = sums[0]; + + return this; + } + + hex(): string { + return `${intAsHex(this.buffer[3])} ${intAsHex(this.buffer[2])} ${intAsHex(this.buffer[1])} ${intAsHex(this.buffer[0])}`; + } + + static multiply(left: Int128, right: Int128): Int128 { + let rtrn = new Int128(new Uint32Array(left.buffer)); + return rtrn.times(right); + } + + static add(left: Int128, right: Int128): Int128 { + let rtrn = new Int128(new Uint32Array(left.buffer)); + return rtrn.plus(right); + } + + static fromString(str: string, out_buffer = new Uint32Array(4)): Int128 { + // TODO: Assert that out_buffer is 0 and length = 4 + const negate = str.startsWith('-'); + const length = str.length; + + let out = new Int128(out_buffer); + for (let posn = negate ? 1 : 0; posn < length;) { + const group = kInt32DecimalDigits < length - posn ? + kInt32DecimalDigits : length - posn; + const chunk = new Int128(new Uint32Array([parseInt(str.substr(posn, group), 10), 0, 0, 0])); + const multiple = new Int128(new Uint32Array([kPowersOfTen[group], 0, 0, 0])); + + out.times(multiple); + out.plus(chunk); + + posn += group; + } + + return negate ? out.negate() : out; + } +} diff --git a/js/src/util/layout.ts b/js/src/util/layout.ts new file mode 100644 index 0000000000000..c064ee9d7d0b0 --- /dev/null +++ b/js/src/util/layout.ts @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { TextEncoder } from 'text-encoding-utf-8'; +import { TypedArrayConstructor, TypedArray } from '../vector/types'; + +export function align(value: number, alignment: number) { + return value + padding(value, alignment); +} + +export function padding(value: number, alignment: number) { + return (value % alignment === 0 ? 0 : alignment - value % alignment); +} + +export type NullableLayout = { nullCount: number, validity: Uint8Array }; +export type BufferLayout> = { data: TArray }; +export type DictionaryLayout> = { data: TArray, keys: number[] }; +export type VariableWidthLayout> = { data: TArray, offsets: number[] }; +export type VariableWidthDictionaryLayout> = { data: TArray, keys: number[], offsets: number[] }; + +export type values = ArrayLike; +export type BufferValueWriter = (src: ArrayLike, dst: number[], index: number) => boolean | void; +export type BufferWriter = (values: values, nulls?: ArrayLike) => BufferLayout; +export type BufferLayoutWriter = (write: BufferValueWriter, values: values, nulls?: ArrayLike) => BufferLayout; + +const writeNumeric64Value = writeFixedWidthValue.bind(null, 64); +const writeNumeric128Value = writeFixedWidthValue.bind(null, 128); +const utf8Encoder = new TextEncoder() as { encode: (s: string) => Uint8Array }; + +const stride1Encode = writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 1)); +const stride1FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, stride1Encode)); +const stride2FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 2))); +const stride4FixedWidth = writeFixedWidthLayout.bind(null, writeValidityLayout.bind(null, writeFixedWidthLayoutWithStride.bind(null, 4))); + +export const writeBools = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeBooleanValue), Uint8Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeInt8s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int8Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeInt16s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int16Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeInt32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Int32Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeInt64s = writeTypedLayout.bind(null, stride2FixedWidth.bind(null, writeNumeric64Value), Int32Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeUint8s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint8Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeUint16s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint16Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeUint32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Uint32Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeUint64s = writeTypedLayout.bind(null, stride2FixedWidth.bind(null, writeNumeric64Value), Uint32Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeDecimals = writeTypedLayout.bind(null, stride4FixedWidth.bind(null, writeNumeric128Value), Uint32Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeFloat32s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Float32Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeFloat64s = writeTypedLayout.bind(null, stride1FixedWidth.bind(null, writeNumericValue), Float64Array) as (values: values, nulls?: ArrayLike) => BufferLayout; +export const writeVariableWidth = writeVariableWidthLayout.bind(null, stride1Encode) as (writeValue: BufferValueWriter, values: values, nulls?: ArrayLike) => VariableWidthLayout; +export const writeBinary = writeTypedLayout.bind(null, writeVariableWidth.bind(null, writeBinaryValue)) as (values: values, TNull>, nulls?: ArrayLike) => VariableWidthLayout; +export const writeUtf8s = writeTypedLayout.bind(null, writeVariableWidth.bind(null, writeUtf8Value), Uint8Array) as (values: values, nulls?: ArrayLike) => VariableWidthLayout; +export const writeDictionaryEncoded = writeDictionaryLayout.bind(null, stride1Encode) as (writeValue: BufferValueWriter, values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedBools = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeBooleanValue), Uint8Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedInt8s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int8Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedInt16s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int16Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedInt32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Int32Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedInt64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride2FixedWidth, writeNumeric64Value), Int32Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedUint8s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint8Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedUint16s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint16Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedUint32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Uint32Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedUint64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride2FixedWidth, writeNumeric64Value), Uint32Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedDecimals = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride4FixedWidth, writeNumeric128Value), Uint32Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedFloat32s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Float32Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedFloat64s = writeTypedLayout.bind(null, writeDictionaryLayout.bind(null, stride1FixedWidth, writeNumericValue), Float64Array) as (values: values, nulls?: ArrayLike) => DictionaryLayout; +export const writeDictionaryEncodedVariableWidth = writeDictionaryLayout.bind(null, writeVariableWidth) as (writeValue: BufferValueWriter, values: values, nulls?: ArrayLike) => VariableWidthDictionaryLayout; +export const writeDictionaryEncodedBinary = writeTypedLayout.bind(null, writeDictionaryEncodedVariableWidth.bind(null, writeBinaryValue)) as (values: values, TNull>, nulls?: ArrayLike) => VariableWidthDictionaryLayout; +export const writeDictionaryEncodedUtf8s = writeTypedLayout.bind(null, writeDictionaryEncodedVariableWidth.bind(null, writeUtf8Value), Uint8Array) as (values: values, nulls?: ArrayLike) => VariableWidthDictionaryLayout; + +function writeFixedWidthLayoutWithStride( + stride: number, + writeValue: BufferValueWriter, + values: values +) { + let index = -stride; + const data = [] as number[]; + const length = values.length; + while ((index += stride) < length) { + writeValue(values as ArrayLike, data, index); + } + return { data: data as ArrayLike }; +} + +function writeFixedWidthLayout( + writeLayout: BufferLayoutWriter, + writeValue: BufferValueWriter, + values: values, + nulls?: ArrayLike +) { + return writeLayout(writeValue, values, nulls); +} + +function writeValidityLayout( + writeLayout: BufferLayoutWriter, + writeValue: BufferValueWriter, + values: values, + nulls?: ArrayLike +) { + let nullCount = 0; + let nullsLength = nulls && nulls.length || 0; + let validity = new Uint8Array(align(values.length >>> 3, 8)).fill(255); + return { + ...writeLayout(writeValueOrValidity, values), + nullCount, validity: (nullCount > 0 && validity) || new Uint8Array(0) + } as BufferLayout & NullableLayout; + function writeValueOrValidity(src: ArrayLike, dst: number[], index: number) { + writeValue(src, dst, index); + let i = -1, x = src[index] as T | TNull; + let isNull = x === null || x === undefined; + while (!isNull && ++i < nullsLength) { + isNull = x === nulls![i]; + } + if (isNull) { + nullCount++; + validity[index >> 3] &= ~(1 << (index % 8)); + } + } +} + +function writeVariableWidthLayout( + writeLayout: BufferLayoutWriter, + writeValue: BufferValueWriter, + values: values, + nulls?: ArrayLike +) { + let offsets = [0], offsetsIndex = 0; + return { ...writeLayout(writeValueAndOffset, values, nulls), offsets } as VariableWidthLayout; + function writeValueAndOffset(src: ArrayLike, dst: number[], index: number) { + if (!writeValue(src, dst, index)) { + offsets[++offsetsIndex] = dst.length; + } + } +} + +function writeDictionaryLayout( + writeLayout: BufferLayoutWriter, + writeValue: BufferValueWriter, + values: values, + nulls?: ArrayLike +) { + let keys = [] as number[], keysIndex = 0, keysMap = Object.create(null); + return { ...writeLayout(writeKeysOrValues, values, nulls), keys }; + function writeKeysOrValues(src: ArrayLike, dst: number[], index: number) { + const x: any = src[index]; + if (x in keysMap) { + return (keys[index] = keysMap[x]) || true; + } else if (!writeValue(src, dst, index)) { + keys[index] = keysMap[x] = keysIndex++; + } + } +} + +function writeTypedLayout( + writeBuffers: BufferWriter, + ArrayBufferView: TypedArrayConstructor, + values: values, + nulls?: ArrayLike +) { + const result = writeBuffers(values, nulls); + result.data = new ArrayBufferView(result.data); + return result as BufferLayout; +} + +function writeBooleanValue(src: ArrayLike, dst: number[], index: number) { + if (src[index]) { + let i = index >>> 3; + let b = dst[i] || 0; + dst[i] = b | 1 << (index % 8); + } +} + +function writeNumericValue(src: ArrayLike, dst: number[], index: number) { + dst[index] = +src[index]; +} + +function writeFixedWidthValue(bitWidth: number, src: ArrayLike, dst: number[], index: number) { + const bytesLen = bitWidth / 32; + for (let i = -1; ++i < bytesLen;) { + dst[index + i] = src[index + i]; + } +} + +function writeUtf8Value(src: ArrayLike, dst: number[], index: number) { + dst.push(...utf8Encoder.encode(src[index])); +} + +function writeBinaryValue(src: ArrayLike>, dst: number[], index: number) { + dst.push(...src[index]); +} diff --git a/js/src/vector/arrow.ts b/js/src/vector/arrow.ts index 88fec7e44542b..64a43bdab1c6b 100644 --- a/js/src/vector/arrow.ts +++ b/js/src/vector/arrow.ts @@ -15,49 +15,19 @@ // specific language governing permissions and limitations // under the License. -import * as Schema_ from '../format/Schema'; -import * as Message_ from '../format/Message'; -import Field = Schema_.org.apache.arrow.flatbuf.Field; -import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; - import { Vector } from './vector'; -import { Utf8Vector as Utf8VectorBase } from './utf8'; -import { StructVector as StructVectorBase } from './struct'; -import { DictionaryVector as DictionaryVectorBase } from './dictionary'; -import { - ListVector as ListVectorBase, - BinaryVector as BinaryVectorBase, - FixedSizeListVector as FixedSizeListVectorBase -} from './list'; - -import { - BoolVector as BoolVectorBase, - Int8Vector as Int8VectorBase, - Int16Vector as Int16VectorBase, - Int32Vector as Int32VectorBase, - Int64Vector as Int64VectorBase, - Uint8Vector as Uint8VectorBase, - Uint16Vector as Uint16VectorBase, - Uint32Vector as Uint32VectorBase, - Uint64Vector as Uint64VectorBase, - Float16Vector as Float16VectorBase, - Float32Vector as Float32VectorBase, - Float64Vector as Float64VectorBase, - Date32Vector as Date32VectorBase, - Date64Vector as Date64VectorBase, - Time32Vector as Time32VectorBase, - Time64Vector as Time64VectorBase, - DecimalVector as DecimalVectorBase, - TimestampVector as TimestampVectorBase, -} from './numeric'; - -import { nullableMixin, fieldMixin } from './traits'; +import * as vectors from './traits/vectors'; +import * as fieldVectors from './traits/field'; +import * as nullableVectors from './traits/nullable'; +import * as nullableFieldVectors from './traits/nullablefield'; +import { Field, FieldNode } from '../format/arrow'; +import { isFieldArgv, isNullableArgv } from './traits/mixins'; function MixinArrowTraits, TArgv>( Base: new (argv: TArgv) => T, Field: new (argv: TArgv & { field: Field, fieldNode: FieldNode }) => T, Nullable: new (argv: TArgv & { validity: Uint8Array }) => T, - NullableField: new (argv: TArgv & { validity: Uint8Array, field: Field, fieldNode: FieldNode }) => T, + NullableField: new (argv: TArgv & { validity: Uint8Array, field: Field, fieldNode: FieldNode }) => T ) { return function(argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })) { return new (!isFieldArgv(argv) @@ -67,179 +37,52 @@ function MixinArrowTraits, TArgv>( } as any as { new (argv: TArgv | (TArgv & { validity: Uint8Array }) | (TArgv & { field: Field, fieldNode: FieldNode })): T }; } -function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } { - return x && x.field instanceof Field && x.fieldNode instanceof FieldNode; -} - -function isNullableArgv(x: any): x is { validity: Uint8Array } { - return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array; -} - export { Vector }; -export class ListVector extends MixinArrowTraits( - ListVectorBase, - class ListVector extends fieldMixin(ListVectorBase) {} as any, - class ListVector extends nullableMixin(ListVectorBase) {} as any, - class ListVector extends nullableMixin(fieldMixin(ListVectorBase)) {} as any -) {} - -export class BinaryVector extends MixinArrowTraits( - BinaryVectorBase, - class BinaryVector extends fieldMixin(BinaryVectorBase) {} as any, - class BinaryVector extends nullableMixin(BinaryVectorBase) {} as any, - class BinaryVector extends nullableMixin(fieldMixin(BinaryVectorBase)) {} as any -) {} - -export class Utf8Vector extends MixinArrowTraits( - Utf8VectorBase, - class Utf8Vector extends fieldMixin(Utf8VectorBase) {} as any, - class Utf8Vector extends nullableMixin(Utf8VectorBase) {} as any, - class Utf8Vector extends nullableMixin(fieldMixin(Utf8VectorBase)) {} as any -) {} - -export class BoolVector extends MixinArrowTraits( - BoolVectorBase, - class BoolVector extends fieldMixin(BoolVectorBase) {} as any, - class BoolVector extends nullableMixin(BoolVectorBase) {} as any, - class BoolVector extends nullableMixin(fieldMixin(BoolVectorBase)) {} as any -) {} - -export class Int8Vector extends MixinArrowTraits( - Int8VectorBase, - class Int8Vector extends fieldMixin(Int8VectorBase) {} as any, - class Int8Vector extends nullableMixin(Int8VectorBase) {} as any, - class Int8Vector extends nullableMixin(fieldMixin(Int8VectorBase)) {} as any -) {} - -export class Int16Vector extends MixinArrowTraits( - Int16VectorBase, - class Int16Vector extends fieldMixin(Int16VectorBase) {} as any, - class Int16Vector extends nullableMixin(Int16VectorBase) {} as any, - class Int16Vector extends nullableMixin(fieldMixin(Int16VectorBase)) {} as any -) {} - -export class Int32Vector extends MixinArrowTraits( - Int32VectorBase, - class Int32Vector extends fieldMixin(Int32VectorBase) {} as any, - class Int32Vector extends nullableMixin(Int32VectorBase) {} as any, - class Int32Vector extends nullableMixin(fieldMixin(Int32VectorBase)) {} as any -) {} - -export class Int64Vector extends MixinArrowTraits( - Int64VectorBase, - class Int64Vector extends fieldMixin(Int64VectorBase) {} as any, - class Int64Vector extends nullableMixin(Int64VectorBase) {} as any, - class Int64Vector extends nullableMixin(fieldMixin(Int64VectorBase)) {} as any -) {} - -export class Uint8Vector extends MixinArrowTraits( - Uint8VectorBase, - class Uint8Vector extends fieldMixin(Uint8VectorBase) {} as any, - class Uint8Vector extends nullableMixin(Uint8VectorBase) {} as any, - class Uint8Vector extends nullableMixin(fieldMixin(Uint8VectorBase)) {} as any -) {} - -export class Uint16Vector extends MixinArrowTraits( - Uint16VectorBase, - class Uint16Vector extends fieldMixin(Uint16VectorBase) {} as any, - class Uint16Vector extends nullableMixin(Uint16VectorBase) {} as any, - class Uint16Vector extends nullableMixin(fieldMixin(Uint16VectorBase)) {} as any -) {} - -export class Uint32Vector extends MixinArrowTraits( - Uint32VectorBase, - class Uint32Vector extends fieldMixin(Uint32VectorBase) {} as any, - class Uint32Vector extends nullableMixin(Uint32VectorBase) {} as any, - class Uint32Vector extends nullableMixin(fieldMixin(Uint32VectorBase)) {} as any -) {} - -export class Uint64Vector extends MixinArrowTraits( - Uint64VectorBase, - class Uint64Vector extends fieldMixin(Uint64VectorBase) {} as any, - class Uint64Vector extends nullableMixin(Uint64VectorBase) {} as any, - class Uint64Vector extends nullableMixin(fieldMixin(Uint64VectorBase)) {} as any -) {} - -export class Date32Vector extends MixinArrowTraits( - Date32VectorBase, - class Date32Vector extends fieldMixin(Date32VectorBase) {} as any, - class Date32Vector extends nullableMixin(Date32VectorBase) {} as any, - class Date32Vector extends nullableMixin(fieldMixin(Date32VectorBase)) {} as any -) {} - -export class Date64Vector extends MixinArrowTraits( - Date64VectorBase, - class Date64Vector extends fieldMixin(Date64VectorBase) {} as any, - class Date64Vector extends nullableMixin(Date64VectorBase) {} as any, - class Date64Vector extends nullableMixin(fieldMixin(Date64VectorBase)) {} as any -) {} - -export class Time32Vector extends MixinArrowTraits( - Time32VectorBase, - class Time32Vector extends fieldMixin(Time32VectorBase) {} as any, - class Time32Vector extends nullableMixin(Time32VectorBase) {} as any, - class Time32Vector extends nullableMixin(fieldMixin(Time32VectorBase)) {} as any -) {} - -export class Time64Vector extends MixinArrowTraits( - Time64VectorBase, - class Time64Vector extends fieldMixin(Time64VectorBase) {} as any, - class Time64Vector extends nullableMixin(Time64VectorBase) {} as any, - class Time64Vector extends nullableMixin(fieldMixin(Time64VectorBase)) {} as any -) {} - -export class Float16Vector extends MixinArrowTraits( - Float16VectorBase, - class Float16Vector extends fieldMixin(Float16VectorBase) {} as any, - class Float16Vector extends nullableMixin(Float16VectorBase) {} as any, - class Float16Vector extends nullableMixin(fieldMixin(Float16VectorBase)) {} as any -) {} - -export class Float32Vector extends MixinArrowTraits( - Float32VectorBase, - class Float32Vector extends fieldMixin(Float32VectorBase) {} as any, - class Float32Vector extends nullableMixin(Float32VectorBase) {} as any, - class Float32Vector extends nullableMixin(fieldMixin(Float32VectorBase)) {} as any -) {} - -export class Float64Vector extends MixinArrowTraits( - Float64VectorBase, - class Float64Vector extends fieldMixin(Float64VectorBase) {} as any, - class Float64Vector extends nullableMixin(Float64VectorBase) {} as any, - class Float64Vector extends nullableMixin(fieldMixin(Float64VectorBase)) {} as any -) {} - -export class StructVector extends MixinArrowTraits( - StructVectorBase, - class StructVector extends fieldMixin(StructVectorBase) {} as any, - class StructVector extends nullableMixin(StructVectorBase) {} as any, - class StructVector extends nullableMixin(fieldMixin(StructVectorBase)) {} as any -) {} - -export class DecimalVector extends MixinArrowTraits( - DecimalVectorBase, - class DecimalVector extends fieldMixin(DecimalVectorBase) {} as any, - class DecimalVector extends nullableMixin(DecimalVectorBase) {} as any, - class DecimalVector extends nullableMixin(fieldMixin(DecimalVectorBase)) {} as any -) {} - -export class TimestampVector extends MixinArrowTraits( - TimestampVectorBase, - class TimestampVector extends fieldMixin(TimestampVectorBase) {} as any, - class TimestampVector extends nullableMixin(TimestampVectorBase) {} as any, - class TimestampVector extends nullableMixin(fieldMixin(TimestampVectorBase)) {} as any -) {} - -export class DictionaryVector extends MixinArrowTraits( - DictionaryVectorBase, - class DictionaryVector extends fieldMixin(DictionaryVectorBase) {} as any, - class DictionaryVector extends nullableMixin(DictionaryVectorBase) {} as any, - class DictionaryVector extends nullableMixin(fieldMixin(DictionaryVectorBase)) {} as any -) {} - -export class FixedSizeListVector extends MixinArrowTraits( - FixedSizeListVectorBase, - class FixedSizeListVector extends fieldMixin(FixedSizeListVectorBase) {} as any, - class FixedSizeListVector extends nullableMixin(FixedSizeListVectorBase) {} as any, - class FixedSizeListVector extends nullableMixin(fieldMixin(FixedSizeListVectorBase)) {} as any -) {} +export const MixinListVector = MixinArrowTraits(vectors.ListVector as any, fieldVectors.ListVector as any, nullableVectors.ListVector as any, nullableFieldVectors.ListVector as any); +export class ListVector extends MixinListVector {} +export const MixinBinaryVector = MixinArrowTraits(vectors.BinaryVector as any, fieldVectors.BinaryVector as any, nullableVectors.BinaryVector as any, nullableFieldVectors.BinaryVector as any); +export class BinaryVector extends MixinBinaryVector {} +export const MixinUtf8Vector = MixinArrowTraits(vectors.Utf8Vector as any, fieldVectors.Utf8Vector as any, nullableVectors.Utf8Vector as any, nullableFieldVectors.Utf8Vector as any); +export class Utf8Vector extends MixinUtf8Vector {} +export const MixinBoolVector = MixinArrowTraits(vectors.BoolVector as any, fieldVectors.BoolVector as any, nullableVectors.BoolVector as any, nullableFieldVectors.BoolVector as any); +export class BoolVector extends MixinBoolVector {} +export const MixinInt8Vector = MixinArrowTraits(vectors.Int8Vector as any, fieldVectors.Int8Vector as any, nullableVectors.Int8Vector as any, nullableFieldVectors.Int8Vector as any); +export class Int8Vector extends MixinInt8Vector {} +export const MixinInt16Vector = MixinArrowTraits(vectors.Int16Vector as any, fieldVectors.Int16Vector as any, nullableVectors.Int16Vector as any, nullableFieldVectors.Int16Vector as any); +export class Int16Vector extends MixinInt16Vector {} +export const MixinInt32Vector = MixinArrowTraits(vectors.Int32Vector as any, fieldVectors.Int32Vector as any, nullableVectors.Int32Vector as any, nullableFieldVectors.Int32Vector as any); +export class Int32Vector extends MixinInt32Vector {} +export const MixinInt64Vector = MixinArrowTraits(vectors.Int64Vector as any, fieldVectors.Int64Vector as any, nullableVectors.Int64Vector as any, nullableFieldVectors.Int64Vector as any); +export class Int64Vector extends MixinInt64Vector {} +export const MixinUint8Vector = MixinArrowTraits(vectors.Uint8Vector as any, fieldVectors.Uint8Vector as any, nullableVectors.Uint8Vector as any, nullableFieldVectors.Uint8Vector as any); +export class Uint8Vector extends MixinUint8Vector {} +export const MixinUint16Vector = MixinArrowTraits(vectors.Uint16Vector as any, fieldVectors.Uint16Vector as any, nullableVectors.Uint16Vector as any, nullableFieldVectors.Uint16Vector as any); +export class Uint16Vector extends MixinUint16Vector {} +export const MixinUint32Vector = MixinArrowTraits(vectors.Uint32Vector as any, fieldVectors.Uint32Vector as any, nullableVectors.Uint32Vector as any, nullableFieldVectors.Uint32Vector as any); +export class Uint32Vector extends MixinUint32Vector {} +export const MixinUint64Vector = MixinArrowTraits(vectors.Uint64Vector as any, fieldVectors.Uint64Vector as any, nullableVectors.Uint64Vector as any, nullableFieldVectors.Uint64Vector as any); +export class Uint64Vector extends MixinUint64Vector {} +export const MixinDate32Vector = MixinArrowTraits(vectors.Date32Vector as any, fieldVectors.Date32Vector as any, nullableVectors.Date32Vector as any, nullableFieldVectors.Date32Vector as any); +export class Date32Vector extends MixinDate32Vector {} +export const MixinDate64Vector = MixinArrowTraits(vectors.Date64Vector as any, fieldVectors.Date64Vector as any, nullableVectors.Date64Vector as any, nullableFieldVectors.Date64Vector as any); +export class Date64Vector extends MixinDate64Vector {} +export const MixinTime32Vector = MixinArrowTraits(vectors.Time32Vector as any, fieldVectors.Time32Vector as any, nullableVectors.Time32Vector as any, nullableFieldVectors.Time32Vector as any); +export class Time32Vector extends MixinTime32Vector {} +export const MixinTime64Vector = MixinArrowTraits(vectors.Time64Vector as any, fieldVectors.Time64Vector as any, nullableVectors.Time64Vector as any, nullableFieldVectors.Time64Vector as any); +export class Time64Vector extends MixinTime64Vector {} +export const MixinFloat16Vector = MixinArrowTraits(vectors.Float16Vector as any, fieldVectors.Float16Vector as any, nullableVectors.Float16Vector as any, nullableFieldVectors.Float16Vector as any); +export class Float16Vector extends MixinFloat16Vector {} +export const MixinFloat32Vector = MixinArrowTraits(vectors.Float32Vector as any, fieldVectors.Float32Vector as any, nullableVectors.Float32Vector as any, nullableFieldVectors.Float32Vector as any); +export class Float32Vector extends MixinFloat32Vector {} +export const MixinFloat64Vector = MixinArrowTraits(vectors.Float64Vector as any, fieldVectors.Float64Vector as any, nullableVectors.Float64Vector as any, nullableFieldVectors.Float64Vector as any); +export class Float64Vector extends MixinFloat64Vector {} +export const MixinStructVector = MixinArrowTraits(vectors.StructVector as any, fieldVectors.StructVector as any, nullableVectors.StructVector as any, nullableFieldVectors.StructVector as any); +export class StructVector extends MixinStructVector {} +export const MixinDecimalVector = MixinArrowTraits(vectors.DecimalVector as any, fieldVectors.DecimalVector as any, nullableVectors.DecimalVector as any, nullableFieldVectors.DecimalVector as any); +export class DecimalVector extends MixinDecimalVector {} +export const MixinTimestampVector = MixinArrowTraits(vectors.TimestampVector as any, fieldVectors.TimestampVector as any, nullableVectors.TimestampVector as any, nullableFieldVectors.TimestampVector as any); +export class TimestampVector extends MixinTimestampVector {} +export const MixinDictionaryVector = MixinArrowTraits(vectors.DictionaryVector as any, fieldVectors.DictionaryVector as any, nullableVectors.DictionaryVector as any, nullableFieldVectors.DictionaryVector as any); +export class DictionaryVector extends MixinDictionaryVector {} +export const MixinFixedSizeListVector = MixinArrowTraits(vectors.FixedSizeListVector as any, fieldVectors.FixedSizeListVector as any, nullableVectors.FixedSizeListVector as any, nullableFieldVectors.FixedSizeListVector as any); +export class FixedSizeListVector extends MixinFixedSizeListVector {} diff --git a/js/src/vector/table.ts b/js/src/vector/table.ts index b15092a72df24..ca2b66a22da80 100644 --- a/js/src/vector/table.ts +++ b/js/src/vector/table.ts @@ -17,27 +17,29 @@ import { Vector } from './vector'; import { StructVector, StructRow } from './struct'; -import { readVectors, readVectorsAsync } from '../reader/arrow'; +import { read, readAsync } from '../reader/arrow'; + +function concatVectors(tableVectors: Vector[], batchVectors: Vector[]) { + return tableVectors.length === 0 ? batchVectors : batchVectors.map((vec, i, _vs, col = tableVectors[i]) => + vec && col && col.concat(vec) || col || vec + ) as Vector[]; +} export class Table extends StructVector { - static from(buffers?: Iterable) { + static from(sources?: Iterable | object | string) { let columns: Vector[] = []; - if (buffers) { - for (let vectors of readVectors(buffers)) { - columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) => - vec && col && col.concat(vec) || col || vec - ) as Vector[]; + if (sources) { + for (let vectors of read(sources)) { + columns = concatVectors(columns, vectors); } } return new Table({ columns }); } - static async fromAsync(buffers?: AsyncIterable) { + static async fromAsync(sources?: AsyncIterable) { let columns: Vector[] = []; - if (buffers) { - for await (let vectors of readVectorsAsync(buffers)) { - columns = columns.length === 0 ? vectors : vectors.map((vec, i, _vs, col = columns[i]) => - vec && col && col.concat(vec) || col || vec - ) as Vector[]; + if (sources) { + for await (let vectors of readAsync(sources)) { + columns = columns = concatVectors(columns, vectors); } } return new Table({ columns }); diff --git a/js/src/vector/traits/field.ts b/js/src/vector/traits/field.ts new file mode 100644 index 0000000000000..9f68f507c6f10 --- /dev/null +++ b/js/src/vector/traits/field.ts @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import * as vectors from './vectors'; +import { fieldMixin } from './mixins'; +import { Field, FieldNode } from '../../format/arrow'; +export { Field, FieldNode, Vector }; + +export const FieldListVector = fieldMixin(vectors.ListVector); +export class ListVector extends FieldListVector {} +export const FieldBinaryVector = fieldMixin(vectors.BinaryVector); +export class BinaryVector extends FieldBinaryVector {} +export const FieldUtf8Vector = fieldMixin(vectors.Utf8Vector); +export class Utf8Vector extends FieldUtf8Vector {} +export const FieldBoolVector = fieldMixin(vectors.BoolVector); +export class BoolVector extends FieldBoolVector {} +export const FieldInt8Vector = fieldMixin(vectors.Int8Vector); +export class Int8Vector extends FieldInt8Vector {} +export const FieldInt16Vector = fieldMixin(vectors.Int16Vector); +export class Int16Vector extends FieldInt16Vector {} +export const FieldInt32Vector = fieldMixin(vectors.Int32Vector); +export class Int32Vector extends FieldInt32Vector {} +export const FieldInt64Vector = fieldMixin(vectors.Int64Vector); +export class Int64Vector extends FieldInt64Vector {} +export const FieldUint8Vector = fieldMixin(vectors.Uint8Vector); +export class Uint8Vector extends FieldUint8Vector {} +export const FieldUint16Vector = fieldMixin(vectors.Uint16Vector); +export class Uint16Vector extends FieldUint16Vector {} +export const FieldUint32Vector = fieldMixin(vectors.Uint32Vector); +export class Uint32Vector extends FieldUint32Vector {} +export const FieldUint64Vector = fieldMixin(vectors.Uint64Vector); +export class Uint64Vector extends FieldUint64Vector {} +export const FieldDate32Vector = fieldMixin(vectors.Date32Vector); +export class Date32Vector extends FieldDate32Vector {} +export const FieldDate64Vector = fieldMixin(vectors.Date64Vector); +export class Date64Vector extends FieldDate64Vector {} +export const FieldTime32Vector = fieldMixin(vectors.Time32Vector); +export class Time32Vector extends FieldTime32Vector {} +export const FieldTime64Vector = fieldMixin(vectors.Time64Vector); +export class Time64Vector extends FieldTime64Vector {} +export const FieldFloat16Vector = fieldMixin(vectors.Float16Vector); +export class Float16Vector extends FieldFloat16Vector {} +export const FieldFloat32Vector = fieldMixin(vectors.Float32Vector); +export class Float32Vector extends FieldFloat32Vector {} +export const FieldFloat64Vector = fieldMixin(vectors.Float64Vector); +export class Float64Vector extends FieldFloat64Vector {} +export const FieldStructVector = fieldMixin(vectors.StructVector); +export class StructVector extends FieldStructVector {} +export const FieldDecimalVector = fieldMixin(vectors.DecimalVector); +export class DecimalVector extends FieldDecimalVector {} +export const FieldTimestampVector = fieldMixin(vectors.TimestampVector); +export class TimestampVector extends FieldTimestampVector {} +export const FieldDictionaryVector = fieldMixin(vectors.DictionaryVector); +export class DictionaryVector extends FieldDictionaryVector {} +export const FieldFixedSizeListVector = fieldMixin(vectors.FixedSizeListVector); +export class FixedSizeListVector extends FieldFixedSizeListVector {} \ No newline at end of file diff --git a/js/src/vector/traits.ts b/js/src/vector/traits/mixins.ts similarity index 67% rename from js/src/vector/traits.ts rename to js/src/vector/traits/mixins.ts index ca933f1605957..011ac31352db6 100644 --- a/js/src/vector/traits.ts +++ b/js/src/vector/traits/mixins.ts @@ -15,13 +15,27 @@ // specific language governing permissions and limitations // under the License. -import { Vector } from './vector'; -import { BoolVector } from './numeric'; -import * as Schema_ from '../format/Schema'; -import * as Message_ from '../format/Message'; +import { Vector } from '../vector'; +import { BoolVector } from '../numeric'; +import * as Schema_ from '../../format/fb/Schema'; +import { Field, FieldNode } from '../../format/arrow'; import Type = Schema_.org.apache.arrow.flatbuf.Type; -import Field = Schema_.org.apache.arrow.flatbuf.Field; -import FieldNode = Message_.org.apache.arrow.flatbuf.FieldNode; + +function isField(x: any): x is Field { + return x instanceof Field; +} + +function isFieldNode(x: any): x is FieldNode { + return x instanceof FieldNode; +} + +export function isFieldArgv(x: any): x is { field: Field, fieldNode: FieldNode } { + return x && isField(x.field) && isFieldNode(x.fieldNode); +} + +export function isNullableArgv(x: any): x is { validity: Uint8Array } { + return x && x.validity && ArrayBuffer.isView(x.validity) && x.validity instanceof Uint8Array; +} type Ctor = new (argv: TArgv) => Vector; @@ -51,19 +65,11 @@ export const fieldMixin = (superclass: new (argv: TArgv const { field, fieldNode } = argv; this.field = field; this.fieldNode = fieldNode; - this.nullable = field.nullable(); - this.type = Type[field.typeType()]; - this.length = fieldNode.length().low | 0; - this.nullCount = fieldNode.nullCount().low; - } - get name() { return this.field.name()!; } - get metadata() { - const { field } = this, data = new Map(); - for (let entry, key, i = -1, n = field && field.customMetadataLength() | 0; ++i < n;) { - if ((entry = field.customMetadata(i)) && (key = entry.key()) != null) { - data.set(key, entry.value()!); - } - } - return data; + this.nullable = field.nullable; + this.type = Type[field.typeType]; + this.length = fieldNode.length.low | 0; + this.nullCount = fieldNode.nullCount.low; } + get name() { return this.field.name!; } + get metadata() { return this.field.metadata!; } }; diff --git a/js/src/vector/traits/nullable.ts b/js/src/vector/traits/nullable.ts new file mode 100644 index 0000000000000..1393e5fd1bf68 --- /dev/null +++ b/js/src/vector/traits/nullable.ts @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import * as vectors from './vectors'; +import { nullableMixin } from './mixins'; + +export { Vector }; +export const NullableListVector = nullableMixin(vectors.ListVector); +export class ListVector extends NullableListVector {} +export const NullableBinaryVector = nullableMixin(vectors.BinaryVector); +export class BinaryVector extends NullableBinaryVector {} +export const NullableUtf8Vector = nullableMixin(vectors.Utf8Vector); +export class Utf8Vector extends NullableUtf8Vector {} +export const NullableBoolVector = nullableMixin(vectors.BoolVector); +export class BoolVector extends NullableBoolVector {} +export const NullableInt8Vector = nullableMixin(vectors.Int8Vector); +export class Int8Vector extends NullableInt8Vector {} +export const NullableInt16Vector = nullableMixin(vectors.Int16Vector); +export class Int16Vector extends NullableInt16Vector {} +export const NullableInt32Vector = nullableMixin(vectors.Int32Vector); +export class Int32Vector extends NullableInt32Vector {} +export const NullableInt64Vector = nullableMixin(vectors.Int64Vector); +export class Int64Vector extends NullableInt64Vector {} +export const NullableUint8Vector = nullableMixin(vectors.Uint8Vector); +export class Uint8Vector extends NullableUint8Vector {} +export const NullableUint16Vector = nullableMixin(vectors.Uint16Vector); +export class Uint16Vector extends NullableUint16Vector {} +export const NullableUint32Vector = nullableMixin(vectors.Uint32Vector); +export class Uint32Vector extends NullableUint32Vector {} +export const NullableUint64Vector = nullableMixin(vectors.Uint64Vector); +export class Uint64Vector extends NullableUint64Vector {} +export const NullableDate32Vector = nullableMixin(vectors.Date32Vector); +export class Date32Vector extends NullableDate32Vector {} +export const NullableDate64Vector = nullableMixin(vectors.Date64Vector); +export class Date64Vector extends NullableDate64Vector {} +export const NullableTime32Vector = nullableMixin(vectors.Time32Vector); +export class Time32Vector extends NullableTime32Vector {} +export const NullableTime64Vector = nullableMixin(vectors.Time64Vector); +export class Time64Vector extends NullableTime64Vector {} +export const NullableFloat16Vector = nullableMixin(vectors.Float16Vector); +export class Float16Vector extends NullableFloat16Vector {} +export const NullableFloat32Vector = nullableMixin(vectors.Float32Vector); +export class Float32Vector extends NullableFloat32Vector {} +export const NullableFloat64Vector = nullableMixin(vectors.Float64Vector); +export class Float64Vector extends NullableFloat64Vector {} +export const NullableStructVector = nullableMixin(vectors.StructVector); +export class StructVector extends NullableStructVector {} +export const NullableDecimalVector = nullableMixin(vectors.DecimalVector); +export class DecimalVector extends NullableDecimalVector {} +export const NullableTimestampVector = nullableMixin(vectors.TimestampVector); +export class TimestampVector extends NullableTimestampVector {} +export const NullableDictionaryVector = nullableMixin(vectors.DictionaryVector); +export class DictionaryVector extends NullableDictionaryVector {} +export const NullableFixedSizeListVector = nullableMixin(vectors.FixedSizeListVector); +export class FixedSizeListVector extends NullableFixedSizeListVector {} \ No newline at end of file diff --git a/js/src/vector/traits/nullablefield.ts b/js/src/vector/traits/nullablefield.ts new file mode 100644 index 0000000000000..8cbee62e43dc3 --- /dev/null +++ b/js/src/vector/traits/nullablefield.ts @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import * as vectors from './vectors'; +import { nullableMixin, fieldMixin } from './mixins'; +import { Field, FieldNode } from '../../format/arrow'; +export { Vector, Field, FieldNode }; + +export const NullableFieldListVector = nullableMixin(fieldMixin(vectors.ListVector)); +export class ListVector extends NullableFieldListVector {} +export const NullableFieldBinaryVector = nullableMixin(fieldMixin(vectors.BinaryVector)); +export class BinaryVector extends NullableFieldBinaryVector {} +export const NullableFieldUtf8Vector = nullableMixin(fieldMixin(vectors.Utf8Vector)); +export class Utf8Vector extends NullableFieldUtf8Vector {} +export const NullableFieldBoolVector = nullableMixin(fieldMixin(vectors.BoolVector)); +export class BoolVector extends NullableFieldBoolVector {} +export const NullableFieldInt8Vector = nullableMixin(fieldMixin(vectors.Int8Vector)); +export class Int8Vector extends NullableFieldInt8Vector {} +export const NullableFieldInt16Vector = nullableMixin(fieldMixin(vectors.Int16Vector)); +export class Int16Vector extends NullableFieldInt16Vector {} +export const NullableFieldInt32Vector = nullableMixin(fieldMixin(vectors.Int32Vector)); +export class Int32Vector extends NullableFieldInt32Vector {} +export const NullableFieldInt64Vector = nullableMixin(fieldMixin(vectors.Int64Vector)); +export class Int64Vector extends NullableFieldInt64Vector {} +export const NullableFieldUint8Vector = nullableMixin(fieldMixin(vectors.Uint8Vector)); +export class Uint8Vector extends NullableFieldUint8Vector {} +export const NullableFieldUint16Vector = nullableMixin(fieldMixin(vectors.Uint16Vector)); +export class Uint16Vector extends NullableFieldUint16Vector {} +export const NullableFieldUint32Vector = nullableMixin(fieldMixin(vectors.Uint32Vector)); +export class Uint32Vector extends NullableFieldUint32Vector {} +export const NullableFieldUint64Vector = nullableMixin(fieldMixin(vectors.Uint64Vector)); +export class Uint64Vector extends NullableFieldUint64Vector {} +export const NullableFieldDate32Vector = nullableMixin(fieldMixin(vectors.Date32Vector)); +export class Date32Vector extends NullableFieldDate32Vector {} +export const NullableFieldDate64Vector = nullableMixin(fieldMixin(vectors.Date64Vector)); +export class Date64Vector extends NullableFieldDate64Vector {} +export const NullableFieldTime32Vector = nullableMixin(fieldMixin(vectors.Time32Vector)); +export class Time32Vector extends NullableFieldTime32Vector {} +export const NullableFieldTime64Vector = nullableMixin(fieldMixin(vectors.Time64Vector)); +export class Time64Vector extends NullableFieldTime64Vector {} +export const NullableFieldFloat16Vector = nullableMixin(fieldMixin(vectors.Float16Vector)); +export class Float16Vector extends NullableFieldFloat16Vector {} +export const NullableFieldFloat32Vector = nullableMixin(fieldMixin(vectors.Float32Vector)); +export class Float32Vector extends NullableFieldFloat32Vector {} +export const NullableFieldFloat64Vector = nullableMixin(fieldMixin(vectors.Float64Vector)); +export class Float64Vector extends NullableFieldFloat64Vector {} +export const NullableFieldStructVector = nullableMixin(fieldMixin(vectors.StructVector)); +export class StructVector extends NullableFieldStructVector {} +export const NullableFieldDecimalVector = nullableMixin(fieldMixin(vectors.DecimalVector)); +export class DecimalVector extends NullableFieldDecimalVector {} +export const NullableFieldTimestampVector = nullableMixin(fieldMixin(vectors.TimestampVector)); +export class TimestampVector extends NullableFieldTimestampVector {} +export const NullableFieldDictionaryVector = nullableMixin(fieldMixin(vectors.DictionaryVector)); +export class DictionaryVector extends NullableFieldDictionaryVector {} +export const NullableFieldFixedSizeListVector = nullableMixin(fieldMixin(vectors.FixedSizeListVector)); +export class FixedSizeListVector extends NullableFieldFixedSizeListVector {} \ No newline at end of file diff --git a/js/src/vector/traits/vectors.ts b/js/src/vector/traits/vectors.ts new file mode 100644 index 0000000000000..f9e05fd4eff1b --- /dev/null +++ b/js/src/vector/traits/vectors.ts @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { Utf8Vector } from '../utf8'; +import { StructVector } from '../struct'; +import { DictionaryVector } from '../dictionary'; +import { + ListVector, + BinaryVector, + FixedSizeListVector +} from '../list'; + +import { + BoolVector, + Int8Vector, + Int16Vector, + Int32Vector, + Int64Vector, + Uint8Vector, + Uint16Vector, + Uint32Vector, + Uint64Vector, + Float16Vector, + Float32Vector, + Float64Vector, + Date32Vector, + Date64Vector, + Time32Vector, + Time64Vector, + DecimalVector, + TimestampVector, +} from '../numeric'; + +export { + Vector, + BoolVector, + ListVector, + Utf8Vector, + Int8Vector, + Int16Vector, + Int32Vector, + Int64Vector, + Uint8Vector, + Uint16Vector, + Uint32Vector, + Uint64Vector, + Date32Vector, + Date64Vector, + Time32Vector, + Time64Vector, + BinaryVector, + StructVector, + Float16Vector, + Float32Vector, + Float64Vector, + DecimalVector, + TimestampVector, + DictionaryVector, + FixedSizeListVector, +}; diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts index 8047c89923223..9565478ad969b 100644 --- a/js/src/vector/vector.ts +++ b/js/src/vector/vector.ts @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -import * as Schema_ from '../format/Schema'; +import * as Schema_ from '../format/fb/Schema'; import Type = Schema_.org.apache.arrow.flatbuf.Type; export interface Vector extends Iterable { @@ -29,6 +29,7 @@ export interface Vector extends Iterable { concat(...vectors: Vector[]): Vector; slice(start?: number, end?: number): R; } + export class Vector implements Vector { slice(start?: number, end?: number): R { let { length } = this, from = start! | 0; diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts index 66a5c21ac2dff..87641e52bf3f8 100644 --- a/js/test/Arrow.ts +++ b/js/test/Arrow.ts @@ -39,6 +39,7 @@ else if (!~targets.indexOf(target)) throwInvalidImportError('target', target, ta else if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats); else modulePath = path.join(target, format); +export { Int64, Uint64, Int128 } from '../src/Arrow'; export { List } from '../src/Arrow'; export { TypedArray } from '../src/Arrow'; export { TypedArrayConstructor } from '../src/Arrow'; @@ -46,4 +47,4 @@ export { NumericVectorConstructor } from '../src/Arrow'; import * as Arrow_ from '../src/Arrow'; export let Arrow: typeof Arrow_ = require(path.resolve(`./targets`, modulePath, `Arrow`)); -export default Arrow; \ No newline at end of file +export default Arrow; diff --git a/js/test/test-config.ts b/js/test/integration/test-config.ts similarity index 100% rename from js/test/test-config.ts rename to js/test/integration/test-config.ts diff --git a/js/test/integration-tests.ts b/js/test/integration/validate-tests.ts similarity index 58% rename from js/test/integration-tests.ts rename to js/test/integration/validate-tests.ts index 4147e862b19a4..c8778ba2b33c2 100644 --- a/js/test/integration-tests.ts +++ b/js/test/integration/validate-tests.ts @@ -15,11 +15,31 @@ // specific language governing permissions and limitations // under the License. -import Arrow from './Arrow'; +import * as fs from 'fs'; +import * as path from 'path'; + +if (!process.env.JSON_PATH || !process.env.ARROW_PATH) { + throw new Error('Integration tests need paths to both json and arrow files'); +} + +const jsonPath = path.resolve(process.env.JSON_PATH + ''); +const arrowPath = path.resolve(process.env.ARROW_PATH + ''); + +if (!fs.existsSync(jsonPath) || !fs.existsSync(arrowPath)) { + throw new Error('Integration tests need both json and arrow files to exist'); +} + +/* tslint:disable */ +const { parse } = require('json-bignum'); + +const jsonData = parse(fs.readFileSync(jsonPath, 'utf8')); +const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)]; + +import Arrow from '../Arrow'; import { zip } from 'ix/iterable/zip'; -import { config, formats } from './test-config'; +import { toArray } from 'ix/iterable/toArray'; -const { Table, readVectors } = Arrow; +const { Table, read } = Arrow; expect.extend({ toEqualVector(v1: any, v2: any) { @@ -65,7 +85,7 @@ expect.extend({ return { pass: allFailures.every(({ failures }) => failures.length === 0), message: () => [ - `${v1.name}: (${format('cpp', 'java', ' !== ')})\n`, + `${v1.name}: (${format('json', 'arrow', ' !== ')})\n`, ...allFailures.map(({ failures, title }) => !failures.length ? `` : [`${title}:`, ...failures].join(`\n`)) ].join('\n') @@ -74,41 +94,35 @@ expect.extend({ }); describe(`Integration`, () => { - for (const format of formats) { - describe(format, () => { - for (const [cppArrow, javaArrow] of zip(config.cpp[format], config.java[format])) { - describe(`${cppArrow.name}`, () => { - testReaderIntegration(cppArrow.buffers, javaArrow.buffers); - testTableFromBuffersIntegration(cppArrow.buffers, javaArrow.buffers); - }); - } - }); - } + testReaderIntegration(); + testTableFromBuffersIntegration(); }); -function testReaderIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) { - test(`cpp and java vectors report the same values`, () => { +function testReaderIntegration() { + test(`json and arrow buffers report the same values`, () => { expect.hasAssertions(); - for (const [cppVectors, javaVectors] of zip(readVectors(cppBuffers), readVectors(javaBuffers))) { - expect(cppVectors.length).toEqual(javaVectors.length); - for (let i = -1, n = cppVectors.length; ++i < n;) { - (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]); + const jsonVectors = toArray(read(jsonData)); + const binaryVectors = toArray(read(arrowBuffers)); + for (const [jVectors, bVectors] of zip(jsonVectors, binaryVectors)) { + expect(jVectors.length).toEqual(bVectors.length); + for (let i = -1, n = jVectors.length; ++i < n;) { + (expect(jVectors[i]) as any).toEqualVector(bVectors[i]); } } }); } -function testTableFromBuffersIntegration(cppBuffers: Uint8Array[], javaBuffers: Uint8Array[]) { - test(`cpp and java tables report the same values`, () => { +function testTableFromBuffersIntegration() { + test(`json and arrow buffers report the same values`, () => { expect.hasAssertions(); - const cppTable = Table.from(cppBuffers); - const javaTable = Table.from(javaBuffers); - const cppVectors = cppTable.columns; - const javaVectors = javaTable.columns; - expect(cppTable.length).toEqual(javaTable.length); - expect(cppVectors.length).toEqual(javaVectors.length); - for (let i = -1, n = cppVectors.length; ++i < n;) { - (expect(cppVectors[i]) as any).toEqualVector(javaVectors[i]); + const jsonTable = Table.from(jsonData); + const binaryTable = Table.from(arrowBuffers); + const jsonVectors = jsonTable.columns; + const binaryVectors = binaryTable.columns; + expect(jsonTable.length).toEqual(binaryTable.length); + expect(jsonVectors.length).toEqual(binaryVectors.length); + for (let i = -1, n = jsonVectors.length; ++i < n;) { + (expect(jsonVectors[i]) as any).toEqualVector(binaryVectors[i]); } }); } diff --git a/js/test/reader-tests.ts b/js/test/reader-tests.ts deleted file mode 100644 index 309cec9e23b14..0000000000000 --- a/js/test/reader-tests.ts +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import Arrow from './Arrow'; -const { readVectors } = Arrow; -import { config, sources, formats } from './test-config'; - -describe(`readBuffers`, () => { - for (const source of sources) { - describe(source, () => { - for (const format of formats) { - describe(format, () => { - for (const { name, buffers } of config[source][format]) { - describe(name, () => { - testReaderIterator(buffers); - testVectorIterator(buffers); - }); - } - }); - } - }); - } -}); - -function testReaderIterator(buffers: Uint8Array[]) { - test(`reads each batch as an Array of Vectors`, () => { - expect.hasAssertions(); - for (const vectors of readVectors(buffers)) { - for (const vector of vectors) { - expect(vector.name).toMatchSnapshot(); - expect(vector.type).toMatchSnapshot(); - expect(vector.length).toMatchSnapshot(); - for (let i = -1, n = vector.length; ++i < n;) { - expect(vector.get(i)).toMatchSnapshot(); - } - } - } - }); -} - -function testVectorIterator(buffers: Uint8Array[]) { - test(`vector iterators report the same values as get`, () => { - expect.hasAssertions(); - for (const vectors of readVectors(buffers)) { - for (const vector of vectors) { - let i = -1, n = vector.length; - for (let v of vector) { - expect(++i).toBeLessThan(n); - expect(v).toEqual(vector.get(i)); - } - expect(++i).toEqual(n); - } - } - }); -} diff --git a/js/test/table-tests.ts b/js/test/table-tests.ts deleted file mode 100644 index 5ec04a72ff33b..0000000000000 --- a/js/test/table-tests.ts +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import Arrow from './Arrow'; -const { Table, readVectors } = Arrow; -import { config, sources, formats } from './test-config'; - -describe(`Table`, () => { - for (const source of sources) { - describe(source, () => { - for (const format of formats) { - describe(format, () => { - for (const { name, buffers } of config[source][format]) { - describe(name, () => { - testTableFromBuffers(buffers); - testColumnIterators(buffers); - testReaderVectorsAndTableColumns(buffers); - testTableRowIterator(buffers); - testTableRowIteratorCompact(buffers); - testEmptyTableToString(); - testTableToStringPretty(buffers); - testTableToStringPrettyWithIndex(buffers); - }); - } - }); - } - }); - } -}); - -function testTableFromBuffers(buffers: Uint8Array[]) { - test(`creates a Table from Arrow buffers`, () => { - expect.hasAssertions(); - const table = Table.from(buffers); - for (const vector of table.columns) { - expect(vector.name).toMatchSnapshot(); - expect(vector.type).toMatchSnapshot(); - expect(vector.length).toMatchSnapshot(); - for (let i = -1, n = vector.length; ++i < n;) { - expect(vector.get(i)).toMatchSnapshot(); - } - } - }); -} - -function testColumnIterators(buffers: Uint8Array[]) { - test(`vector iterators report the same values as get`, () => { - expect.hasAssertions(); - const table = Table.from(buffers); - for (const vector of table.columns) { - let i = -1, n = vector.length; - for (let v of vector) { - expect(++i).toBeLessThan(n); - expect(v).toEqual(vector.get(i)); - } - expect(++i).toEqual(n); - } - }); -} - -function testReaderVectorsAndTableColumns(buffers: Uint8Array[]) { - test(`batch and Table Vectors report the same values`, () => { - expect.hasAssertions(); - let rowsTotal = 0; - let table = Table.from(buffers); - for (let vectors of readVectors(buffers)) { - let rowsNow = Math.max(...vectors.map((v) => v.length)); - for (let vi = -1, vn = vectors.length; ++vi < vn;) { - let v1 = vectors[vi]; - let v2 = table.columns[vi]; - expect(v1.name).toEqual(v2.name); - expect(v1.type).toEqual(v2.type); - for (let i = -1, n = v1.length; ++i < n;) { - expect(v1.get(i)).toEqual(v2.get(i + rowsTotal)); - } - } - rowsTotal += rowsNow; - } - }); -} - -function testTableRowIterator(buffers: Uint8Array[]) { - test(`enumerates Table rows`, () => { - expect.hasAssertions(); - const table = Table.from(buffers); - expect(table.length).toMatchSnapshot(); - expect(table.columns.length).toMatchSnapshot(); - for (const row of table) { - expect(row!.toObject()).toMatchSnapshot(); - } - }); -} - -function testTableRowIteratorCompact(buffers: Uint8Array[]) { - test(`enumerates Table rows compact`, () => { - expect.hasAssertions(); - const table = Table.from(buffers); - expect(table.length).toMatchSnapshot(); - expect(table.columns.length).toMatchSnapshot(); - for (const row of table) { - expect(row!.toArray()).toMatchSnapshot(); - } - }); -} - -function testEmptyTableToString() { - test(`toString() prints an empty Table`, () => { - expect(Table.from().toString()).toMatchSnapshot(); - }); -} - -function testTableToStringPretty(buffers: Uint8Array[]) { - test(`toString() prints a pretty Table`, () => { - expect(Table.from(buffers).toString()).toMatchSnapshot(); - }); -} - -function testTableToStringPrettyWithIndex(buffers: Uint8Array[]) { - test(`toString({ index: true }) prints a pretty Table with an Index column`, () => { - expect(Table.from(buffers).toString({ index: true })).toMatchSnapshot(); - }); -} \ No newline at end of file diff --git a/js/test/unit/int-tests.ts b/js/test/unit/int-tests.ts new file mode 100644 index 0000000000000..74c96e8efbb6d --- /dev/null +++ b/js/test/unit/int-tests.ts @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + Int64, + Uint64, + Int128 +} from '../Arrow'; + +describe(`Uint64`, () => { + test(`gets expected high/low bytes`, () => { + let i = new Uint64(new Uint32Array([5, 0])); + expect(i.high()).toEqual(0); + expect(i.low()).toEqual(5); + }); + test(`adds 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([5, 0])); + let b = new Uint64(new Uint32Array([9, 0])); + let expected = new Uint64(new Uint32Array([14, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`addition overflows 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([0xffffffff, 0])); + let b = new Uint64(new Uint32Array([9, 0])); + let expected = new Uint64(new Uint32Array([8, 1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`multiplies 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([5, 0])); + let b = new Uint64(new Uint32Array([9, 0])); + let expected = new Uint64(new Uint32Array([45, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication overflows 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([0x80000000, 0])); + let b = new Uint64(new Uint32Array([3, 0])); + let expected = new Uint64(new Uint32Array([0x80000000, 1])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication is associative`, () => { + let a = new Uint64(new Uint32Array([0x80000000, 0])); + let b = new Uint64(new Uint32Array([3, 0])); + expect(Uint64.multiply(a, b)).toEqual(Uint64.multiply(b,a)); + }); + test(`lessThan works on 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([0x0000abcd, 0])); + let b = new Uint64(new Uint32Array([0x0000abcf, 0])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on 64-bit numbers`, () => { + let a = new Uint64(new Uint32Array([123, 32])); + let b = new Uint64(new Uint32Array([568, 32])); + expect(a.lessThan(b)).toBeTruthy(); + }); +}); + +describe(`Int64`, () => { + test(`gets expected high/low bytes`, () => { + let i = new Int64(new Uint32Array([5, 0])); + expect(i.high()).toEqual(0); + expect(i.low()).toEqual(5); + }); + test(`adds 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([5, 0])); + let b = new Int64(new Uint32Array([9, 0])); + let expected = new Int64(new Uint32Array([14, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`adds negative 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([56789 , 0])); + let b = new Int64(new Uint32Array([-66789, -1])); + let expected = new Int64(new Uint32Array([-10000, -1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`addition overflows 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([0xffffffff, 0])); + let b = new Int64(new Uint32Array([9, 0])); + let expected = new Int64(new Uint32Array([8, 1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`multiplies 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([5, 0])); + let b = new Int64(new Uint32Array([9, 0])); + let expected = new Int64(new Uint32Array([45, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication overflows 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([0x80000000, 0])); + let b = new Int64(new Uint32Array([3, 0])); + let expected = new Int64(new Uint32Array([0x80000000, 1])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication works on negative numbers`, () => { + let a = new Int64(new Uint32Array([-5, -1])); + let b = new Int64(new Uint32Array([-100, -1])); + expect(a.times(b)).toEqual(new Int64(new Uint32Array([ 500, 0]))); + expect(a.times(b)).toEqual(new Int64(new Uint32Array([ -50000, -1]))); + expect(a.times(b)).toEqual(new Int64(new Uint32Array([5000000, 0]))); + }); + test(`multiplication is associative`, () => { + let a = new Int64(new Uint32Array([0x80000000, 0])); + let b = new Int64(new Uint32Array([3, 0])); + expect(Int64.multiply(a, b)).toEqual(Int64.multiply(b,a)); + }); + test(`lessThan works on 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([0x0000abcd, 0])); + let b = new Int64(new Uint32Array([0x0000abcf, 0])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on 64-bit numbers`, () => { + let a = new Int64(new Uint32Array([123, 32])); + let b = new Int64(new Uint32Array([568, 32])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on negative numbers`, () => { + let a = new Int64(new Uint32Array([0, -158])); + let b = new Int64(new Uint32Array([-3, -1])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on mixed numbers`, () => { + let a = new Int64(new Uint32Array([-3, -1])); + let b = new Int64(new Uint32Array([ 0, 3])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`negate works on 32-bit number`, () => { + expect (new Int64(new Uint32Array([123456, 0])).negate()).toEqual(new Int64(new Uint32Array([-123456, -1]))); + }); + test(`double negation is noop`, () => { + let test = new Int64(new Uint32Array([6789, 12345])); + let expected = new Int64(new Uint32Array([6789, 12345])); + expect(test.negate().negate()).toEqual(expected); + }); + test(`negate works on 64-bit number`, () => { + expect (new Int64(new Uint32Array([0xb74abf15, 0x62c])).negate()).toEqual(new Int64(new Uint32Array([0x48b540eb, 0xfffff9d3]))); + }); + test(`fromString parses string`, () => { + expect(Int64.fromString('6789123456789')).toEqual(new Int64(new Uint32Array([0xb74abf15, 0x62c]))); + }); + test(`fromString parses negative string`, () => { + expect(Int64.fromString('-6789123456789')).toEqual(new Int64(new Uint32Array([0x48b540eb, 0xfffff9d3]))); + }); +}); + +describe(`Int128`, () => { + test(`gets expected bytes`, () => { + let i = new Int128(new Uint32Array([4, 3, 2, 1])); + expect(i.high().high()).toEqual(1); + expect(i.high().low() ).toEqual(2); + expect(i.low().high() ).toEqual(3); + expect(i.low().low() ).toEqual(4); + }); + test(`adds 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([5, 0, 0, 0])); + let b = new Int128(new Uint32Array([9, 0, 0, 0])); + let expected = new Int128(new Uint32Array([14, 0, 0, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`adds negative 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([56789 , 0, 0, 0])); + let b = new Int128(new Uint32Array([-66789, -1, -1, -1])); + let expected = new Int128(new Uint32Array([-10000, -1, -1, -1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`addition overflows 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([0xffffffff, 0, 0, 0])); + let b = new Int128(new Uint32Array([9, 0, 0, 0])); + let expected = new Int128(new Uint32Array([8, 1, 0, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`multiplies 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([5, 0, 0, 0])); + let b = new Int128(new Uint32Array([9, 0, 0, 0])); + let expected = new Int128(new Uint32Array([45, 0, 0, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication overflows 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([0x80000000, 0, 0, 0])); + let b = new Int128(new Uint32Array([3, 0, 0, 0])); + let expected = new Int128(new Uint32Array([0x80000000, 1, 0, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication works on negative numbers`, () => { + let a = new Int128(new Uint32Array([-5, -1, -1, -1])); + let b = new Int128(new Uint32Array([-100, -1, -1, -1])); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([ 500, 0, 0, 0]))); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([ -50000, -1, -1, -1]))); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([5000000, 0, 0, 0]))); + }); + test(`multiplication is associative`, () => { + let a = new Int128(new Uint32Array([4, 3, 2, 1])); + let b = new Int128(new Uint32Array([3, 0, 0, 0])); + expect(Int128.multiply(a, b)).toEqual(Int128.multiply(b,a)); + }); + test(`multiplication can produce 128-bit number`, () => { + let a = new Int128(new Uint32Array([0, 0xf0000000, 0, 0])); + let b = new Int128(new Uint32Array([0, 0x10000000, 0, 0])); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([0x00000000, 0x00000000, 0x00000000, 0xf000000]))); + }); + test(`fromString parses string`, () => { + expect(Int128.fromString('1002111867823618826746863804903129070')) + .toEqual(new Int64(new Uint32Array([0x00c0ffee, + 0x00c0ffee, + 0x00c0ffee, + 0x00c0ffee]))); + }); + test(`fromString parses negative string`, () => { + expect(Int128.fromString('-12345678901234567890123456789012345678')) + .toEqual(new Int64(new Uint32Array([0x21c70cb2, + 0x3bb66faf, + 0x0ffdccec, + 0xf6b64f09]))); + }); +}); diff --git a/js/test/vector-tests.ts b/js/test/unit/vector-tests.ts similarity index 99% rename from js/test/vector-tests.ts rename to js/test/unit/vector-tests.ts index 0aaba19ee9144..75706229ab172 100644 --- a/js/test/vector-tests.ts +++ b/js/test/unit/vector-tests.ts @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -import Arrow from './Arrow'; -import { +import Arrow, { TypedArray, TypedArrayConstructor, NumericVectorConstructor, -} from './Arrow'; +} from '../Arrow'; const { BoolVector, diff --git a/js/tsconfig/tsconfig.es5.cls.json b/js/tsconfig/tsconfig.es5.cls.json index 502432da04348..4df18aa595d92 100644 --- a/js/tsconfig/tsconfig.es5.cls.json +++ b/js/tsconfig/tsconfig.es5.cls.json @@ -2,8 +2,10 @@ { "extends": "./tsconfig.base.json", "compilerOptions": { - "target": "ES5", + "target": "es2015", "module": "es2015", - "declaration": false + "declaration": false, + "noEmitHelpers": true, + "importHelpers": true } } From d023b40dc81ef43b9b033e88add1859c437bb700 Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 18 Dec 2017 23:36:19 -0500 Subject: [PATCH 146/177] ARROW-1922: Blog post on JAVA vector changes cc @wesm , @jacques-n , @BryanCutler , @icexelloss A small post on recent improvements in JAVA vectors. Suggestions are welcome :) Author: siddharth Author: Wes McKinney Closes #1419 from siddharthteotia/ARROW-1922 and squashes the following commits: ebdd986a [Wes McKinney] Minor tweaks to post, add Dremio link eaedd879 [siddharth] review comments 57050195 [siddharth] correct typo c2af13c2 [siddharth] ARROW-1922: Blog post on JAVA vector changes --- site/_data/contributors.yml | 3 + .../2017-12-19-java-vector-improvements.md | 104 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 site/_posts/2017-12-19-java-vector-improvements.md diff --git a/site/_data/contributors.yml b/site/_data/contributors.yml index 2a8d95b77b239..19ca53ad48599 100644 --- a/site/_data/contributors.yml +++ b/site/_data/contributors.yml @@ -32,4 +32,7 @@ - name: Michael König apacheId: MathMagique githubId: MathMagique +- name: Siddharth Teotia + apacheId: siddharthteotia + githubId: siddharthteotia # End contributors.yml diff --git a/site/_posts/2017-12-19-java-vector-improvements.md b/site/_posts/2017-12-19-java-vector-improvements.md new file mode 100644 index 0000000000000..2854861fe090b --- /dev/null +++ b/site/_posts/2017-12-19-java-vector-improvements.md @@ -0,0 +1,104 @@ +--- +layout: post +title: "Improvements to Java Vector API in Apache Arrow 0.8.0" +excerpt: "This post describes the recent improvements in Java Vector code" +date: 2017-12-19 00:00:00 +author: Siddharth Teotia +categories: [application] +--- + + + +This post gives insight into the major improvements in the Java implementation +of vectors. We undertook this work over the last 10 weeks since the last Arrow +release. + +## Design Goals + +1. Improved maintainability and extensibility +2. Improved heap memory usage +3. No performance overhead on hot code paths + +## Background + +### Improved maintainability and extensibility + +We use templates in several places for compile time Java code generation for +different vector classes, readers, writers etc. Templates are helpful as the +developers don't have to write a lot of duplicate code. + +However, we realized that over a period of time some specific Java +templates became extremely complex with giant if-else blocks, poor code indentation +and documentation. All this impacted the ability to easily extend these templates +for adding new functionality or improving the existing infrastructure. + +So we evaluated the usage of templates for compile time code generation and +decided not to use complex templates in some places by writing small amount of +duplicate code which is elegant, well documented and extensible. + +### Improved heap usage + +We did extensive memory analysis downstream in [Dremio][1] where Arrow is used +heavily for in-memory query execution on columnar data. The general conclusion +was that Arrow's Java vector classes have non-negligible heap overhead and +volume of objects was too high. There were places in code where we were +creating objects unnecessarily and using structures that could be substituted +with better alternatives. + +### No performance overhead on hot code paths + +Java vectors used delegation and abstraction heavily throughout the object +hierarchy. The performance critical get/set methods of vectors went through a +chain of function calls back and forth between different objects before doing +meaningful work. We also evaluated the usage of branches in vector APIs and +reimplemented some of them by avoiding branches completely. + +We took inspiration from how the Java memory code in `ArrowBuf` works. For all +the performance critical methods, `ArrowBuf` bypasses all the netty object +hierarchy, grabs the target virtual address and directly interacts with the +memory. + +There were cases where branches could be avoided all together. + +In case of nullable vectors, we were doing multiple checks to confirm if +the value at a given position in the vector is null or not. + +## Our implementation approach + +- For scalars, the inheritance tree was simplified by writing different +abstract base classes for fixed and variable width scalars. +- The base classes contained all the common functionality across different +types. +- The individual subclasses implemented type specific APIs for fixed and +variable width scalar vectors. +- For the performance critical methods, all the work is done either in +the vector class or corresponding ArrowBuf. There is no delegation to any +internal object. +- The mutator and accessor based access to vector APIs is removed. These +objects led to unnecessary heap overhead and complicated the use of APIs. +- Both scalar and complex vectors directly interact with underlying buffers +that manage the offsets, data and validity. Earlier we were creating different +inner vectors for each vector and delegating all the functionality to inner +vectors. This introduced a lot of bugs in memory management, excessive heap +overhead and performance penalty due to chain of delegations. +- We reduced the number of vector classes by removing non-nullable vectors. +In the new implementation, all vectors in Java are nullable in nature. + +[1]: https://www.dremio.com/ \ No newline at end of file From bcfd83d5046d76dbe3714fd3c6e8e1e2d794490e Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 19 Dec 2017 09:39:06 -0500 Subject: [PATCH 147/177] ARROW-1935: [Website] Remove link to nightly builds. Fix signature / checksum links, add verification instructions Website fixes per ASF policies and feedback in ARROW-1935, ARROW-1936 Author: Wes McKinney Closes #1434 from wesm/ARROW-1935 and squashes the following commits: 41f128ec [Wes McKinney] Remove link to nightly builds. Fix signature / checksum links, add KEYS file, verification instructions --- site/install.md | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/site/install.md b/site/install.md index 6e67d790a9d31..ec30e0469cdc1 100644 --- a/site/install.md +++ b/site/install.md @@ -29,8 +29,9 @@ See the [release notes][10] for more about what's new. ### Source release * **Source Release**: [apache-arrow-0.8.0.tar.gz][6] -* **Verification**: [sha512][3], [asc][7] +* **Verification**: [sha512][3], [asc][7] ([verification instructions][12]) * [Git tag 1d689e5][2] +* [PGP keys for release signatures][11] ### Java Packages @@ -144,26 +145,15 @@ These repositories are managed at [red-data-tools/arrow-packages][9]. If you have any feedback, please send it to the project instead of Apache Arrow project. -### Nightly Development Builds - -To assist with development and debugging, some nightly builds are -available. These builds are not releases and not necessarily produced on ASF -infrastructure. They are to be used strictly for development. - -* **conda packages** for C++ and Python (Linux only) - -``` -conda install arrow-cpp -c twosigma -conda install pyarrow -c twosigma -``` - [1]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/ [2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.8.0 -[3]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512 +[3]: https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.sha512 [4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.8.0%22 [5]: http://conda-forge.github.io [6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz -[7]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc +[7]: https://www.apache.org/dist/arrow/arrow-0.8.0/apache-arrow-0.8.0.tar.gz.asc [8]: https://github.com/red-data-tools/parquet-glib [9]: https://github.com/red-data-tools/arrow-packages [10]: http://arrow.apache.org/release/0.8.0.html +[11]: http://www.apache.org/dist/arrow/KEYS +[12]: https://www.apache.org/dyn/closer.cgi#verify \ No newline at end of file From ac789bafee598326ca9aeaaaa84468017b9f2598 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 19 Dec 2017 09:40:11 -0500 Subject: [PATCH 148/177] ARROW-232: [Python] Add unit test for writing Parquet file from chunked table This requires PARQUET-1092 https://github.com/apache/parquet-cpp/pull/426 Author: Wes McKinney Closes #1425 from wesm/ARROW-232 and squashes the following commits: da8d9998 [Wes McKinney] Add unit test to validate PARQUET-1092 --- python/pyarrow/tests/test_parquet.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index fc8c8f0c98da9..c2bb31c9bcf51 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -118,6 +118,24 @@ def test_pandas_parquet_2_0_rountrip(tmpdir): tm.assert_frame_equal(df, df_read) +@parquet +def test_chunked_table_write(tmpdir): + # ARROW-232 + df = alltypes_sample(size=10) + + # The nanosecond->ms conversion is a nuisance, so we just avoid it here + del df['datetime'] + + batch = pa.RecordBatch.from_pandas(df) + table = pa.Table.from_batches([batch] * 3) + _check_roundtrip(table, version='2.0') + + df, _ = dataframe_with_lists() + batch = pa.RecordBatch.from_pandas(df) + table = pa.Table.from_batches([batch] * 3) + _check_roundtrip(table, version='2.0') + + @parquet def test_pandas_parquet_datetime_tz(): import pyarrow.parquet as pq From 9895181610c0a5ef1ba836300ea2036e1a3e5621 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 19 Dec 2017 10:29:30 -0500 Subject: [PATCH 149/177] ARROW-1934: [Website] 0.8.0 release highlights blog post I took a hack at this after poring through the changelog, others please let me know if you'd like to add or change anything. I need to incorporate Sidd's blog post and add a link to that here. I can publish all of this sometime tomorrow morning New York time and post to social media etc. Author: Wes McKinney Closes #1432 from wesm/ARROW-1934 and squashes the following commits: 4aa6bc6a [Wes McKinney] Tweaks for publication da9e65d4 [Wes McKinney] Start drafting 0.8.0 release blog post --- site/_posts/2017-12-19-0.8.0-release.md | 192 ++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 site/_posts/2017-12-19-0.8.0-release.md diff --git a/site/_posts/2017-12-19-0.8.0-release.md b/site/_posts/2017-12-19-0.8.0-release.md new file mode 100644 index 0000000000000..c65d30832b7b8 --- /dev/null +++ b/site/_posts/2017-12-19-0.8.0-release.md @@ -0,0 +1,192 @@ +--- +layout: post +title: "Apache Arrow 0.8.0 Release" +date: "2017-12-19 00:01:00 -0400" +author: wesm +categories: [release] +--- + + +The Apache Arrow team is pleased to announce the 0.8.0 release. It is the +product of 10 weeks of development andincludes [**286 resolved JIRAs**][1] with +many new features and bug fixes to the various language implementations. This +is the largest release since 0.3.0 earlier this year. + +As part of work towards a stabilizing the Arrow format and making a 1.0.0 +release sometime in 2018, we made a series of backwards-incompatible changes to +the serialized Arrow metadata that requires Arrow readers and writers (0.7.1 +and earlier) to upgrade in order to be compatible with 0.8.0 and higher. We +expect future backwards-incompatible changes to be rare going forward. + +See the [Install Page][2] to learn how to get the libraries for your +platform. The [complete changelog][3] is also available. + +We discuss some highlights from the release and other project news in this +post. + +## Projects "Powered By" Apache Arrow + +A growing ecosystem of projects are using Arrow to solve in-memory analytics +and data interchange problems. We have added a new [Powered By][18] page to the +Arrow website where we can acknowledge open source projects and companies which +are using Arrow. If you would like to add your project to the list as an Arrow +user, please let us know. + +## New Arrow committers + +Since the last release, we have added 5 new Apache committers: + +* [Phillip Cloud][5], who has mainly contributed to C++ and Python +* [Bryan Cutler][13], who has mainly contributed to Java and Spark integration +* [Li Jin][14], who has mainly contributed to Java and Spark integration +* [Paul Taylor][4], who has mainly contributed to JavaScript +* [Siddharth Teotia][15], who has mainly contributed to Java + +Welcome to the Arrow team, and thank you for your contributions! + +## Improved Java vector API, performance improvements + +Siddharth Teotia led efforts to revamp the Java vector API to make things +simpler and faster. As part of this, we removed the dichotomy between nullable +and non-nullable vectors. + +See [Sidd's blog post][10] for more about these changes. + +## Decimal support in C++, Python, consistency with Java + +[Phillip Cloud][5] led efforts this release to harden details about exact +decimal values in the Arrow specification and ensure a consistent +implementation across Java, C++, and Python. + +Arrow now supports decimals represented internally as a 128-bit little-endian +integer, with a set precision and scale (as defined in many SQL-based +systems). As part of this work, we needed to change Java's internal +representation from big- to little-endian. + +We are now integration testing decimals between Java, C++, and Python, which +will facilitate Arrow adoption in Apache Spark and other systems that use both +Java and Python. + +Decimal data can now be read and written by the [Apache Parquet C++ +library][6], including via pyarrow. + +In the future, we may implement support for smaller-precision decimals +represented by 32- or 64-bit integers. + +## C++ improvements: expanded kernels library and more + +In C++, we have continued developing the new `arrow::compute` submodule +consisting of native computation fuctions for Arrow data. New contributor +[Licht Takeuchi][7] helped expand the supported types for type casting in +`compute::Cast`. We have also implemented new kernels `Unique` and +`DictionaryEncode` for computing the distinct elements of an array and +dictionary encoding (conversion to categorical), respectively. + +We expect the C++ computation "kernel" library to be a major expansion area for +the project over the next year and beyond. Here, we can also implement SIMD- +and GPU-accelerated versions of basic in-memory analytics functionality. + +As minor breaking API change in C++, we have made the `RecordBatch` and `Table` +APIs "virtual" or abstract interfaces, to enable different implementations of a +record batch or table which conform to the standard interface. This will help +enable features like lazy IO or column loading. + +There was significant work improving the C++ library generally and supporting +work happening in Python and C. See the change log for full details. + +## GLib C improvements: Meson build, GPU support + +Developing of the GLib-based C bindings has generally tracked work happening in +the C++ library. These bindings are being used to develop [data science tools +for Ruby users][8] and elsewhere. + +The C bindings now support the [Meson build system][9] in addition to +autotools, which enables them to be built on Windows. + +The Arrow GPU extension library is now also supported in the C bindings. + +## JavaScript: first independent release on NPM + +[Brian Hulette][11] and [Paul Taylor][4] have been continuing to drive efforts +on the TypeScript-based JavaScript implementation. + +Since the last release, we made a first JavaScript-only Apache release, version +0.2.0, which is [now available on NPM][12]. We decided to make separate +JavaScript releases to enable the JS library to release more frequently than +the rest of the project. + +## Python improvements + +In addition to some of the new features mentioned above, we have made a variety +of usability and performance improvements for integrations with pandas, NumPy, +Dask, and other Python projects which may make use of pyarrow, the Arrow Python +library. + +Some of these improvements include: + +* [Component-based serialization][16] for more flexible and memory-efficient + transport of large or complex Python objects +* Substantially improved serialization performance for pandas objects when + using `pyarrow.serialize` and `pyarrow.deserialize`. This includes a special + `pyarrow.pandas_serialization_context` which further accelerates certain + internal details of pandas serialization * Support zero-copy reads for +* `pandas.DataFrame` using `pyarrow.deserialize` for objects without Python + objects +* Multithreaded conversions from `pandas.DataFrame` to `pyarrow.Table` (we + already supported multithreaded conversions from Arrow back to pandas) +* More efficient conversion from 1-dimensional NumPy arrays to Arrow format +* New generic buffer compression and decompression APIs `pyarrow.compress` and + `pyarrow.decompress` +* Enhanced Parquet cross-compatibility with [fastparquet][17] and improved Dask + support +* Python support for accessing Parquet row group column statistics + +## Upcoming Roadmap + +The 0.8.0 release includes some API and format changes, but upcoming releases +will focus on ompleting and stabilizing critical functionality to move the +project closer to a 1.0.0 release. + +With the ecosystem of projects using Arrow expanding rapidly, we will be +working to improve and expand the libraries in support of downstream use cases. + +We continue to look for more JavaScript, Julia, R, Rust, and other programming +language developers to join the project and expand the available +implementations and bindings to more languages. + +[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.8.0 +[2]: https://arrow.apache.org/install +[3]: https://arrow.apache.org/release/0.7.0.html +[3]: https://github.com/kou +[4]: https://github.com/trxcllnt +[5]: https://github.com/cpcloud +[6]: https://github.com/apache/parquet-cpp +[7]: https://github.com/licht-t +[8]: https://github.com/red-data-tools +[9]: https://mesonbuild.com +[10]: https://arrow.apache.org/blog/2017/12/19/java-vector-improvements/ +[11]: https://github.com/TheNeuralBit +[12]: http://npmjs.org/package/apache-arrow +[13]: https://github.com/BryanCutler +[14]: https://github.com/icexelloss +[15]: https://github.com/siddharthteotia +[16]: http://arrow.apache.org/docs/python/ipc.html +[17]: https://github.com/dask/fastparquet +[18]: http://arrow.apache.org/powered_by/ From ba009564d32b5b681128cff345ae9177ef39f82a Mon Sep 17 00:00:00 2001 From: siddharth Date: Wed, 20 Dec 2017 11:52:17 -0800 Subject: [PATCH 150/177] ARROW-1939: Correct links in release blog post cc @wesm Author: siddharth Closes #1436 from siddharthteotia/ARROW-1939 and squashes the following commits: 31f1be14 [siddharth] ARROW-1939: Correct links in release blog post --- site/_posts/2017-12-19-0.8.0-release.md | 66 ++++++++++++------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/site/_posts/2017-12-19-0.8.0-release.md b/site/_posts/2017-12-19-0.8.0-release.md index c65d30832b7b8..8c1f464ed7200 100644 --- a/site/_posts/2017-12-19-0.8.0-release.md +++ b/site/_posts/2017-12-19-0.8.0-release.md @@ -44,7 +44,7 @@ post. ## Projects "Powered By" Apache Arrow A growing ecosystem of projects are using Arrow to solve in-memory analytics -and data interchange problems. We have added a new [Powered By][18] page to the +and data interchange problems. We have added a new [Powered By][19] page to the Arrow website where we can acknowledge open source projects and companies which are using Arrow. If you would like to add your project to the list as an Arrow user, please let us know. @@ -53,11 +53,11 @@ user, please let us know. Since the last release, we have added 5 new Apache committers: -* [Phillip Cloud][5], who has mainly contributed to C++ and Python -* [Bryan Cutler][13], who has mainly contributed to Java and Spark integration -* [Li Jin][14], who has mainly contributed to Java and Spark integration -* [Paul Taylor][4], who has mainly contributed to JavaScript -* [Siddharth Teotia][15], who has mainly contributed to Java +* [Phillip Cloud][6], who has mainly contributed to C++ and Python +* [Bryan Cutler][14], who has mainly contributed to Java and Spark integration +* [Li Jin][15], who has mainly contributed to Java and Spark integration +* [Paul Taylor][5], who has mainly contributed to JavaScript +* [Siddharth Teotia][16], who has mainly contributed to Java Welcome to the Arrow team, and thank you for your contributions! @@ -67,11 +67,11 @@ Siddharth Teotia led efforts to revamp the Java vector API to make things simpler and faster. As part of this, we removed the dichotomy between nullable and non-nullable vectors. -See [Sidd's blog post][10] for more about these changes. +See [Sidd's blog post][11] for more about these changes. ## Decimal support in C++, Python, consistency with Java -[Phillip Cloud][5] led efforts this release to harden details about exact +[Phillip Cloud][6] led efforts this release to harden details about exact decimal values in the Arrow specification and ensure a consistent implementation across Java, C++, and Python. @@ -85,7 +85,7 @@ will facilitate Arrow adoption in Apache Spark and other systems that use both Java and Python. Decimal data can now be read and written by the [Apache Parquet C++ -library][6], including via pyarrow. +library][7], including via pyarrow. In the future, we may implement support for smaller-precision decimals represented by 32- or 64-bit integers. @@ -94,7 +94,7 @@ represented by 32- or 64-bit integers. In C++, we have continued developing the new `arrow::compute` submodule consisting of native computation fuctions for Arrow data. New contributor -[Licht Takeuchi][7] helped expand the supported types for type casting in +[Licht Takeuchi][8] helped expand the supported types for type casting in `compute::Cast`. We have also implemented new kernels `Unique` and `DictionaryEncode` for computing the distinct elements of an array and dictionary encoding (conversion to categorical), respectively. @@ -115,20 +115,20 @@ work happening in Python and C. See the change log for full details. Developing of the GLib-based C bindings has generally tracked work happening in the C++ library. These bindings are being used to develop [data science tools -for Ruby users][8] and elsewhere. +for Ruby users][9] and elsewhere. -The C bindings now support the [Meson build system][9] in addition to +The C bindings now support the [Meson build system][10] in addition to autotools, which enables them to be built on Windows. The Arrow GPU extension library is now also supported in the C bindings. ## JavaScript: first independent release on NPM -[Brian Hulette][11] and [Paul Taylor][4] have been continuing to drive efforts +[Brian Hulette][12] and [Paul Taylor][5] have been continuing to drive efforts on the TypeScript-based JavaScript implementation. Since the last release, we made a first JavaScript-only Apache release, version -0.2.0, which is [now available on NPM][12]. We decided to make separate +0.2.0, which is [now available on NPM][13]. We decided to make separate JavaScript releases to enable the JS library to release more frequently than the rest of the project. @@ -141,7 +141,7 @@ library. Some of these improvements include: -* [Component-based serialization][16] for more flexible and memory-efficient +* [Component-based serialization][17] for more flexible and memory-efficient transport of large or complex Python objects * Substantially improved serialization performance for pandas objects when using `pyarrow.serialize` and `pyarrow.deserialize`. This includes a special @@ -154,7 +154,7 @@ Some of these improvements include: * More efficient conversion from 1-dimensional NumPy arrays to Arrow format * New generic buffer compression and decompression APIs `pyarrow.compress` and `pyarrow.decompress` -* Enhanced Parquet cross-compatibility with [fastparquet][17] and improved Dask +* Enhanced Parquet cross-compatibility with [fastparquet][18] and improved Dask support * Python support for accessing Parquet row group column statistics @@ -173,20 +173,20 @@ implementations and bindings to more languages. [1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.8.0 [2]: https://arrow.apache.org/install -[3]: https://arrow.apache.org/release/0.7.0.html -[3]: https://github.com/kou -[4]: https://github.com/trxcllnt -[5]: https://github.com/cpcloud -[6]: https://github.com/apache/parquet-cpp -[7]: https://github.com/licht-t -[8]: https://github.com/red-data-tools -[9]: https://mesonbuild.com -[10]: https://arrow.apache.org/blog/2017/12/19/java-vector-improvements/ -[11]: https://github.com/TheNeuralBit -[12]: http://npmjs.org/package/apache-arrow -[13]: https://github.com/BryanCutler -[14]: https://github.com/icexelloss -[15]: https://github.com/siddharthteotia -[16]: http://arrow.apache.org/docs/python/ipc.html -[17]: https://github.com/dask/fastparquet -[18]: http://arrow.apache.org/powered_by/ +[3]: https://arrow.apache.org/release/0.8.0.html +[4]: https://github.com/kou +[5]: https://github.com/trxcllnt +[6]: https://github.com/cpcloud +[7]: https://github.com/apache/parquet-cpp +[8]: https://github.com/licht-t +[9]: https://github.com/red-data-tools +[10]: https://mesonbuild.com +[11]: https://arrow.apache.org/blog/2017/12/19/java-vector-improvements/ +[12]: https://github.com/TheNeuralBit +[13]: http://npmjs.org/package/apache-arrow +[14]: https://github.com/BryanCutler +[15]: https://github.com/icexelloss +[16]: https://github.com/siddharthteotia +[17]: http://arrow.apache.org/docs/python/ipc.html +[18]: https://github.com/dask/fastparquet +[19]: http://arrow.apache.org/powered_by/ From ce95db3e3a678ac391125bd899c7ec4f1bbf4b97 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 21 Dec 2017 17:11:52 -0500 Subject: [PATCH 151/177] ARROW-1931: [C++] Suppress C4996 deprecation warning in MSVC builds for now It's reasonably harmless to suppress these warnings for the time being. When we upgrade to a new release of googletest, we can remove this again Author: Wes McKinney Closes #1433 from wesm/ARROW-1931 and squashes the following commits: 09c3722f [Wes McKinney] Add tr1 define to CMAKE_CXX_FLAGS 6fe636ec [Wes McKinney] Rearrange appveyor build jobs for faster feedback 6ace398e [Wes McKinney] Use CXX_COMMON_FLAGS, all in one place b786a405 [Wes McKinney] Silence std::tr1 tuple warning everywhere 549e0833 [Wes McKinney] Silence std::tr1 namespace warning only when building gtest 13a77e6b [Wes McKinney] Silence tr1 deprecation warning in MSVC 2017 ccf13188 [Wes McKinney] Add /bigobj flag, suppress C4996 deprecation warning for now --- appveyor.yml | 18 +++++++++--------- cpp/cmake_modules/SetupCxxFlags.cmake | 11 +++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e647b8b77080f..ea7922bf658ef 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,16 +20,13 @@ os: Visual Studio 2015 environment: matrix: - - JOB: "Cmake_Script_Tests" - GENERATOR: NMake Makefiles - PYTHON: "3.5" - ARCH: "64" - CONFIGURATION: "Release" - JOB: "Build" - GENERATOR: NMake Makefiles + GENERATOR: Visual Studio 15 2017 Win64 PYTHON: "3.5" ARCH: "64" CONFIGURATION: "Release" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + BOOST_ROOT: C:\Libraries\boost_1_64_0 - JOB: "Build_Debug" GENERATOR: Visual Studio 14 2015 Win64 PYTHON: "3.5" @@ -49,13 +46,16 @@ environment: PYTHON: "3.5" ARCH: "64" CONFIGURATION: "Release" + - JOB: "Cmake_Script_Tests" + GENERATOR: NMake Makefiles + PYTHON: "3.5" + ARCH: "64" + CONFIGURATION: "Release" - JOB: "Build" - GENERATOR: Visual Studio 15 2017 Win64 + GENERATOR: NMake Makefiles PYTHON: "3.5" ARCH: "64" CONFIGURATION: "Release" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - BOOST_ROOT: C:\Libraries\boost_1_64_0 MSVC_DEFAULT_OPTIONS: ON BOOST_ROOT: C:\Libraries\boost_1_63_0 diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 4e0ace0baea0c..97aed6b274976 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -34,6 +34,14 @@ if (MSVC) # headers will see dllimport add_definitions(-DARROW_EXPORTING) + # ARROW-1931 See https://github.com/google/googletest/issues/1318 + # + # This is added to CMAKE_CXX_FLAGS instead of CXX_COMMON_FLAGS since only the + # former is passed into the external projects + if (MSVC_VERSION VERSION_GREATER 1900) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_SILENCE_TR1_NAMESPACE_DEPRECATION_WARNING") + endif() + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # clang-cl set(CXX_COMMON_FLAGS "-EHsc") @@ -56,6 +64,9 @@ if (MSVC) string(REPLACE "/MD" "-MT" ${c_flag} "${${c_flag}}") endforeach() endif() + + # Support large object code + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /bigobj") else() # Common flags set below with warning level set(CXX_COMMON_FLAGS "") From 65f5add61829bd413aa8175ad2644cdd362d0c50 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 21 Dec 2017 18:05:15 -0500 Subject: [PATCH 152/177] ARROW-1944: [C++] Fix ARROW_STATIC_LIB in FindArrow Author: Philipp Moritz Closes #1440 from pcmoritz/findarrow-libs-fix and squashes the following commits: bb161b4d [Philipp Moritz] fix static lib path in FindArrow --- cpp/cmake_modules/FindArrow.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/FindArrow.cmake b/cpp/cmake_modules/FindArrow.cmake index 12f76b6c2b3e8..bce4404a47809 100644 --- a/cpp/cmake_modules/FindArrow.cmake +++ b/cpp/cmake_modules/FindArrow.cmake @@ -97,8 +97,8 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS) set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib) set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib) else() - set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_LIB_NAME}.a) - set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_PYTHON_LIB_NAME}.a) + set(ARROW_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}.a) + set(ARROW_PYTHON_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}.a) set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) From 8986521255f48a2aa775921eac0175b4e7afaa16 Mon Sep 17 00:00:00 2001 From: siddharth Date: Fri, 22 Dec 2017 11:24:43 -0800 Subject: [PATCH 153/177] ARROW-1943: [JAVA] handle setInitialCapacity for deeply nested lists The current implementation of setInitialCapacity() uses a factor of 5 for every level we go into list: So if the schema is LIST (LIST (LIST (LIST (LIST (LIST (LIST (BIGINT)))))) and we start with an initial capacity of 128, we end up throwing OversizedAllocationException from the BigIntVector because at every level we increased the capacity by 5 and by the time we reached inner scalar that actually stores the data, we were well over max size limit per vector (1MB). We saw this problem downstream when we failed to read deeply nested JSON data. The potential fix is to use the factor of 5 only when we are down to the leaf vector. As the depth increases and we are still working with complex/list, we don't use the factor of 5. cc @jacques-n , @BryanCutler , @icexelloss Author: siddharth Closes #1439 from siddharthteotia/ARROW-1943 and squashes the following commits: d0adbade [siddharth] unit tests e2f21a8b [siddharth] fix imports d103436b [siddharth] ARROW-1943: handle setInitialCapacity for deeply nested lists --- .../complex/BaseRepeatedValueVector.java | 8 +- .../apache/arrow/vector/TestListVector.java | 128 ++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index a9221f2f6ea56..9a23fd8c356cb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -29,6 +29,8 @@ import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.BaseVariableWidthVector; +import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; @@ -134,7 +136,11 @@ public FieldVector getDataVector() { @Override public void setInitialCapacity(int numRecords) { offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; - vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); + if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) { + vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); + } else { + vector.setInitialCapacity(numRecords); + } } @Override diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1acce7e0b6654..e2023f4461879 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -559,6 +559,134 @@ public void testNestedListVector() throws Exception { } } + @Test + public void testNestedListVector1() throws Exception { + try (ListVector listVector = ListVector.empty("sourceVector", allocator)) { + + MinorType listType = MinorType.LIST; + MinorType scalarType = MinorType.BIGINT; + + listVector.addOrGetVector(FieldType.nullable(listType.getType())); + + ListVector innerList1 = (ListVector)listVector.getDataVector(); + innerList1.addOrGetVector(FieldType.nullable(listType.getType())); + + ListVector innerList2 = (ListVector)innerList1.getDataVector(); + innerList2.addOrGetVector(FieldType.nullable(listType.getType())); + + ListVector innerList3 = (ListVector)innerList2.getDataVector(); + innerList3.addOrGetVector(FieldType.nullable(listType.getType())); + + ListVector innerList4 = (ListVector)innerList3.getDataVector(); + innerList4.addOrGetVector(FieldType.nullable(listType.getType())); + + ListVector innerList5 = (ListVector)innerList4.getDataVector(); + innerList5.addOrGetVector(FieldType.nullable(listType.getType())); + + ListVector innerList6 = (ListVector)innerList5.getDataVector(); + innerList6.addOrGetVector(FieldType.nullable(scalarType.getType())); + + listVector.setInitialCapacity(128); + } + } + + @Test + public void testNestedListVector2() throws Exception { + try (ListVector listVector = ListVector.empty("sourceVector", allocator)) { + listVector.setInitialCapacity(1); + UnionListWriter listWriter = listVector.getWriter(); + /* allocate memory */ + listWriter.allocate(); + + /* write one or more inner lists at index 0 */ + listWriter.setPosition(0); + listWriter.startList(); + + listWriter.list().startList(); + listWriter.list().bigInt().writeBigInt(50); + listWriter.list().bigInt().writeBigInt(100); + listWriter.list().bigInt().writeBigInt(200); + listWriter.list().endList(); + + listWriter.list().startList(); + listWriter.list().bigInt().writeBigInt(75); + listWriter.list().bigInt().writeBigInt(125); + listWriter.list().endList(); + + listWriter.endList(); + + /* write one or more inner lists at index 1 */ + listWriter.setPosition(1); + listWriter.startList(); + + listWriter.list().startList(); + listWriter.list().bigInt().writeBigInt(15); + listWriter.list().bigInt().writeBigInt(20); + listWriter.list().endList(); + + listWriter.list().startList(); + listWriter.list().bigInt().writeBigInt(25); + listWriter.list().bigInt().writeBigInt(30); + listWriter.list().bigInt().writeBigInt(35); + listWriter.list().endList(); + + listWriter.endList(); + + assertEquals(2, listVector.getLastSet()); + + listVector.setValueCount(2); + + assertEquals(2, listVector.getValueCount()); + + /* get listVector value at index 0 -- the value itself is a listvector */ + Object result = listVector.getObject(0); + ArrayList> resultSet = (ArrayList>) result; + ArrayList list; + + assertEquals(2, resultSet.size()); /* 2 inner lists at index 0 */ + assertEquals(3, resultSet.get(0).size()); /* size of first inner list */ + assertEquals(2, resultSet.get(1).size()); /* size of second inner list */ + + list = resultSet.get(0); + assertEquals(new Long(50), list.get(0)); + assertEquals(new Long(100), list.get(1)); + assertEquals(new Long(200), list.get(2)); + + list = resultSet.get(1); + assertEquals(new Long(75), list.get(0)); + assertEquals(new Long(125), list.get(1)); + + /* get listVector value at index 1 -- the value itself is a listvector */ + result = listVector.getObject(1); + resultSet = (ArrayList>) result; + + assertEquals(2, resultSet.size()); /* 3 inner lists at index 1 */ + assertEquals(2, resultSet.get(0).size()); /* size of first inner list */ + assertEquals(3, resultSet.get(1).size()); /* size of second inner list */ + + list = resultSet.get(0); + assertEquals(new Long(15), list.get(0)); + assertEquals(new Long(20), list.get(1)); + + list = resultSet.get(1); + assertEquals(new Long(25), list.get(0)); + assertEquals(new Long(30), list.get(1)); + assertEquals(new Long(35), list.get(2)); + + /* check underlying bitVector */ + assertFalse(listVector.isNull(0)); + assertFalse(listVector.isNull(1)); + + /* check underlying offsets */ + final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); + + /* listVector has 2 lists at index 0 and 3 lists at index 1 */ + assertEquals(0, offsetBuffer.getInt(0 * ListVector.OFFSET_WIDTH)); + assertEquals(2, offsetBuffer.getInt(1 * ListVector.OFFSET_WIDTH)); + assertEquals(4, offsetBuffer.getInt(2 * ListVector.OFFSET_WIDTH)); + } + } + @Test public void testGetBufferAddress() throws Exception { try (ListVector listVector = ListVector.empty("vector", allocator)) { From 16c79cc94e2440321bcad1ebbef53ea1266b94e8 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Thu, 28 Dec 2017 13:34:32 -0800 Subject: [PATCH 154/177] ARROW-1951: [Python] Add memcopy threads argument to PlasmaClient put. Author: Robert Nishihara Closes #1451 from robertnishihara/numthreads and squashes the following commits: 5e2c7ee [Robert Nishihara] Fix tests. 55cb8ac [Robert Nishihara] Revert old change 0903726 [Robert Nishihara] Move memcopy_threads from serialization context to put. 9281de1 [Robert Nishihara] Expose memcopy threads to serialization context. --- python/pyarrow/plasma.pyx | 7 +++++-- python/pyarrow/tests/test_plasma.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx index f2e8653d8a828..9b3e4093a2f2b 100644 --- a/python/pyarrow/plasma.pyx +++ b/python/pyarrow/plasma.pyx @@ -370,7 +370,7 @@ cdef class PlasmaClient: object_buffers[i].metadata_size)) return result - def put(self, object value, ObjectID object_id=None, + def put(self, object value, ObjectID object_id=None, int memcopy_threads=6, serialization_context=None): """ Store a Python value into the object store. @@ -382,6 +382,9 @@ cdef class PlasmaClient: object_id : ObjectID, default None If this is provided, the specified object ID will be used to refer to the object. + memcopy_threads : int, default 6 + The number of threads to use to write the serialized object into + the object store for large objects. serialization_context : pyarrow.SerializationContext, default None Custom serialization and deserialization context. @@ -394,7 +397,7 @@ cdef class PlasmaClient: serialized = pyarrow.serialize(value, serialization_context) buffer = self.create(target_id, serialized.total_bytes) stream = pyarrow.FixedSizeBufferWriter(buffer) - stream.set_memcopy_threads(4) + stream.set_memcopy_threads(memcopy_threads) serialized.write_to(stream) self.seal(target_id) return target_id diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py index b28bd60c48ad4..decdc73d911b4 100644 --- a/python/pyarrow/tests/test_plasma.py +++ b/python/pyarrow/tests/test_plasma.py @@ -310,12 +310,14 @@ def __init__(self, val): serialization_context = pa.SerializationContext() serialization_context.register_type(CustomType, 20*b"\x00") - object_id = self.plasma_client.put(val, None, serialization_context) + object_id = self.plasma_client.put( + val, None, serialization_context=serialization_context) with pytest.raises(pa.ArrowSerializationError): result = self.plasma_client.get(object_id) - result = self.plasma_client.get(object_id, -1, serialization_context) + result = self.plasma_client.get( + object_id, -1, serialization_context=serialization_context) assert result.val == val.val def test_store_arrow_objects(self): From 2ca69d1dc2ff8d794639ba059455a21462eb0675 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Thu, 28 Dec 2017 15:27:50 -0800 Subject: [PATCH 155/177] ARROW-1948: [Java] Load ListVector validity buffer with BitVectorHelper to handle all non-null Need to properly set the ListVector validity buffer for the case when the field has all non-nulls. This is done already in `BitVectorHelper.loadValidityBuffer`, so just need to build the buffer with a call to that function. Author: Bryan Cutler Closes #1447 from BryanCutler/java-ListVector-non-null-validity-buffer-ARROW-1948 and squashes the following commits: 0d82345 [Bryan Cutler] used BitVectorHelper to properly set the validity buffer --- .../main/java/org/apache/arrow/vector/complex/ListVector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index d50d4c447fefd..0c1daf4905222 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -124,7 +124,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers ArrowBuf offBuffer = ownBuffers.get(1); validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); offsetBuffer.release(); offsetBuffer = offBuffer.retain(allocator); From 0401e6d7f8f1aa8f62b1f783fd65e1bd1ced7c50 Mon Sep 17 00:00:00 2001 From: Viktor Gal Date: Fri, 29 Dec 2017 17:49:55 +0100 Subject: [PATCH 156/177] ARROW-1945: [C++] Fix doxygen documentation of array.h this is just a small fix in doxygen documentation of the c++ api Author: Viktor Gal Closes #1442 from vigsterkr/doxygen_fix and squashes the following commits: 3557aef [Viktor Gal] fix doxygen example in array.h --- cpp/src/arrow/array.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index 6721c7d2717f3..0ae1ddd8ea221 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -65,9 +65,9 @@ struct Decimal; /// could cast from int64 to float64 like so: /// /// Int64Array arr = GetMyData(); -/// auto new_data = arr->data()->ShallowCopy(); +/// auto new_data = arr.data()->ShallowCopy(); /// new_data->type = arrow::float64(); -/// Float64Array double_arr(new_data); +/// DoubleArray double_arr(new_data); /// /// This object is also useful in an analytics setting where memory may be /// reused. For example, if we had a group of operations all returning doubles, From d86fdc71b659958878fb100e811c66f7a1cd1b08 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Fri, 29 Dec 2017 08:54:05 -0800 Subject: [PATCH 157/177] Fix a comment typo (#1448) --- .../java/org/apache/arrow/vector/util/DictionaryUtility.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java index cf0596c8c1fb4..6b46dbae385bb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DictionaryUtility.java @@ -73,7 +73,7 @@ public static Field toMessageFormat(Field field, DictionaryProvider provider, Se } /** - * Convert field and child fields that have a dictionary encoding to message format, so fields + * Convert field and child fields that have a dictionary encoding to memory format, so fields * have the index type */ public static Field toMemoryFormat(Field field, BufferAllocator allocator, Map dictionaries) { From 4daffadb92fc527e982d8011b4ebc483503c97ff Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Fri, 29 Dec 2017 14:47:05 -0500 Subject: [PATCH 158/177] ARROW-1953: [JS] Fix JS build this is necessary until gulp publishes 4.0.0 builds to npm Author: Paul Taylor Closes #1453 from trxcllnt/fix-js-gulp-build and squashes the following commits: 0dddbcc [Paul Taylor] Merge branch 'master' into fix-js-gulp-build dc329fa [Paul Taylor] Merge branch 'master' into fix-js-gulp-build d9c1b0c [Paul Taylor] set gulp dependency to specific commit --- js/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/package.json b/js/package.json index 179107b63db11..3903d1eedc442 100644 --- a/js/package.json +++ b/js/package.json @@ -75,7 +75,7 @@ "esdoc-standard-plugin": "1.0.0", "glob": "7.1.2", "google-closure-compiler": "20171203.0.0", - "gulp": "github:gulpjs/gulp#4.0", + "gulp": "github:gulpjs/gulp#6d71a658c61edb3090221579d8f97dbe086ba2ed", "gulp-json-transform": "0.4.5", "gulp-rename": "1.2.2", "gulp-sourcemaps": "2.6.1", From 0f38a9503bcde872b705976bd314ccb6c0d0f8d5 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Fri, 29 Dec 2017 14:19:38 -0800 Subject: [PATCH 159/177] ARROW-1947: [Plasma] Change Client Create and Get to use Buffers - Create now takes in a pointer to a shared pointer of Buffer and returns a MutableBuffer. - Object Buffers data and metadata are pointers to shared pointers of Buffer. Author: Philipp Moritz Author: William Paul Closes #1444 from Wapaul1/plasma_buffer_api and squashes the following commits: 7fe1cee [Philipp Moritz] fix size of MutableBuffer returned by plasma::Create aeed751 [Philipp Moritz] more linting b3274e0 [Philipp Moritz] fix 463dbeb [Philipp Moritz] fix plasma python extension a055fa8 [Philipp Moritz] fix linting fc62dda [William Paul] Added metadata buffer 4d8cbb8 [William Paul] Create and Get use Buffers now --- cpp/src/plasma/client.cc | 44 ++++++++++++++++++----------- cpp/src/plasma/client.h | 18 +++++++----- cpp/src/plasma/test/client_tests.cc | 36 ++++++++++++++--------- python/pyarrow/plasma.pyx | 14 ++++----- 4 files changed, 67 insertions(+), 45 deletions(-) diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc index 9bbafac38004a..0dd1c44d71c74 100644 --- a/cpp/src/plasma/client.cc +++ b/cpp/src/plasma/client.cc @@ -40,6 +40,7 @@ #include #include +#include "arrow/buffer.h" #include "plasma/common.h" #include "plasma/fling.h" #include "plasma/io.h" @@ -53,6 +54,8 @@ namespace plasma { +using arrow::MutableBuffer; + // Number of threads used for memcopy and hash computations. constexpr int64_t kThreadPoolSize = 8; constexpr int64_t kBytesInMB = 1 << 20; @@ -145,7 +148,8 @@ void PlasmaClient::increment_object_count(const ObjectID& object_id, PlasmaObjec } Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size, - uint8_t* metadata, int64_t metadata_size, uint8_t** data) { + uint8_t* metadata, int64_t metadata_size, + std::shared_ptr* data) { ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn_ << " with size " << data_size << " and metadata size " << metadata_size; RETURN_NOT_OK(SendCreateRequest(store_conn_, object_id, data_size, metadata_size)); @@ -162,14 +166,16 @@ Status PlasmaClient::Create(const ObjectID& object_id, int64_t data_size, ARROW_CHECK(object.metadata_size == metadata_size); // The metadata should come right after the data. ARROW_CHECK(object.metadata_offset == object.data_offset + data_size); - *data = lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) + - object.data_offset; + *data = std::make_shared( + lookup_or_mmap(fd, object.handle.store_fd, object.handle.mmap_size) + + object.data_offset, + data_size); // If plasma_create is being called from a transfer, then we will not copy the // metadata here. The metadata will be written along with the data streamed // from the transfer. if (metadata != NULL) { // Copy the metadata to the buffer. - memcpy(*data + object.data_size, metadata, metadata_size); + memcpy((*data)->mutable_data() + object.data_size, metadata, metadata_size); } // Increment the count of the number of instances of this object that this // client is using. A call to PlasmaClient::Release is required to decrement @@ -203,10 +209,12 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects, ARROW_CHECK(object_entry->second->is_sealed) << "Plasma client called get on an unsealed object that it created"; PlasmaObject* object = &object_entry->second->object; - object_buffers[i].data = lookup_mmapped_file(object->handle.store_fd); - object_buffers[i].data = object_buffers[i].data + object->data_offset; + uint8_t* data = lookup_mmapped_file(object->handle.store_fd); + object_buffers[i].data = + std::make_shared(data + object->data_offset, object->data_size); + object_buffers[i].metadata = std::make_shared( + data + object->data_offset + object->data_size, object->metadata_size); object_buffers[i].data_size = object->data_size; - object_buffers[i].metadata = object_buffers[i].data + object->data_size; object_buffers[i].metadata_size = object->metadata_size; // Increment the count of the number of instances of this object that this // client is using. A call to PlasmaClient::Release is required to @@ -254,13 +262,15 @@ Status PlasmaClient::Get(const ObjectID* object_ids, int64_t num_objects, // The object was retrieved. The user will be responsible for releasing // this object. int fd = recv_fd(store_conn_); - ARROW_CHECK(fd >= 0); - object_buffers[i].data = + uint8_t* data = lookup_or_mmap(fd, object->handle.store_fd, object->handle.mmap_size); + ARROW_CHECK(fd >= 0); // Finish filling out the return values. - object_buffers[i].data = object_buffers[i].data + object->data_offset; + object_buffers[i].data = + std::make_shared(data + object->data_offset, object->data_size); + object_buffers[i].metadata = std::make_shared( + data + object->data_offset + object->data_size, object->metadata_size); object_buffers[i].data_size = object->data_size; - object_buffers[i].metadata = object_buffers[i].data + object->data_size; object_buffers[i].metadata_size = object->metadata_size; // Increment the count of the number of instances of this object that this // client is using. A call to PlasmaClient::Release is required to @@ -438,14 +448,16 @@ static uint64_t compute_object_hash(const ObjectBuffer& obj_buffer) { XXH64_state_t hash_state; XXH64_reset(&hash_state, XXH64_DEFAULT_SEED); if (obj_buffer.data_size >= kBytesInMB) { - compute_object_hash_parallel(&hash_state, - reinterpret_cast(obj_buffer.data), - obj_buffer.data_size); + compute_object_hash_parallel( + &hash_state, reinterpret_cast(obj_buffer.data->data()), + obj_buffer.data_size); } else { - XXH64_update(&hash_state, reinterpret_cast(obj_buffer.data), + XXH64_update(&hash_state, + reinterpret_cast(obj_buffer.data->data()), obj_buffer.data_size); } - XXH64_update(&hash_state, reinterpret_cast(obj_buffer.metadata), + XXH64_update(&hash_state, + reinterpret_cast(obj_buffer.metadata->data()), obj_buffer.metadata_size); return XXH64_digest(&hash_state); } diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h index cfd11c16dbace..78793f1a73a64 100644 --- a/cpp/src/plasma/client.h +++ b/cpp/src/plasma/client.h @@ -26,11 +26,13 @@ #include #include +#include "arrow/buffer.h" #include "arrow/status.h" #include "arrow/util/visibility.h" #include "plasma/common.h" using arrow::Status; +using arrow::Buffer; namespace plasma { @@ -41,14 +43,16 @@ constexpr int64_t kL3CacheSizeBytes = 100000000; /// Object buffer data structure. struct ObjectBuffer { + /// The data buffer. + std::shared_ptr data; /// The size in bytes of the data object. int64_t data_size; - /// The address of the data object. - uint8_t* data; + /// The metadata buffer. + std::shared_ptr metadata; /// The metadata size in bytes. int64_t metadata_size; - /// The address of the metadata. - uint8_t* metadata; + /// The device number. + int device_num; }; /// Configuration options for the plasma client. @@ -107,11 +111,11 @@ class ARROW_EXPORT PlasmaClient { /// should be NULL. /// \param metadata_size The size in bytes of the metadata. If there is no /// metadata, this should be 0. - /// \param data The address of the newly created object will be written here. + /// \param data A buffer containing the address of the newly created object + /// will be written here. /// \return The return status. Status Create(const ObjectID& object_id, int64_t data_size, uint8_t* metadata, - int64_t metadata_size, uint8_t** data); - + int64_t metadata_size, std::shared_ptr* data); /// Get some objects from the Plasma Store. This function will block until the /// objects have all been created and sealed in the Plasma Store or the /// timeout diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc index f44ed2510c582..5cd3063bb4309 100644 --- a/cpp/src/plasma/test/client_tests.cc +++ b/cpp/src/plasma/test/client_tests.cc @@ -71,7 +71,7 @@ TEST_F(TestPlasmaStore, ContainsTest) { int64_t data_size = 100; uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); - uint8_t* data; + std::shared_ptr data; ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); ARROW_CHECK_OK(client_.Seal(object_id)); // Avoid race condition of Plasma Manager waiting for notification. @@ -94,16 +94,20 @@ TEST_F(TestPlasmaStore, GetTest) { int64_t data_size = 4; uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); + std::shared_ptr data_buffer; uint8_t* data; - ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); + ARROW_CHECK_OK( + client_.Create(object_id, data_size, metadata, metadata_size, &data_buffer)); + data = data_buffer->mutable_data(); for (int64_t i = 0; i < data_size; i++) { data[i] = static_cast(i % 4); } ARROW_CHECK_OK(client_.Seal(object_id)); ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer)); + const uint8_t* object_data = object_buffer.data->data(); for (int64_t i = 0; i < data_size; i++) { - ASSERT_EQ(data[i], object_buffer.data[i]); + ASSERT_EQ(data[i], object_data[i]); } } @@ -116,18 +120,18 @@ TEST_F(TestPlasmaStore, MultipleGetTest) { int64_t data_size = 4; uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); - uint8_t* data; + std::shared_ptr data; ARROW_CHECK_OK(client_.Create(object_id1, data_size, metadata, metadata_size, &data)); - data[0] = 1; + data->mutable_data()[0] = 1; ARROW_CHECK_OK(client_.Seal(object_id1)); ARROW_CHECK_OK(client_.Create(object_id2, data_size, metadata, metadata_size, &data)); - data[0] = 2; + data->mutable_data()[0] = 2; ARROW_CHECK_OK(client_.Seal(object_id2)); ARROW_CHECK_OK(client_.Get(object_ids, 2, -1, object_buffer)); - ASSERT_EQ(object_buffer[0].data[0], 1); - ASSERT_EQ(object_buffer[1].data[0], 2); + ASSERT_EQ(object_buffer[0].data->data()[0], 1); + ASSERT_EQ(object_buffer[1].data->data()[0], 2); } TEST_F(TestPlasmaStore, AbortTest) { @@ -143,11 +147,13 @@ TEST_F(TestPlasmaStore, AbortTest) { int64_t data_size = 4; uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); - uint8_t* data; + std::shared_ptr data; + uint8_t* data_ptr; ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); + data_ptr = data->mutable_data(); // Write some data. for (int64_t i = 0; i < data_size / 2; i++) { - data[i] = static_cast(i % 4); + data_ptr[i] = static_cast(i % 4); } // Attempt to abort. Test that this fails before the first release. Status status = client_.Abort(object_id); @@ -162,15 +168,17 @@ TEST_F(TestPlasmaStore, AbortTest) { // Create the object successfully this time. ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); + data_ptr = data->mutable_data(); for (int64_t i = 0; i < data_size; i++) { - data[i] = static_cast(i % 4); + data_ptr[i] = static_cast(i % 4); } ARROW_CHECK_OK(client_.Seal(object_id)); // Test that we can get the object. ARROW_CHECK_OK(client_.Get(&object_id, 1, -1, &object_buffer)); + const uint8_t* buffer_ptr = object_buffer.data->data(); for (int64_t i = 0; i < data_size; i++) { - ASSERT_EQ(data[i], object_buffer.data[i]); + ASSERT_EQ(data_ptr[i], buffer_ptr[i]); } } @@ -187,7 +195,7 @@ TEST_F(TestPlasmaStore, MultipleClientTest) { int64_t data_size = 100; uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); - uint8_t* data; + std::shared_ptr data; ARROW_CHECK_OK(client2_.Create(object_id, data_size, metadata, metadata_size, &data)); ARROW_CHECK_OK(client2_.Seal(object_id)); // Test that the first client can get the object. @@ -227,7 +235,7 @@ TEST_F(TestPlasmaStore, ManyObjectTest) { int64_t data_size = 100; uint8_t metadata[] = {5}; int64_t metadata_size = sizeof(metadata); - uint8_t* data; + std::shared_ptr data; ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); if (i % 3 == 0) { diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx index 9b3e4093a2f2b..abeec32783398 100644 --- a/python/pyarrow/plasma.pyx +++ b/python/pyarrow/plasma.pyx @@ -81,7 +81,7 @@ cdef extern from "plasma/client.h" nogil: CStatus Create(const CUniqueID& object_id, int64_t data_size, const uint8_t* metadata, int64_t metadata_size, - uint8_t** data) + const shared_ptr[CBuffer]* data) CStatus Get(const CUniqueID* object_ids, int64_t num_objects, int64_t timeout_ms, CObjectBuffer* object_buffers) @@ -118,9 +118,9 @@ cdef extern from "plasma/client.h" nogil: cdef struct CObjectBuffer" plasma::ObjectBuffer": int64_t data_size - uint8_t* data + shared_ptr[CBuffer] data int64_t metadata_size - uint8_t* metadata + shared_ptr[CBuffer] metadata def make_object_id(object_id): @@ -245,10 +245,8 @@ cdef class PlasmaClient: check_status(self.client.get().Get(ids.data(), ids.size(), timeout_ms, result[0].data())) - cdef _make_plasma_buffer(self, ObjectID object_id, uint8_t* data, + cdef _make_plasma_buffer(self, ObjectID object_id, shared_ptr[CBuffer] buffer, int64_t size): - cdef shared_ptr[CBuffer] buffer - buffer.reset(new CBuffer(data, size)) result = PlasmaBuffer(object_id, self) result.init(buffer) return result @@ -296,12 +294,12 @@ cdef class PlasmaClient: not be created because the plasma store is unable to evict enough objects to create room for it. """ - cdef uint8_t* data + cdef shared_ptr[CBuffer] data with nogil: check_status(self.client.get().Create(object_id.data, data_size, (metadata.data()), metadata.size(), &data)) - return self._make_mutable_plasma_buffer(object_id, data, data_size) + return self._make_mutable_plasma_buffer(object_id, data.get().mutable_data(), data_size) def get_buffers(self, object_ids, timeout_ms=-1): """ From 37c3265b8a7a9e3ccb3063faeafa537d5b334830 Mon Sep 17 00:00:00 2001 From: siddharth Date: Sat, 30 Dec 2017 20:05:08 -0800 Subject: [PATCH 160/177] ARROW-1946: [JAVA] Add APIs to decimal vector for writing big endian data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We recently moved Dremio to LE Decimal format (similar to Arrow). As part of that we introduced some APIs in decimal vector which take a big endian data and swap the bytes while writing into the ArrowBuf of decimal vector. The advantage of these APIs is that caller would not have to allocate an additional memory and write( and read) source big endian twice for swapping into new memory and using that to write into the vector. We can directly swap bytes while writing into the vector – just read once and swap while writing. cc @jacques-n , @BryanCutler , @icexelloss Author: siddharth Closes #1443 from siddharthteotia/ARROW-1946 and squashes the following commits: 7805b62b [siddharth] unit tests c89efbf8 [siddharth] ARROW-1946: Add APIs to decimal vector for writing big endian data --- .../templates/AbstractFieldWriter.java | 4 + .../AbstractPromotableFieldWriter.java | 8 ++ .../codegen/templates/ComplexWriters.java | 7 ++ .../apache/arrow/vector/DecimalVector.java | 47 +++++++++++ .../arrow/vector/TestDecimalVector.java | 79 +++++++++++++++++++ 5 files changed, 145 insertions(+) diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java index 853f67fd0dd56..fce6876025a91 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -67,6 +67,10 @@ public void write(${name}Holder holder) { public void write${minor.class}(${friendlyType} value) { fail("${name}"); } + + public void writeBigEndianBytesToDecimal(byte[] value) { + fail("${name}"); + } diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 228c2c531f98f..7f4a13d4f06e8 100644 --- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -16,6 +16,8 @@ * limitations under the License. */ +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.vector.types.Types; import org.apache.drill.common.types.TypeProtos.MinorType; <@pp.dropOutputFile /> @@ -82,6 +84,12 @@ public void write(${name}Holder holder) { getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, ); } + <#if minor.class == "Decimal"> + public void writeBigEndianBytesToDecimal(byte[] value) { + getWriter(Types.MinorType.DECIMAL).writeBigEndianBytesToDecimal(value); + } + + public void writeNull() { } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 4a1545a173dd3..24994d72ac1d8 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -120,6 +120,11 @@ public void write(Nullable${minor.class}Holder h) { vector.setSafe(idx(), value); vector.setValueCount(idx()+1); } + + public void writeBigEndianBytesToDecimal(byte[] value) { + vector.setBigEndianSafe(idx(), value); + vector.setValueCount(idx()+1); + } public void writeNull() { @@ -146,6 +151,8 @@ public interface ${eName}Writer extends BaseWriter { <#if minor.class == "Decimal"> public void write${minor.class}(${friendlyType} value); + + public void writeBigEndianBytesToDecimal(byte[] value); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 71a74cb9f3338..a043575081fb3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -195,6 +195,43 @@ public void set(int index, ArrowBuf buffer) { valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); } + /** + * Set the decimal element at given index to the provided array of bytes. + * Decimal is now implemented as Little Endian. This API allows the user + * to pass a decimal value in the form of byte array in BE byte order. + * + * Consumers of Arrow code can use this API instead of first swapping + * the source bytes (doing a write and read) and then finally writing to + * ArrowBuf of decimal vector. + * + * This method takes care of adding the necessary padding if the length + * of byte array is less then 16 (length of decimal type). + * + * @param index position of element + * @param value array of bytes containing decimal in big endian byte order. + */ + public void setBigEndian(int index, byte[] value) { + assert value.length <= TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int length = value.length; + int startIndex = index * TYPE_WIDTH; + if (length == TYPE_WIDTH) { + for (int i = TYPE_WIDTH - 1; i >= 3; i-=4) { + valueBuffer.setByte(startIndex, value[i]); + valueBuffer.setByte(startIndex + 1, value[i-1]); + valueBuffer.setByte(startIndex + 2, value[i-2]); + valueBuffer.setByte(startIndex + 3, value[i-3]); + startIndex += 4; + } + } else { + for (int i = length - 1; i >= 0; i--) { + valueBuffer.setByte(startIndex, value[i]); + startIndex++; + } + valueBuffer.setZero(startIndex, TYPE_WIDTH - length); + } + } + /** * Set the element at the given index to the given value. * @@ -262,6 +299,16 @@ public void setSafe(int index, ArrowBuf buffer) { set(index, buffer); } + /** + * Same as {@link #setBigEndian(int, byte[])} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + */ + public void setBigEndianSafe(int index, byte[] value) { + handleSafe(index); + setBigEndian(index, value); + } + /** * Same as {@link #set(int, int, ArrowBuf)} except that it handles the * case when index is greater than or equal to existing diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index a48c035718990..8c86452fcc3bf 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -111,4 +111,83 @@ public void testBigDecimalDifferentScaleAndPrecision() { } } } + + @Test + public void testWriteBigEndian() { + try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38, 9), allocator);) { + decimalVector.allocateNew(); + BigDecimal decimal1 = new BigDecimal("123456789.000000000"); + BigDecimal decimal2 = new BigDecimal("11.123456789"); + BigDecimal decimal3 = new BigDecimal("1.000000000"); + BigDecimal decimal4 = new BigDecimal("0.111111111"); + BigDecimal decimal5 = new BigDecimal("987654321.123456789"); + BigDecimal decimal6 = new BigDecimal("222222222222.222222222"); + BigDecimal decimal7 = new BigDecimal("7777777777777.666666667"); + BigDecimal decimal8 = new BigDecimal("1212121212.343434343"); + + byte[] decimalValue1 = decimal1.unscaledValue().toByteArray(); + byte[] decimalValue2 = decimal2.unscaledValue().toByteArray(); + byte[] decimalValue3 = decimal3.unscaledValue().toByteArray(); + byte[] decimalValue4 = decimal4.unscaledValue().toByteArray(); + byte[] decimalValue5 = decimal5.unscaledValue().toByteArray(); + byte[] decimalValue6 = decimal6.unscaledValue().toByteArray(); + byte[] decimalValue7 = decimal7.unscaledValue().toByteArray(); + byte[] decimalValue8 = decimal8.unscaledValue().toByteArray(); + + decimalVector.setBigEndian(0, decimalValue1); + decimalVector.setBigEndian(1, decimalValue2); + decimalVector.setBigEndian(2, decimalValue3); + decimalVector.setBigEndian(3, decimalValue4); + decimalVector.setBigEndian(4, decimalValue5); + decimalVector.setBigEndian(5, decimalValue6); + decimalVector.setBigEndian(6, decimalValue7); + decimalVector.setBigEndian(7, decimalValue8); + + decimalVector.setValueCount(8); + assertEquals(8, decimalVector.getValueCount()); + assertEquals(decimal1, decimalVector.getObject(0)); + assertEquals(decimal2, decimalVector.getObject(1)); + assertEquals(decimal3, decimalVector.getObject(2)); + assertEquals(decimal4, decimalVector.getObject(3)); + assertEquals(decimal5, decimalVector.getObject(4)); + assertEquals(decimal6, decimalVector.getObject(5)); + assertEquals(decimal7, decimalVector.getObject(6)); + assertEquals(decimal8, decimalVector.getObject(7)); + } + } + + @Test + public void testBigDecimalReadWrite() { + try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38, 9), allocator);) { + decimalVector.allocateNew(); + BigDecimal decimal1 = new BigDecimal("123456789.000000000"); + BigDecimal decimal2 = new BigDecimal("11.123456789"); + BigDecimal decimal3 = new BigDecimal("1.000000000"); + BigDecimal decimal4 = new BigDecimal("-0.111111111"); + BigDecimal decimal5 = new BigDecimal("-987654321.123456789"); + BigDecimal decimal6 = new BigDecimal("-222222222222.222222222"); + BigDecimal decimal7 = new BigDecimal("7777777777777.666666667"); + BigDecimal decimal8 = new BigDecimal("1212121212.343434343"); + + decimalVector.set(0, decimal1); + decimalVector.set(1, decimal2); + decimalVector.set(2, decimal3); + decimalVector.set(3, decimal4); + decimalVector.set(4, decimal5); + decimalVector.set(5, decimal6); + decimalVector.set(6, decimal7); + decimalVector.set(7, decimal8); + + decimalVector.setValueCount(8); + assertEquals(8, decimalVector.getValueCount()); + assertEquals(decimal1, decimalVector.getObject(0)); + assertEquals(decimal2, decimalVector.getObject(1)); + assertEquals(decimal3, decimalVector.getObject(2)); + assertEquals(decimal4, decimalVector.getObject(3)); + assertEquals(decimal5, decimalVector.getObject(4)); + assertEquals(decimal6, decimalVector.getObject(5)); + assertEquals(decimal7, decimalVector.getObject(6)); + assertEquals(decimal8, decimalVector.getObject(7)); + } + } } From 4a3e6e361685cc3350e679ef3461103ffc4df0a2 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Tue, 2 Jan 2018 11:30:45 -0500 Subject: [PATCH 161/177] ARROW-1941: [Python] Fix empty list roundtrip in to_pandas This closes [ARROW-1941](https://issues.apache.org/jira/browse/ARROW-1941). Author: Licht-T Closes #1449 from Licht-T/fix-empty-list-roundtrip and squashes the following commits: 165dc6f [Licht-T] TST: Add test for the empty list roundtrip 0ddfd87 [Licht-T] BUG: Fix empty list roundtrip --- cpp/src/arrow/python/arrow_to_pandas.cc | 2 ++ python/pyarrow/tests/test_convert_pandas.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 08ce37cda6351..e21bbda055953 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -90,6 +90,7 @@ struct WrapBytes { static inline bool ListTypeSupported(const DataType& type) { switch (type.id()) { + case Type::NA: case Type::UINT8: case Type::INT8: case Type::UINT16: @@ -695,6 +696,7 @@ class ObjectBlock : public PandasBlock { } else if (type == Type::LIST) { auto list_type = std::static_pointer_cast(col->type()); switch (list_type->value_type()->id()) { + CONVERTLISTSLIKE_CASE(FloatType, NA) CONVERTLISTSLIKE_CASE(UInt8Type, UINT8) CONVERTLISTSLIKE_CASE(Int8Type, INT8) CONVERTLISTSLIKE_CASE(UInt16Type, UINT16) diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index 7609d3488c6c0..76b55cf90431c 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -1317,6 +1317,18 @@ def test_table_column_subset_metadata(self): result = table_subset2.to_pandas() tm.assert_frame_equal(result, df[['a']].reset_index(drop=True)) + def test_empty_list_roundtrip(self): + empty_list_array = np.empty((3,), dtype=object) + empty_list_array.fill([]) + + df = pd.DataFrame({'a': np.array(['1', '2', '3']), + 'b': empty_list_array}) + tbl = pa.Table.from_pandas(df) + + result = tbl.to_pandas() + + tm.assert_frame_equal(result, df) + def _fully_loaded_dataframe_example(): from distutils.version import LooseVersion From 3a69efce59f807c60b8d218abf533e4f1b45ed62 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Thu, 4 Jan 2018 10:11:13 -0500 Subject: [PATCH 162/177] ARROW-1920 [C++/Python] Add experimental reader for Apache ORC files This adds support for reading ORC files in the C++ library, as well as python bindings for this functionality. Author: Jim Crist Author: Uwe L. Korn Closes #1418 from jcrist/orc-adapter and squashes the following commits: 7e0400eb [Jim Crist] lint d6d32b50 [Uwe L. Korn] Hide symbols introduced by orc static lib a2966406 [Jim Crist] Tweak error message f45ac3dd [Jim Crist] Read reads as a table 57bc63dd [Jim Crist] Use `vector` instead of `list` 1d539276 [Jim Crist] date32 instead of date64 4b7a3a5d [Jim Crist] Add brief docs e7835445 [Jim Crist] More fixups 33f5b10f [Jim Crist] Turn off ARROW_ORC on windows 86a23550 [Jim Crist] Cleanups 2cfdd924 [Jim Crist] Fix build when dependencies aren't already installed 876c3a36 [Jim Crist] Use fPIC on protobuf as well f4a29f87 [Jim Crist] Ensure -fPIC on orc build 7cf1659e [Jim Crist] Build python orc support on travis 2adf938a [Jim Crist] Add ORC support 5c791046 [Jim Crist] Add cmake support for liborc --- ci/travis_script_python.sh | 2 +- cpp/CMakeLists.txt | 22 + cpp/cmake_modules/FindProtobuf.cmake | 89 +++ cpp/cmake_modules/ThirdpartyToolchain.cmake | 73 ++ cpp/src/arrow/CMakeLists.txt | 5 + cpp/src/arrow/adapters/orc/CMakeLists.txt | 25 + cpp/src/arrow/adapters/orc/adapter.cc | 697 ++++++++++++++++++ cpp/src/arrow/adapters/orc/adapter.h | 105 +++ cpp/src/arrow/symbols.map | 4 + python/CMakeLists.txt | 11 + .../scripts/check_arrow_visibility.sh | 5 +- python/pyarrow/_orc.pxd | 50 ++ python/pyarrow/_orc.pyx | 111 +++ python/pyarrow/orc.py | 149 ++++ python/setup.py | 9 + 15 files changed, 1355 insertions(+), 2 deletions(-) create mode 100644 cpp/cmake_modules/FindProtobuf.cmake create mode 100644 cpp/src/arrow/adapters/orc/CMakeLists.txt create mode 100644 cpp/src/arrow/adapters/orc/adapter.cc create mode 100644 cpp/src/arrow/adapters/orc/adapter.h create mode 100644 python/pyarrow/_orc.pxd create mode 100644 python/pyarrow/_orc.pyx create mode 100644 python/pyarrow/orc.py diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 5f7b0a9a1afed..444386fdf955e 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -82,7 +82,7 @@ fi export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE pip install -r requirements.txt -python setup.py build_ext --with-parquet --with-plasma \ +python setup.py build_ext --with-parquet --with-plasma --with-orc\ install --single-version-externally-managed --record=record.text popd diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dd159ced29193..ede13af5d413b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -119,6 +119,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Build the Arrow GPU extensions (requires CUDA installation)" OFF) + option(ARROW_ORC + "Build the Arrow ORC adapter" + ON) + option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" OFF) @@ -226,6 +230,17 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) set(ARROW_WITH_ZSTD ON) endif() +if (MSVC) + # ORC doesn't build on windows + set(ARROW_ORC OFF) +endif() + +if(ARROW_ORC) + set(ARROW_WITH_LZ4 ON) + set(ARROW_WITH_SNAPPY ON) + set(ARROW_WITH_ZLIB ON) +endif() + if(NOT ARROW_BUILD_TESTS) set(NO_TESTS 1) endif() @@ -526,6 +541,13 @@ if (ARROW_WITH_GRPC) ${ARROW_STATIC_LINK_LIBS}) endif() +if (ARROW_ORC) + SET(ARROW_STATIC_LINK_LIBS + orc + protobuf + ${ARROW_STATIC_LINK_LIBS}) +endif() + if (ARROW_STATIC_LINK_LIBS) add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS}) endif() diff --git a/cpp/cmake_modules/FindProtobuf.cmake b/cpp/cmake_modules/FindProtobuf.cmake new file mode 100644 index 0000000000000..a42f4493af494 --- /dev/null +++ b/cpp/cmake_modules/FindProtobuf.cmake @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# PROTOBUF_HOME environmental variable is used to check for Protobuf headers and static library + +# PROTOBUF_INCLUDE_DIR: directory containing headers +# PROTOBUF_LIBS: directory containing Protobuf libraries +# PROTOBUF_STATIC_LIB: location of protobuf.a +# PROTOC_STATIC_LIB: location of protoc.a +# PROTOBUF_EXECUTABLE: location of protoc +# PROTOBUF_FOUND is set if Protobuf is found + + +if( NOT "${PROTOBUF_HOME}" STREQUAL "") + file (TO_CMAKE_PATH "${PROTOBUF_HOME}" _protobuf_path) +endif() + +message (STATUS "PROTOBUF_HOME: ${PROTOBUF_HOME}") + +find_path (PROTOBUF_INCLUDE_DIR google/protobuf/io/zero_copy_stream.h HINTS + ${_protobuf_path} + NO_DEFAULT_PATH + PATH_SUFFIXES "include") + +find_path (PROTOBUF_INCLUDE_DIR google/protobuf/io/coded_stream.h HINTS + ${_protobuf_path} + NO_DEFAULT_PATH + PATH_SUFFIXES "include") + +find_library (PROTOBUF_LIBRARY NAMES protobuf PATHS + ${_protobuf_path} + NO_DEFAULT_PATH + PATH_SUFFIXES "lib") + +find_library (PROTOC_LIBRARY NAMES protoc PATHS + ${_protobuf_path} + NO_DEFAULT_PATH + PATH_SUFFIXES "lib") + +find_program(PROTOBUF_EXECUTABLE protoc HINTS + ${_protobuf_path} + NO_DEFAULT_PATH + PATH_SUFFIXES "bin") + +if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOC_LIBRARY AND PROTOBUF_EXECUTABLE) + set (PROTOBUF_FOUND TRUE) + get_filename_component (PROTOBUF_LIBS ${PROTOBUF_LIBRARY} PATH) + set (PROTOBUF_LIB_NAME protobuf) + set (PROTOC_LIB_NAME protoc) + set (PROTOBUF_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOBUF_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) + set (PROTOC_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) +else () + set (PROTOBUF_FOUND FALSE) +endif () + +if (PROTOBUF_FOUND) + message (STATUS "Found the Protobuf headers: ${PROTOBUF_INCLUDE_DIR}") + message (STATUS "Found the Protobuf library: ${PROTOBUF_STATIC_LIB}") + message (STATUS "Found the Protoc library: ${PROTOC_STATIC_LIB}") + message (STATUS "Found the Protoc executable: ${PROTOBUF_EXECUTABLE}") +else() + if (_protobuf_path) + set (PROTOBUF_ERR_MSG "Could not find Protobuf. Looked in ${_protobuf_path}.") + else () + set (PROTOBUF_ERR_MSG "Could not find Protobuf in system search paths.") + endif() + + if (Protobuf_FIND_REQUIRED) + message (FATAL_ERROR "${PROTOBUF_ERR_MSG}") + else () + message (STATUS "${PROTOBUF_ERR_MSG}") + endif () +endif() + +mark_as_advanced ( + PROTOBUF_INCLUDE_DIR + PROTOBUF_LIBS + PROTOBUF_STATIC_LIB + PROTOC_STATIC_LIB +) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b706aab45265d..4f64434170655 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -29,7 +29,9 @@ set(SNAPPY_VERSION "1.1.3") set(BROTLI_VERSION "v0.6.0") set(LZ4_VERSION "1.7.5") set(ZSTD_VERSION "1.2.0") +set(PROTOBUF_VERSION "2.6.0") set(GRPC_VERSION "94582910ad7f82ad447ecc72e6548cb669e4f7a9") # v1.6.5 +set(ORC_VERSION "cf00b67795717ab3eb04e950780ed6d104109017") string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) @@ -721,6 +723,7 @@ if (ARROW_WITH_LZ4) if("${LZ4_HOME}" STREQUAL "") set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") + set(LZ4_HOME "${LZ4_BUILD_DIR}") set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib") if (MSVC) @@ -865,3 +868,73 @@ if (ARROW_WITH_GRPC) endif() endif() + +if (ARROW_ORC) + # protobuf + if ("${PROTOBUF_HOME}" STREQUAL "") + set (PROTOBUF_PREFIX "${THIRDPARTY_DIR}/protobuf_ep-install") + set (PROTOBUF_HOME "${PROTOBUF_PREFIX}") + set (PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") + set (PROTOBUF_STATIC_LIB "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") + set (PROTOBUF_SRC_URL "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz") + + ExternalProject_Add(protobuf_ep + CONFIGURE_COMMAND "./configure" "--disable-shared" "--prefix=${PROTOBUF_PREFIX}" "CXXFLAGS=${EP_CXX_FLAGS}" + BUILD_IN_SOURCE 1 + URL ${PROTOBUF_SRC_URL} + LOG_DOWNLOAD 1 + LOG_CONFIGURE 1 + LOG_BUILD 1 + LOG_INSTALL 1 + BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}") + + set (PROTOBUF_VENDORED 1) + else () + find_package (Protobuf REQUIRED) + set (PROTOBUF_VENDORED 0) + endif () + + include_directories (SYSTEM ${PROTOBUF_INCLUDE_DIR}) + ADD_THIRDPARTY_LIB(protobuf + STATIC_LIB ${PROTOBUF_STATIC_LIB}) + + if (PROTOBUF_VENDORED) + add_dependencies (protobuf protobuf_ep) + endif () + + # orc + set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-install") + set(ORC_HOME "${ORC_PREFIX}") + set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include") + set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}orc${CMAKE_STATIC_LIBRARY_SUFFIX}") + + # Since LZ4 isn't installed, the header file is in ${LZ4_HOME}/lib instead of + # ${LZ4_HOME}/include, which forces us to specify the include directory + # manually as well. + set (ORC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${ORC_PREFIX} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} + -DBUILD_LIBHDFSPP=OFF + -DBUILD_JAVA=OFF + -DBUILD_TOOLS=OFF + -DBUILD_CPP_TESTS=OFF + -DINSTALL_VENDORED_LIBS=OFF + -DPROTOBUF_HOME=${PROTOBUF_HOME} + -DLZ4_HOME=${LZ4_HOME} + -DLZ4_INCLUDE_DIR=${LZ4_INCLUDE_DIR} + -DSNAPPY_HOME=${SNAPPY_HOME} + -DZLIB_HOME=${ZLIB_HOME}) + + ExternalProject_Add(orc_ep + GIT_REPOSITORY "https://github.com/apache/orc" + GIT_TAG ${ORC_VERSION} + BUILD_BYPRODUCTS ${ORC_STATIC_LIB} + CMAKE_ARGS ${ORC_CMAKE_ARGS}) + + include_directories(SYSTEM ${ORC_INCLUDE_DIR}) + ADD_THIRDPARTY_LIB(orc + STATIC_LIB ${ORC_STATIC_LIB}) + + add_dependencies(orc_ep protobuf lz4_static snappy zlib) + add_dependencies(orc orc_ep) +endif() diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index d645cca224480..ad86256e0be34 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -90,6 +90,11 @@ if (ARROW_WITH_ZSTD) SET(ARROW_SRCS util/compression_zstd.cc ${ARROW_SRCS}) endif() +if (ARROW_ORC) + add_subdirectory(adapters/orc) + SET(ARROW_SRCS adapters/orc/adapter.cc ${ARROW_SRCS}) +endif() + if (NOT ARROW_BOOST_HEADER_ONLY) set(ARROW_SRCS ${ARROW_SRCS} io/hdfs.cc diff --git a/cpp/src/arrow/adapters/orc/CMakeLists.txt b/cpp/src/arrow/adapters/orc/CMakeLists.txt new file mode 100644 index 0000000000000..eb7194cd42113 --- /dev/null +++ b/cpp/src/arrow/adapters/orc/CMakeLists.txt @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +####################################### +# arrow_orc +####################################### + +# Headers: top level +install(FILES + adapter.h + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/adapters/orc") diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc new file mode 100644 index 0000000000000..473c90f925124 --- /dev/null +++ b/cpp/src/arrow/adapters/orc/adapter.cc @@ -0,0 +1,697 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/adapters/orc/adapter.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/buffer.h" +#include "arrow/builder.h" +#include "arrow/io/interfaces.h" +#include "arrow/memory_pool.h" +#include "arrow/record_batch.h" +#include "arrow/status.h" +#include "arrow/table.h" +#include "arrow/table_builder.h" +#include "arrow/type.h" +#include "arrow/type_traits.h" +#include "arrow/util/bit-util.h" +#include "arrow/util/decimal.h" +#include "arrow/util/macros.h" +#include "arrow/util/visibility.h" + +#include "orc/OrcFile.hh" + +// alias to not interfere with nested orc namespace +namespace liborc = orc; + +namespace arrow { +namespace adapters { +namespace orc { + +#define ORC_THROW_NOT_OK(s) \ + do { \ + Status _s = (s); \ + if (!_s.ok()) { \ + std::stringstream ss; \ + ss << "Arrow error: " << _s.ToString(); \ + throw liborc::ParseError(ss.str()); \ + } \ + } while (0) + +class ArrowInputFile : public liborc::InputStream { + public: + explicit ArrowInputFile(const std::shared_ptr& file) + : file_(file) {} + + uint64_t getLength() const override { + int64_t size; + ORC_THROW_NOT_OK(file_->GetSize(&size)); + return static_cast(size); + } + + uint64_t getNaturalReadSize() const override { return 128 * 1024; } + + void read(void* buf, uint64_t length, uint64_t offset) override { + int64_t bytes_read; + + ORC_THROW_NOT_OK(file_->ReadAt(offset, length, &bytes_read, buf)); + + if (static_cast(bytes_read) != length) { + throw liborc::ParseError("Short read from arrow input file"); + } + } + + const std::string& getName() const override { + static const std::string filename("ArrowInputFile"); + return filename; + } + + private: + std::shared_ptr file_; +}; + +struct StripeInformation { + uint64_t offset; + uint64_t length; + uint64_t num_rows; +}; + +Status GetArrowType(const liborc::Type* type, std::shared_ptr* out) { + // When subselecting fields on read, liborc will set some nodes to nullptr, + // so we need to check for nullptr before progressing + if (type == nullptr) { + *out = null(); + return Status::OK(); + } + liborc::TypeKind kind = type->getKind(); + switch (kind) { + case liborc::BOOLEAN: + *out = boolean(); + break; + case liborc::BYTE: + *out = int8(); + break; + case liborc::SHORT: + *out = int16(); + break; + case liborc::INT: + *out = int32(); + break; + case liborc::LONG: + *out = int64(); + break; + case liborc::FLOAT: + *out = float32(); + break; + case liborc::DOUBLE: + *out = float64(); + break; + case liborc::VARCHAR: + case liborc::STRING: + *out = utf8(); + break; + case liborc::BINARY: + *out = binary(); + break; + case liborc::CHAR: + *out = fixed_size_binary(type->getMaximumLength()); + break; + case liborc::TIMESTAMP: + *out = timestamp(TimeUnit::NANO); + break; + case liborc::DATE: + *out = date32(); + break; + case liborc::DECIMAL: { + if (type->getPrecision() == 0) { + // In HIVE 0.11/0.12 precision is set as 0, but means max precision + *out = decimal(38, 6); + } else { + *out = decimal(type->getPrecision(), type->getScale()); + } + break; + } + case liborc::LIST: { + if (type->getSubtypeCount() != 1) { + return Status::Invalid("Invalid Orc List type"); + } + std::shared_ptr elemtype; + RETURN_NOT_OK(GetArrowType(type->getSubtype(0), &elemtype)); + *out = list(elemtype); + break; + } + case liborc::MAP: { + if (type->getSubtypeCount() != 2) { + return Status::Invalid("Invalid Orc Map type"); + } + std::shared_ptr keytype; + std::shared_ptr valtype; + RETURN_NOT_OK(GetArrowType(type->getSubtype(0), &keytype)); + RETURN_NOT_OK(GetArrowType(type->getSubtype(1), &valtype)); + *out = list(struct_({field("key", keytype), field("value", valtype)})); + break; + } + case liborc::STRUCT: { + int size = type->getSubtypeCount(); + std::vector> fields; + for (int child = 0; child < size; ++child) { + std::shared_ptr elemtype; + RETURN_NOT_OK(GetArrowType(type->getSubtype(child), &elemtype)); + std::string name = type->getFieldName(child); + fields.push_back(field(name, elemtype)); + } + *out = struct_(fields); + break; + } + case liborc::UNION: { + int size = type->getSubtypeCount(); + std::vector> fields; + std::vector type_codes; + for (int child = 0; child < size; ++child) { + std::shared_ptr elemtype; + RETURN_NOT_OK(GetArrowType(type->getSubtype(child), &elemtype)); + fields.push_back(field("_union_" + std::to_string(child), elemtype)); + type_codes.push_back(static_cast(child)); + } + *out = union_(fields, type_codes); + break; + } + default: { + std::stringstream ss; + ss << "Unknown Orc type kind: " << kind; + return Status::Invalid(ss.str()); + } + } + return Status::OK(); +} + +// The number of rows to read in a ColumnVectorBatch +constexpr int64_t kReadRowsBatch = 1000; + +// The numer of nanoseconds in a second +constexpr int64_t kOneSecondNanos = 1000000000LL; + +class ORCFileReader::Impl { + public: + Impl() {} + ~Impl() {} + + Status Open(const std::shared_ptr& file, MemoryPool* pool) { + std::unique_ptr io_wrapper(new ArrowInputFile(file)); + liborc::ReaderOptions options; + std::unique_ptr liborc_reader; + try { + liborc_reader = createReader(std::move(io_wrapper), options); + } catch (const liborc::ParseError& e) { + return Status::IOError(e.what()); + } + pool_ = pool; + reader_ = std::move(liborc_reader); + + return Init(); + } + + Status Init() { + int64_t nstripes = reader_->getNumberOfStripes(); + stripes_.resize(nstripes); + std::unique_ptr stripe; + for (int i = 0; i < nstripes; ++i) { + stripe = reader_->getStripe(i); + stripes_[i] = StripeInformation( + {stripe->getOffset(), stripe->getLength(), stripe->getNumberOfRows()}); + } + return Status::OK(); + } + + int64_t NumberOfStripes() { return stripes_.size(); } + + int64_t NumberOfRows() { return reader_->getNumberOfRows(); } + + Status ReadSchema(std::shared_ptr* out) { + const liborc::Type& type = reader_->getType(); + return GetArrowSchema(type, out); + } + + Status GetArrowSchema(const liborc::Type& type, std::shared_ptr* out) { + if (type.getKind() != liborc::STRUCT) { + return Status::NotImplemented( + "Only ORC files with a top-level struct " + "can be handled"); + } + int size = type.getSubtypeCount(); + std::vector> fields; + for (int child = 0; child < size; ++child) { + std::shared_ptr elemtype; + RETURN_NOT_OK(GetArrowType(type.getSubtype(child), &elemtype)); + std::string name = type.getFieldName(child); + fields.push_back(field(name, elemtype)); + } + std::list keys = reader_->getMetadataKeys(); + std::shared_ptr metadata; + if (!keys.empty()) { + metadata = std::make_shared(); + for (auto it = keys.begin(); it != keys.end(); ++it) { + metadata->Append(*it, reader_->getMetadataValue(*it)); + } + } + + *out = std::make_shared(fields, metadata); + return Status::OK(); + } + + Status Read(std::shared_ptr* out) { + liborc::RowReaderOptions opts; + return ReadTable(opts, out); + } + + Status Read(const std::vector& include_indices, std::shared_ptr
    * out) { + liborc::RowReaderOptions opts; + RETURN_NOT_OK(SelectIndices(&opts, include_indices)); + return ReadTable(opts, out); + } + + Status ReadStripe(int64_t stripe, std::shared_ptr* out) { + liborc::RowReaderOptions opts; + RETURN_NOT_OK(SelectStripe(&opts, stripe)); + return ReadBatch(opts, stripes_[stripe].num_rows, out); + } + + Status ReadStripe(int64_t stripe, const std::vector& include_indices, + std::shared_ptr* out) { + liborc::RowReaderOptions opts; + RETURN_NOT_OK(SelectIndices(&opts, include_indices)); + RETURN_NOT_OK(SelectStripe(&opts, stripe)); + return ReadBatch(opts, stripes_[stripe].num_rows, out); + } + + Status SelectStripe(liborc::RowReaderOptions* opts, int64_t stripe) { + if (stripe < 0 || stripe >= NumberOfStripes()) { + std::stringstream ss; + ss << "Out of bounds stripe: " << stripe; + return Status::Invalid(ss.str()); + } + opts->range(stripes_[stripe].offset, stripes_[stripe].length); + return Status::OK(); + } + + Status SelectIndices(liborc::RowReaderOptions* opts, + const std::vector& include_indices) { + std::list include_indices_list; + for (auto it = include_indices.begin(); it != include_indices.end(); ++it) { + if (*it < 0) { + return Status::Invalid("Negative field index"); + } + include_indices_list.push_back(*it); + } + opts->includeTypes(include_indices_list); + return Status::OK(); + } + + Status ReadTable(const liborc::RowReaderOptions& row_opts, + std::shared_ptr
    * out) { + liborc::RowReaderOptions opts(row_opts); + std::vector> batches(stripes_.size()); + for (size_t stripe = 0; stripe < stripes_.size(); stripe++) { + opts.range(stripes_[stripe].offset, stripes_[stripe].length); + RETURN_NOT_OK(ReadBatch(opts, stripes_[stripe].num_rows, &batches[stripe])); + } + return Table::FromRecordBatches(batches, out); + } + + Status ReadBatch(const liborc::RowReaderOptions& opts, int64_t nrows, + std::shared_ptr* out) { + std::unique_ptr rowreader; + std::unique_ptr batch; + try { + rowreader = reader_->createRowReader(opts); + batch = rowreader->createRowBatch(std::min(nrows, kReadRowsBatch)); + } catch (const liborc::ParseError& e) { + return Status::Invalid(e.what()); + } + const liborc::Type& type = rowreader->getSelectedType(); + std::shared_ptr schema; + RETURN_NOT_OK(GetArrowSchema(type, &schema)); + + std::unique_ptr builder; + RETURN_NOT_OK(RecordBatchBuilder::Make(schema, pool_, nrows, &builder)); + + // The top-level type must be a struct to read into an arrow table + const auto& struct_batch = static_cast(*batch); + + while (rowreader->next(*batch)) { + for (int i = 0; i < builder->num_fields(); i++) { + RETURN_NOT_OK(AppendBatch(type.getSubtype(i), struct_batch.fields[i], 0, + batch->numElements, builder->GetField(i))); + } + } + RETURN_NOT_OK(builder->Flush(out)); + return Status::OK(); + } + + Status AppendBatch(const liborc::Type* type, liborc::ColumnVectorBatch* batch, + int64_t offset, int64_t length, ArrayBuilder* builder) { + if (type == nullptr) { + return Status::OK(); + } + liborc::TypeKind kind = type->getKind(); + switch (kind) { + case liborc::STRUCT: + return AppendStructBatch(type, batch, offset, length, builder); + case liborc::LIST: + return AppendListBatch(type, batch, offset, length, builder); + case liborc::MAP: + return AppendMapBatch(type, batch, offset, length, builder); + case liborc::LONG: + return AppendNumericBatch( + batch, offset, length, builder); + case liborc::INT: + return AppendNumericBatchCast(batch, offset, length, builder); + case liborc::SHORT: + return AppendNumericBatchCast(batch, offset, length, builder); + case liborc::BYTE: + return AppendNumericBatchCast(batch, offset, length, builder); + case liborc::DOUBLE: + return AppendNumericBatch( + batch, offset, length, builder); + case liborc::FLOAT: + return AppendNumericBatchCast(batch, offset, length, builder); + case liborc::BOOLEAN: + return AppendBoolBatch(batch, offset, length, builder); + case liborc::VARCHAR: + case liborc::STRING: + return AppendBinaryBatch(batch, offset, length, builder); + case liborc::BINARY: + return AppendBinaryBatch(batch, offset, length, builder); + case liborc::CHAR: + return AppendFixedBinaryBatch(batch, offset, length, builder); + case liborc::DATE: + return AppendNumericBatchCast(batch, offset, length, builder); + case liborc::TIMESTAMP: + return AppendTimestampBatch(batch, offset, length, builder); + case liborc::DECIMAL: + return AppendDecimalBatch(type, batch, offset, length, builder); + default: + std::stringstream ss; + ss << "Not implemented type kind: " << kind; + return Status::NotImplemented(ss.str()); + } + } + + Status AppendStructBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch, + int64_t offset, int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + + const uint8_t* valid_bytes = nullptr; + if (batch->hasNulls) { + valid_bytes = reinterpret_cast(batch->notNull.data()) + offset; + } + RETURN_NOT_OK(builder->Append(length, valid_bytes)); + + for (int i = 0; i < builder->num_fields(); i++) { + RETURN_NOT_OK(AppendBatch(type->getSubtype(i), batch->fields[i], offset, length, + builder->field_builder(i))); + } + return Status::OK(); + } + + Status AppendListBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch, + int64_t offset, int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + liborc::ColumnVectorBatch* elements = batch->elements.get(); + const liborc::Type* elemtype = type->getSubtype(0); + + const bool has_nulls = batch->hasNulls; + for (int i = offset; i < length + offset; i++) { + if (!has_nulls || batch->notNull[i]) { + int64_t start = batch->offsets[i]; + int64_t end = batch->offsets[i + 1]; + RETURN_NOT_OK(builder->Append()); + RETURN_NOT_OK(AppendBatch(elemtype, elements, start, end - start, + builder->value_builder())); + } else { + RETURN_NOT_OK(builder->AppendNull()); + } + } + return Status::OK(); + } + + Status AppendMapBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch, + int64_t offset, int64_t length, ArrayBuilder* abuilder) { + auto list_builder = static_cast(abuilder); + auto struct_builder = static_cast(list_builder->value_builder()); + auto batch = static_cast(cbatch); + liborc::ColumnVectorBatch* keys = batch->keys.get(); + liborc::ColumnVectorBatch* vals = batch->elements.get(); + const liborc::Type* keytype = type->getSubtype(0); + const liborc::Type* valtype = type->getSubtype(1); + + const bool has_nulls = batch->hasNulls; + for (int i = offset; i < length + offset; i++) { + RETURN_NOT_OK(list_builder->Append()); + int64_t start = batch->offsets[i]; + int64_t list_length = batch->offsets[i + 1] - start; + if (list_length && (!has_nulls || batch->notNull[i])) { + RETURN_NOT_OK(struct_builder->Append(list_length, nullptr)); + RETURN_NOT_OK(AppendBatch(keytype, keys, start, list_length, + struct_builder->field_builder(0))); + RETURN_NOT_OK(AppendBatch(valtype, vals, start, list_length, + struct_builder->field_builder(1))); + } + } + return Status::OK(); + } + + template + Status AppendNumericBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset, + int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + + if (length == 0) { + return Status::OK(); + } + const uint8_t* valid_bytes = nullptr; + if (batch->hasNulls) { + valid_bytes = reinterpret_cast(batch->notNull.data()) + offset; + } + const elem_type* source = batch->data.data() + offset; + RETURN_NOT_OK(builder->Append(source, length, valid_bytes)); + return Status::OK(); + } + + template + Status AppendNumericBatchCast(liborc::ColumnVectorBatch* cbatch, int64_t offset, + int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + + if (length == 0) { + return Status::OK(); + } + int start = builder->length(); + + const uint8_t* valid_bytes = nullptr; + if (batch->hasNulls) { + valid_bytes = reinterpret_cast(batch->notNull.data()) + offset; + } + RETURN_NOT_OK(builder->AppendNulls(valid_bytes, length)); + + const source_type* source = batch->data.data() + offset; + target_type* target = reinterpret_cast(builder->data()->mutable_data()); + + std::copy(source, source + length, target + start); + + return Status::OK(); + } + + Status AppendBoolBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset, + int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + + if (length == 0) { + return Status::OK(); + } + int start = builder->length(); + + const uint8_t* valid_bytes = nullptr; + if (batch->hasNulls) { + valid_bytes = reinterpret_cast(batch->notNull.data()) + offset; + } + RETURN_NOT_OK(builder->AppendNulls(valid_bytes, length)); + + const int64_t* source = batch->data.data() + offset; + uint8_t* target = reinterpret_cast(builder->data()->mutable_data()); + + for (int i = 0; i < length; i++) { + if (source[i]) { + BitUtil::SetBit(target, start + i); + } else { + BitUtil::ClearBit(target, start + i); + } + } + return Status::OK(); + } + + Status AppendTimestampBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset, + int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + + if (length == 0) { + return Status::OK(); + } + int start = builder->length(); + + const uint8_t* valid_bytes = nullptr; + if (batch->hasNulls) { + valid_bytes = reinterpret_cast(batch->notNull.data()) + offset; + } + RETURN_NOT_OK(builder->AppendNulls(valid_bytes, length)); + + const int64_t* seconds = batch->data.data() + offset; + const int64_t* nanos = batch->nanoseconds.data() + offset; + int64_t* target = reinterpret_cast(builder->data()->mutable_data()); + + for (int i = 0; i < length; i++) { + // TODO: boundscheck this, as ORC supports higher resolution timestamps + // than arrow for nanosecond resolution + target[start + i] = seconds[i] * kOneSecondNanos + nanos[i]; + } + return Status::OK(); + } + + template + Status AppendBinaryBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset, + int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + + const bool has_nulls = batch->hasNulls; + for (int i = offset; i < length + offset; i++) { + if (!has_nulls || batch->notNull[i]) { + RETURN_NOT_OK(builder->Append(batch->data[i], batch->length[i])); + } else { + RETURN_NOT_OK(builder->AppendNull()); + } + } + return Status::OK(); + } + + Status AppendFixedBinaryBatch(liborc::ColumnVectorBatch* cbatch, int64_t offset, + int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + auto batch = static_cast(cbatch); + + const bool has_nulls = batch->hasNulls; + for (int i = offset; i < length + offset; i++) { + if (!has_nulls || batch->notNull[i]) { + RETURN_NOT_OK(builder->Append(batch->data[i])); + } else { + RETURN_NOT_OK(builder->AppendNull()); + } + } + return Status::OK(); + } + + Status AppendDecimalBatch(const liborc::Type* type, liborc::ColumnVectorBatch* cbatch, + int64_t offset, int64_t length, ArrayBuilder* abuilder) { + auto builder = static_cast(abuilder); + + const bool has_nulls = cbatch->hasNulls; + if (type->getPrecision() == 0 || type->getPrecision() > 18) { + auto batch = static_cast(cbatch); + for (int i = offset; i < length + offset; i++) { + if (!has_nulls || batch->notNull[i]) { + RETURN_NOT_OK(builder->Append( + Decimal128(batch->values[i].getHighBits(), batch->values[i].getLowBits()))); + } else { + RETURN_NOT_OK(builder->AppendNull()); + } + } + } else { + auto batch = static_cast(cbatch); + for (int i = offset; i < length + offset; i++) { + if (!has_nulls || batch->notNull[i]) { + RETURN_NOT_OK(builder->Append(Decimal128(batch->values[i]))); + } else { + RETURN_NOT_OK(builder->AppendNull()); + } + } + } + return Status::OK(); + } + + private: + MemoryPool* pool_; + std::unique_ptr reader_; + std::vector stripes_; +}; + +ORCFileReader::ORCFileReader() { impl_.reset(new ORCFileReader::Impl()); } + +ORCFileReader::~ORCFileReader() {} + +Status ORCFileReader::Open(const std::shared_ptr& file, + MemoryPool* pool, std::unique_ptr* reader) { + auto result = std::unique_ptr(new ORCFileReader()); + RETURN_NOT_OK(result->impl_->Open(file, pool)); + *reader = std::move(result); + return Status::OK(); +} + +Status ORCFileReader::ReadSchema(std::shared_ptr* out) { + return impl_->ReadSchema(out); +} + +Status ORCFileReader::Read(std::shared_ptr
    * out) { return impl_->Read(out); } + +Status ORCFileReader::Read(const std::vector& include_indices, + std::shared_ptr
    * out) { + return impl_->Read(include_indices, out); +} + +Status ORCFileReader::ReadStripe(int64_t stripe, std::shared_ptr* out) { + return impl_->ReadStripe(stripe, out); +} + +Status ORCFileReader::ReadStripe(int64_t stripe, const std::vector& include_indices, + std::shared_ptr* out) { + return impl_->ReadStripe(stripe, include_indices, out); +} + +int64_t ORCFileReader::NumberOfStripes() { return impl_->NumberOfStripes(); } + +int64_t ORCFileReader::NumberOfRows() { return impl_->NumberOfRows(); } + +} // namespace orc +} // namespace adapters +} // namespace arrow diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h new file mode 100644 index 0000000000000..6438658fd45bb --- /dev/null +++ b/cpp/src/arrow/adapters/orc/adapter.h @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_ORC_CONVERTER_H +#define ARROW_ORC_CONVERTER_H + +#include +#include +#include + +#include "arrow/io/interfaces.h" +#include "arrow/memory_pool.h" +#include "arrow/record_batch.h" +#include "arrow/status.h" +#include "arrow/type.h" +#include "arrow/util/visibility.h" + +namespace arrow { + +namespace adapters { + +namespace orc { + +/// \class ORCFileReader +/// \brief Read an Arrow Table or RecordBatch from an ORC file. +class ARROW_EXPORT ORCFileReader { + public: + ~ORCFileReader(); + + /// \brief Create a new ORC reader + /// + /// \param[in] file the data source + /// \param[in] pool a MemoryPool to use for buffer allocations + /// \param[out] reader the returned reader object + /// \return Status + static Status Open(const std::shared_ptr& file, + MemoryPool* pool, std::unique_ptr* reader); + + /// \brief Return the schema read from the ORC file + /// + /// \param[out] out the returned Schema object + Status ReadSchema(std::shared_ptr* out); + + /// \brief Read the file as a Table + /// + /// The table will be composed of one record batch per stripe. + /// + /// \param[out] out the returned RecordBatch + Status Read(std::shared_ptr
    * out); + + /// \brief Read the file as a Table + /// + /// The table will be composed of one record batch per stripe. + /// + /// \param[in] include_indices the selected field indices to read + /// \param[out] out the returned RecordBatch + Status Read(const std::vector& include_indices, std::shared_ptr
    * out); + + /// \brief Read a single stripe as a RecordBatch + /// + /// \param[in] stripe the stripe index + /// \param[out] out the returned RecordBatch + Status ReadStripe(int64_t stripe, std::shared_ptr* out); + + /// \brief Read a single stripe as a RecordBatch + /// + /// \param[in] stripe the stripe index + /// \param[in] include_indices the selected field indices to read + /// \param[out] out the returned RecordBatch + Status ReadStripe(int64_t stripe, const std::vector& include_indices, + std::shared_ptr* out); + + /// \brief The number of stripes in the file + int64_t NumberOfStripes(); + + /// \brief The number of rows in the file + int64_t NumberOfRows(); + + private: + class Impl; + std::unique_ptr impl_; + ORCFileReader(); +}; + +} // namespace orc + +} // namespace adapters + +} // namespace arrow + +#endif // ARROW_ORC_CONVERTER_H diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map index f216d865001d2..c5d23793ccbae 100644 --- a/cpp/src/arrow/symbols.map +++ b/cpp/src/arrow/symbols.map @@ -55,6 +55,8 @@ ERR_getErrorString; # jemalloc je_arrow_*; + # ORC destructors + _ZThn8_N3orc*; extern "C++" { # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically @@ -65,6 +67,8 @@ # Statically linked C++ dependencies boost::*; + google::*; + orc::*; snappy::*; }; }; diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index cbbb464d05a76..e9de08ba19730 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -70,6 +70,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") option(PYARROW_BUILD_PLASMA "Build the PyArrow Plasma integration" OFF) + option(PYARROW_BUILD_ORC + "Build the PyArrow ORC integration" + OFF) option(PYARROW_BUNDLE_ARROW_CPP "Bundle the Arrow C++ libraries" OFF) @@ -357,6 +360,14 @@ if (PYARROW_BUILD_PLASMA) file(COPY ${PLASMA_EXECUTABLE} DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}) endif() + +if (PYARROW_BUILD_ORC) + ## ORC + set(CYTHON_EXTENSIONS + ${CYTHON_EXTENSIONS} + _orc) +endif() + ############################################################ # Setup and build Cython modules ############################################################ diff --git a/python/manylinux1/scripts/check_arrow_visibility.sh b/python/manylinux1/scripts/check_arrow_visibility.sh index 27a30f7479bb6..bed357edf664a 100755 --- a/python/manylinux1/scripts/check_arrow_visibility.sh +++ b/python/manylinux1/scripts/check_arrow_visibility.sh @@ -17,10 +17,13 @@ # under the License. nm -D -C /arrow-dist/lib64/libarrow.so > nm_arrow.log +grep ' T ' nm_arrow.log | grep -v arrow > visible_symbols.log -if [[ `grep ' T ' nm_arrow.log | grep -v arrow | wc -l` -eq 2 ]] +if [[ `cat visible_symbols.log | wc -l` -eq 2 ]] then exit 0 fi +cat visible_symbols.log + exit 1 diff --git a/python/pyarrow/_orc.pxd b/python/pyarrow/_orc.pxd new file mode 100644 index 0000000000000..411691510423c --- /dev/null +++ b/python/pyarrow/_orc.pxd @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libc.string cimport const_char +from libcpp.vector cimport vector as std_vector +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus, + CTable, CMemoryPool, + CKeyValueMetadata, + CRecordBatch, + CTable, + RandomAccessFile, OutputStream, + TimeUnit) + + +cdef extern from "arrow/adapters/orc/adapter.h" namespace "arrow::adapters::orc" nogil: + cdef cppclass ORCFileReader: + + @staticmethod + CStatus Open(const shared_ptr[RandomAccessFile]& file, + CMemoryPool* pool, + unique_ptr[ORCFileReader]* reader) + + CStatus ReadSchema(shared_ptr[CSchema]* out) + + CStatus ReadStripe(int64_t stripe, shared_ptr[CRecordBatch]* out) + CStatus ReadStripe(int64_t stripe, std_vector[int], shared_ptr[CRecordBatch]* out) + + CStatus Read(shared_ptr[CTable]* out) + CStatus Read(std_vector[int], shared_ptr[CTable]* out) + + int64_t NumberOfStripes() + + int64_t NumberOfRows() diff --git a/python/pyarrow/_orc.pyx b/python/pyarrow/_orc.pyx new file mode 100644 index 0000000000000..7ff4bac6dc95f --- /dev/null +++ b/python/pyarrow/_orc.pyx @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: profile=False +# distutils: language = c++ +# cython: embedsignature = True + +from cython.operator cimport dereference as deref +from libcpp.vector cimport vector as std_vector +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.lib cimport (check_status, + MemoryPool, maybe_unbox_memory_pool, + Schema, pyarrow_wrap_schema, + RecordBatch, + pyarrow_wrap_table, + get_reader) +import six + + +cdef class ORCReader: + cdef: + object source + CMemoryPool* allocator + unique_ptr[ORCFileReader] reader + + def __cinit__(self, MemoryPool memory_pool=None): + self.allocator = maybe_unbox_memory_pool(memory_pool) + + def open(self, object source): + cdef: + shared_ptr[RandomAccessFile] rd_handle + + self.source = source + + get_reader(source, &rd_handle) + with nogil: + check_status(ORCFileReader.Open(rd_handle, self.allocator, + &self.reader)) + + def schema(self): + """ + The arrow schema for this file. + + Returns + ------- + schema : pyarrow.Schema + """ + cdef: + shared_ptr[CSchema] sp_arrow_schema + + with nogil: + check_status(deref(self.reader).ReadSchema(&sp_arrow_schema)) + + return pyarrow_wrap_schema(sp_arrow_schema) + + def nrows(self): + return deref(self.reader).NumberOfRows(); + + def nstripes(self): + return deref(self.reader).NumberOfStripes(); + + def read_stripe(self, n, include_indices=None): + cdef: + shared_ptr[CRecordBatch] sp_record_batch + RecordBatch batch + int64_t stripe + std_vector[int] indices + + stripe = n + + if include_indices is None: + with nogil: + check_status(deref(self.reader).ReadStripe(stripe, &sp_record_batch)) + else: + indices = include_indices + with nogil: + check_status(deref(self.reader).ReadStripe(stripe, indices, &sp_record_batch)) + + batch = RecordBatch() + batch.init(sp_record_batch) + return batch + + def read(self, include_indices=None): + cdef: + shared_ptr[CTable] sp_table + std_vector[int] indices + + if include_indices is None: + with nogil: + check_status(deref(self.reader).Read(&sp_table)) + else: + indices = include_indices + with nogil: + check_status(deref(self.reader).Read(indices, &sp_table)) + + return pyarrow_wrap_table(sp_table) diff --git a/python/pyarrow/orc.py b/python/pyarrow/orc.py new file mode 100644 index 0000000000000..22451d521825e --- /dev/null +++ b/python/pyarrow/orc.py @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from itertools import count +from numbers import Integral + +from pyarrow import _orc +from pyarrow import types +from pyarrow.lib import Schema + + +def _is_map(typ): + return (types.is_list(typ) and + types.is_struct(typ.value_type) and + typ.value_type.num_children == 2 and + typ.value_type[0].name == 'key' and + typ.value_type[1].name == 'value') + + +def _traverse(typ, counter): + if isinstance(typ, Schema) or types.is_struct(typ): + for field in typ: + path = (field.name,) + yield path, next(counter) + for sub, c in _traverse(field.type, counter): + yield path + sub, c + elif _is_map(typ): + for sub_c in _traverse(typ.value_type, counter): + yield sub_c + elif types.is_list(typ): + # Skip one index for list type, since this can never be selected + # directly + next(counter) + for sub_c in _traverse(typ.value_type, counter): + yield sub_c + elif types.is_union(typ): + # Union types not supported, just skip the indexes + for dtype in typ: + next(counter) + for sub_c in _traverse(dtype, counter): + pass + + +def _schema_to_indices(schema): + return {'.'.join(i): c for i, c in _traverse(schema, count(1))} + + +class ORCFile(object): + """ + Reader interface for a single ORC file + + Parameters + ---------- + source : str or pyarrow.io.NativeFile + Readable source. For passing Python file objects or byte buffers, + see pyarrow.io.PythonFileInterface or pyarrow.io.BufferReader. + """ + def __init__(self, source): + self.reader = _orc.ORCReader() + self.reader.open(source) + self._column_index_lookup = _schema_to_indices(self.schema) + + @property + def schema(self): + """The file schema, as an arrow schema""" + return self.reader.schema() + + @property + def nrows(self): + """The number of rows in the file""" + return self.reader.nrows() + + @property + def nstripes(self): + """The number of stripes in the file""" + return self.reader.nstripes() + + def _select_indices(self, columns=None): + if columns is None: + return None + + schema = self.schema + indices = [] + for col in columns: + if isinstance(col, Integral): + col = int(col) + if 0 <= col < len(schema): + col = schema[col].name + else: + raise ValueError("Column indices must be in 0 <= ind < %d," + " got %d" % (len(schema), col)) + if col in self._column_index_lookup: + indices.append(self._column_index_lookup[col]) + else: + raise ValueError("Unknown column name %r" % col) + + return indices + + def read_stripe(self, n, columns=None): + """Read a single stripe from the file. + + Parameters + ---------- + n : int + The stripe index + columns : list + If not None, only these columns will be read from the stripe. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e' + + Returns + ------- + pyarrow.lib.RecordBatch + Content of the stripe as a RecordBatch. + """ + include_indices = self._select_indices(columns) + return self.reader.read_stripe(n, include_indices=include_indices) + + def read(self, columns=None): + """Read the whole file. + + Parameters + ---------- + columns : list + If not None, only these columns will be read from the file. A + column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e' + + Returns + ------- + pyarrow.lib.Table + Content of the file as a Table. + """ + include_indices = self._select_indices(columns) + return self.reader.read(include_indices=include_indices) diff --git a/python/setup.py b/python/setup.py index 32e76ab05d836..3d3831dc048c6 100644 --- a/python/setup.py +++ b/python/setup.py @@ -84,6 +84,7 @@ def run(self): ('with-parquet', None, 'build the Parquet extension'), ('with-static-parquet', None, 'link parquet statically'), ('with-plasma', None, 'build the Plasma extension'), + ('with-orc', None, 'build the ORC extension'), ('bundle-arrow-cpp', None, 'bundle the Arrow C++ libraries')] + _build_ext.user_options) @@ -109,12 +110,15 @@ def initialize_options(self): os.environ.get('PYARROW_WITH_STATIC_BOOST', '1')) self.with_plasma = strtobool( os.environ.get('PYARROW_WITH_PLASMA', '0')) + self.with_orc = strtobool( + os.environ.get('PYARROW_WITH_ORC', '0')) self.bundle_arrow_cpp = strtobool( os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0')) CYTHON_MODULE_NAMES = [ 'lib', '_parquet', + '_orc', 'plasma'] def _run_cmake(self): @@ -157,6 +161,9 @@ def _run_cmake(self): if self.with_plasma: cmake_options.append('-DPYARROW_BUILD_PLASMA=on') + if self.with_orc: + cmake_options.append('-DPYARROW_BUILD_ORC=on') + if len(self.cmake_cxxflags) > 0: cmake_options.append('-DPYARROW_CXXFLAGS="{0}"' .format(self.cmake_cxxflags)) @@ -284,6 +291,8 @@ def _failure_permitted(self, name): return True if name == 'plasma' and not self.with_plasma: return True + if name == '_orc' and not self.with_orc: + return True return False def _get_inplace_dir(self): From cb9c52e4b1d81446ddcfaf4485a8284006dc68d0 Mon Sep 17 00:00:00 2001 From: Adam Seibert Date: Thu, 4 Jan 2018 20:38:32 -0500 Subject: [PATCH 163/177] ARROW-1909: [C++] Enables building with benchmarks on windows These changes were necessary to compile on Windows with "-DARROW_BUILD_BENCHMARKS=ON". I added Shwlapi based on https://github.com/google/benchmark/issues/202. Author: Adam Seibert Closes #1406 from seibs/ARROW-1909 and squashes the following commits: 98602cd0 [Adam Seibert] ARROW-1909: [C++] Enables building with benchmarks on windows --- cpp/src/arrow/compute/compute-benchmark.cc | 2 +- cpp/src/arrow/util/CMakeLists.txt | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/compute/compute-benchmark.cc b/cpp/src/arrow/compute/compute-benchmark.cc index aa7d899c864e7..44df441394ad4 100644 --- a/cpp/src/arrow/compute/compute-benchmark.cc +++ b/cpp/src/arrow/compute/compute-benchmark.cc @@ -191,7 +191,7 @@ static void BM_UniqueString100bytes(benchmark::State& state) { BENCHMARK(BM_BuildDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond); BENCHMARK(BM_BuildStringDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond); -constexpr int64_t kHashBenchmarkLength = 1 << 24; +constexpr int kHashBenchmarkLength = 1 << 24; #define ADD_HASH_ARGS(WHAT) \ WHAT->Args({kHashBenchmarkLength, 50}) \ diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 42613d6a5f56b..a36dffb52e294 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -57,6 +57,11 @@ if (ARROW_BUILD_BENCHMARKS) target_link_libraries(arrow_benchmark_main benchmark ) + elseif(MSVC) + target_link_libraries(arrow_benchmark_main + benchmark + Shlwapi.lib + ) else() target_link_libraries(arrow_benchmark_main benchmark From 0115e93c9d891a1973f023645435866eb77cb420 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 4 Jan 2018 22:33:04 -0500 Subject: [PATCH 164/177] ARROW-1919: [Plasma] Test that object ids are 20 bytes Author: Philipp Moritz Closes #1421 from pcmoritz/plasma-object-ids and squashes the following commits: fc779087 [Philipp Moritz] fixes 9f613c0d [Philipp Moritz] fix windows test f1d7ca05 [Philipp Moritz] fix linting 6be7f4a9 [Philipp Moritz] Test that object ids are 20 bytes --- python/pyarrow/plasma.pyx | 3 +++ python/pyarrow/tests/test_plasma.py | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/plasma.pyx b/python/pyarrow/plasma.pyx index abeec32783398..29e233b6e4e67 100644 --- a/python/pyarrow/plasma.pyx +++ b/python/pyarrow/plasma.pyx @@ -136,6 +136,9 @@ cdef class ObjectID: CUniqueID data def __cinit__(self, object_id): + if not isinstance(object_id, bytes) or len(object_id) != 20: + raise ValueError("Object ID must by 20 bytes," + " is " + str(object_id)) self.data = CUniqueID.from_binary(object_id) def __richcmp__(ObjectID self, ObjectID object_id, operation): diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py index decdc73d911b4..9ea6476670441 100644 --- a/python/pyarrow/tests/test_plasma.py +++ b/python/pyarrow/tests/test_plasma.py @@ -362,7 +362,7 @@ def test_store_pandas_dataframe(self): # Read the DataFrame. [data] = self.plasma_client.get_buffers([object_id]) reader = pa.RecordBatchStreamReader(pa.BufferReader(data)) - result = reader.get_next_batch().to_pandas() + result = reader.read_next_batch().to_pandas() pd.util.testing.assert_frame_equal(df, result) @@ -747,3 +747,11 @@ def test_use_one_memory_mapped_file(self): with pytest.raises(pa.lib.PlasmaStoreFull): create_object(self.plasma_client, DEFAULT_PLASMA_STORE_MEMORY + 1, 0) + + +@pytest.mark.plasma +def test_object_id_size(): + import pyarrow.plasma as plasma + with pytest.raises(ValueError): + plasma.ObjectID("hello") + plasma.ObjectID(20 * b"0") From 4dbce607d50031a405af39d36e08cd03c5ffc764 Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Fri, 5 Jan 2018 13:08:59 -0800 Subject: [PATCH 165/177] ARROW-1962: [Java] Adding reset to ValueVector interface Adding `reset()` to the ValueVector interface and implementing where it is not done already. Removing unused abstract class BaseDataValueVector that is not used anymore by the UnionVector. Expanded reset tests to check that valueCount is 0, and buffers have same capacity and zeroed out. Author: Bryan Cutler Closes #1455 from BryanCutler/java-reset-ValueVector-ARROW-1962 and squashes the following commits: da994e1 [Bryan Cutler] typo a52e7db [Bryan Cutler] expanded reset documentations 1526a83 [Bryan Cutler] improved vector reset testing a251d10 [Bryan Cutler] reset should zero data buffer and set value count to 0 bf2a16a [Bryan Cutler] add reset to NullableMapVector to zero validity buffer 7fbde5b [Bryan Cutler] need to zero out vector buffers when reset b59addf [Bryan Cutler] adding reset to ValueVector interface, removing BaseDataValueVector --- .../main/codegen/templates/UnionVector.java | 8 +- .../arrow/vector/BaseDataValueVector.java | 129 ------------------ .../arrow/vector/BaseFixedWidthVector.java | 2 + .../arrow/vector/BaseVariableWidthVector.java | 2 + .../org/apache/arrow/vector/ValueVector.java | 11 +- .../org/apache/arrow/vector/ZeroVector.java | 4 + .../complex/BaseRepeatedValueVector.java | 7 + .../vector/complex/FixedSizeListVector.java | 7 + .../arrow/vector/complex/ListVector.java | 7 + .../arrow/vector/complex/MapVector.java | 8 ++ .../vector/complex/NullableMapVector.java | 13 +- .../apache/arrow/vector/TestVectorReset.java | 109 ++++++++++++++- 12 files changed, 165 insertions(+), 142 deletions(-) delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 9377bd0da211d..aa8178a92f80a 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; -import org.apache.arrow.vector.BaseDataValueVector; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -318,6 +317,13 @@ public void clear() { internalMap.clear(); } + @Override + public void reset() { + valueCount = 0; + typeBuffer.setZero(0, typeBuffer.capacity()); + internalMap.reset(); + } + @Override public Field getField() { List childFields = new ArrayList<>(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java deleted file mode 100644 index 80675135e3c1c..0000000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.ipc.message.ArrowFieldNode; - -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.TransferPair; - - -public abstract class BaseDataValueVector extends BaseValueVector implements BufferBacked { - - public static void load(ArrowFieldNode fieldNode, List vectors, List buffers) { - int expectedSize = vectors.size(); - if (buffers.size() != expectedSize) { - throw new IllegalArgumentException("Illegal buffer count, expected " + expectedSize + ", got: " + buffers.size()); - } - for (int i = 0; i < expectedSize; i++) { - vectors.get(i).load(fieldNode, buffers.get(i)); - } - } - - public static void truncateBufferBasedOnSize(List buffers, int bufferIndex, int byteSize) { - if (bufferIndex >= buffers.size()) { - throw new IllegalArgumentException("no buffer at index " + bufferIndex + ": " + buffers); - } - ArrowBuf buffer = buffers.get(bufferIndex); - if (buffer.writerIndex() < byteSize) { - throw new IllegalArgumentException("can not truncate buffer to a larger size " + byteSize + ": " + buffer.writerIndex()); - } - buffer.writerIndex(byteSize); - } - - public static List unload(List vectors) { - List result = new ArrayList<>(vectors.size()); - for (BufferBacked vector : vectors) { - result.add(vector.unLoad()); - } - return result; - } - - protected ArrowBuf data; - - public BaseDataValueVector(String name, BufferAllocator allocator) { - super(name, allocator); - data = allocator.getEmpty(); - } - - @Override - public void clear() { - data.release(); - data = allocator.getEmpty(); - super.clear(); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return getTransferPair(ref, allocator); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - ArrowBuf[] out; - if (getBufferSize() == 0) { - out = new ArrowBuf[0]; - } else { - out = new ArrowBuf[]{data}; - data.readerIndex(0); - if (clear) { - data.retain(1); - } - } - if (clear) { - clear(); - } - return out; - } - - @Override - public int getBufferSize() { - if (getValueCount() == 0) { - return 0; - } - return data.writerIndex(); - } - - public ArrowBuf getBuffer() { - return data; - } - - @Override - public void load(ArrowFieldNode fieldNode, ArrowBuf data) { - this.data.release(); - this.data = data.retain(allocator); - } - - @Override - public ArrowBuf unLoad() { - return this.data.readerIndex(0); - } - - /** - * This method has a similar effect of allocateNew() without actually clearing and reallocating - * the value vector. The purpose is to move the value vector to a "mutate" state - */ - public void reset() { - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 77026d4e5a112..702db9f528152 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -208,7 +208,9 @@ private void initValueBuffer() { * Reset the vector to initial state. Same as {@link #zeroVector()}. * Note that this method doesn't release any memory. */ + @Override public void reset() { + valueCount = 0; zeroVector(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index fbadb35d7556a..fff329a9b9d66 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -193,6 +193,7 @@ private int getOffsetBufferValueCapacity() { public void zeroVector() { initValidityBuffer(); initOffsetBuffer(); + valueBuffer.setZero(0, valueBuffer.capacity()); } /* zero out the validity buffer */ @@ -212,6 +213,7 @@ private void initOffsetBuffer() { public void reset() { zeroVector(); lastSet = -1; + valueCount = 0; } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index e77c1b12703ec..24cf59a0da81d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -100,10 +100,19 @@ public interface ValueVector extends Closeable, Iterable { void close(); /** - * Release the underlying ArrowBuf and reset the ValueVector to empty. + * Release any owned ArrowBuf and reset the ValueVector to the initial state. If the + * vector has any child vectors, they will also be cleared. */ void clear(); + /** + * Reset the ValueVector to the initial state without releasing any owned ArrowBuf. + * Buffer capacities will remain unchanged and any previous data will be zeroed out. + * This includes buffers for data, validity, offset, etc. If the vector has any + * child vectors, they will also be reset. + */ + void reset(); + /** * Get information about how this field is materialized. * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 962a1c947071f..2d3c543d03ffa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -72,6 +72,10 @@ public void close() { public void clear() { } + @Override + public void reset() { + } + @Override public Field getField() { return new Field(DATA_VECTOR_NAME, FieldType.nullable(new Null()), null); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 9a23fd8c356cb..d0a664ac01da2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -186,6 +186,13 @@ public void clear() { super.clear(); } + @Override + public void reset() { + offsetBuffer.setZero(0, offsetBuffer.capacity()); + vector.reset(); + valueCount = 0; + } + @Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 93a8127cfd3b4..9314a2566b124 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -271,6 +271,13 @@ public void clear() { super.clear(); } + @Override + public void reset() { + validityBuffer.setZero(0, validityBuffer.capacity()); + vector.reset(); + valueCount = 0; + } + @Override public ArrowBuf[] getBuffers(boolean clear) { setReaderAndWriterIndex(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 0c1daf4905222..8aeeb7e5a2886 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -513,6 +513,13 @@ public void clear() { lastSet = 0; } + @Override + public void reset() { + super.reset(); + validityBuffer.setZero(0, validityBuffer.capacity()); + lastSet = 0; + } + /** * Return the underlying buffers associated with this vector. Note that this doesn't * impact the reference counts for this buffer so it only should be used for in-context diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index e13084588ce0b..6eab6ef3dcdcc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -302,6 +302,14 @@ public void clear() { valueCount = 0; } + @Override + public void reset() { + for (final ValueVector v : getChildren()) { + v.reset(); + } + valueCount = 0; + } + @Override public Field getField() { List children = new ArrayList<>(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index d887b73064643..fb84d23b5074e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -305,6 +305,15 @@ public void clear() { super.clear(); } + /** + * Reset this vector to empty, does not release buffers + */ + @Override + public void reset() { + super.reset(); + validityBuffer.setZero(0, validityBuffer.capacity()); + } + /** * Release the validity buffer */ @@ -493,8 +502,4 @@ public void setValueCount(int valueCount) { super.setValueCount(valueCount); this.valueCount = valueCount; } - - public void reset() { - valueCount = 0; - } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java index 28903b1170e31..84ea9657f8c9f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java @@ -19,13 +19,23 @@ package org.apache.arrow.vector; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.*; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; +import org.apache.arrow.vector.types.pojo.ArrowType.Int; +import org.apache.arrow.vector.types.pojo.FieldType; + import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.nio.charset.StandardCharsets; + public class TestVectorReset { private BufferAllocator allocator; @@ -40,15 +50,100 @@ public void terminate() throws Exception { allocator.close(); } + private void resetVectorAndVerify(ValueVector vector, ArrowBuf[] bufs) { + int[] sizeBefore = new int[bufs.length]; + for (int i = 0; i < bufs.length; i++) { + sizeBefore[i] = bufs[i].capacity(); + } + vector.reset(); + for (int i = 0; i < bufs.length; i++) { + assertEquals(sizeBefore[i], bufs[i].capacity()); + verifyBufferZeroed(bufs[i]); + } + assertEquals(0, vector.getValueCount()); + } + + private void verifyBufferZeroed(ArrowBuf buf) { + for (int i = 0; i < buf.capacity(); i++) { + assertTrue((byte) 0 == buf.getByte(i)); + } + } + @Test public void testFixedTypeReset() { - try (final UInt4Vector vector = new UInt4Vector("", allocator)) { - vector.allocateNew(); - final int sizeBefore = vector.getBufferSize(); - vector.reAlloc(); - vector.reset(); - final int sizeAfter = vector.getBufferSize(); - assertEquals(sizeBefore, sizeAfter); + try (final UInt4Vector vector = new UInt4Vector("UInt4", allocator)) { + vector.allocateNewSafe(); + vector.setNull(0); + vector.setValueCount(1); + resetVectorAndVerify(vector, vector.getBuffers(false)); + } + } + + @Test + public void testVariableTypeReset() { + try (final VarCharVector vector = new VarCharVector("VarChar", allocator)) { + vector.allocateNewSafe(); + vector.set(0, "a".getBytes(StandardCharsets.UTF_8)); + vector.setLastSet(0); + vector.setValueCount(1); + resetVectorAndVerify(vector, vector.getBuffers(false)); + assertEquals(-1, vector.getLastSet()); + } + } + + @Test + public void testListTypeReset() { + try (final ListVector variableList = new ListVector("VarList", allocator, FieldType.nullable(MinorType.INT.getType()), null); + final FixedSizeListVector fixedList = new FixedSizeListVector("FixedList", allocator, FieldType.nullable(new FixedSizeList(2)), null) + ) { + // ListVector + variableList.allocateNewSafe(); + variableList.startNewValue(0); + variableList.endValue(0, 0); + variableList.setValueCount(1); + resetVectorAndVerify(variableList, variableList.getBuffers(false)); + assertEquals(0, variableList.getLastSet()); + + // FixedSizeListVector + fixedList.allocateNewSafe(); + fixedList.setNull(0); + fixedList.setValueCount(1); + resetVectorAndVerify(fixedList, fixedList.getBuffers(false)); + } + } + + @Test + public void testMapTypeReset() { + try (final MapVector mapVector = new MapVector("Map", allocator, FieldType.nullable(MinorType.INT.getType()), null); + final NullableMapVector nullableMapVector = new NullableMapVector("NullableMap", allocator, FieldType.nullable(MinorType.INT.getType()), null) + ) { + // MapVector + mapVector.allocateNewSafe(); + IntVector mapChild = mapVector.addOrGet("child", FieldType.nullable(new Int(32, true)), IntVector.class); + mapChild.setNull(0); + mapVector.setValueCount(1); + resetVectorAndVerify(mapVector, mapVector.getBuffers(false)); + + // NullableMapVector + nullableMapVector.allocateNewSafe(); + nullableMapVector.setNull(0); + nullableMapVector.setValueCount(1); + resetVectorAndVerify(nullableMapVector, nullableMapVector.getBuffers(false)); + } + } + + @Test + public void testUnionTypeReset() { + try (final UnionVector vector = new UnionVector("Union", allocator, null); + final IntVector dataVector = new IntVector("Int", allocator) + ) { + vector.getBufferSize(); + vector.allocateNewSafe(); + dataVector.allocateNewSafe(); + vector.addVector(dataVector); + dataVector.setNull(0); + vector.setValueCount(1); + resetVectorAndVerify(vector, vector.getBuffers(false)); } } } From bda85bb1d58b71b822b358653665407de0cd4cad Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Sun, 7 Jan 2018 12:19:29 -0500 Subject: [PATCH 166/177] ARROW-1970: [GLib] Add garrow_chunked_array_get_value_data_type() and garrow_chunked_array_get_value_type() Author: Kouhei Sutou Closes #1458 from kou/glib-add-chunked-array-get-value-type and squashes the following commits: 4d99a079 [Kouhei Sutou] [GLib] Add garrow_chunked_array_get_value_data_type() and garrow_chunked_array_get_value_type() --- c_glib/arrow-glib/chunked-array.cpp | 35 +++++++++++++++++++++++++++++ c_glib/arrow-glib/chunked-array.h | 5 +++++ c_glib/test/test-chunked-array.rb | 18 +++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/c_glib/arrow-glib/chunked-array.cpp b/c_glib/arrow-glib/chunked-array.cpp index 62d666fbcaaba..69195c57b1768 100644 --- a/c_glib/arrow-glib/chunked-array.cpp +++ b/c_glib/arrow-glib/chunked-array.cpp @@ -23,6 +23,8 @@ #include #include +#include +#include G_BEGIN_DECLS @@ -163,6 +165,39 @@ garrow_chunked_array_equal(GArrowChunkedArray *chunked_array, return arrow_chunked_array->Equals(arrow_other_chunked_array); } +/** + * garrow_chunked_array_get_value_data_type: + * @chunked_array: A #GArrowChunkedArray. + * + * Returns: (transfer full): The #GArrowDataType of the value of + * the chunked array. + * + * Since: 0.9.0 + */ +GArrowDataType * +garrow_chunked_array_get_value_data_type(GArrowChunkedArray *chunked_array) +{ + auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array); + auto arrow_type = arrow_chunked_array->type(); + return garrow_data_type_new_raw(&arrow_type); +} + +/** + * garrow_chunked_array_get_value_type: + * @chunked_array: A #GArrowChunkedArray. + * + * Returns: The #GArrowType of the value of the chunked array. + * + * Since: 0.9.0 + */ +GArrowType +garrow_chunked_array_get_value_type(GArrowChunkedArray *chunked_array) +{ + auto arrow_chunked_array = garrow_chunked_array_get_raw(chunked_array); + auto arrow_type = arrow_chunked_array->type(); + return garrow_type_from_raw(arrow_type->id()); +} + /** * garrow_chunked_array_get_length: * @chunked_array: A #GArrowChunkedArray. diff --git a/c_glib/arrow-glib/chunked-array.h b/c_glib/arrow-glib/chunked-array.h index c5f986a631835..0c3c81a744ceb 100644 --- a/c_glib/arrow-glib/chunked-array.h +++ b/c_glib/arrow-glib/chunked-array.h @@ -70,6 +70,11 @@ GArrowChunkedArray *garrow_chunked_array_new(GList *chunks); gboolean garrow_chunked_array_equal(GArrowChunkedArray *chunked_array, GArrowChunkedArray *other_chunked_array); +GArrowDataType * +garrow_chunked_array_get_value_data_type(GArrowChunkedArray *chunked_array); +GArrowType +garrow_chunked_array_get_value_type(GArrowChunkedArray *chunked_array); + guint64 garrow_chunked_array_get_length (GArrowChunkedArray *chunked_array); guint64 garrow_chunked_array_get_n_nulls(GArrowChunkedArray *chunked_array); guint garrow_chunked_array_get_n_chunks (GArrowChunkedArray *chunked_array); diff --git a/c_glib/test/test-chunked-array.rb b/c_glib/test/test-chunked-array.rb index cde7a8b0c61f1..9287058e1abc6 100644 --- a/c_glib/test/test-chunked-array.rb +++ b/c_glib/test/test-chunked-array.rb @@ -31,6 +31,24 @@ def test_equal Arrow::ChunkedArray.new(chunks2)) end + def test_value_data_type + chunks = [ + build_boolean_array([true, false]), + build_boolean_array([true]), + ] + assert_equal(Arrow::BooleanDataType.new, + Arrow::ChunkedArray.new(chunks).value_data_type) + end + + def test_value_type + chunks = [ + build_boolean_array([true, false]), + build_boolean_array([true]), + ] + assert_equal(Arrow::Type::BOOL, + Arrow::ChunkedArray.new(chunks).value_type) + end + def test_length chunks = [ build_boolean_array([true, false]), From 234ff713dd9ecbcc2824d336b4e4f9d059a490dd Mon Sep 17 00:00:00 2001 From: Albert Shieh Date: Sun, 7 Jan 2018 12:28:19 -0500 Subject: [PATCH 167/177] ARROW-1958: [Python] Error in pandas conversion for datetimetz row index Fix conversion of datetimetz row index for non-UTC time zones in to_pandas. Author: Albert Shieh Closes #1454 from adshieh/master and squashes the following commits: 6f413027 [Albert Shieh] Fix pandas conversion for datetimetz row index. --- python/pyarrow/pandas_compat.py | 9 ++++++++- python/pyarrow/tests/test_convert_pandas.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index f08049f6adba0..f3089d2a012a6 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -459,6 +459,8 @@ def _make_datetimetz(tz): def table_to_blockmanager(options, table, memory_pool, nthreads=1, categoricals=None): + from pyarrow.compat import DatetimeTZDtype + index_columns = [] columns = [] column_indexes = [] @@ -517,7 +519,12 @@ def table_to_blockmanager(options, table, memory_pool, nthreads=1, # non-writeable arrays when calling MultiIndex.from_arrays values = values.copy() - index_arrays.append(pd.Series(values, dtype=col_pandas.dtype)) + if isinstance(col_pandas.dtype, DatetimeTZDtype): + index_array = (pd.Series(values).dt.tz_localize('utc') + .dt.tz_convert(col_pandas.dtype.tz)) + else: + index_array = pd.Series(values, dtype=col_pandas.dtype) + index_arrays.append(index_array) index_names.append( _backwards_compatible_index_name(raw_name, logical_name) ) diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py index 76b55cf90431c..83b1da135eea4 100644 --- a/python/pyarrow/tests/test_convert_pandas.py +++ b/python/pyarrow/tests/test_convert_pandas.py @@ -254,6 +254,16 @@ def test_datetimetz_column_index(self): md = column_indexes['metadata'] assert md['timezone'] == 'America/New_York' + def test_datetimetz_row_index(self): + df = pd.DataFrame({ + 'a': pd.date_range( + start='2017-01-01', periods=3, tz='America/New_York' + ) + }) + df = df.set_index('a') + + _check_pandas_roundtrip(df, preserve_index=True) + def test_categorical_row_index(self): df = pd.DataFrame({'a': [1, 2, 3], 'b': [1, 2, 3]}) df['a'] = df.a.astype('category') From 95d894da250ee7d5dfccb09e2d349e5748cc5fb8 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Sun, 7 Jan 2018 13:48:08 -0800 Subject: [PATCH 168/177] ARROW-1972: [Python] Import pyarrow in DeserializeObject. Author: Robert Nishihara Author: Philipp Moritz Closes #1463 from robertnishihara/segfaultfix and squashes the following commits: ec8a6c5 [Robert Nishihara] Add comment. 6222340 [Philipp Moritz] fix tests, linting, add license 3e969db [Robert Nishihara] Simplify tests. 8aa3fca [Philipp Moritz] add regression test bfa0851 [Robert Nishihara] Import pyarrow in DeserializeObject. --- cpp/src/arrow/python/arrow_to_python.cc | 1 + python/pyarrow/tests/deserialize_buffer.py | 26 ++++++++++++++++++++++ python/pyarrow/tests/test_serialization.py | 14 ++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 python/pyarrow/tests/deserialize_buffer.py diff --git a/cpp/src/arrow/python/arrow_to_python.cc b/cpp/src/arrow/python/arrow_to_python.cc index ce539a597be37..c060ab8bfd6db 100644 --- a/cpp/src/arrow/python/arrow_to_python.cc +++ b/cpp/src/arrow/python/arrow_to_python.cc @@ -284,6 +284,7 @@ Status DeserializeObject(PyObject* context, const SerializedPyObject& obj, PyObj PyObject** out) { PyAcquireGIL lock; PyDateTime_IMPORT; + import_pyarrow(); return DeserializeList(context, *obj.batch->column(0), 0, obj.batch->num_rows(), base, obj, out); } diff --git a/python/pyarrow/tests/deserialize_buffer.py b/python/pyarrow/tests/deserialize_buffer.py new file mode 100644 index 0000000000000..982dc6695d590 --- /dev/null +++ b/python/pyarrow/tests/deserialize_buffer.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This file is called from a test in test_serialization.py. + +import sys + +import pyarrow as pa + +with open(sys.argv[1], 'rb') as f: + data = f.read() + pa.deserialize(data) diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py index f245dc299cb25..6116556386b1a 100644 --- a/python/pyarrow/tests/test_serialization.py +++ b/python/pyarrow/tests/test_serialization.py @@ -541,3 +541,17 @@ def deserialize_regex(serialized, q): p.start() assert q.get().pattern == regex.pattern p.join() + + +def test_deserialize_buffer_in_different_process(): + import tempfile + import subprocess + + f = tempfile.NamedTemporaryFile(delete=False) + b = pa.serialize(pa.frombuffer(b'hello')).to_buffer() + f.write(b.to_pybytes()) + f.close() + + dir_path = os.path.dirname(os.path.realpath(__file__)) + python_file = os.path.join(dir_path, 'deserialize_buffer.py') + subprocess.check_call(['python', python_file, f.name]) From ee134f0e4b285e17f33a56e7750836dd82a13735 Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Mon, 8 Jan 2018 15:17:12 -0500 Subject: [PATCH 169/177] ARROW-1969: [C++] Don't build ORC extension by default - Turns off building optional ORC extension by default - Fixes travis builds to turn on ORC extension for a few branches - Adds trivial import test to python build - Adds documentation on how to build optional ORC extension Author: Jim Crist Closes #1457 from jcrist/orc-off-by-default and squashes the following commits: fc9898d8 [Jim Crist] Document how to build ORC integration 950ae38e [Jim Crist] ORC integration is off by default --- .travis.yml | 2 ++ ci/travis_before_script_cpp.sh | 4 ++++ ci/travis_script_python.sh | 2 ++ cpp/CMakeLists.txt | 2 +- cpp/README.md | 7 +++++++ python/doc/source/development.rst | 6 ++++++ python/manylinux1/build_arrow.sh | 2 +- 7 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ddadf739aab25..58d6786aab16a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,7 @@ matrix: - export ARROW_TRAVIS_USE_TOOLCHAIN=1 - export ARROW_TRAVIS_VALGRIND=1 - export ARROW_TRAVIS_PLASMA=1 + - export ARROW_TRAVIS_ORC=1 - export ARROW_TRAVIS_CLANG_FORMAT=1 - export ARROW_BUILD_WARNING_LEVEL=CHECKIN - export CC="clang-4.0" @@ -75,6 +76,7 @@ matrix: before_script: - export ARROW_TRAVIS_USE_TOOLCHAIN=1 - export ARROW_TRAVIS_PLASMA=1 + - export ARROW_TRAVIS_ORC=1 - export ARROW_BUILD_WARNING_LEVEL=CHECKIN - travis_wait 50 $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index 664f7ce5fed2d..fd2c1644638c4 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -84,6 +84,10 @@ if [ $ARROW_TRAVIS_PLASMA == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON" fi +if [ $ARROW_TRAVIS_ORC == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_ORC=ON" +fi + if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON" fi diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 444386fdf955e..9e74906d03739 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -63,6 +63,7 @@ cmake -GNinja \ -DARROW_BUILD_UTILITIES=off \ -DARROW_PLASMA=on \ -DARROW_PYTHON=on \ + -DARROW_ORC=on \ -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ $ARROW_CPP_DIR @@ -88,6 +89,7 @@ popd python -c "import pyarrow.parquet" python -c "import pyarrow.plasma" +python -c "import pyarrow.orc" if [ $TRAVIS_OS_NAME == "linux" ]; then export PLASMA_VALGRIND=1 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ede13af5d413b..0558cba495a83 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -121,7 +121,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") option(ARROW_ORC "Build the Arrow ORC adapter" - ON) + OFF) option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" diff --git a/cpp/README.md b/cpp/README.md index 92e87cb436ada..39a1ccac64818 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -145,6 +145,13 @@ The CUDA toolchain used to build the library can be customized by using the This library is still in Alpha stages, and subject to API changes without deprecation warnings. +### Building Apache ORC integration (optional) + +The optional arrow reader for the Apache ORC format (found in the +`arrow::adapters::orc` namespace) can be built by passing `-DARROW_ORC=on`. +This is currently not supported on windows. Note that this functionality is +still in Alpha stages, and subject to API changes without deprecation warnings. + ### API documentation To generate the (html) API documentation, run the following command in the apidoc diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst index 1c1a308fc4a56..01844fa18d133 100644 --- a/python/doc/source/development.rst +++ b/python/doc/source/development.rst @@ -175,6 +175,9 @@ Now build and install the Arrow C++ libraries: If you don't want to build and install the Plasma in-memory object store, you can omit the ``-DARROW_PLASMA=on`` flag. +To add support for the experimental Apache ORC integration, include +``-DARROW_ORC=on`` in these flags. + Now, optionally build and install the Apache Parquet libraries in your toolchain: @@ -205,6 +208,9 @@ Now, build pyarrow: If you did not build parquet-cpp, you can omit ``--with-parquet`` and if you did not build with plasma, you can omit ``--with-plasma``. +If you built with the experimental Apache ORC integration, include +``--with-orc`` in these flags. + You should be able to run the unit tests with: .. code-block:: shell diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index ced4556e92ad5..4d816bec9b6a4 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -58,7 +58,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON} mkdir -p "${ARROW_BUILD_DIR}" pushd "${ARROW_BUILD_DIR}" - PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON .. + PATH="$(cpython_path $PYTHON)/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON .. make -j5 install popd From 1b0732cde515f9ec9b42f98061f9204796c9e154 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Wed, 10 Jan 2018 23:34:47 +0900 Subject: [PATCH 170/177] ARROW-1965: [GLib] Add garrow_array_builder_get_value_data_type() garrow_array_builder_get_value_type() is also added for convenient. Author: Kouhei Sutou Closes #1456 from kou/glib-add-builder-get-value-type and squashes the following commits: 5a4b1229 [Kouhei Sutou] [GLib] Add garrow_array_builder_get_value_data_type() --- c_glib/arrow-glib/array-builder.cpp | 34 ++++++ c_glib/arrow-glib/array-builder.h | 4 + c_glib/test/test-array-builder.rb | 156 ++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+) diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp index a5df681421c25..7625bcd51ee0f 100644 --- a/c_glib/arrow-glib/array-builder.cpp +++ b/c_glib/arrow-glib/array-builder.cpp @@ -24,6 +24,7 @@ #include #include #include +#include template gboolean @@ -327,6 +328,39 @@ garrow_array_builder_release_ownership(GArrowArrayBuilder *builder) priv->have_ownership = FALSE; } +/** + * garrow_array_builder_get_value_data_type: + * @builder: A #GArrowArrayBuilder. + * + * Returns: (transfer full): The #GArrowDataType of the value of + * the array builder. + * + * Since: 0.9.0 + */ +GArrowDataType * +garrow_array_builder_get_value_data_type(GArrowArrayBuilder *builder) +{ + auto arrow_builder = garrow_array_builder_get_raw(builder); + auto arrow_type = arrow_builder->type(); + return garrow_data_type_new_raw(&arrow_type); +} + +/** + * garrow_array_builder_get_value_type: + * @builder: A #GArrowArrayBuilder. + * + * Returns: The #GArrowType of the value of the array builder. + * + * Since: 0.9.0 + */ +GArrowType +garrow_array_builder_get_value_type(GArrowArrayBuilder *builder) +{ + auto arrow_builder = garrow_array_builder_get_raw(builder); + auto arrow_type = arrow_builder->type(); + return garrow_type_from_raw(arrow_type->id()); +} + /** * garrow_array_builder_finish: * @builder: A #GArrowArrayBuilder. diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h index 19dadb30999bd..ea95f31e8fae1 100644 --- a/c_glib/arrow-glib/array-builder.h +++ b/c_glib/arrow-glib/array-builder.h @@ -37,6 +37,10 @@ struct _GArrowArrayBuilderClass void garrow_array_builder_release_ownership(GArrowArrayBuilder *builder); +GArrowDataType * +garrow_array_builder_get_value_data_type(GArrowArrayBuilder *builder); +GArrowType garrow_array_builder_get_value_type(GArrowArrayBuilder *builder); + GArrowArray *garrow_array_builder_finish (GArrowArrayBuilder *builder, GError **error); diff --git a/c_glib/test/test-array-builder.rb b/c_glib/test/test-array-builder.rb index 92976a424ccad..a773131e15b5d 100644 --- a/c_glib/test/test-array-builder.rb +++ b/c_glib/test/test-array-builder.rb @@ -76,6 +76,18 @@ def test_negative end end +module ArrayBuilderValueTypeTests + def test_value_data_type + assert_equal(value_data_type, + build_array(sample_values).value_data_type) + end + + def test_value_type + assert_equal(value_data_type.id, + build_array(sample_values).value_type) + end +end + class TestArrayBuilder < Test::Unit::TestCase include Helper::Buildable include Helper::Omittable @@ -93,6 +105,10 @@ def create_builder Arrow::BooleanArrayBuilder.new end + def value_data_type + Arrow::BooleanDataType.new + end + def builder_class_name "boolean-array-builder" end @@ -101,6 +117,10 @@ def sample_values [true, false, true] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -115,6 +135,10 @@ def create_builder Arrow::IntArrayBuilder.new end + def value_data_type + Arrow::Int8DataType.new + end + def builder_class_name "int-array-builder" end @@ -123,6 +147,10 @@ def sample_values [1, -2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -137,6 +165,10 @@ def create_builder Arrow::UIntArrayBuilder.new end + def value_data_type + Arrow::UInt8DataType.new + end + def builder_class_name "uint-array-builder" end @@ -145,6 +177,10 @@ def sample_values [1, 2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -159,6 +195,10 @@ def create_builder Arrow::Int8ArrayBuilder.new end + def value_data_type + Arrow::Int8DataType.new + end + def builder_class_name "int8-array-builder" end @@ -167,6 +207,10 @@ def sample_values [1, -2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -181,6 +225,10 @@ def create_builder Arrow::UInt8ArrayBuilder.new end + def value_data_type + Arrow::UInt8DataType.new + end + def builder_class_name "uint8-array-builder" end @@ -189,6 +237,10 @@ def sample_values [1, 2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -203,6 +255,10 @@ def create_builder Arrow::Int16ArrayBuilder.new end + def value_data_type + Arrow::Int16DataType.new + end + def builder_class_name "int16-array-builder" end @@ -211,6 +267,10 @@ def sample_values [1, -2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -225,6 +285,10 @@ def create_builder Arrow::UInt16ArrayBuilder.new end + def value_data_type + Arrow::UInt16DataType.new + end + def builder_class_name "uint16-array-builder" end @@ -233,6 +297,10 @@ def sample_values [1, 2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -247,6 +315,10 @@ def create_builder Arrow::Int32ArrayBuilder.new end + def value_data_type + Arrow::Int32DataType.new + end + def builder_class_name "int32-array-builder" end @@ -255,6 +327,10 @@ def sample_values [1, -2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -269,6 +345,10 @@ def create_builder Arrow::UInt32ArrayBuilder.new end + def value_data_type + Arrow::UInt32DataType.new + end + def builder_class_name "uint32-array-builder" end @@ -277,6 +357,10 @@ def sample_values [1, 2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -291,6 +375,10 @@ def create_builder Arrow::Int64ArrayBuilder.new end + def value_data_type + Arrow::Int64DataType.new + end + def builder_class_name "int64-array-builder" end @@ -299,6 +387,10 @@ def sample_values [1, -2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -313,6 +405,10 @@ def create_builder Arrow::UInt64ArrayBuilder.new end + def value_data_type + Arrow::UInt64DataType.new + end + def builder_class_name "uint64-array-builder" end @@ -321,6 +417,10 @@ def sample_values [1, 2, 3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -335,6 +435,10 @@ def create_builder Arrow::FloatArrayBuilder.new end + def value_data_type + Arrow::FloatDataType.new + end + def builder_class_name "float-array-builder" end @@ -343,6 +447,10 @@ def sample_values [1.1, -2.2, 3.3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -357,6 +465,10 @@ def create_builder Arrow::DoubleArrayBuilder.new end + def value_data_type + Arrow::DoubleDataType.new + end + def builder_class_name "double-array-builder" end @@ -365,6 +477,10 @@ def sample_values [1.1, -2.2, 3.3] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -379,6 +495,10 @@ def create_builder Arrow::Date32ArrayBuilder.new end + def value_data_type + Arrow::Date32DataType.new + end + def builder_class_name "date32-array-builder" end @@ -391,6 +511,10 @@ def sample_values ] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -405,6 +529,10 @@ def create_builder Arrow::Date64ArrayBuilder.new end + def value_data_type + Arrow::Date64DataType.new + end + def builder_class_name "date64-array-builder" end @@ -417,6 +545,10 @@ def sample_values ] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -432,6 +564,10 @@ def create_builder Arrow::TimestampArrayBuilder.new(data_type) end + def value_data_type + Arrow::TimestampDataType.new(:milli) + end + def builder_class_name "timestamp-array-builder" end @@ -444,6 +580,10 @@ def sample_values ] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -459,6 +599,10 @@ def create_builder Arrow::Time32ArrayBuilder.new(data_type) end + def value_data_type + Arrow::Time32DataType.new(:second) + end + def builder_class_name "time32-array-builder" end @@ -471,6 +615,10 @@ def sample_values ] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end @@ -486,6 +634,10 @@ def create_builder Arrow::Time64ArrayBuilder.new(data_type) end + def value_data_type + Arrow::Time64DataType.new(:micro) + end + def builder_class_name "time64-array-builder" end @@ -498,6 +650,10 @@ def sample_values ] end + sub_test_case("value type") do + include ArrayBuilderValueTypeTests + end + sub_test_case("#append_values") do include ArrayBuilderAppendValuesTests end From d0a93ba3fac91fe12b83d2ef7fccf97435c4f4ec Mon Sep 17 00:00:00 2001 From: Jim Crist Date: Wed, 10 Jan 2018 17:04:15 -0500 Subject: [PATCH 171/177] ARROW-1980: [Python] Fix race condition in write_to_dataset One race condition was already fixed, but another one exists when writing by partition. Author: Jim Crist Closes #1468 from jcrist/fix-race-condition-parquet-partition and squashes the following commits: 180c4a24 [Jim Crist] Fix race condition in write_to_dataset --- python/pyarrow/parquet.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py index d9f1bd2c36430..151e0df8a22d0 100644 --- a/python/pyarrow/parquet.py +++ b/python/pyarrow/parquet.py @@ -966,6 +966,14 @@ def write_table(table, where, row_group_size=None, version='1.0', """.format(_parquet_writer_arg_docs) +def _mkdir_if_not_exists(fs, path): + if fs._isfilestore() and not fs.exists(path): + try: + fs.mkdir(path) + except OSError: + assert fs.exists(path) + + def write_to_dataset(table, root_path, partition_cols=None, filesystem=None, preserve_index=True, **kwargs): """ @@ -1012,11 +1020,7 @@ def write_to_dataset(table, root_path, partition_cols=None, else: fs = _ensure_filesystem(filesystem) - if fs._isfilestore() and not fs.exists(root_path): - try: - fs.mkdir(root_path) - except OSError: - assert fs.exists(root_path) + _mkdir_if_not_exists(fs, root_path) if partition_cols is not None and len(partition_cols) > 0: df = table.to_pandas() @@ -1034,8 +1038,7 @@ def write_to_dataset(table, root_path, partition_cols=None, subtable = Table.from_pandas(subgroup, preserve_index=preserve_index) prefix = "/".join([root_path, subdir]) - if fs._isfilestore() and not fs.exists(prefix): - fs.mkdir(prefix) + _mkdir_if_not_exists(fs, prefix) outfile = compat.guid() + ".parquet" full_path = "/".join([prefix, outfile]) with fs.open(full_path, 'wb') as f: From 3e14e12427bf9b81b8c43e54c2def509c1ca75f0 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 10 Jan 2018 17:05:47 -0500 Subject: [PATCH 172/177] ARROW-1978: [Website] Consolidate Powered By project list, add more visibly to front page There can be a fine line between what is an open source project and what is a company and what is a product or organization, so to keep things simple I have consolidated things into a single list without passing any judgments. I have also added a link to the Powered By page that is visible without digging into any menus on the front page Author: Wes McKinney Closes #1465 from wesm/website-powered-by-simplify and squashes the following commits: 7c43e0bd [Wes McKinney] Suggest adding to Powered By by pull request instead of e-mailing 9a61eed0 [Wes McKinney] Consolidate Powered By project list, add more visibly to front page --- site/index.html | 4 ++++ site/powered_by.md | 45 ++++++++++++++++++--------------------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/site/index.html b/site/index.html index ffa8d54af2f4f..87995cbabed48 100644 --- a/site/index.html +++ b/site/index.html @@ -38,6 +38,10 @@

    Standard

    projects, including Calcite, Cassandra, Drill, Hadoop, HBase, Ibis, Impala, Kudu, Pandas, Parquet, Phoenix, Spark, and Storm making it the de-facto standard for columnar in-memory analytics.

    + +

    Learn more about projects that are Powered By Apache Arrow

    + diff --git a/site/powered_by.md b/site/powered_by.md index 56f6e2bcee41a..4033202679817 100644 --- a/site/powered_by.md +++ b/site/powered_by.md @@ -43,11 +43,9 @@ names, etc.) like "arrow-foo". These are permitted. Nominative use of trademarks in descriptions is also always allowed, as in "BigCoProduct is a widget for Apache Arrow". -### Open Source Projects - -To add yourself to the list, please email dev@arrow.apache.org with your +To add yourself to the list, please open a pull request adding your organization name, URL, a list of which Arrow components you are using, and a -short description of your use case. +short description of your use case. See the following for some examples. * **[Apache Parquet][3]:** A columnar storage format available to any project in the Hadoop ecosystem, regardless of the choice of data processing @@ -61,10 +59,23 @@ short description of your use case. * **[Dask][15]:** Python library for parallel and distributed execution of dynamic task graphs. Dask supports using pyarrow for accessing Parquet files +* **[Dremio][9]:** A self-service data platform. Dremio makes it easy for + users to discover, curate, accelerate, and share data from any source. + It includes a distributed SQL execution engine based on Apache Arrow. + Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow + buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI, + Python, R, and more (all backed by Apache Arrow). * **[GeoMesa][8]:** A suite of tools that enables large-scale geospatial query and analytics on distributed computing systems. GeoMesa supports query results in the Arrow IPC format, which can then be used for in-browser visualizations and/or further analytics. +* **[GOAI][19]:** Open GPU-Accelerated Analytics Initiative for Arrow-powered + analytics across GPU tools and vendors +* **[Graphistry][18]:** Supercharged Visual Investigation Platform used by + teams for security, anti-fraud, and related investigations. The Graphistry + team uses Arrow in its NodeJS GPU backend and client libraries, and is an + early contributing member to GOAI and Arrow\[JS\] focused on bringing these + technologies to the enterprise. * **[libgdf][14]:** A C library of CUDA-based analytics functions and GPU IPC support for structured data. Uses the Arrow IPC format and targets the Arrow memory layout in its analytic functions. This work is part of the [GPU Open @@ -75,6 +86,9 @@ short description of your use case. * **[pandas][12]:** data analysis toolkit for Python programmers. pandas supports reading and writing Parquet files using pyarrow. Several pandas core developers are also contributors to Apache Arrow. +* **[Quilt Data][13]:** Quilt is a data package manager, designed to make + managing data as easy as managing code. It supports Parquet format via + pyarrow for data access. * **[Ray][5]:** A flexible, high-performance distributed execution framework with a focus on machine learning and AI applications. Uses Arrow to efficiently store Python data structures containing large arrays of numerical @@ -91,29 +105,6 @@ short description of your use case. Arrow Tables and RecordBatches in addition to the Python Database API Specification 2.0. -### Companies and Organizations - -To add yourself to the list, please email dev@arrow.apache.org with your -organization name, URL, a list of which Arrow components you are using, and a -short description of your use case. - -* **[Dremio][9]:** A self-service data platform. Dremio makes it easy for - users to discover, curate, accelerate, and share data from any source. - It includes a distributed SQL execution engine based on Apache Arrow. - Dremio reads data from any source (RDBMS, HDFS, S3, NoSQL) into Arrow - buffers, and provides fast SQL access via ODBC, JDBC, and REST for BI, - Python, R, and more (all backed by Apache Arrow). -* **[GOAI][19]:** Open GPU-Accelerated Analytics Initiative for Arrow-powered - analytics across GPU tools and vendors -* **[Graphistry][18]:** Supercharged Visual Investigation Platform used by - teams for security, anti-fraud, and related investigations. The Graphistry - team uses Arrow in its NodeJS GPU backend and client libraries, and is an - early contributing member to GOAI and Arrow\[JS\] focused on bringing these - technologies to the enterprise. -* **[Quilt Data][13]:** Quilt is a data package manager, designed to make - managing data as easy as managing code. It supports Parquet format via - pyarrow for data access. - [1]: https://www.apache.org/foundation/marks/ [2]: https://www.apache.org/foundation/marks/faq/ [3]: https://parquet.apache.org/ From 289e205c16bdeb5a63b629ce5df05e4164b78e72 Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Wed, 10 Jan 2018 17:07:14 -0500 Subject: [PATCH 173/177] ARROW-1977: [C++] Update windows dev docs Author: Chris Bartak Closes #1466 from chris-b1/windows-cpp-devdoc and squashes the following commits: 0990ddc0 [Chris Bartak] remove python from cpp conda env 4dab8b05 [Chris Bartak] updates windows c++ env docs --- cpp/apidoc/Windows.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md index e7e83f1946b56..aa3d31f1f7b90 100644 --- a/cpp/apidoc/Windows.md +++ b/cpp/apidoc/Windows.md @@ -55,20 +55,16 @@ previous step: activate arrow-dev ``` -We are using [cmake][4] tool to support Windows builds. +We are using the [cmake][4] tool to support Windows builds. To allow cmake to pick up 3rd party dependencies, you should set `ARROW_BUILD_TOOLCHAIN` environment variable to contain `Library` folder path of new created on previous step `arrow-dev` conda environment. -For instance, if `Miniconda` was installed to default destination, `Library` -folder path for `arrow-dev` conda environment will be as following: +To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal +session you can run following. `%CONDA_PREFIX` is set by conda to the current environment +root by the `activate` script. ```shell -C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library -``` - -To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal session you can run following: -```shell -set ARROW_BUILD_TOOLCHAIN=C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library +set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library ``` To validate value of `ARROW_BUILD_TOOLCHAIN` environment variable you can run following terminal command: From 9eae5089b5099548f67bbeca9927ca81df9d5de8 Mon Sep 17 00:00:00 2001 From: Adam Seibert Date: Wed, 10 Jan 2018 17:10:39 -0500 Subject: [PATCH 174/177] ARROW-764: [C++] Improves performance of CopyBitmap and adds benchmarks I took a swing at improving the CopyBitmap performance (benchmarks below). I'm a C/C++ novice, so I thought I'd get some feedback before I went too much further. **Starting Point** ``` Run on (4 X 2208 MHz CPU s) 12/13/17 21:15:18 Benchmark Time CPU Iterations ------------------------------------------------------------------------------ BM_CopyBitmap/97.6563k/0/min_time:1.000 4779 us 4758 us 289 20.0445MB/s BM_CopyBitmap/976.563k/0/min_time:1.000 47740 us 47476 us 26 20.0875MB/s BM_CopyBitmap/97.6563k/4/min_time:1.000 4858 us 4866 us 289 19.5991MB/s BM_CopyBitmap/976.563k/4/min_time:1.000 48117 us 47953 us 29 19.8879MB/s ``` **Using stanford bithacks for SetBitTo** ``` Run on (4 X 2208 MHz CPU s) 12/13/17 21:22:05 Benchmark Time CPU Iterations ------------------------------------------------------------------------------ BM_CopyBitmap/97.6563k/0/min_time:1.000 1647 us 1649 us 815 57.8415MB/s BM_CopyBitmap/976.563k/0/min_time:1.000 16368 us 16397 us 81 58.1629MB/s BM_CopyBitmap/97.6563k/4/min_time:1.000 1599 us 1610 us 815 59.2186MB/s BM_CopyBitmap/976.563k/4/min_time:1.000 16026 us 16011 us 81 59.5644MB/s ``` **memcpy + shifting** *This solution provides varying performance depending on whether or not the bit offset is a multiple of 8* ``` Run on (4 X 2208 MHz CPU s) 12/13/17 21:23:44 Benchmark Time CPU Iterations ------------------------------------------------------------------------------ BM_CopyBitmap/97.6563k/0/min_time:1.000 5 us 5 us 280000 18.9651GB/s BM_CopyBitmap/976.563k/0/min_time:1.000 62 us 61 us 22400 15.1721GB/s BM_CopyBitmap/97.6563k/4/min_time:1.000 171 us 170 us 6892 560.872MB/s BM_CopyBitmap/976.563k/4/min_time:1.000 1639 us 1639 us 896 581.782MB/s ``` Author: Adam Seibert Closes #1422 from seibs/ARROW-764 and squashes the following commits: c813c8e2 [Adam Seibert] ARROW-764: [C++] Improves performance of CopyBitmap and adds benchmarks --- cpp/src/arrow/util/CMakeLists.txt | 6 ++- cpp/src/arrow/util/bit-util-benchmark.cc | 58 ++++++++++++++++++++++++ cpp/src/arrow/util/bit-util-test.cc | 17 +++---- cpp/src/arrow/util/bit-util.cc | 32 ++++++++++++- cpp/src/arrow/util/bit-util.h | 9 ++-- 5 files changed, 104 insertions(+), 18 deletions(-) create mode 100644 cpp/src/arrow/util/bit-util-benchmark.cc diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index a36dffb52e294..8b61a3acfe709 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -63,10 +63,10 @@ if (ARROW_BUILD_BENCHMARKS) Shlwapi.lib ) else() - target_link_libraries(arrow_benchmark_main + target_link_libraries(arrow_benchmark_main benchmark pthread - ) + ) endif() # TODO(wesm): Some benchmarks include gtest.h @@ -80,4 +80,6 @@ ADD_ARROW_TEST(key-value-metadata-test) ADD_ARROW_TEST(rle-encoding-test) ADD_ARROW_TEST(stl-util-test) +ADD_ARROW_BENCHMARK(bit-util-benchmark) + add_subdirectory(variant) diff --git a/cpp/src/arrow/util/bit-util-benchmark.cc b/cpp/src/arrow/util/bit-util-benchmark.cc new file mode 100644 index 0000000000000..8969dd80b157e --- /dev/null +++ b/cpp/src/arrow/util/bit-util-benchmark.cc @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "benchmark/benchmark.h" + +#include + +#include "arrow/buffer.h" +#include "arrow/builder.h" +#include "arrow/memory_pool.h" +#include "arrow/test-util.h" +#include "arrow/util/bit-util.h" + +namespace arrow { +namespace BitUtil { + +static void BM_CopyBitmap(benchmark::State& state) { // NOLINT non-const reference + const int kBufferSize = state.range(0); + + std::shared_ptr buffer; + ASSERT_OK(AllocateBuffer(default_memory_pool(), kBufferSize, &buffer)); + memset(buffer->mutable_data(), 0, kBufferSize); + test::random_bytes(kBufferSize, 0, buffer->mutable_data()); + + const int num_bits = kBufferSize * 8; + const uint8_t* src = buffer->data(); + + std::shared_ptr copy; + while (state.KeepRunning()) { + ABORT_NOT_OK(CopyBitmap(default_memory_pool(), src, state.range(1), num_bits, ©)); + } + state.SetBytesProcessed(state.iterations() * kBufferSize * sizeof(int8_t)); +} + +BENCHMARK(BM_CopyBitmap) + ->Args({100000, 0}) + ->Args({1000000, 0}) + ->Args({100000, 4}) + ->Args({1000000, 4}) + ->MinTime(1.0) + ->Unit(benchmark::kMicrosecond); + +} // namespace BitUtil +} // namespace arrow diff --git a/cpp/src/arrow/util/bit-util-test.cc b/cpp/src/arrow/util/bit-util-test.cc index 92bdcb5fc0846..4c64dea374d01 100644 --- a/cpp/src/arrow/util/bit-util-test.cc +++ b/cpp/src/arrow/util/bit-util-test.cc @@ -165,19 +165,20 @@ TEST(BitUtilTests, TestCopyBitmap) { memset(buffer->mutable_data(), 0, kBufferSize); test::random_bytes(kBufferSize, 0, buffer->mutable_data()); - const int num_bits = kBufferSize * 8; - const uint8_t* src = buffer->data(); + std::vector lengths = {kBufferSize * 8 - 4, kBufferSize * 8}; std::vector offsets = {0, 12, 16, 32, 37, 63, 64, 128}; - for (int64_t offset : offsets) { - const int64_t copy_length = num_bits - offset; + for (int64_t num_bits : lengths) { + for (int64_t offset : offsets) { + const int64_t copy_length = num_bits - offset; - std::shared_ptr copy; - ASSERT_OK(CopyBitmap(default_memory_pool(), src, offset, copy_length, ©)); + std::shared_ptr copy; + ASSERT_OK(CopyBitmap(default_memory_pool(), src, offset, copy_length, ©)); - for (int64_t i = 0; i < copy_length; ++i) { - ASSERT_EQ(BitUtil::GetBit(src, i + offset), BitUtil::GetBit(copy->data(), i)); + for (int64_t i = 0; i < copy_length; ++i) { + ASSERT_EQ(BitUtil::GetBit(src, i + offset), BitUtil::GetBit(copy->data(), i)); + } } } } diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc index 4dd91e99ad9da..c77f0d008b502 100644 --- a/cpp/src/arrow/util/bit-util.cc +++ b/cpp/src/arrow/util/bit-util.cc @@ -109,9 +109,37 @@ Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t std::shared_ptr buffer; RETURN_NOT_OK(GetEmptyBitmap(pool, length, &buffer)); uint8_t* dest = buffer->mutable_data(); - for (int64_t i = 0; i < length; ++i) { - BitUtil::SetBitTo(dest, i, BitUtil::GetBit(data, i + offset)); + + int64_t byte_offset = offset / 8; + int64_t bit_offset = offset % 8; + int64_t num_bytes = BitUtil::BytesForBits(length); + int64_t bits_to_zero = num_bytes * 8 - length; + + if (bit_offset > 0) { + uint32_t carry_mask = BitUtil::kBitmask[bit_offset] - 1U; + uint32_t carry_shift = 8U - static_cast(bit_offset); + + uint32_t carry = 0U; + if (BitUtil::BytesForBits(length + bit_offset) > num_bytes) { + carry = (data[byte_offset + num_bytes] & carry_mask) << carry_shift; + } + + int64_t i = num_bytes - 1; + while (i + 1 > 0) { + uint8_t cur_byte = data[byte_offset + i]; + dest[i] = static_cast((cur_byte >> bit_offset) | carry); + carry = (cur_byte & carry_mask) << carry_shift; + --i; + } + } else { + std::memcpy(dest, data + byte_offset, static_cast(num_bytes)); + } + + for (int64_t i = length; i < length + bits_to_zero; ++i) { + // Both branches may copy extra bits - unsetting to match specification. + BitUtil::SetBitTo(dest, i, false); } + *out = buffer; return Status::OK(); } diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h index cab3c9ee703c9..86c17d16801b9 100644 --- a/cpp/src/arrow/util/bit-util.h +++ b/cpp/src/arrow/util/bit-util.h @@ -139,13 +139,10 @@ static inline void SetArrayBit(uint8_t* bits, int i, bool is_set) { } static inline void SetBitTo(uint8_t* bits, int64_t i, bool bit_is_set) { - // TODO: speed up. See https://graphics.stanford.edu/~seander/bithacks.html + // https://graphics.stanford.edu/~seander/bithacks.html // "Conditionally set or clear bits without branching" - if (bit_is_set) { - SetBit(bits, i); - } else { - ClearBit(bits, i); - } + bits[i / 8] ^= static_cast(-static_cast(bit_is_set) ^ bits[i / 8]) & + kBitmask[i % 8]; } // Returns the minimum number of bits needed to represent the value of 'x' From f82b7e4f57bf6d5aa283823f483fabdda59d56ad Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Wed, 10 Jan 2018 15:32:11 -0800 Subject: [PATCH 175/177] ARROW-1927: [Plasma] Add delete function Hi, I just add the delete function for Plasma and tested. JIRA ticked: https://issues.apache.org/jira/browse/ARROW-1927 Author: Jin Hai Author: Philipp Moritz Closes #1427 from JinHai-CN/plasma-delete and squashes the following commits: c6df5be [Philipp Moritz] rebase 424c1b7 [Philipp Moritz] fix linting 1d76437 [Philipp Moritz] fix tests 0ca115a [Jin Hai] Fixed two bugs according to the comments ce27077 [Jin Hai] Update the unit test cases 8b6804e [Jin Hai] ARROW-1927: [Plasma] Try to fix unit-test fault be88990 [Jin Hai] ARROW-1927: [Plasma] Add 3 test cases for delete function baf82b9 [Jin Hai] ARROW-1927: [Plasma] Update according to the CI error 53e24eb [Jin Hai] ARROW-1927: [Plasma] Update according to the comments and CI error c9984a4 [Jin Hai] ARROW-1927: [Plasma] Add delete function --- cpp/src/plasma/client.cc | 17 +++++++++-- cpp/src/plasma/client.h | 3 +- cpp/src/plasma/eviction_policy.cc | 10 +++++++ cpp/src/plasma/eviction_policy.h | 5 ++++ cpp/src/plasma/format/plasma.fbs | 6 +++- cpp/src/plasma/store.cc | 46 ++++++++++++++++++++++++++--- cpp/src/plasma/store.h | 9 ++++++ cpp/src/plasma/test/client_tests.cc | 25 ++++++++++++++++ 8 files changed, 112 insertions(+), 9 deletions(-) diff --git a/cpp/src/plasma/client.cc b/cpp/src/plasma/client.cc index 0dd1c44d71c74..d74c0f412d97f 100644 --- a/cpp/src/plasma/client.cc +++ b/cpp/src/plasma/client.cc @@ -513,9 +513,20 @@ Status PlasmaClient::Abort(const ObjectID& object_id) { } Status PlasmaClient::Delete(const ObjectID& object_id) { - // TODO(rkn): In the future, we can use this method to give hints to the - // eviction policy about when an object will no longer be needed. - return Status::NotImplemented("PlasmaClient::Delete is not implemented."); + RETURN_NOT_OK(FlushReleaseHistory()); + // If the object is in used, client can't send the remove message. + if (objects_in_use_.count(object_id) > 0) { + return Status::UnknownError("PlasmaClient::Object is in use."); + } else { + // If we don't already have a reference to the object, we can try to remove the object + RETURN_NOT_OK(SendDeleteRequest(store_conn_, object_id)); + std::vector buffer; + RETURN_NOT_OK(PlasmaReceive(store_conn_, MessageType_PlasmaDeleteReply, &buffer)); + ObjectID object_id2; + DCHECK_GT(buffer.size(), 0); + RETURN_NOT_OK(ReadDeleteReply(buffer.data(), buffer.size(), &object_id2)); + return Status::OK(); + } } Status PlasmaClient::Evict(int64_t num_bytes, int64_t& num_bytes_evicted) { diff --git a/cpp/src/plasma/client.h b/cpp/src/plasma/client.h index 78793f1a73a64..35182f8403201 100644 --- a/cpp/src/plasma/client.h +++ b/cpp/src/plasma/client.h @@ -174,7 +174,8 @@ class ARROW_EXPORT PlasmaClient { Status Seal(const ObjectID& object_id); /// Delete an object from the object store. This currently assumes that the - /// object is present and has been sealed. + /// object is present, has been sealed and not used by another client. Otherwise, + /// it is a no operation. /// /// @todo We may want to allow the deletion of objects that are not present or /// haven't been sealed. diff --git a/cpp/src/plasma/eviction_policy.cc b/cpp/src/plasma/eviction_policy.cc index a7758fd2c0e74..66a3b2ea298ac 100644 --- a/cpp/src/plasma/eviction_policy.cc +++ b/cpp/src/plasma/eviction_policy.cc @@ -102,4 +102,14 @@ void EvictionPolicy::end_object_access(const ObjectID& object_id, cache_.add(object_id, entry->info.data_size + entry->info.metadata_size); } +void EvictionPolicy::remove_object(const ObjectID& object_id) { + /* If the object is in the LRU cache, remove it. */ + cache_.remove(object_id); + + auto entry = store_info_->objects[object_id].get(); + int64_t size = entry->info.data_size + entry->info.metadata_size; + ARROW_CHECK(memory_used_ >= size); + memory_used_ -= size; +} + } // namespace plasma diff --git a/cpp/src/plasma/eviction_policy.h b/cpp/src/plasma/eviction_policy.h index cebf35b1c1b08..b076309552952 100644 --- a/cpp/src/plasma/eviction_policy.h +++ b/cpp/src/plasma/eviction_policy.h @@ -120,6 +120,11 @@ class EvictionPolicy { int64_t choose_objects_to_evict(int64_t num_bytes_required, std::vector* objects_to_evict); + /// This method will be called when an object is going to be removed + /// + /// @param object_id The ID of the object that is now being used. + void remove_object(const ObjectID& object_id); + private: /// The amount of memory (in bytes) currently being used. int64_t memory_used_; diff --git a/cpp/src/plasma/format/plasma.fbs b/cpp/src/plasma/format/plasma.fbs index b6d03b8a3c10d..ea6dc8bb98da5 100644 --- a/cpp/src/plasma/format/plasma.fbs +++ b/cpp/src/plasma/format/plasma.fbs @@ -76,7 +76,11 @@ enum PlasmaError:int { // Trying to access an object that doesn't exist. ObjectNonexistent, // Trying to create an object but there isn't enough space in the store. - OutOfMemory + OutOfMemory, + // Trying to delete an object but it's not sealed. + ObjectNotSealed, + // Trying to delete an object but it's in use. + ObjectInUse } // Plasma store messages diff --git a/cpp/src/plasma/store.cc b/cpp/src/plasma/store.cc index c6a19a5471883..dde7f9cdfa8eb 100644 --- a/cpp/src/plasma/store.cc +++ b/cpp/src/plasma/store.cc @@ -411,6 +411,39 @@ int PlasmaStore::abort_object(const ObjectID& object_id, Client* client) { } } +int PlasmaStore::delete_object(ObjectID& object_id) { + auto entry = get_object_table_entry(&store_info_, object_id); + // TODO(rkn): This should probably not fail, but should instead throw an + // error. Maybe we should also support deleting objects that have been + // created but not sealed. + if (entry == NULL) { + // To delete an object it must be in the object table. + return PlasmaError_ObjectNonexistent; + } + + if (entry->state != PLASMA_SEALED) { + // To delete an object it must have been sealed. + return PlasmaError_ObjectNotSealed; + } + + if (entry->clients.size() != 0) { + // To delete an object, there must be no clients currently using it. + return PlasmaError_ObjectInUse; + } + + eviction_policy_.remove_object(object_id); + + dlfree(entry->pointer); + store_info_.objects.erase(object_id); + // Inform all subscribers that the object has been deleted. + ObjectInfoT notification; + notification.object_id = object_id.binary(); + notification.is_deletion = true; + push_notification(¬ification); + + return PlasmaError_OK; +} + void PlasmaStore::delete_objects(const std::vector& object_ids) { for (const auto& object_id : object_ids) { ARROW_LOG(DEBUG) << "deleting object " << object_id.hex(); @@ -626,18 +659,23 @@ Status PlasmaStore::process_message(Client* client) { RETURN_NOT_OK(ReadGetRequest(input, input_size, object_ids_to_get, &timeout_ms)); process_get_request(client, object_ids_to_get, timeout_ms); } break; - case MessageType_PlasmaReleaseRequest: + case MessageType_PlasmaReleaseRequest: { RETURN_NOT_OK(ReadReleaseRequest(input, input_size, &object_id)); release_object(object_id, client); - break; - case MessageType_PlasmaContainsRequest: + } break; + case MessageType_PlasmaDeleteRequest: { + RETURN_NOT_OK(ReadDeleteRequest(input, input_size, &object_id)); + int error_code = delete_object(object_id); + HANDLE_SIGPIPE(SendDeleteReply(client->fd, object_id, error_code), client->fd); + } break; + case MessageType_PlasmaContainsRequest: { RETURN_NOT_OK(ReadContainsRequest(input, input_size, &object_id)); if (contains_object(object_id) == OBJECT_FOUND) { HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd); } else { HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd); } - break; + } break; case MessageType_PlasmaSealRequest: { unsigned char digest[kDigestSize]; RETURN_NOT_OK(ReadSealRequest(input, input_size, &object_id, &digest[0])); diff --git a/cpp/src/plasma/store.h b/cpp/src/plasma/store.h index a72c6259a9cea..7eada5a126991 100644 --- a/cpp/src/plasma/store.h +++ b/cpp/src/plasma/store.h @@ -83,6 +83,15 @@ class PlasmaStore { /// @return 1 if the abort succeeds, else 0. int abort_object(const ObjectID& object_id, Client* client); + /// Delete an specific object by object_id that have been created in the hash table. + /// + /// @param object_id Object ID of the object to be deleted. + /// @return One of the following error codes: + /// - PlasmaError_OK, if the object was delete successfully. + /// - PlasmaError_ObjectNonexistent, if ths object isn't existed. + /// - PlasmaError_ObjectInUse, if the object is in use. + int delete_object(ObjectID& object_id); + /// Delete objects that have been created in the hash table. This should only /// be called on objects that are returned by the eviction policy to evict. /// diff --git a/cpp/src/plasma/test/client_tests.cc b/cpp/src/plasma/test/client_tests.cc index 5cd3063bb4309..f19c2bfbdb380 100644 --- a/cpp/src/plasma/test/client_tests.cc +++ b/cpp/src/plasma/test/client_tests.cc @@ -58,6 +58,31 @@ class TestPlasmaStore : public ::testing::Test { PlasmaClient client2_; }; +TEST_F(TestPlasmaStore, DeleteTest) { + ObjectID object_id = ObjectID::from_random(); + + // Test for deleting non-existance object. + Status result = client_.Delete(object_id); + ASSERT_EQ(result.IsPlasmaObjectNonexistent(), true); + + // Test for the object being in local Plasma store. + // First create object. + int64_t data_size = 100; + uint8_t metadata[] = {5}; + int64_t metadata_size = sizeof(metadata); + std::shared_ptr data; + ARROW_CHECK_OK(client_.Create(object_id, data_size, metadata, metadata_size, &data)); + ARROW_CHECK_OK(client_.Seal(object_id)); + + // Object is in use, can't be delete. + result = client_.Delete(object_id); + ASSERT_EQ(result.IsUnknownError(), true); + + // Avoid race condition of Plasma Manager waiting for notification. + ARROW_CHECK_OK(client_.Release(object_id)); + ARROW_CHECK_OK(client_.Delete(object_id)); +} + TEST_F(TestPlasmaStore, ContainsTest) { ObjectID object_id = ObjectID::from_random(); From b49e8f3765350f9b22d631fcd49181c2fbefbc75 Mon Sep 17 00:00:00 2001 From: devin-petersohn Date: Wed, 10 Jan 2018 18:47:45 -0500 Subject: [PATCH 176/177] ARROW-1971: [Python] Add pandas serialization to the default Moving pandas register into default register. Author: devin-petersohn Closes #1462 from devin-petersohn/jira/1971_pandas_serialization and squashes the following commits: b3dfd5b6 [devin-petersohn] Removing slower codepath 2ed31371 [devin-petersohn] Moving pandas register into default register --- python/pyarrow/serialization.py | 124 ++++++++++++-------------------- 1 file changed, 44 insertions(+), 80 deletions(-) diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py index 689ec15d3292d..61f2e83f3193d 100644 --- a/python/pyarrow/serialization.py +++ b/python/pyarrow/serialization.py @@ -21,7 +21,6 @@ import numpy as np -from pyarrow import serialize_pandas, deserialize_pandas from pyarrow.compat import builtin_pickle from pyarrow.lib import _default_serialization_context, frombuffer @@ -61,6 +60,48 @@ def _load_pickle_from_buffer(data): _deserialize_numpy_array_pickle = _load_pickle_from_buffer +# ---------------------------------------------------------------------- +# pandas-specific serialization matters + +def _register_custom_pandas_handlers(context): + # ARROW-1784, faster path for pandas-only visibility + + try: + import pandas as pd + except ImportError: + return + + import pyarrow.pandas_compat as pdcompat + + def _serialize_pandas_dataframe(obj): + return pdcompat.dataframe_to_serialized_dict(obj) + + def _deserialize_pandas_dataframe(data): + return pdcompat.serialized_dict_to_dataframe(data) + + def _serialize_pandas_series(obj): + return _serialize_pandas_dataframe(pd.DataFrame({obj.name: obj})) + + def _deserialize_pandas_series(data): + deserialized = _deserialize_pandas_dataframe(data) + return deserialized[deserialized.columns[0]] + + context.register_type( + pd.Series, 'pd.Series', + custom_serializer=_serialize_pandas_series, + custom_deserializer=_deserialize_pandas_series) + + context.register_type( + pd.Index, 'pd.Index', + custom_serializer=_pickle_to_buffer, + custom_deserializer=_load_pickle_from_buffer) + + context.register_type( + pd.DataFrame, 'pd.DataFrame', + custom_serializer=_serialize_pandas_dataframe, + custom_deserializer=_deserialize_pandas_dataframe) + + def register_default_serialization_handlers(serialization_context): # ---------------------------------------------------------------------- @@ -136,90 +177,13 @@ def _deserialize_torch_tensor(data): # no torch pass - -register_default_serialization_handlers(_default_serialization_context) + _register_custom_pandas_handlers(serialization_context) -# ---------------------------------------------------------------------- -# pandas-specific serialization matters - +register_default_serialization_handlers(_default_serialization_context) pandas_serialization_context = _default_serialization_context.clone() - -def _register_pandas_arrow_handlers(context): - try: - import pandas as pd - except ImportError: - return - - def _serialize_pandas_series(obj): - return serialize_pandas(pd.DataFrame({obj.name: obj})) - - def _deserialize_pandas_series(data): - deserialized = deserialize_pandas(data) - return deserialized[deserialized.columns[0]] - - def _serialize_pandas_dataframe(obj): - return serialize_pandas(obj) - - def _deserialize_pandas_dataframe(data): - return deserialize_pandas(data) - - context.register_type( - pd.Series, 'pd.Series', - custom_serializer=_serialize_pandas_series, - custom_deserializer=_deserialize_pandas_series) - - context.register_type( - pd.DataFrame, 'pd.DataFrame', - custom_serializer=_serialize_pandas_dataframe, - custom_deserializer=_deserialize_pandas_dataframe) - - -def _register_custom_pandas_handlers(context): - # ARROW-1784, faster path for pandas-only visibility - - try: - import pandas as pd - except ImportError: - return - - import pyarrow.pandas_compat as pdcompat - - def _serialize_pandas_dataframe(obj): - return pdcompat.dataframe_to_serialized_dict(obj) - - def _deserialize_pandas_dataframe(data): - return pdcompat.serialized_dict_to_dataframe(data) - - def _serialize_pandas_series(obj): - return _serialize_pandas_dataframe(pd.DataFrame({obj.name: obj})) - - def _deserialize_pandas_series(data): - deserialized = _deserialize_pandas_dataframe(data) - return deserialized[deserialized.columns[0]] - - context.register_type( - pd.Series, 'pd.Series', - custom_serializer=_serialize_pandas_series, - custom_deserializer=_deserialize_pandas_series) - - context.register_type( - pd.Index, 'pd.Index', - custom_serializer=_pickle_to_buffer, - custom_deserializer=_load_pickle_from_buffer) - - context.register_type( - pd.DataFrame, 'pd.DataFrame', - custom_serializer=_serialize_pandas_dataframe, - custom_deserializer=_deserialize_pandas_dataframe) - - -_register_pandas_arrow_handlers(_default_serialization_context) -_register_custom_pandas_handlers(pandas_serialization_context) - - pandas_serialization_context.register_type( np.ndarray, 'np.array', custom_serializer=_serialize_numpy_array_pickle, From 99e58da5d58b8587657413b4bcca14a773d6de36 Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Thu, 11 Jan 2018 18:35:05 -0500 Subject: [PATCH 177/177] ARROW-1979: [JS] Fix JS builds hanging in es2015 Also fixes [ARROW-1903](https://issues.apache.org/jira/browse/ARROW-1903) Author: Paul Taylor Closes #1471 from trxcllnt/fix-js-es2015-builds and squashes the following commits: 62db3381 [Paul Taylor] update dependencies and add es6+ umd targets to jest transform ignore patterns to fix ci 6ff18e94 [Paul Taylor] ship es2015 commonJS in main package to avoid confusion 74e828af [Paul Taylor] fix typings issues (ARROW-1903) --- js/README.md | 2 +- js/gulp/arrow-task.js | 8 +++--- js/gulp/package-task.js | 6 ++--- js/gulp/test-task.js | 2 +- js/package.json | 38 +++++++++++++-------------- js/src/text-encoding-utf-8.d.ts | 4 --- js/src/vector/numeric.ts | 7 ++--- js/src/vector/virtual.ts | 2 +- js/test/Arrow.ts | 2 +- js/test/integration/validate-tests.ts | 2 +- 10 files changed, 35 insertions(+), 38 deletions(-) delete mode 100644 js/src/text-encoding-utf-8.d.ts diff --git a/js/README.md b/js/README.md index e58d335cd0d42..b427923e37ea1 100644 --- a/js/README.md +++ b/js/README.md @@ -178,7 +178,7 @@ The base `apache-arrow` package includes all the compilation targets for conveni The targets are also published under the `@apache-arrow` namespace: ```sh -npm install apache-arrow # <-- combined es5/CommonJS + UMD, es2015/ESModules + UMD, and TypeScript package +npm install apache-arrow # <-- combined es5/UMD, es2015/CommonJS/ESModules/UMD, and TypeScript package npm install @apache-arrow/ts # standalone TypeScript package npm install @apache-arrow/es5-cjs # standalone es5/CommonJS package npm install @apache-arrow/es5-esm # standalone es5/ESModules package diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js index cc33ee14497b6..d1e8046e67ab9 100644 --- a/js/gulp/arrow-task.js +++ b/js/gulp/arrow-task.js @@ -28,8 +28,8 @@ const { Observable, ReplaySubject } = require('rxjs'); const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, format) { const out = targetDir(target); - const srcGlob = `src/**/*.ts`; - const es5Glob = `${targetDir(`es5`, `cjs`)}/**/*.js`; + const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`; + const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`; const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`; const es5UmdGlob = `${targetDir(`es5`, `umd`)}/**/*.js`; const es5UmdMaps = `${targetDir(`es5`, `umd`)}/**/*.map`; @@ -38,8 +38,8 @@ const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target, forma const ch_ext = (ext) => gulpRename((p) => { p.extname = ext; }); const append = (ap) => gulpRename((p) => { p.basename += ap; }); return Observable.forkJoin( - observableFromStreams(gulp.src(srcGlob), gulp.dest(out)), // copy src ts files - observableFromStreams(gulp.src(es5Glob), gulp.dest(out)), // copy es5 cjs files + observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts files + observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy es2015 cjs files observableFromStreams(gulp.src(esmGlob), ch_ext(`.mjs`), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs` observableFromStreams(gulp.src(es5UmdGlob), append(`.es5.min`), gulp.dest(out)), // copy es5 umd files and add `.min` observableFromStreams(gulp.src(es5UmdMaps), gulp.dest(out)), // copy es5 umd sourcemap files, but don't rename diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js index fc959643503bd..2976d0ad45d09 100644 --- a/js/gulp/package-task.js +++ b/js/gulp/package-task.js @@ -46,8 +46,8 @@ const createMainPackageJson = (target, format) => (orig) => ({ name: npmPkgName, main: mainExport, module: `${mainExport}.mjs`, - browser: `${mainExport}.es5.min.js`, - [`browser:es2015`]: `${mainExport}.es2015.min.js`, + dist: `${mainExport}.es5.min.js`, + [`dist:es2015`]: `${mainExport}.es2015.min.js`, [`@std/esm`]: { esm: `mjs` } }); @@ -67,7 +67,7 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) => (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }), { name: `${npmOrgName}/${packageName(target, format)}`, version: undefined, main: `${mainExport}.js`, types: `${mainExport}.d.ts`, - browser: undefined, [`browser:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined } + dist: undefined, [`dist:es2015`]: undefined, module: undefined, [`@std/esm`]: undefined } ) ) ); diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js index f21aaf2364d03..ab280b092635c 100644 --- a/js/gulp/test-task.js +++ b/js/gulp/test-task.js @@ -34,7 +34,7 @@ argv.update && jestArgv.push(`-u`); argv.verbose && jestArgv.push(`--verbose`); argv.coverage && jestArgv.push(`--coverage`); -const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`]; +const debugArgv = [`--runInBand`, `--env`, `node-debug`]; const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`)); const testOptions = { env: { ...process.env }, diff --git a/js/package.json b/js/package.json index 3903d1eedc442..d68e7a6279e61 100644 --- a/js/package.json +++ b/js/package.json @@ -49,10 +49,8 @@ "gulpfile.js", "npm-release.sh" ], - "peerDependencies": { - "command-line-usage": "4.0.1" - }, "dependencies": { + "@types/text-encoding-utf-8": "1.0.1", "command-line-args": "4.0.7", "command-line-usage": "4.0.2", "flatbuffers": "trxcllnt/flatbuffers-esm", @@ -61,45 +59,44 @@ "tslib": "1.8.1" }, "devDependencies": { - "@std/esm": "0.18.0", + "@std/esm": "0.19.1", "@types/flatbuffers": "1.6.5", "@types/glob": "5.0.34", - "@types/jest": "21.1.8", - "@types/node": "8.5.0", - "@types/text-encoding": "0.0.32", + "@types/jest": "22.0.1", + "@types/node": "9.3.0", "ast-types": "0.10.1", "benchmark": "2.1.4", "coveralls": "3.0.0", "del": "3.0.0", - "esdoc": "1.0.3", + "esdoc": "1.0.4", "esdoc-standard-plugin": "1.0.0", "glob": "7.1.2", - "google-closure-compiler": "20171203.0.0", + "google-closure-compiler": "20180101.0.0", "gulp": "github:gulpjs/gulp#6d71a658c61edb3090221579d8f97dbe086ba2ed", "gulp-json-transform": "0.4.5", "gulp-rename": "1.2.2", - "gulp-sourcemaps": "2.6.1", + "gulp-sourcemaps": "2.6.3", "gulp-transform-js-ast": "1.0.2", "gulp-typescript": "3.2.3", "ix": "2.3.4", - "jest": "21.2.1", + "jest": "22.0.5", "jest-environment-node-debug": "2.0.0", "json": "9.0.6", - "lerna": "2.5.1", + "lerna": "2.6.0", "lint-staged": "6.0.0", - "merge2": "1.2.0", + "merge2": "1.2.1", "mkdirp": "0.5.1", "npm-run-all": "4.1.2", "pump": "1.0.2", "rimraf": "2.6.2", - "rxjs": "5.5.5", + "rxjs": "5.5.6", "shx": "0.2.2", "source-map-loader": "0.2.3", "trash": "4.2.1", - "ts-jest": "21.2.4", - "tslint": "5.8.0", + "ts-jest": "22.0.1", + "tslint": "5.9.1", "typescript": "2.6.2", - "uglifyjs-webpack-plugin": "1.1.2", + "uglifyjs-webpack-plugin": "1.1.6", "webpack": "3.10.0", "xml2js": "0.4.19" }, @@ -134,9 +131,12 @@ "/node_modules/" ], "transform": { - ".(ts|tsx)": "/node_modules/ts-jest/preprocessor.js", - ".(js|jsx)": "/node_modules/babel-jest/build/index.js" + ".(ts|tsx)": "./node_modules/ts-jest/preprocessor.js", + ".(js|jsx)": "./node_modules/babel-jest/build/index.js" }, + "transformIgnorePatterns": [ + "/node_modules/", "/(es2015|esnext)\/umd/" + ], "testRegex": "(.*(-|\\.)(test|spec)s?)\\.(ts|tsx|js)$" } } diff --git a/js/src/text-encoding-utf-8.d.ts b/js/src/text-encoding-utf-8.d.ts deleted file mode 100644 index 68ba4dfd9a346..0000000000000 --- a/js/src/text-encoding-utf-8.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -declare module 'text-encoding-utf-8' { - import * as TextEncoding from 'text-encoding'; - export = TextEncoding; -} diff --git a/js/src/vector/numeric.ts b/js/src/vector/numeric.ts index fe4767809f465..830d6082bcc4a 100644 --- a/js/src/vector/numeric.ts +++ b/js/src/vector/numeric.ts @@ -34,10 +34,10 @@ export class NumericVector extends Vector { concat(...vectors: Vector[]): Vector { return new VirtualVector(this.data.constructor as TypedArrayConstructor, this, ...vectors); } - slice(start?: number, end?: number) { + slice(start?: number, end?: number): R { const { data, stride } = this, from = start! | 0; const to = end === undefined ? data.length : Math.max(end | 0, from); - return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0); + return data.subarray(Math.min(from, to) * stride | 0, to * stride | 0) as any as R; } } @@ -49,7 +49,8 @@ export class FixedWidthNumericVector extends Numer export class BoolVector extends NumericVector { static pack(values: Iterable) { - let xs = [], n, i = 0; + let n = 0, i = 0; + let xs: number[] = []; let bit = 0, byte = 0; for (const value of values) { value && (byte |= 1 << bit); diff --git a/js/src/vector/virtual.ts b/js/src/vector/virtual.ts index 6ec3a8eef9f4d..42db78706db51 100644 --- a/js/src/vector/virtual.ts +++ b/js/src/vector/virtual.ts @@ -93,7 +93,7 @@ export class VirtualVector implements Vector { // this is a significant improvement as we avoid the memcpy 🎉 if ((source.length / vector.stride | 0) < total) { let vectorsLength = vectors.length; - let count = 0, length = 0, sources = []; + let count = 0, length = 0, sources = [] as any[]; do { sources.push(source); length += source.length; diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts index 87641e52bf3f8..f2c4e930f92e4 100644 --- a/js/test/Arrow.ts +++ b/js/test/Arrow.ts @@ -16,7 +16,7 @@ // under the License. /* tslint:disable */ -// Dynamically load an Ix target build based on command line arguments +// Dynamically load an Arrow target build based on command line arguments const path = require('path'); const target = process.env.TEST_TARGET!; diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts index c8778ba2b33c2..c612d62ad0c04 100644 --- a/js/test/integration/validate-tests.ts +++ b/js/test/integration/validate-tests.ts @@ -37,7 +37,7 @@ const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)]; import Arrow from '../Arrow'; import { zip } from 'ix/iterable/zip'; -import { toArray } from 'ix/iterable/toArray'; +import { toArray } from 'ix/iterable/toarray'; const { Table, read } = Arrow;