From f9c4e1915c34dfff229c4d425fc926fc6b6df78a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 9 Oct 2024 16:44:52 +0900 Subject: [PATCH] GH-35589: [Ruby] Add support or JRuby This is not a complete support. This just can create int8 and int32 arrays by using the Java implementation not the C++ implementation. We can improve this step by step. Note that we can build gem for JRuby but we'll not release it for now. We need to build our gems as artifacts by CI in release process and publish approved gems after release vote. If we use the current "gem build && gem push" for JRuby gems, we need JRuby on release. It's not desired because it increases release complexity. --- c_glib/arrow-glib/array-builder.cpp | 2 +- c_glib/arrow-glib/array-builder.h | 2 +- ruby/red-arrow/lib/arrow.rb | 9 +- ruby/red-arrow/lib/arrow/array.rb | 11 +- ruby/red-arrow/lib/arrow/jruby.rb | 52 ++++++++ .../lib/arrow/jruby/array-builder.rb | 114 ++++++++++++++++ ruby/red-arrow/lib/arrow/jruby/array.rb | 109 +++++++++++++++ .../lib/arrow/jruby/chunked-array.rb | 36 +++++ .../lib/arrow/jruby/compression-type.rb | 26 ++++ .../lib/arrow/jruby/csv-read-options.rb | 32 +++++ ruby/red-arrow/lib/arrow/jruby/data-type.rb | 48 +++++++ ruby/red-arrow/lib/arrow/jruby/decimal128.rb | 28 ++++ ruby/red-arrow/lib/arrow/jruby/decimal256.rb | 28 ++++ ruby/red-arrow/lib/arrow/jruby/error.rb | 23 ++++ ruby/red-arrow/lib/arrow/jruby/file-system.rb | 24 ++++ ruby/red-arrow/lib/arrow/jruby/function.rb | 24 ++++ .../lib/arrow/jruby/record-batch-iterator.rb | 24 ++++ .../red-arrow/lib/arrow/jruby/record-batch.rb | 24 ++++ ruby/red-arrow/lib/arrow/jruby/sort-key.rb | 24 ++++ .../red-arrow/lib/arrow/jruby/sort-options.rb | 24 ++++ .../lib/arrow/jruby/stream-listener-raw.rb | 25 ++++ ruby/red-arrow/lib/arrow/jruby/table.rb | 40 ++++++ ruby/red-arrow/lib/arrow/jruby/writable.rb | 24 ++++ ruby/red-arrow/lib/arrow/libraries.rb | 126 ++++++++++++++++++ ruby/red-arrow/lib/arrow/loader.rb | 114 +--------------- ruby/red-arrow/lib/arrow/ruby.rb | 22 +++ ruby/red-arrow/red-arrow.gemspec | 21 ++- ruby/red-arrow/test/run-test.rb | 2 +- 28 files changed, 907 insertions(+), 131 deletions(-) create mode 100644 ruby/red-arrow/lib/arrow/jruby.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/array-builder.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/array.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/chunked-array.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/compression-type.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/csv-read-options.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/data-type.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/decimal128.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/decimal256.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/error.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/file-system.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/function.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/record-batch-iterator.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/record-batch.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/sort-key.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/sort-options.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/stream-listener-raw.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/table.rb create mode 100644 ruby/red-arrow/lib/arrow/jruby/writable.rb create mode 100644 ruby/red-arrow/lib/arrow/libraries.rb create mode 100644 ruby/red-arrow/lib/arrow/ruby.rb diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp index 9b7c608ca8a5b..1897562e13286 100644 --- a/c_glib/arrow-glib/array-builder.cpp +++ b/c_glib/arrow-glib/array-builder.cpp @@ -6320,7 +6320,7 @@ garrow_union_array_builder_class_init(GArrowUnionArrayBuilderClass *klass) * garrow_union_array_builder_append_child: * @builder: A #GArrowUnionArrayBuilder. * @child: A #GArrowArrayBuilder for new child. - * @filed_name: (nullable): A field name for new child. + * @field_name: (nullable): A field name for new child. * * Returns: The type ID for the appended child. * diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h index 6a0d0154833a7..da9e8748ee387 100644 --- a/c_glib/arrow-glib/array-builder.h +++ b/c_glib/arrow-glib/array-builder.h @@ -1820,7 +1820,7 @@ GARROW_AVAILABLE_IN_12_0 gint8 garrow_union_array_builder_append_child(GArrowUnionArrayBuilder *builder, GArrowArrayBuilder *child, - const gchar *filed_name); + const gchar *field_name); GARROW_AVAILABLE_IN_12_0 gboolean diff --git a/ruby/red-arrow/lib/arrow.rb b/ruby/red-arrow/lib/arrow.rb index 8fbc537bc2088..4cfd570d16091 100644 --- a/ruby/red-arrow/lib/arrow.rb +++ b/ruby/red-arrow/lib/arrow.rb @@ -15,16 +15,11 @@ # specific language governing permissions and limitations # under the License. -require "extpp/setup" -require "gio2" - require "arrow/version" -require "arrow/loader" - module Arrow class Error < StandardError end - - Loader.load end + +require_relative "arrow/#{RUBY_ENGINE}" diff --git a/ruby/red-arrow/lib/arrow/array.rb b/ruby/red-arrow/lib/arrow/array.rb index 2c5e5cf2754eb..7bdbbeaec4e5a 100644 --- a/ruby/red-arrow/lib/arrow/array.rb +++ b/ruby/red-arrow/lib/arrow/array.rb @@ -33,9 +33,10 @@ def new(*args) end def builder_class - builder_class_name = "#{name}Builder" - return nil unless const_defined?(builder_class_name) - const_get(builder_class_name) + local_name = name.split("::").last + builder_class_name = "#{local_name}Builder" + return nil unless Arrow.const_defined?(builder_class_name) + Arrow.const_get(builder_class_name) end # @api private @@ -92,6 +93,8 @@ def equal_array?(other, options=nil) equal_options(other, options) end + alias_method :size, :length + def each return to_enum(__method__) unless block_given? @@ -250,7 +253,7 @@ def resolve(other_array) "[array][resolve] need to implement " + "a feature that building #{value_data_type} array " + "from raw Ruby Array" - raise NotImplemented, message + raise NotImplementedError, message end other_array elsif other_array.respond_to?(:value_data_type) diff --git a/ruby/red-arrow/lib/arrow/jruby.rb b/ruby/red-arrow/lib/arrow/jruby.rb new file mode 100644 index 0000000000000..685f6631bb284 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby.rb @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if File.exist?("../red-arrow_jars") + # installed gems + require_relative "../red-arrow_jars" +else + # local development + require "red-arrow_jars" +end + +module Arrow + class << self + def allocator + @allocator ||= org.apache.arrow.memory.RootAllocator.new + end + end +end + +require_relative "jruby/array" +require_relative "jruby/array-builder" +require_relative "jruby/chunked-array" +require_relative "jruby/compression-type" +require_relative "jruby/csv-read-options" +require_relative "jruby/decimal128" +require_relative "jruby/decimal256" +require_relative "jruby/error" +require_relative "jruby/file-system" +require_relative "jruby/function" +require_relative "jruby/record-batch" +require_relative "jruby/record-batch-iterator" +require_relative "jruby/sort-key" +require_relative "jruby/sort-options" +require_relative "jruby/stream-listener-raw" +require_relative "jruby/table" +require_relative "jruby/writable" + +require_relative "libraries" diff --git a/ruby/red-arrow/lib/arrow/jruby/array-builder.rb b/ruby/red-arrow/lib/arrow/jruby/array-builder.rb new file mode 100644 index 0000000000000..99f605260e95b --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/array-builder.rb @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative "array" + +module Arrow + module ArrayBuildable + ValueVector = org.apache.arrow.vector.ValueVector + def buildable?(args) + return false if args.size == 1 and args.first.is_a?(ValueVector) + super + end + end + + class ArrayBuilder + class << self + prepend ArrayBuildable + end + + def initialize + @vector = self.class::Array::Vector.new("", Arrow.allocator) + @vector.allocate_new + @index = 0 + end + + def append_value(value) + @vector.set(@index, value) + @index += 1 + end + + def append_values(values, is_valids=nil) + if is_valids + values.zip(is_valids) do |value, is_valid| + if is_valid + @vector.set(@index, value) + else + @vector.set_null(@index) + end + @index += 1 + end + else + values.each do |value| + @vector.set(@index, value) + @index += 1 + end + end + end + + def append_nulls(n) + n.times do + @vector.set_null(@index) + @index += 1 + end + end + + def finish + @vector.set_value_count(@index) + vector, @vector = @vector, nil + self.class::Array.new(vector) + end + end + + class Int8ArrayBuilder < ArrayBuilder + Array = Int8Array + end + + class Int32ArrayBuilder < ArrayBuilder + Array = Int32Array + end + + class FixedSizeBinaryArrayBuilder < ArrayBuilder + end + + class Decimal128ArrayBuilder < FixedSizeBinaryArrayBuilder + end + + class Decimal256ArrayBuilder < FixedSizeBinaryArrayBuilder + end + + class ListArrayBuilder < ArrayBuilder + end + + class MapArrayBuilder < ArrayBuilder + end + + class StructArrayBuilder < ArrayBuilder + end + + class UnionArrayBuilder < ArrayBuilder + def append_child(child, filed_name) + raise NotImplementedError + end + end + + class DenseUnionArrayBuilder < UnionArrayBuilder + end + + class SparseUnionArrayBuilder < UnionArrayBuilder + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/array.rb b/ruby/red-arrow/lib/arrow/jruby/array.rb new file mode 100644 index 0000000000000..90135a4297681 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/array.rb @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative "data-type" + +module Arrow + class Array + VectorAppender = org.apache.arrow.vector.util.VectorAppender + VectorEqualsVisitor = org.apache.arrow.vector.compare.VectorEqualsVisitor + + attr_reader :vector + + def initialize(vector) + @vector = vector + end + + def ==(other_array) + return false unless other_array.is_a?(self.class) + VectorEqualsVisitor.vector_equals(@vector, other_array.vector) + end + + def null?(i) + @vector.null?(i) + end + + def get_value(i) + @vector.get_object(i) + end + + def to_s + @vector.to_s + end + + def inspect + super.sub(/>\z/) do + " #{to_s}>" + end + end + + def close + @vector.close + end + + def length + @vector.value_count + end + + def value_data_type + self.class::ValueDataType.new + end + + def values + each.to_a + end + + def cast(other_value_data_type) + other_value_data_type.build_array(to_a) + end + + def is_in(values) + raise NotImplementedError + end + + def concatenate(other_arrays) + total_size = length + other_arrays.sum(&:length) + vector = self.class::Vector.new("", Arrow.allocator) + vector.allocate_new(total_size) + appender = VectorAppender.new(vector) + @vector.accept(appender, nil) + other_arrays.each do |other_array| + other_array.vector.accept(appender, nil) + end + self.class.new(vector) + end + end + + class Int8Array < Array + Vector = org.apache.arrow.vector.SmallIntVector + ValueDataType = Int8DataType + end + + class Int32Array < Array + Vector = org.apache.arrow.vector.IntVector + ValueDataType = Int32DataType + end + + class FixedSizeBinaryArray < Array + end + + class StructArray < Array + def fields + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/chunked-array.rb b/ruby/red-arrow/lib/arrow/jruby/chunked-array.rb new file mode 100644 index 0000000000000..3f08fc3caa83c --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/chunked-array.rb @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class ChunkedArray + def initialize(arrays) + @arrays = arrays + end + + def n_rows + @arrays.sum(&:size) + end + + def chunks + @arrays + end + + def get_chunk(i) + @arrays[i] + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/compression-type.rb b/ruby/red-arrow/lib/arrow/jruby/compression-type.rb new file mode 100644 index 0000000000000..90ed7f12c2aea --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/compression-type.rb @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class CompressionType + class << self + def values + [] + end + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/csv-read-options.rb b/ruby/red-arrow/lib/arrow/jruby/csv-read-options.rb new file mode 100644 index 0000000000000..6aeb62c3fd509 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/csv-read-options.rb @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class CSVReadOptions + def add_column_type(name, type) + raise NotImplementedError + end + + def delimiter + raise NotImplementedError + end + + def delimiter=(delimiter) + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/data-type.rb b/ruby/red-arrow/lib/arrow/jruby/data-type.rb new file mode 100644 index 0000000000000..861e3b7dc35ba --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/data-type.rb @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class DataType + def initialize + @minor_type = self.class::MinorType + end + end + + class Int8DataType < DataType + MinorType = org.apache.arrow.vector.types.Types::MinorType::SMALLINT + end + + class Int32DataType < DataType + MinorType = org.apache.arrow.vector.types.Types::MinorType::INT + end + + class Decimal128DataType < DataType + class << self + def max_precision + 38 + end + end + end + + class Decimal256DataType < DataType + class << self + def max_precision + 76 + end + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/decimal128.rb b/ruby/red-arrow/lib/arrow/jruby/decimal128.rb new file mode 100644 index 0000000000000..3ac14f2d9c680 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/decimal128.rb @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class Decimal128 + def abs + raise NotImplementedError + end + + def negate + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/decimal256.rb b/ruby/red-arrow/lib/arrow/jruby/decimal256.rb new file mode 100644 index 0000000000000..e22c9bdde3d00 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/decimal256.rb @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class Decimal256 + def abs + raise NotImplementedError + end + + def negate + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/error.rb b/ruby/red-arrow/lib/arrow/jruby/error.rb new file mode 100644 index 0000000000000..07892aec1ae18 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/error.rb @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class Error < StandardError + class Invalid < Error + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/file-system.rb b/ruby/red-arrow/lib/arrow/jruby/file-system.rb new file mode 100644 index 0000000000000..bd219bf037881 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/file-system.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class FileSystem + def open_output_stream(path) + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/function.rb b/ruby/red-arrow/lib/arrow/jruby/function.rb new file mode 100644 index 0000000000000..1aa1fb761c98d --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/function.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class Function + def execute(args, options=nil, context=nil) + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/record-batch-iterator.rb b/ruby/red-arrow/lib/arrow/jruby/record-batch-iterator.rb new file mode 100644 index 0000000000000..05a82f9fab290 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/record-batch-iterator.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class RecordBatchIterator + def to_list + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/record-batch.rb b/ruby/red-arrow/lib/arrow/jruby/record-batch.rb new file mode 100644 index 0000000000000..28c3a375202d3 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/record-batch.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class RecordBatch + def n_rows + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/sort-key.rb b/ruby/red-arrow/lib/arrow/jruby/sort-key.rb new file mode 100644 index 0000000000000..ca667c0fb9848 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/sort-key.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class SortKey + def target + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/sort-options.rb b/ruby/red-arrow/lib/arrow/jruby/sort-options.rb new file mode 100644 index 0000000000000..13eaf23b7156b --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/sort-options.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class SortOptions + def add_sort_key(target, order=nil) + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/stream-listener-raw.rb b/ruby/red-arrow/lib/arrow/jruby/stream-listener-raw.rb new file mode 100644 index 0000000000000..2327422d8dec8 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/stream-listener-raw.rb @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class StreamListenerRaw + class << self + def type_register + end + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/table.rb b/ruby/red-arrow/lib/arrow/jruby/table.rb new file mode 100644 index 0000000000000..d829280848cae --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/table.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + class Table + def filter(filter, options=nil) + raise NotImplementedError + end + + def take(indices) + raise NotImplementedError + end + + def n_rows + raise NotImplementedError + end + + def slice(from, length) + raise NotImplementedError + end + + def remove_column(index) + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/jruby/writable.rb b/ruby/red-arrow/lib/arrow/jruby/writable.rb new file mode 100644 index 0000000000000..e3e7d9e9172e0 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/jruby/writable.rb @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Arrow + module Writable + def write(data) + raise NotImplementedError + end + end +end diff --git a/ruby/red-arrow/lib/arrow/libraries.rb b/ruby/red-arrow/lib/arrow/libraries.rb new file mode 100644 index 0000000000000..c01e0bf60c436 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/libraries.rb @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative "array-computable" +require_relative "column-containable" +require_relative "field-containable" +require_relative "generic-filterable" +require_relative "generic-takeable" +require_relative "input-referable" +require_relative "record-containable" +require_relative "symbol-values-appendable" + +require_relative "aggregate-node-options" +require_relative "aggregation" +require_relative "array" +require_relative "array-builder" +require_relative "bigdecimal-extension" +require_relative "binary-dictionary-array-builder" +require_relative "buffer" +require_relative "chunked-array" +require_relative "column" +require_relative "compression-type" +require_relative "csv-loader" +require_relative "csv-read-options" +require_relative "data-type" +require_relative "date32-array" +require_relative "date32-array-builder" +require_relative "date64-array" +require_relative "date64-array-builder" +require_relative "datum" +require_relative "day-time-interval-array-builder" +require_relative "decimal128" +require_relative "decimal128-array" +require_relative "decimal128-array-builder" +require_relative "decimal128-data-type" +require_relative "decimal256" +require_relative "decimal256-array" +require_relative "decimal256-array-builder" +require_relative "decimal256-data-type" +require_relative "dense-union-array" +require_relative "dense-union-array-builder" +require_relative "dense-union-data-type" +require_relative "dictionary-array" +require_relative "dictionary-data-type" +require_relative "equal-options" +require_relative "expression" +require_relative "field" +require_relative "file-output-stream" +require_relative "file-system" +require_relative "fixed-size-binary-array" +require_relative "fixed-size-binary-array-builder" +require_relative "function" +require_relative "group" +require_relative "half-float" +require_relative "half-float-array" +require_relative "half-float-array-builder" +require_relative "list-array-builder" +require_relative "list-data-type" +require_relative "map-array" +require_relative "map-array-builder" +require_relative "map-data-type" +require_relative "month-day-nano-interval-array-builder" +require_relative "null-array" +require_relative "null-array-builder" +require_relative "path-extension" +require_relative "record" +require_relative "record-batch" +require_relative "record-batch-builder" +require_relative "record-batch-file-reader" +require_relative "record-batch-iterator" +require_relative "record-batch-reader" +require_relative "record-batch-stream-reader" +require_relative "rolling-window" +require_relative "s3-global-options" +require_relative "scalar" +require_relative "schema" +require_relative "slicer" +require_relative "sort-key" +require_relative "sort-options" +require_relative "source-node-options" +require_relative "sparse-union-array" +require_relative "sparse-union-array-builder" +require_relative "sparse-union-data-type" +require_relative "string-dictionary-array-builder" +require_relative "string-array-builder" +require_relative "stream-decoder" +require_relative "stream-listener" +require_relative "struct-array" +require_relative "struct-array-builder" +require_relative "struct-data-type" +require_relative "table" +require_relative "table-concatenate-options" +require_relative "table-formatter" +require_relative "table-list-formatter" +require_relative "table-table-formatter" +require_relative "table-loader" +require_relative "table-saver" +require_relative "tensor" +require_relative "time" +require_relative "time-unit" +require_relative "time32-array" +require_relative "time32-array-builder" +require_relative "time32-data-type" +require_relative "time64-array" +require_relative "time64-array-builder" +require_relative "time64-data-type" +require_relative "timestamp-array" +require_relative "timestamp-array-builder" +require_relative "timestamp-data-type" +require_relative "timestamp-parser" +require_relative "union-array-builder" +require_relative "writable" diff --git a/ruby/red-arrow/lib/arrow/loader.rb b/ruby/red-arrow/lib/arrow/loader.rb index 5468b0c78cc99..b56350ddac2c9 100644 --- a/ruby/red-arrow/lib/arrow/loader.rb +++ b/ruby/red-arrow/lib/arrow/loader.rb @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -require "arrow/block-closable" +require_relative "block-closable" module Arrow class Loader < GObjectIntrospection::Loader @@ -34,115 +34,7 @@ def post_load(repository, namespace) end def require_libraries - require "arrow/array-computable" - require "arrow/column-containable" - require "arrow/field-containable" - require "arrow/generic-filterable" - require "arrow/generic-takeable" - require "arrow/input-referable" - require "arrow/record-containable" - require "arrow/symbol-values-appendable" - - require "arrow/aggregate-node-options" - require "arrow/aggregation" - require "arrow/array" - require "arrow/array-builder" - require "arrow/bigdecimal-extension" - require "arrow/binary-dictionary-array-builder" - require "arrow/buffer" - require "arrow/chunked-array" - require "arrow/column" - require "arrow/compression-type" - require "arrow/csv-loader" - require "arrow/csv-read-options" - require "arrow/data-type" - require "arrow/date32-array" - require "arrow/date32-array-builder" - require "arrow/date64-array" - require "arrow/date64-array-builder" - require "arrow/datum" - require "arrow/day-time-interval-array-builder" - require "arrow/decimal128" - require "arrow/decimal128-array" - require "arrow/decimal128-array-builder" - require "arrow/decimal128-data-type" - require "arrow/decimal256" - require "arrow/decimal256-array" - require "arrow/decimal256-array-builder" - require "arrow/decimal256-data-type" - require "arrow/dense-union-array" - require "arrow/dense-union-array-builder" - require "arrow/dense-union-data-type" - require "arrow/dictionary-array" - require "arrow/dictionary-data-type" - require "arrow/equal-options" - require "arrow/expression" - require "arrow/field" - require "arrow/file-output-stream" - require "arrow/file-system" - require "arrow/fixed-size-binary-array" - require "arrow/fixed-size-binary-array-builder" - require "arrow/function" - require "arrow/group" - require "arrow/half-float" - require "arrow/half-float-array" - require "arrow/half-float-array-builder" - require "arrow/list-array-builder" - require "arrow/list-data-type" - require "arrow/map-array" - require "arrow/map-array-builder" - require "arrow/map-data-type" - require "arrow/month-day-nano-interval-array-builder" - require "arrow/null-array" - require "arrow/null-array-builder" - require "arrow/path-extension" - require "arrow/record" - require "arrow/record-batch" - require "arrow/record-batch-builder" - require "arrow/record-batch-file-reader" - require "arrow/record-batch-iterator" - require "arrow/record-batch-reader" - require "arrow/record-batch-stream-reader" - require "arrow/rolling-window" - require "arrow/s3-global-options" - require "arrow/scalar" - require "arrow/schema" - require "arrow/slicer" - require "arrow/sort-key" - require "arrow/sort-options" - require "arrow/source-node-options" - require "arrow/sparse-union-array" - require "arrow/sparse-union-array-builder" - require "arrow/sparse-union-data-type" - require "arrow/string-dictionary-array-builder" - require "arrow/string-array-builder" - require "arrow/stream-decoder" - require "arrow/stream-listener" - require "arrow/struct-array" - require "arrow/struct-array-builder" - require "arrow/struct-data-type" - require "arrow/table" - require "arrow/table-concatenate-options" - require "arrow/table-formatter" - require "arrow/table-list-formatter" - require "arrow/table-table-formatter" - require "arrow/table-loader" - require "arrow/table-saver" - require "arrow/tensor" - require "arrow/time" - require "arrow/time-unit" - require "arrow/time32-array" - require "arrow/time32-array-builder" - require "arrow/time32-data-type" - require "arrow/time64-array" - require "arrow/time64-array-builder" - require "arrow/time64-data-type" - require "arrow/timestamp-array" - require "arrow/timestamp-array-builder" - require "arrow/timestamp-data-type" - require "arrow/timestamp-parser" - require "arrow/union-array-builder" - require "arrow/writable" + require_relative "libraries" end def require_extension_library @@ -150,7 +42,7 @@ def require_extension_library end def gc_guard - require "arrow/constructor-arguments-gc-guardable" + require_relative "constructor-arguments-gc-guardable" [ @base_module::BinaryScalar, diff --git a/ruby/red-arrow/lib/arrow/ruby.rb b/ruby/red-arrow/lib/arrow/ruby.rb new file mode 100644 index 0000000000000..34d20fc7901a7 --- /dev/null +++ b/ruby/red-arrow/lib/arrow/ruby.rb @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "extpp/setup" +require "gio2" + +require_relative "loader" +Arrow::Loader.load diff --git a/ruby/red-arrow/red-arrow.gemspec b/ruby/red-arrow/red-arrow.gemspec index 67fec2e0907c1..345fd97353b63 100644 --- a/ruby/red-arrow/red-arrow.gemspec +++ b/ruby/red-arrow/red-arrow.gemspec @@ -20,7 +20,11 @@ require_relative "lib/arrow/version" Gem::Specification.new do |spec| + is_jruby = RUBY_ENGINE == "jruby" + spec.name = "red-arrow" + spec.platform = "java" if is_jruby + version_components = [ Arrow::Version::MAJOR.to_s, Arrow::Version::MINOR.to_s, @@ -43,15 +47,20 @@ Gem::Specification.new do |spec| spec.files += Dir.glob("lib/**/*.rb") spec.files += Dir.glob("image/*.*") spec.files += Dir.glob("doc/text/*") - spec.test_files += Dir.glob("test/**/*") - spec.extensions = ["ext/arrow/extconf.rb"] + spec.extensions = ["ext/arrow/extconf.rb"] unless is_jruby spec.add_runtime_dependency("bigdecimal", ">= 3.1.0") spec.add_runtime_dependency("csv") - spec.add_runtime_dependency("extpp", ">= 0.1.1") - spec.add_runtime_dependency("gio2", ">= 4.2.3") - spec.add_runtime_dependency("native-package-installer") - spec.add_runtime_dependency("pkg-config") + if is_jruby + spec.add_runtime_dependency("jar-dependencies") + spec.requirements << "jar org.apache.arrow, arrow-vector, #{spec.version}" + spec.requirements << "jar org.apache.arrow, arrow-memory-netty, #{spec.version}" + else + spec.add_runtime_dependency("extpp", ">= 0.1.1") + spec.add_runtime_dependency("gio2", ">= 4.2.3") + spec.add_runtime_dependency("native-package-installer") + spec.add_runtime_dependency("pkg-config") + end required_msys2_package_version = version_components[0, 3].join(".") spec.metadata["msys2_mingw_dependencies"] = diff --git a/ruby/red-arrow/test/run-test.rb b/ruby/red-arrow/test/run-test.rb index 41ab73cb6999d..97a2fa4fb89cc 100755 --- a/ruby/red-arrow/test/run-test.rb +++ b/ruby/red-arrow/test/run-test.rb @@ -59,9 +59,9 @@ end system("#{make} > #{File::NULL}") or exit(false) end + $LOAD_PATH.unshift(build_dir.to_s) end -$LOAD_PATH.unshift(build_dir.to_s) $LOAD_PATH.unshift(lib_dir.to_s) require_relative "helper"