From c69bb079d2da784cf6f9045588f004c06a877be3 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Tue, 12 Mar 2024 21:10:37 -0800 Subject: [PATCH 01/12] Add Ruby client example --- http/get_simple/ruby/client/Gemfile | 20 ++++++++ http/get_simple/ruby/client/Gemfile.lock | 34 +++++++++++++ http/get_simple/ruby/client/README.md | 37 ++++++++++++++ http/get_simple/ruby/client/client.rb | 61 ++++++++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100644 http/get_simple/ruby/client/Gemfile create mode 100644 http/get_simple/ruby/client/Gemfile.lock create mode 100644 http/get_simple/ruby/client/README.md create mode 100644 http/get_simple/ruby/client/client.rb diff --git a/http/get_simple/ruby/client/Gemfile b/http/get_simple/ruby/client/Gemfile new file mode 100644 index 0000000..3b8f9e6 --- /dev/null +++ b/http/get_simple/ruby/client/Gemfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source "https://rubygems.org" + +gem "red-arrow" diff --git a/http/get_simple/ruby/client/Gemfile.lock b/http/get_simple/ruby/client/Gemfile.lock new file mode 100644 index 0000000..b134e50 --- /dev/null +++ b/http/get_simple/ruby/client/Gemfile.lock @@ -0,0 +1,34 @@ +GEM + remote: https://rubygems.org/ + specs: + bigdecimal (3.1.6) + csv (3.2.8) + extpp (0.1.1) + fiddle (1.1.2) + gio2 (4.2.1) + fiddle + gobject-introspection (= 4.2.1) + glib2 (4.2.1) + native-package-installer (>= 1.0.3) + pkg-config (>= 1.3.5) + gobject-introspection (4.2.1) + glib2 (= 4.2.1) + native-package-installer (1.1.9) + pkg-config (1.5.6) + red-arrow (15.0.1) + bigdecimal (>= 3.1.0) + csv + extpp (>= 0.1.1) + gio2 (>= 3.5.0) + native-package-installer + pkg-config + +PLATFORMS + arm64-darwin-23 + ruby + +DEPENDENCIES + red-arrow + +BUNDLED WITH + 2.5.3 diff --git a/http/get_simple/ruby/client/README.md b/http/get_simple/ruby/client/README.md new file mode 100644 index 0000000..2f8c2bd --- /dev/null +++ b/http/get_simple/ruby/client/README.md @@ -0,0 +1,37 @@ + + +# HTTP GET Arrow Data: Simple Ruby Client Example + +This directory contains a minimal example of an HTTP client implemented in Ruby. + +The client: + +1. Sends an HTTP GET request to a server. +2. Receives an HTTP 200 response from the server, converts the response to a RecordBatchReader, and consumes it. +3. Sends a second HTTP GET request to a server. +with the response body containing an Arrow IPC stream of record batches. +4. Streams the response, creating and printing record batches as it goes + +To run this example, first start one of the server examples in the parent directory, then: + +```sh +bundle install +bundle exec ruby client.rb +``` diff --git a/http/get_simple/ruby/client/client.rb b/http/get_simple/ruby/client/client.rb new file mode 100644 index 0000000..40d736d --- /dev/null +++ b/http/get_simple/ruby/client/client.rb @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "net/http" +require "stringio" +require "arrow" + +host = "localhost" +port = 8008 +url = URI("http://#{host}:#{port}") + +# Non-streaming +resp = Net::HTTP.get(url) + +StringIO.open(resp) do |stringio_input| + Gio::RubyInputStream.open(stringio_input) do |gio_input| + Arrow::GIOInputStream.open(gio_input) do |arrow_input| + reader = Arrow::RecordBatchStreamReader.new(arrow_input) + + p reader.schema + p reader.read_all + end + end +end + +# Streaming (WIP) +# +# Current error: +# /Users/bryce/.gem/ruby/3.3.0/gems/gobject-introspection-4.2.1/lib/gobject-introspection/loader.rb:713:in `invoke': +# [record-batch-stream-reader][open]: IOError: Invalid IPC stream: negative continuation token (Arrow::Error::Io) +Net::HTTP.start(host, port) do |http| + req = Net::HTTP::Get.new(url) + + http.request(req) do |res| + res.read_body do |chunk| + StringIO.open(chunk) do |stringio_input| + Gio::RubyInputStream.open(stringio_input) do |gio_input| + Arrow::GIOInputStream.open(gio_input) do |arrow_input| + reader = Arrow::RecordBatchStreamReader.new(arrow_input) + + p reader.schema + end + end + end + end + end +end From 25a02622cd7d4d8f91f5f19a292e2f009aed47de Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Tue, 12 Mar 2024 21:24:14 -0800 Subject: [PATCH 02/12] Update http/get_simple/ruby/client/README.md --- http/get_simple/ruby/client/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/get_simple/ruby/client/README.md b/http/get_simple/ruby/client/README.md index 2f8c2bd..7743c8b 100644 --- a/http/get_simple/ruby/client/README.md +++ b/http/get_simple/ruby/client/README.md @@ -27,7 +27,7 @@ The client: 2. Receives an HTTP 200 response from the server, converts the response to a RecordBatchReader, and consumes it. 3. Sends a second HTTP GET request to a server. with the response body containing an Arrow IPC stream of record batches. -4. Streams the response, creating and printing record batches as it goes +4. Streams the response, creating and printing record batches as it goes. To run this example, first start one of the server examples in the parent directory, then: From 8e3c4a7856a2493a78b7b8567dd7b069afcbcfc9 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Tue, 12 Mar 2024 21:35:49 -0800 Subject: [PATCH 03/12] Fix streaming example --- http/get_simple/ruby/client/client.rb | 29 +++++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/http/get_simple/ruby/client/client.rb b/http/get_simple/ruby/client/client.rb index 40d736d..3bca7a9 100644 --- a/http/get_simple/ruby/client/client.rb +++ b/http/get_simple/ruby/client/client.rb @@ -37,25 +37,32 @@ end end -# Streaming (WIP) -# -# Current error: -# /Users/bryce/.gem/ruby/3.3.0/gems/gobject-introspection-4.2.1/lib/gobject-introspection/loader.rb:713:in `invoke': -# [record-batch-stream-reader][open]: IOError: Invalid IPC stream: negative continuation token (Arrow::Error::Io) +# Streaming + +nrows = 0 +batches = [] + Net::HTTP.start(host, port) do |http| req = Net::HTTP::Get.new(url) http.request(req) do |res| - res.read_body do |chunk| - StringIO.open(chunk) do |stringio_input| - Gio::RubyInputStream.open(stringio_input) do |gio_input| - Arrow::GIOInputStream.open(gio_input) do |arrow_input| - reader = Arrow::RecordBatchStreamReader.new(arrow_input) + StringIO.open(res.read_body) do |stringio_input| + Gio::RubyInputStream.open(stringio_input) do |gio_input| + Arrow::GIOInputStream.open(gio_input) do |arrow_input| + reader = Arrow::RecordBatchStreamReader.new(arrow_input) - p reader.schema + p reader.schema + + reader.each do |batch| + puts "Got batch of #{batch.length} rows" + + nrows += batch.length + batches << batch end end end end end end + +puts "Streamed a total of #{batches.length} batches and #{nrows} total rows" From 17218fbb329c1a2a648de31c9366ffc059f63e20 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 13 Mar 2024 12:46:52 -0800 Subject: [PATCH 04/12] Delete Gemfile.lock --- http/get_simple/ruby/client/Gemfile.lock | 34 ------------------------ 1 file changed, 34 deletions(-) delete mode 100644 http/get_simple/ruby/client/Gemfile.lock diff --git a/http/get_simple/ruby/client/Gemfile.lock b/http/get_simple/ruby/client/Gemfile.lock deleted file mode 100644 index b134e50..0000000 --- a/http/get_simple/ruby/client/Gemfile.lock +++ /dev/null @@ -1,34 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - bigdecimal (3.1.6) - csv (3.2.8) - extpp (0.1.1) - fiddle (1.1.2) - gio2 (4.2.1) - fiddle - gobject-introspection (= 4.2.1) - glib2 (4.2.1) - native-package-installer (>= 1.0.3) - pkg-config (>= 1.3.5) - gobject-introspection (4.2.1) - glib2 (= 4.2.1) - native-package-installer (1.1.9) - pkg-config (1.5.6) - red-arrow (15.0.1) - bigdecimal (>= 3.1.0) - csv - extpp (>= 0.1.1) - gio2 (>= 3.5.0) - native-package-installer - pkg-config - -PLATFORMS - arm64-darwin-23 - ruby - -DEPENDENCIES - red-arrow - -BUNDLED WITH - 2.5.3 From 85c1e681607df51ff344129884fd6e884ce9b0ff Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 13 Mar 2024 12:47:23 -0800 Subject: [PATCH 05/12] Create .gitignore --- http/get_simple/ruby/client/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 http/get_simple/ruby/client/.gitignore diff --git a/http/get_simple/ruby/client/.gitignore b/http/get_simple/ruby/client/.gitignore new file mode 100644 index 0000000..b844b14 --- /dev/null +++ b/http/get_simple/ruby/client/.gitignore @@ -0,0 +1 @@ +Gemfile.lock From 4aef851e3621c6955d5962a1a4966a08cf336702 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 13 Mar 2024 12:50:14 -0800 Subject: [PATCH 06/12] Update http/get_simple/ruby/client/client.rb Co-authored-by: Sutou Kouhei --- http/get_simple/ruby/client/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/get_simple/ruby/client/client.rb b/http/get_simple/ruby/client/client.rb index 3bca7a9..27fade5 100644 --- a/http/get_simple/ruby/client/client.rb +++ b/http/get_simple/ruby/client/client.rb @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -require "net/http" +require "open-uri" require "stringio" require "arrow" From 6075b57f101140aeb31f5e810b2fa134b039b21f Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 13 Mar 2024 12:50:22 -0800 Subject: [PATCH 07/12] Update http/get_simple/ruby/client/client.rb Co-authored-by: Sutou Kouhei --- http/get_simple/ruby/client/client.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/http/get_simple/ruby/client/client.rb b/http/get_simple/ruby/client/client.rb index 27fade5..8647d90 100644 --- a/http/get_simple/ruby/client/client.rb +++ b/http/get_simple/ruby/client/client.rb @@ -16,7 +16,6 @@ # under the License. require "open-uri" -require "stringio" require "arrow" host = "localhost" From dcc5772bd95246493846561b1cea6b90775634c3 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 13 Mar 2024 12:50:43 -0800 Subject: [PATCH 08/12] Update http/get_simple/ruby/client/client.rb Co-authored-by: Sutou Kouhei --- http/get_simple/ruby/client/client.rb | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/http/get_simple/ruby/client/client.rb b/http/get_simple/ruby/client/client.rb index 8647d90..42c5a9a 100644 --- a/http/get_simple/ruby/client/client.rb +++ b/http/get_simple/ruby/client/client.rb @@ -23,17 +23,11 @@ url = URI("http://#{host}:#{port}") # Non-streaming -resp = Net::HTTP.get(url) - -StringIO.open(resp) do |stringio_input| - Gio::RubyInputStream.open(stringio_input) do |gio_input| - Arrow::GIOInputStream.open(gio_input) do |arrow_input| - reader = Arrow::RecordBatchStreamReader.new(arrow_input) - - p reader.schema - p reader.read_all - end - end +url.open do |input| + input = Arrow::BufferInputStream.new(input.read) + reader = Arrow::RecordBatchStreamReader.new(input) + p reader.schema + p reader.read_all end # Streaming From bcd9c181bfe08dbb9c2f046437dff6f70a11d9cc Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 14 Mar 2024 14:30:08 -0800 Subject: [PATCH 09/12] Update http/get_simple/ruby/client/.gitignore Co-authored-by: Sutou Kouhei --- http/get_simple/ruby/client/.gitignore | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/http/get_simple/ruby/client/.gitignore b/http/get_simple/ruby/client/.gitignore index b844b14..f07a468 100644 --- a/http/get_simple/ruby/client/.gitignore +++ b/http/get_simple/ruby/client/.gitignore @@ -1 +1,18 @@ -Gemfile.lock +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +/Gemfile.lock From 8a25d1edfa0058dd222680ef55d4a983039a506f Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 14 Mar 2024 14:30:22 -0800 Subject: [PATCH 10/12] Update http/get_simple/ruby/client/client.rb Co-authored-by: Sutou Kouhei --- http/get_simple/ruby/client/client.rb | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/http/get_simple/ruby/client/client.rb b/http/get_simple/ruby/client/client.rb index 42c5a9a..2d4d144 100644 --- a/http/get_simple/ruby/client/client.rb +++ b/http/get_simple/ruby/client/client.rb @@ -15,20 +15,25 @@ # specific language governing permissions and limitations # under the License. -require "open-uri" +require "net/http" + require "arrow" host = "localhost" port = 8008 -url = URI("http://#{host}:#{port}") - -# Non-streaming -url.open do |input| - input = Arrow::BufferInputStream.new(input.read) - reader = Arrow::RecordBatchStreamReader.new(input) - p reader.schema - p reader.read_all -end +uri = URI("http://#{host}:#{port}") + +start = Time.now +arrows_data = Net::HTTP.get(uri) +input = Arrow::BufferInputStream.new(arrows_data) +reader = Arrow::RecordBatchStreamReader.new(input) +schema = reader.schema +table = reader.read_all +elapsed_time = Time.now - start + +n_received_record_batches = table[0].data.n_chunks +puts("#{n_received_record_batches} record batches received") +puts("%.2f seconds elapsed" % elapsed_time) # Streaming From 1297057c05e68d4a55e8e2caab48a08e38707a0b Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 14 Mar 2024 14:41:41 -0800 Subject: [PATCH 11/12] Clean up client example --- http/get_simple/ruby/client/client.rb | 35 +-------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/http/get_simple/ruby/client/client.rb b/http/get_simple/ruby/client/client.rb index 2d4d144..a608bb5 100644 --- a/http/get_simple/ruby/client/client.rb +++ b/http/get_simple/ruby/client/client.rb @@ -16,12 +16,9 @@ # under the License. require "net/http" - require "arrow" -host = "localhost" -port = 8008 -uri = URI("http://#{host}:#{port}") +uri = URI("http://localhost:8008") start = Time.now arrows_data = Net::HTTP.get(uri) @@ -34,33 +31,3 @@ n_received_record_batches = table[0].data.n_chunks puts("#{n_received_record_batches} record batches received") puts("%.2f seconds elapsed" % elapsed_time) - -# Streaming - -nrows = 0 -batches = [] - -Net::HTTP.start(host, port) do |http| - req = Net::HTTP::Get.new(url) - - http.request(req) do |res| - StringIO.open(res.read_body) do |stringio_input| - Gio::RubyInputStream.open(stringio_input) do |gio_input| - Arrow::GIOInputStream.open(gio_input) do |arrow_input| - reader = Arrow::RecordBatchStreamReader.new(arrow_input) - - p reader.schema - - reader.each do |batch| - puts "Got batch of #{batch.length} rows" - - nrows += batch.length - batches << batch - end - end - end - end - end -end - -puts "Streamed a total of #{batches.length} batches and #{nrows} total rows" From 164aefdce0aea55eb88dff76db8148eeec3af409 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 14 Mar 2024 14:47:27 -0800 Subject: [PATCH 12/12] Update http/get_simple/ruby/client/README.md Co-authored-by: Ian Cook --- http/get_simple/ruby/client/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/http/get_simple/ruby/client/README.md b/http/get_simple/ruby/client/README.md index 7743c8b..5ae8d30 100644 --- a/http/get_simple/ruby/client/README.md +++ b/http/get_simple/ruby/client/README.md @@ -24,10 +24,8 @@ This directory contains a minimal example of an HTTP client implemented in Ruby. The client: 1. Sends an HTTP GET request to a server. -2. Receives an HTTP 200 response from the server, converts the response to a RecordBatchReader, and consumes it. -3. Sends a second HTTP GET request to a server. -with the response body containing an Arrow IPC stream of record batches. -4. Streams the response, creating and printing record batches as it goes. +2. Receives an HTTP 200 response from the server, with the response body containing an Arrow IPC stream of record batches. +3. Creates an Arrow table from the record batches. To run this example, first start one of the server examples in the parent directory, then: