From f7194d3bf5e66f2418953d7be67c8ba4466daa0d Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Fri, 30 Jul 2021 07:34:55 +0000 Subject: [PATCH] Added support to parquet IO in WASM. --- .github/workflows/test.yml | 2 +- Cargo.toml | 27 +++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fbef76bacc0..4655dd4afc8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -271,7 +271,7 @@ jobs: export CARGO_HOME="/github/home/.cargo" export CARGO_TARGET_DIR="/github/home/target" # no need - cargo build --no-default-features --features=merge_sort,io_ipc,io_csv,io_json --target wasm32-unknown-unknown + cargo build --no-default-features --features=merge_sort,io_ipc,io_csv,io_json,io_parquet --target wasm32-unknown-unknown linux-simd-test: name: SIMD diff --git a/Cargo.toml b/Cargo.toml index 04cb014ae6d..6b290095f94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ futures = { version = "0.3", optional = true } # for faster hashing ahash = { version = "0.7", optional = true } -parquet2 = { version = "0.1", optional = true } +parquet2 = { version = "0.1", optional = true, default_features = false, features = ["stream"] } [dev-dependencies] rand = "0.7" @@ -68,12 +68,33 @@ doc-comment = "0.3" crossbeam-channel = "0.5.1" [features] -default = ["io_csv", "io_json", "io_ipc", "io_ipc_compression", "io_json_integration", "io_print", "io_parquet", "regex", "merge_sort", "ahash", "benchmarks", "compute"] +default = [ + "io_csv", + "io_json", + "io_ipc", + "io_ipc_compression", + "io_json_integration", + "io_print", + "io_parquet", + "io_parquet_compression", + "regex", + "merge_sort", + "ahash", + "benchmarks", + "compute", +] merge_sort = ["itertools"] io_csv = ["csv", "lazy_static", "regex"] io_json = ["serde", "serde_derive", "serde_json", "indexmap"] io_ipc = ["flatbuffers"] io_ipc_compression = ["lz4", "zstd"] +io_parquet_compression = [ + "parquet2/zstd", + "parquet2/snappy", + "parquet2/gzip", + "parquet2/lz4", + "parquet2/brotli", +] io_json_integration = ["io_json", "hex"] io_print = ["prettytable-rs"] # the compute kernels. Disabling this significantly reduces compile time. @@ -88,6 +109,8 @@ skip_feature_sets = [ ["benchmarks"], ["merge_sort"], ["io_json_integration"], + # this does not change the public API + ["io_parquet_compression"], ["simd"], ]