diff --git a/.dockerignore b/.dockerignore index eb71138c679a..9a64a123f735 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,42 +21,6 @@ # This setup requires to all of our docker containers have arrow's source # as a mounted directory. -# exclude everything -** - -# include explicitly -!ci/** -!c_glib/Gemfile -!dev/archery/requirements*.txt -!python/requirements*.txt -!python/manylinux1/** -!python/manylinux2010/** -!r/DESCRIPTION -!ruby/Gemfile -!ruby/red-arrow/Gemfile -!ruby/red-arrow/lib/arrow/version.rb -!ruby/red-arrow/red-arrow.gemspec -!ruby/red-arrow-cuda/Gemfile -!ruby/red-arrow-cuda/lib/arrow-cuda/version.rb -!ruby/red-arrow-cuda/red-arrow-cuda.gemspec -!ruby/red-gandiva/Gemfile -!ruby/red-gandiva/lib/gandiva/version.rb -!ruby/red-gandiva/red-gandiva.gemspec -!ruby/red-parquet/Gemfile -!ruby/red-parquet/lib/parquet/version.rb -!ruby/red-parquet/red-parquet.gemspec -!ruby/red-plasma/Gemfile -!ruby/red-plasma/lib/plasma/version.rb -!ruby/red-plasma/red-plasma.gemspec -!rust/Cargo.toml -!rust/benchmarks/Cargo.toml -!rust/arrow/Cargo.toml -!rust/arrow/benches -!rust/arrow-flight/Cargo.toml -!rust/parquet/Cargo.toml -!rust/parquet/build.rs -!rust/parquet_derive/Cargo.toml -!rust/parquet_derive_test/Cargo.toml -!rust/datafusion/Cargo.toml -!rust/datafusion/benches -!rust/integration-testing/Cargo.toml +ci +dev +**/target/* diff --git a/ballista/.dockerignore b/ballista/.dockerignore deleted file mode 100644 index 3cde49e0a0c4..000000000000 --- a/ballista/.dockerignore +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -rust/**/target diff --git a/ballista/docker/README.md b/ballista/docker/README.md deleted file mode 100644 index 8417d04c4922..000000000000 --- a/ballista/docker/README.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# Ballista Docker Images - -Pre-built docker images are available from [Docker Hub](https://hub.docker.com/orgs/ballistacompute/repositories) but here are the commands to build the images from source. - -Run these commands from the root directory of the project. - -```bash -./dev/build-all.sh -``` - diff --git a/ballista/dev/build-rust-base.sh b/dev/build-rust-base.sh similarity index 96% rename from ballista/dev/build-rust-base.sh rename to dev/build-rust-base.sh index ee4b32c8e690..e424909fb6f1 100755 --- a/ballista/dev/build-rust-base.sh +++ b/dev/build-rust-base.sh @@ -18,4 +18,4 @@ # under the License. BALLISTA_VERSION=0.4.2-SNAPSHOT set -e -docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f docker/rust-base.dockerfile . +docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f dev/docker/rust-base.dockerfile . diff --git a/ballista/dev/build-rust.sh b/dev/build-rust.sh similarity index 96% rename from ballista/dev/build-rust.sh rename to dev/build-rust.sh index 1916f8efbefb..d31c5241c6f1 100755 --- a/ballista/dev/build-rust.sh +++ b/dev/build-rust.sh @@ -21,4 +21,4 @@ BALLISTA_VERSION=0.4.2-SNAPSHOT set -e -docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f docker/rust.dockerfile . +docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f dev/docker/rust.dockerfile . diff --git a/ballista/docker/rust-base.dockerfile b/dev/docker/rust-base.dockerfile similarity index 99% rename from ballista/docker/rust-base.dockerfile rename to dev/docker/rust-base.dockerfile index 4519225d2197..31620b38cf39 100644 --- a/ballista/docker/rust-base.dockerfile +++ b/dev/docker/rust-base.dockerfile @@ -23,7 +23,7 @@ # Base image extends debian:buster-slim -FROM rust:1.49.0-buster AS builder +FROM rust:1.51.0-buster AS builder RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl diff --git a/ballista/docker/rust.dockerfile b/dev/docker/rust.dockerfile similarity index 64% rename from ballista/docker/rust.dockerfile rename to dev/docker/rust.dockerfile index 8b06af3dc78f..19dd4879eab6 100644 --- a/ballista/docker/rust.dockerfile +++ b/dev/docker/rust.dockerfile @@ -22,13 +22,21 @@ # as a mounted directory. ARG RELEASE_FLAG=--release -FROM ballistacompute/rust-base:0.4.0-20210213 AS base +FROM ballistacompute/rust-base:0.4.2-SNAPSHOT AS base WORKDIR /tmp/ballista RUN apt-get -y install cmake RUN cargo install cargo-chef FROM base as planner -COPY rust . +RUN mkdir /tmp/ballista/ballista +RUN mkdir /tmp/ballista/benchmarks +RUN mkdir /tmp/ballista/datafusion +RUN mkdir /tmp/ballista/datafusion-examples +ADD Cargo.toml . +COPY benchmarks ./benchmarks/ +COPY datafusion ./datafusion/ +COPY datafusion-examples ./datafusion-examples/ +COPY ballista ./ballista/ RUN cargo chef prepare --recipe-path recipe.json FROM base as cacher @@ -36,25 +44,35 @@ COPY --from=planner /tmp/ballista/recipe.json recipe.json RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json FROM base as builder -COPY rust . +RUN mkdir /tmp/ballista/ballista +RUN mkdir /tmp/ballista/benchmarks +RUN mkdir /tmp/ballista/datafusion +RUN mkdir /tmp/ballista/datafusion-examples +ADD Cargo.toml . +COPY benchmarks ./benchmarks/ +COPY datafusion ./datafusion/ +COPY ballista ./ballista/ +COPY datafusion-examples ./datafusion-examples/ COPY --from=cacher /tmp/ballista/target target ARG RELEASE_FLAG=--release # force build.rs to run to generate configure_me code. ENV FORCE_REBUILD='true' RUN cargo build $RELEASE_FLAG +RUN cd ballista/rust && \ + cargo build $RELEASE_FLAG # put the executor on /executor (need to be copied from different places depending on FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-executor /executor; else mv /tmp/ballista/ballista/rust/target/release/ballista-executor /executor; fi # put the scheduler on /scheduler (need to be copied from different places depending on FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/ballista/rust/target/release/ballista-scheduler /scheduler; fi # put the tpch on /tpch (need to be copied from different places depending on FLAG) ENV RELEASE_FLAG=${RELEASE_FLAG} -RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi +RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/tpch /tpch; else mv /tmp/ballista/ballista/rust/target/release/tpch /tpch; fi # Copy the binary into a new container for a smaller docker image FROM ballistacompute/rust-base:0.4.0-20210213 diff --git a/ballista/dev/integration-tests.sh b/dev/integration-tests.sh similarity index 93% rename from ballista/dev/integration-tests.sh rename to dev/integration-tests.sh index cc34a5ce91f5..6ed764ecda8a 100755 --- a/ballista/dev/integration-tests.sh +++ b/dev/integration-tests.sh @@ -17,8 +17,9 @@ # specific language governing permissions and limitations # under the License. set -e +./dev/build-rust-base.sh ./dev/build-rust.sh -pushd rust/benchmarks/tpch +pushd ballista/rust/benchmarks/tpch ./tpch-gen.sh docker-compose up -d