Add gsutil to all Dockerfiles (flyteorg#483)

Signed-off-by: Haytham Abuelfutuh <[email protected]>
eapolinario · Nov 11, 2021 · e4495fc · e4495fc
1 parent 10c923f
commit e4495fc
Show file tree

Hide file tree

Showing 19 changed files with 76 additions and 0 deletions.
diff --git a/cookbook/case_studies/feature_engineering/eda/Dockerfile b/cookbook/case_studies/feature_engineering/eda/Dockerfile
@@ -21,6 +21,10 @@ RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Virtual environment
 RUN python3.8 -m venv ${VENV}
 RUN ${VENV}/bin/pip install wheel

diff --git a/cookbook/case_studies/feature_engineering/feast_integration/Dockerfile b/cookbook/case_studies/feature_engineering/feast_integration/Dockerfile
@@ -9,6 +9,10 @@ ENV PYTHONPATH "$PYTHONPATH:/root/feast_integration"
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Virtual environment
 RUN python3 -m venv ${VENV}
 RUN ${VENV}/bin/pip install wheel

diff --git a/cookbook/case_studies/ml_training/house_price_prediction/Dockerfile b/cookbook/case_studies/ml_training/house_price_prediction/Dockerfile
@@ -21,6 +21,10 @@ RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Virtual environment
 RUN python3.8 -m venv ${VENV}
 RUN ${VENV}/bin/pip install wheel

diff --git a/cookbook/case_studies/ml_training/mnist_classifier/Dockerfile b/cookbook/case_studies/ml_training/mnist_classifier/Dockerfile
@@ -13,6 +13,10 @@ ENV PYTHONPATH /root
 # Install the AWS cli for AWS support
 RUN pip install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 RUN apt-get update && apt-get install -y make build-essential libssl-dev curl
 
 # Virtual environment

diff --git a/cookbook/case_studies/ml_training/pima_diabetes/Dockerfile b/cookbook/case_studies/ml_training/pima_diabetes/Dockerfile
@@ -21,6 +21,10 @@ RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Virtual environment
 RUN python3.8 -m venv ${VENV}
 RUN ${VENV}/bin/pip install wheel

diff --git a/cookbook/case_studies/ml_training/spark_horovod/Dockerfile b/cookbook/case_studies/ml_training/spark_horovod/Dockerfile
@@ -37,6 +37,10 @@ ENV PATH="${VENV}/bin:$PATH"
 # in future versions to make it completely portable
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Install wheel after venv is activated
 RUN pip3 install wheel
 

diff --git a/cookbook/core/Dockerfile b/cookbook/core/Dockerfile
@@ -13,6 +13,10 @@ RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg b
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/aws/athena/Dockerfile b/cookbook/integrations/aws/athena/Dockerfile
@@ -13,6 +13,10 @@ RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg b
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/aws/sagemaker_pytorch/Dockerfile b/cookbook/integrations/aws/sagemaker_pytorch/Dockerfile
@@ -16,6 +16,10 @@ RUN apt-get update && apt-get install -y make build-essential libssl-dev curl
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Install gcc , g++ and make
 RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.list \
     && apt-get update && apt-get install --no-install-recommends -y gcc g++

diff --git a/cookbook/integrations/aws/sagemaker_training/Dockerfile b/cookbook/integrations/aws/sagemaker_training/Dockerfile
@@ -9,6 +9,10 @@ ENV PYTHONPATH /root
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Setup a virtual environment
 ENV VENV /opt/venv
 # Virtual environment

diff --git a/cookbook/integrations/flytekit_plugins/dolt/Dockerfile b/cookbook/integrations/flytekit_plugins/dolt/Dockerfile
@@ -17,6 +17,10 @@ RUN sudo bash -c 'curl -L https://github.com/dolthub/dolt/releases/latest/downlo
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/flytekit_plugins/greatexpectations/Dockerfile b/cookbook/integrations/flytekit_plugins/greatexpectations/Dockerfile
@@ -9,6 +9,10 @@ ENV PYTHONPATH /root
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/flytekit_plugins/pandera_examples/Dockerfile b/cookbook/integrations/flytekit_plugins/pandera_examples/Dockerfile
@@ -9,6 +9,10 @@ ENV PYTHONPATH /root
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/flytekit_plugins/papermilltasks/Dockerfile b/cookbook/integrations/flytekit_plugins/papermilltasks/Dockerfile
@@ -9,6 +9,10 @@ ENV PYTHONPATH /root
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/flytekit_plugins/sql/Dockerfile b/cookbook/integrations/flytekit_plugins/sql/Dockerfile
@@ -9,6 +9,10 @@ ENV PYTHONPATH /root
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/kubernetes/k8s_spark/Dockerfile b/cookbook/integrations/kubernetes/k8s_spark/Dockerfile
@@ -15,6 +15,10 @@ RUN apt-get update && apt-get install -y python3.8 python3.8-venv make build-ess
 # in future versions to make it completely portable
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}

diff --git a/cookbook/integrations/kubernetes/kfmpi/Dockerfile b/cookbook/integrations/kubernetes/kfmpi/Dockerfile
@@ -37,6 +37,10 @@ ENV PATH="${VENV}/bin:$PATH"
 # in future versions to make it completely portable
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Install wheel after venv is activated
 RUN pip3 install wheel
 

diff --git a/cookbook/integrations/kubernetes/kfpytorch/Dockerfile b/cookbook/integrations/kubernetes/kfpytorch/Dockerfile
@@ -14,6 +14,10 @@ RUN apt-get update && apt-get install -y make build-essential libssl-dev curl
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 # Install gcc , g++ and make
 RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.list \
     && apt-get update && apt-get install --no-install-recommends -y gcc g++

diff --git a/cookbook/integrations/kubernetes/pod/Dockerfile b/cookbook/integrations/kubernetes/pod/Dockerfile
@@ -10,6 +10,10 @@ ENV PYTHONPATH /root
 # Install the AWS cli separately to prevent issues with boto being written over
 RUN pip3 install awscli
 
+# Install gcloud for GCP
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH="$PATH:/root/google-cloud-sdk/bin"
+
 ENV VENV /opt/venv
 # Virtual environment
 RUN python3 -m venv ${VENV}