From a67eb064bf3923f50bfbbe5f88f37851cf972aa0 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Wed, 13 Apr 2022 10:00:54 -0700 Subject: [PATCH] Spark testing (#722) * updat image Signed-off-by: Yee Hing Tong * update sha again Signed-off-by: Yee Hing Tong * delete flytekitvenv Signed-off-by: Yee Hing Tong * fixes Signed-off-by: Yee Hing Tong * delete bad file Signed-off-by: Yee Hing Tong * revert 4.02 Signed-off-by: Yee Hing Tong * reverttagdeletion Signed-off-by: Yee Hing Tong --- cookbook/integrations/kubernetes/k8s_spark/Dockerfile | 10 +++++----- .../integrations/kubernetes/k8s_spark/sandbox.config | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cookbook/integrations/kubernetes/k8s_spark/Dockerfile b/cookbook/integrations/kubernetes/k8s_spark/Dockerfile index 550b91fe42..21b355798b 100644 --- a/cookbook/integrations/kubernetes/k8s_spark/Dockerfile +++ b/cookbook/integrations/kubernetes/k8s_spark/Dockerfile @@ -44,14 +44,16 @@ ENV PYSPARK_PYTHON ${VENV}/bin/python3 ENV PYSPARK_DRIVER_PYTHON ${VENV}/bin/python3 # Copy the makefile targets to expose on the container. This makes it easier to register. +# Delete this after we update CI COPY in_container.mk /root/Makefile + +# Delete this after we update CI to not serialize inside the container COPY k8s_spark/sandbox.config /root # Copy the actual code COPY k8s_spark/ /root/k8s_spark -# This tag is supplied by the build script and will be used to determine the version -# when registering tasks, workflows, and launch plans +# Delete these after we switch to package ARG tag ENV FLYTE_INTERNAL_IMAGE $tag @@ -62,6 +64,4 @@ RUN chmod a+x /usr/local/bin/flytekit_venv # For spark we want to use the default entrypoint which is part of the # distribution, also enable the virtualenv for this image. # Note this relies on the VENV variable we've set in this image. -ENTRYPOINT ["/usr/local/bin/flytekit_venv", "/opt/entrypoint.sh"] - -ENV FLYTE_SDK_USE_STRUCTURED_DATASET TRUE +ENTRYPOINT ["/opt/entrypoint.sh"] diff --git a/cookbook/integrations/kubernetes/k8s_spark/sandbox.config b/cookbook/integrations/kubernetes/k8s_spark/sandbox.config index 7bf486a95a..451c294f56 100644 --- a/cookbook/integrations/kubernetes/k8s_spark/sandbox.config +++ b/cookbook/integrations/kubernetes/k8s_spark/sandbox.config @@ -1,3 +1,2 @@ [sdk] workflow_packages=k8s_spark -python_venv=flytekit_venv