Spark testing (flyteorg#722)

* updat image Signed-off-by: Yee Hing Tong <[email protected]> * update sha again Signed-off-by: Yee Hing Tong <[email protected]> * delete flytekitvenv Signed-off-by: Yee Hing Tong <[email protected]> * fixes Signed-off-by: Yee Hing Tong <[email protected]> * delete bad file Signed-off-by: Yee Hing Tong <[email protected]> * revert 4.02 Signed-off-by: Yee Hing Tong <[email protected]> * reverttagdeletion Signed-off-by: Yee Hing Tong <[email protected]>
eapolinario · Apr 13, 2022 · a67eb06 · a67eb06
1 parent 682a03f
commit a67eb06
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 6 deletions.
diff --git a/cookbook/integrations/kubernetes/k8s_spark/Dockerfile b/cookbook/integrations/kubernetes/k8s_spark/Dockerfile
@@ -44,14 +44,16 @@ ENV PYSPARK_PYTHON ${VENV}/bin/python3
 ENV PYSPARK_DRIVER_PYTHON ${VENV}/bin/python3
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
+# Delete this after we update CI
 COPY in_container.mk /root/Makefile
+
+# Delete this after we update CI to not serialize inside the container
 COPY k8s_spark/sandbox.config /root
 
 # Copy the actual code
 COPY k8s_spark/ /root/k8s_spark
 
-# This tag is supplied by the build script and will be used to determine the version
-# when registering tasks, workflows, and launch plans
+# Delete these after we switch to package
 ARG tag
 ENV FLYTE_INTERNAL_IMAGE $tag
 
@@ -62,6 +64,4 @@ RUN chmod a+x /usr/local/bin/flytekit_venv
 # For spark we want to use the default entrypoint which is part of the
 # distribution, also enable the virtualenv for this image. 
 # Note this relies on the VENV variable we've set in this image.
-ENTRYPOINT ["/usr/local/bin/flytekit_venv", "/opt/entrypoint.sh"]
-
-ENV FLYTE_SDK_USE_STRUCTURED_DATASET TRUE
+ENTRYPOINT ["/opt/entrypoint.sh"]
diff --git a/cookbook/integrations/kubernetes/k8s_spark/sandbox.config b/cookbook/integrations/kubernetes/k8s_spark/sandbox.config
@@ -1,3 +1,2 @@
 [sdk]
 workflow_packages=k8s_spark
-python_venv=flytekit_venv