Fix serialization failures (flyteorg#407)

* Fix all serialization failures

Signed-off-by: Haytham Abuelfutuh <[email protected]>

* Cleanup athena/Makefile

Signed-off-by: Haytham Abuelfutuh <[email protected]>

cookbook/Makefile

@@ -1,6 +1,6 @@
 .SILENT:
 
-include common/Makefile
+include common/common.mk
 include common/parent.mk
 
 define PIP_COMPILE

cookbook/case_studies/Makefile

@@ -1,2 +1,2 @@
-include ../common/Makefile
+include ../common/common.mk
 include ../common/parent.mk

cookbook/case_studies/feature_engineering/Makefile

@@ -1,2 +1,2 @@
-include ../../common/Makefile
+include ../../common/common.mk
 include ../../common/parent.mk

cookbook/case_studies/feature_engineering/eda/Dockerfile

@@ -29,6 +29,9 @@ RUN ${VENV}/bin/pip install wheel
 COPY eda/requirements.txt /root
 RUN ${VENV}/bin/pip install -r /root/requirements.txt
 
+# Copy the makefile targets to expose on the container. This makes it easier to register.
+COPY in_container.mk /root/Makefile
+
 # Copy the actual code
 COPY eda/ /root/eda/
 

cookbook/case_studies/feature_engineering/eda/Makefile

@@ -1,3 +1,3 @@
 PREFIX=eda
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk

cookbook/case_studies/feature_engineering/sqlite_datacleaning/Makefile

@@ -1,3 +1,3 @@
 PREFIX=sqlite_datacleaning
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk

cookbook/case_studies/ml_training/Makefile

@@ -1,2 +1,2 @@
-include ../../common/Makefile
+include ../../common/common.mk
 include ../../common/parent.mk

cookbook/case_studies/ml_training/house_price_prediction/Makefile

@@ -1,3 +1,3 @@
 PREFIX=house_price_prediction
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk

cookbook/case_studies/ml_training/mnist_classifier/Makefile

@@ -1,3 +1,3 @@
 PREFIX=mnist_classifier
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk

cookbook/case_studies/ml_training/pima_diabetes/Makefile

@@ -1,3 +1,3 @@
 PREFIX=pima_diabetes
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk

cookbook/common/parent.mk

@@ -14,64 +14,75 @@ fast_serialize:
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
 		trimmed=$${dir%/}; \
-		test -f $$dir/Makefile && \
-		PREFIX=$$trimmed $(MAKE) fast_serialize; \
+	    if [[ -f $$dir/Makefile ]]; then \
+    		PREFIX=$$trimmed $(MAKE) fast_serialize || exit; \
+		fi; \
 	done
 
 .PHONY: fast_register
 fast_register: ## Registers new code changes using the last built image (assumes current HEAD refers to a built image).
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
 		trimmed=$${dir%/}; \
-		test -f $$dir/Makefile && \
-		PREFIX=$$trimmed $(MAKE) fast_register; \
+	    if [[ -f $$dir/Makefile ]]; then \
+    		PREFIX=$$trimmed $(MAKE) fast_register || exit; \
+		fi; \
 	done
 
 .PHONY: register
 register: ## Builds, pushes and registers all docker images, workflows and tasks in all sub directories.
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
-		test -f $$dir/Makefile && \
-		$(MAKE) -C $$dir register; \
+	    if [[ -f $$dir/Makefile ]]; then \
+    		$(MAKE) -C $$dir register || exit; \
+		fi; \
 	done
 
 .PHONY: serialize
 serialize: ## Builds and serializes all docker images, workflows and tasks in all sub directories.
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
-		test -f $$dir/Makefile && \
-		$(MAKE) -C $$dir serialize; \
+		echo "$$dir"; \
+	    if [[ -f $$dir/Makefile ]]; then \
+	        if [[ "$$dir" != 'docs/' ]]; then \
+	            $(MAKE) -C $$dir serialize || exit; \
+	        fi; \
+		fi; \
 	done
 
 .PHONY: docker_push
 docker_push: ## Builds and pushes all docker images.
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
 		test -f $$dir/Dockerfile && \
-		test -f $$dir/Makefile && \
-		$(MAKE) -C $$dir docker_push; \
+	    if [[ -f $$dir/Makefile ]]; then \
+    		$(MAKE) -C $$dir docker_push || exit; \
+		fi; \
 	done
 
 .PHONY: requirements
 requirements: ## Makes all requirement files in sub directories.
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
-		test -f $$dir/Makefile && \
-		$(MAKE) -C $$dir requirements; \
+	    if [[ -f $$dir/Makefile ]]; then \
+    		$(MAKE) -C $$dir requirements || exit; \
+		fi; \
 	done
 
 .PHONY: k3d_load_image
 k3d_load_image:
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
-		test -f $$dir/Makefile && \
-		$(MAKE) -C $$dir k3d_load_image; \
+	    if [[ -f $$dir/Makefile ]]; then \
+    		$(MAKE) -C $$dir k3d_load_image || exit; \
+		fi; \
 	done
 
 .PHONY: clean
 clean: ## Deletes build directories (e.g. _pb_output/)
 	@for dir in $(SUBDIRS) ; do \
 		echo "processing ${PWD}/$$dir"; \
-		test -f $$dir/Makefile && \
-		$(MAKE) -C $$dir clean; \
-	done
+	    if [[ -f $$dir/Makefile ]]; then \
+    		$(MAKE) -C $$dir clean || exit; \
+		fi; \
+	done

cookbook/core/Makefile

@@ -1,4 +1,4 @@
 PREFIX=core
 PWD=$(shell pwd)
-include ../common/Makefile
+include ../common/common.mk
 include ../common/leaf.mk

cookbook/integrations/Makefile

@@ -1,2 +1,2 @@
-include ../common/Makefile
+include ../common/common.mk
 include ../common/parent.mk

cookbook/integrations/aws/Makefile

@@ -1,2 +1,2 @@
-include ../../common/Makefile
+include ../../common/common.mk
 include ../../common/parent.mk

cookbook/integrations/aws/athena/Dockerfile

@@ -0,0 +1,35 @@
+FROM python:3.8-slim-buster
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks
+
+WORKDIR /root
+ENV VENV /opt/venv
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV PYTHONPATH /root
+
+# This is necessary for opencv to work
+RUN apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev ffmpeg build-essential
+
+# Install the AWS cli separately to prevent issues with boto being written over
+RUN pip3 install awscli
+
+ENV VENV /opt/venv
+# Virtual environment
+RUN python3 -m venv ${VENV}
+ENV PATH="${VENV}/bin:$PATH"
+
+# Install Python dependencies
+COPY athena/requirements.txt /root
+RUN pip install -r /root/requirements.txt
+
+# Copy the makefile targets to expose on the container. This makes it easier to register
+COPY in_container.mk /root/Makefile
+COPY athena/sandbox.config /root
+
+# Copy the actual code
+COPY athena /root/athena
+
+# This tag is supplied by the build script and will be used to determine the version
+# when registering tasks, workflows, and launch plans
+ARG tag
+ENV FLYTE_INTERNAL_IMAGE $tag

cookbook/integrations/aws/athena/Makefile

@@ -1,3 +1,3 @@
 PREFIX=athena
-include ../../../common/Makefile
-include ../../../common/leaf.mk
+include ../../../common/common.mk
+include ../../../common/leaf.mk

cookbook/integrations/aws/sagemaker_pytorch/Makefile

@@ -1,3 +1,3 @@
 PREFIX=sagemaker_pytorch
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk

cookbook/integrations/aws/sagemaker_pytorch/requirements.in

@@ -1,5 +1,5 @@
 -r ../../../common/requirements-common.in
 flytekitplugins-awssagemaker>=0.16.0
---find-links https://download.pytorch.org/whl/torch_stable.html torch==1.8.1+cpu
---find-links https://download.pytorch.org/whl/torch_stable.html torchvision==0.9.1+cpu
+torch==1.8.1
+torchvision==0.9.1
 tensorboardX

cookbook/integrations/aws/sagemaker_pytorch/requirements.txt

@@ -2,17 +2,15 @@
 # This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    /Library/Developer/CommandLineTools/usr/bin/make requirements.txt
+#    /Applications/Xcode.app/Contents/Developer/usr/bin/make requirements.txt
 #
---find-links https://download.pytorch.org/whl/torch_stable.html
-
 attrs==21.2.0
     # via scantree
 bcrypt==3.2.0
     # via paramiko
-boto3==1.18.38
+boto3==1.18.39
     # via sagemaker-training
-botocore==1.21.38
+botocore==1.21.39
     # via
     #   boto3
     #   s3transfer
@@ -98,14 +96,18 @@ numpy==1.21.2
     #   sagemaker-training
     #   scipy
     #   tensorboardx
+    #   torch
+    #   torchvision
 pandas==1.3.2
     # via flytekit
 paramiko==2.7.2
     # via sagemaker-training
 pathspec==0.9.0
     # via scantree
 pillow==8.3.2
-    # via matplotlib
+    # via
+    #   matplotlib
+    #   torchvision
 protobuf==3.17.3
     # via
     #   flyteidl
@@ -178,8 +180,16 @@ statsd==3.3.0
     # via flytekit
 tensorboardx==2.4
     # via -r requirements.in
+torch==1.8.1
+    # via
+    #   -r requirements.in
+    #   torchvision
+torchvision==0.9.1
+    # via -r requirements.in
 typing-extensions==3.10.0.2
-    # via typing-inspect
+    # via
+    #   torch
+    #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
 urllib3==1.26.6

cookbook/integrations/aws/sagemaker_training/Makefile

@@ -1,3 +1,3 @@
 PREFIX=sagemaker_training
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk

cookbook/integrations/external_services/Makefile

@@ -1,2 +1,2 @@
-include ../../common/Makefile
+include ../../common/common.mk
 include ../../common/parent.mk

cookbook/integrations/external_services/hive/Makefile

@@ -1,5 +1,5 @@
 PREFIX=hive
-include ../../../common/Makefile
+include ../../../common/common.mk
 include ../../../common/leaf.mk
 
 .PHONY: docker_build

cookbook/integrations/flytekit_plugins/Makefile

@@ -1,2 +1,2 @@
-include ../../common/Makefile
+include ../../common/common.mk
 include ../../common/parent.mk

cookbook/integrations/flytekit_plugins/dolt/Dockerfile

@@ -6,6 +6,9 @@ ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 ENV PYTHONPATH /root
 
+RUN apt-get update && \
+    apt-get -y install sudo
+
 # Install dolt
 RUN sudo bash -c 'curl -L https://github.com/dolthub/dolt/releases/latest/download/install.sh | sudo bash' &&\
     dolt config --global --add user.email [email protected] &&\
@@ -31,7 +34,7 @@ COPY dolt/sandbox.config /root
 COPY dolt/ /root/dolt/
 
 RUN mkdir -p /root/foo &&\
-    cd /root/dolt/foo &&\
+    cd /root/foo &&\
     dolt init
 
 # This tag is supplied by the build script and will be used to determine the version

cookbook/integrations/flytekit_plugins/dolt/Makefile

@@ -1,2 +1,3 @@
 PREFIX=dolt
-include ../../../common/Makefile
+include ../../../common/common.mk
+include ../../../common/leaf.mk

cookbook/integrations/flytekit_plugins/dolt/requirements.in

@@ -1,2 +1,3 @@
 -r ../../../common/requirements-common.in
 flytekitplugins.dolt>=0.18.0
+great-expectations>=0.13.31