[docker, scorecard] add docker base image, use in scorecard (#5623)

* add docker base image, use in scorecard

* fix permissions

* remove build which doesn't have pull permissions

* added pyasyncinit

* reorder, install jdk-headless.

docker/Dockerfile.base

@@ -0,0 +1,26 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && \
+  apt-get -y install \
+    htop \
+    unzip bzip2 \
+    wget curl \
+    emacs25-nox \
+    default-jdk-headless \
+    python3 python3-pip && \
+  rm -rf /var/lib/apt/lists/*
+
+# source: https://cloud.google.com/storage/docs/gsutil_install#linux
+RUN /bin/sh -c 'curl https://sdk.cloud.google.com | bash' && \
+    mv /root/google-cloud-sdk / && \
+    /google-cloud-sdk/bin/gcloud components install beta kubectl
+ENV PATH $PATH:/google-cloud-sdk/bin
+
+RUN python3 -m pip install -U \
+  pip decorator pylint pytest flake8 \
+  requests \
+  jinja2 \
+  pyasyncinit aiohttp aiodns aiohttp_jinja2 uvloop>=0.12 \
+  werkzeug flask flask-cors Flask_Sockets \
+  kubernetes google-cloud-storage \
+  PyGithub cerberus humanize libsass authlib

docker/Dockerfile.spark-base

@@ -0,0 +1,11 @@
+FROM base
+
+RUN wget -O spark-2.2.0-bin-hadoop2.7.tgz https://archive.apache.org/dist/spark/spark-2.2.0/spark-2.2.0-bin-hadoop2.7.tgz && \
+  tar xzf spark-2.2.0-bin-hadoop2.7.tgz && \
+  rm spark-2.2.0-bin-hadoop2.7.tgz
+
+RUN wget -O /spark-2.2.0-bin-hadoop2.7/jars/gcs-connector-hadoop2-latest.jar https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-latest.jar
+COPY core-site.xml /spark-2.2.0-bin-hadoop2.7/conf/core-site.xml
+
+ENV SPARK_HOME /spark-2.2.0-bin-hadoop2.7
+ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"

docker/Makefile

@@ -0,0 +1,21 @@
+.PHONY: build push deploy
+
+PROJECT = $(shell gcloud config get-value project)
+
+BASE_IMAGE = gcr.io/$(PROJECT)/base:$(shell docker images -q --no-trunc base:latest | sed -e 's,[^:]*:,,')
+SPARK_BASE_IMAGE = gcr.io/$(PROJECT)/spark-base:$(shell docker images -q --no-trunc spark-base:latest | sed -e 's,[^:]*:,,')
+
+build:
+	-docker pull ubuntu:18.04
+	-docker pull gcr.io/$(PROJECT)/base
+	-docker pull gcr.io/$(PROJECT)/spark-base
+	docker build . -t base -f Dockerfile.base --cache-from base,ubuntu:18.04
+	docker build . -t spark-base -f Dockerfile.spark-base --cache-from spark-base,base,ubuntu:18.04
+
+push: build
+	docker tag base $(BASE_IMAGE)
+	docker push $(BASE_IMAGE)
+	docker tag spark-base $(SPARK_BASE_IMAGE)
+	docker push $(SPARK_BASE_IMAGE)
+
+deploy: push

docker/core-site.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+  <property>
+    <name>google.cloud.auth.service.account.enable</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>google.cloud.auth.service.account.json.keyfile</name>
+    <value>/hail-vdc-sa-key/hail-vdc-sa-key.json</value>
+  </property>
+
+</configuration>

docker/hail-ci-deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -ex
+
+gcloud -q auth activate-service-account \
+  --key-file=/secrets/gcr-push-service-account-key.json
+
+gcloud -q auth configure-docker
+
+make deploy

projects.yaml

@@ -1,3 +1,4 @@
+- project: docker
 - project: auth-gateway
 - project: batch
 - project: ci
@@ -15,6 +16,7 @@
 - project: pipeline
   dependencies: ["batch"]
 - project: scorecard
+  dependencies: ["docker"]
 - project: site
 - project: upload
 - project: vdc

scorecard/Dockerfile

@@ -1,13 +1,7 @@
-FROM continuumio/miniconda
-MAINTAINER Hail Team <[email protected]>
-
-COPY environment.yml .
-RUN conda env create scorecard -f environment.yml && \
-    rm -f environment.yml && \
-    rm -rf /home/root/.conda/pkgs/*
+FROM base
 
 COPY scorecard /scorecard
 
 EXPOSE 5000
 
-CMD ["bash", "-c", "source activate scorecard; python /scorecard/scorecard.py"]
+CMD ["bash", "-c", "python3 /scorecard/scorecard.py"]

scorecard/Makefile

@@ -2,23 +2,24 @@
 
 PROJECT = $(shell gcloud config get-value project)
 
+SCORECARD_IMAGE = gcr.io/$(PROJECT)/scorecard:$(shell docker images -q --no-trunc scorecard:latest | sed -e 's,[^:]*:,,')
+
 build:
-	docker build . -t scorecard
+	-docker pull gcr.io/$(PROJECT)/scorecard
+	docker build . -t scorecard --cache-from scorecard,base,ubuntu:18.04
 
-push: IMAGE = gcr.io/$(PROJECT)/scorecard:$(shell docker images -q --no-trunc scorecard | sed -e 's,[^:]*:,,')
 push: build
-	echo $(IMAGE) > scorecard-image
-	docker tag scorecard $(IMAGE)
-	docker push $(IMAGE)
+	docker tag scorecard $(SCORECARD_IMAGE)
+	docker push $(SCORECARD_IMAGE)
 
 run-docker: build
-	docker run -i -p 5000:5000 -v secrets:/secrets -t scorecard
+	docker run -i -p 5000:5000 -v `pwd`/secrets:/secrets -t scorecard
 
 run:
 	GITHUB_TOKEN_PATH=secrets/scorecard-github-access-token.txt python scorecard/scorecard.py
 
 deploy: push
 	sed -e "s,@sha@,$(shell git rev-parse --short=12 HEAD)," \
-	  -e "s,@image@,$(shell cat scorecard-image)," \
+	  -e "s,@image@,$(SCORECARD_IMAGE)," \
 	  < deployment.yaml.in > deployment.yaml
 	kubectl -n default apply -f deployment.yaml