Rebased OSB backbone

.gitignore

@@ -461,3 +461,4 @@ __pycache__/
 vendor
 *.bak
 **/resources/*
+build

.gitlab-ci.jsonnet

@@ -6,7 +6,6 @@ local images = vars.images;
 local docker = utils.docker;
 local stages_list = [
     // gitlab-ci stages
-    'sanity',
     'docker_base',
     'docker_build',
     'deploy_preview',
@@ -42,38 +41,24 @@ local jobs = {
         only: ["schedules", "tags"],
     },
 
-    'sanity-checks': baseJob.sanityCheck {
-        stage: 'sanity',
-        script: [
-            "make vendor",
-            "make verify-catalog",
-            "make verify-codegen",
-        ],
-    },
-
     'container-build': baseJob.dockerBuild {
         // Build and push the alm container.
         // Docker Tag is the branch/tag name
         stage: stages.docker_build,
-        before_script+: ["mkdir -p $PWD/bin"],
-        script:
-            docker.build_and_push(images.ci.alm.name,
-                                  cache=false,
-                                  extra_opts=["-f alm-ci.Dockerfile"]) +
-            docker.build_and_push(images.ci.catalog.name,
-                                  cache=false,
-                                  extra_opts=["-f catalog-ci.Dockerfile"]) +
-            docker.cp(images.ci.alm.name, src="/bin/alm", dest="bin/alm") +
-            docker.cp(images.ci.catalog.name, src="/bin/catalog", dest="bin/catalog") +
-            docker.build_and_push(images.prerelease.alm.name,
-                                  cache=false,
-                                  extra_opts=["-f alm-pre.Dockerfile"]) +
-            docker.build_and_push(images.prerelease.catalog.name,
-                                  cache=false,
-                                  extra_opts=["-f catalog-pre.Dockerfile"]) +
-            docker.build_and_push(images.e2e.name,
-                                  cache=false,
-                                  extra_opts=["-f e2e-run.Dockerfile"]),
+        before_script+: [
+        	"mkdir -p $PWD/bin",
+        ],
+
+        // builds a single multistage dockerfile and tags images based on labels
+        // on the intermediate builds
+        script: docker.multibuild_and_push("Dockerfile", labelImageMap={
+            'builder': images.ci.alm.name,
+            'olm': images.prerelease.alm.name,
+            'catalog': images.prerelease.catalog.name,
+            'broker': images.prerelease.servicebroker.name,
+            'e2e': images.e2e.name,
+        }) +
+        docker.run(images.ci.alm.name, "make verify-codegen verify-catalog")
     },
 
     'container-release': baseJob.dockerBuild {
@@ -84,6 +69,7 @@ local jobs = {
         script:
             docker.rename(images.prerelease.alm.name, images.release.alm.name) +
             docker.rename(images.prerelease.catalog.name, images.release.catalog.name) +
+            docker.rename(images.prerelease.servicebroker.name, images.release.servicebroker.name) +
             docker.rename(images.e2e.name, images.e2elatest.name),
 
     } + onlyMaster,

.gitlab-ci.yml

@@ -23,18 +23,18 @@ container-build:
   - mkdir -p $PWD/bin
   image: docker:git
   script:
-  - 'docker build  --no-cache -f alm-ci.Dockerfile -t quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG} . '
+  - docker build -f Dockerfile .
+  - "docker tag $(docker images --filter 'label=broker=true' --format '{{.CreatedAt}}\t{{.ID}}' | sort -nr | head -n 1 | cut -f2) quay.io/coreos/alm-service-broker-ci:${CI_COMMIT_REF_SLUG}-pre"
+  - "docker tag $(docker images --filter 'label=builder=true' --format '{{.CreatedAt}}\t{{.ID}}' | sort -nr | head -n 1 | cut -f2) quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG}"
+  - "docker tag $(docker images --filter 'label=catalog=true' --format '{{.CreatedAt}}\t{{.ID}}' | sort -nr | head -n 1 | cut -f2) quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG}-pre"
+  - "docker tag $(docker images --filter 'label=e2e=true' --format '{{.CreatedAt}}\t{{.ID}}' | sort -nr | head -n 1 | cut -f2) quay.io/coreos/alm-e2e:${CI_COMMIT_REF_SLUG}-${SHA8}"
+  - "docker tag $(docker images --filter 'label=olm=true' --format '{{.CreatedAt}}\t{{.ID}}' | sort -nr | head -n 1 | cut -f2) quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG}-pre"
+  - docker push quay.io/coreos/alm-service-broker-ci:${CI_COMMIT_REF_SLUG}-pre
   - docker push quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG}
-  - 'docker build  --no-cache -f catalog-ci.Dockerfile -t quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG} . '
-  - docker push quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG}
-  - docker create quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG} | xargs -I{} docker cp {}:/bin/alm bin/alm
-  - docker create quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG} | xargs -I{} docker cp {}:/bin/catalog bin/catalog
-  - 'docker build  --no-cache -f alm-pre.Dockerfile -t quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG}-pre . '
-  - docker push quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG}-pre
-  - 'docker build  --no-cache -f catalog-pre.Dockerfile -t quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG}-pre . '
   - docker push quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG}-pre
-  - 'docker build  --no-cache -f e2e-run.Dockerfile -t quay.io/coreos/alm-e2e:${CI_COMMIT_REF_SLUG}-${SHA8} . '
   - docker push quay.io/coreos/alm-e2e:${CI_COMMIT_REF_SLUG}-${SHA8}
+  - docker push quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG}-pre
+  - docker run  quay.io/coreos/alm-ci:${CI_COMMIT_REF_SLUG} make verify-codegen verify-catalog
   stage: docker_build
   tags:
   - kubernetes
@@ -56,6 +56,9 @@ container-release:
   - docker pull quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG}-pre
   - docker tag quay.io/coreos/catalog-ci:${CI_COMMIT_REF_SLUG}-pre quay.io/coreos/catalog:${CI_COMMIT_REF_SLUG}-${SHA8}
   - docker push quay.io/coreos/catalog:${CI_COMMIT_REF_SLUG}-${SHA8}
+  - docker pull quay.io/coreos/alm-service-broker-ci:${CI_COMMIT_REF_SLUG}-pre
+  - docker tag quay.io/coreos/alm-service-broker-ci:${CI_COMMIT_REF_SLUG}-pre quay.io/coreos/alm-service-broker:${CI_COMMIT_REF_SLUG}-${SHA8}
+  - docker push quay.io/coreos/alm-service-broker:${CI_COMMIT_REF_SLUG}-${SHA8}
   - docker pull quay.io/coreos/alm-e2e:${CI_COMMIT_REF_SLUG}-${SHA8}
   - docker tag quay.io/coreos/alm-e2e:${CI_COMMIT_REF_SLUG}-${SHA8} quay.io/coreos/alm-e2e:latest
   - docker push quay.io/coreos/alm-e2e:latest
@@ -215,8 +218,8 @@ e2e_tests:
   - 'kubectl -n e2e-${CI_COMMIT_REF_SLUG}-${SHA8} patch serviceaccount default -p ''{"imagePullSecrets": [{"name": "coreos-pull-secret"}]}'' || true'
   - kubectl -n e2e-${CI_COMMIT_REF_SLUG}-${SHA8} create rolebinding e2e-admin-rb --clusterrole=cluster-admin --serviceaccount=e2e-${CI_COMMIT_REF_SLUG}-${SHA8}:default --namespace=e2e-${CI_COMMIT_REF_SLUG}-${SHA8}
     || true
-  - charttmpdir=`mktemp -d 2>/dev/null || mktemp -d -t 'charttmpdir'`;mkdir -p ${charttmpdir};pushd e2e/chart/templates;filenames=$(ls *.yaml);popd;for f in ${filenames};do helm template --set namespace=e2e-${CI_COMMIT_REF_SLUG}-${SHA8}
-    e2e/chart -x templates/${f} --set e2e.image.ref=quay.io/coreos/alm-e2e:${CI_COMMIT_REF_SLUG}-${SHA8} --set job_name=e2e-${CI_COMMIT_REF_SLUG}-${SHA8} --set namespace=e2e-${CI_COMMIT_REF_SLUG}-${SHA8}
+  - charttmpdir=`mktemp -d 2>/dev/null || mktemp -d -t 'charttmpdir'`;mkdir -p ${charttmpdir};pushd test/e2e/chart/templates;filenames=$(ls *.yaml);popd;for f in ${filenames};do helm template --set namespace=e2e-${CI_COMMIT_REF_SLUG}-${SHA8}
+    test/e2e/chart -x templates/${f} --set e2e.image.ref=quay.io/coreos/alm-e2e:${CI_COMMIT_REF_SLUG}-${SHA8} --set job_name=e2e-${CI_COMMIT_REF_SLUG}-${SHA8} --set namespace=e2e-${CI_COMMIT_REF_SLUG}-${SHA8}
     > ${charttmpdir}/${f};done;kubectl apply -f ${charttmpdir}
   - until kubectl -n e2e-${CI_COMMIT_REF_SLUG}-${SHA8} logs job/e2e-${CI_COMMIT_REF_SLUG}-${SHA8} | grep -v 'ContainerCreating'; do echo 'waiting for job to run' && sleep 1; done
   - kubectl -n e2e-${CI_COMMIT_REF_SLUG}-${SHA8} logs job/e2e-${CI_COMMIT_REF_SLUG}-${SHA8} -f
@@ -228,19 +231,7 @@ e2e_tests:
   variables:
     K8S_NAMESPACE: e2e-${CI_COMMIT_REF_SLUG}-${SHA8}
     NAMESPACE: e2e-${CI_COMMIT_REF_SLUG}-${SHA8}
-sanity-checks:
-  before_script:
-  - mkdir -p $GOPATH/src/github.com/operator-framework/operator-lifecycle-manager
-  - cp -a $CI_PROJECT_DIR/. $GOPATH/src/github.com/operator-framework/operator-lifecycle-manager
-  - cd $GOPATH/src/github.com/operator-framework/operator-lifecycle-manager
-  image: quay.io/coreos/alm-ci:base
-  script:
-  - make vendor
-  - make verify-catalog
-  - make verify-codegen
-  stage: sanity
 stages:
-- sanity
 - docker_base
 - docker_build
 - deploy_preview

.gitlab-ci/base_jobs.libsonnet

@@ -8,16 +8,6 @@ local appr = utils.appr;
 {
     local job_tags = { tags: ["kubernetes"] },
 
-    sanityCheck: {
-        before_script: [
-            "mkdir -p %s" % vars.paths.alm.src,
-            "cp -a $CI_PROJECT_DIR/. %s" % vars.paths.alm.src,
-            "cd %s" % vars.paths.alm.src,
-        ],
-        // base job for running sanity checks before building containers
-        image: vars.images.base.name,
-    },
-
     dockerBuild: {
         // base job to manage containers (build / push)
         image: "docker:git",
@@ -47,7 +37,7 @@ local appr = utils.appr;
             appname: self.namespace,
             namespace: "e2e-%s" % "${CI_COMMIT_REF_SLUG}-${SHA8}",
             jobname: "e2e-%s" % "${CI_COMMIT_REF_SLUG}-${SHA8}",
-            chart: "e2e/chart",
+            chart: "test/e2e/chart",
             appversion: "1.0.0-e2e-%s" % self.image.alm.tag,
             helm_opts: [],
             params: {

.gitlab-ci/utils.libsonnet

@@ -54,7 +54,6 @@
         login(server, user, password):: [
             "docker login -u %s -p %s %s" % [user, password, server],
         ],
-
         cp(image, src, dest):: [
             "docker create %s | xargs -I{} docker cp {}:%s %s" % [image, src, dest],
         ],
@@ -69,7 +68,22 @@
             Docker.push(image)
         ),
 
-        build(image, cache=true, args={}, extra_opts=[]):: [
+        multibuild_and_push(dockerfile, labelImageMap={}):: (
+            Docker.build_file(dockerfile) +
+            Docker.tag_from_labels(labelImageMap) +
+            Docker.push_all([labelImageMap[label] for label in std.objectFields(labelImageMap)])
+        ),
+
+        build_file(dockerfile):: [
+            'docker build -f %s .' % [dockerfile],
+        ],
+
+        tag_from_labels(labelImageMap={}):: [
+            "docker tag $(docker images --filter 'label=%s=true' --format '{{.CreatedAt}}\t{{.ID}}' | sort -nr | head -n 1 | cut -f2) %s"
+            % [label, labelImageMap[label]] for label in std.objectFields(labelImageMap)
+        ],
+
+        build(image, cache=true, args={},extra_opts=[]):: [
             local cache_opt = if cache == false
             then '--no-cache'
             else if std.type(cache) == 'boolean'
@@ -86,6 +100,10 @@
         push(image):: [
             'docker push %s' % image,
         ],
+
+        push_all(images=[]):: (
+            ['docker push %s' % image for image in images]
+        ),
 
         rename(src, dest):: [
             'docker pull %s' % src,

.gitlab-ci/vars.libsonnet

@@ -37,6 +37,11 @@ local utils = import "utils.libsonnet";
                 tag: "${CI_COMMIT_REF_SLUG}-${SHA8}",
                 name: utils.containerName(self.repo, self.tag),
             },
+            servicebroker: {
+                repo: "quay.io/coreos/alm-service-broker",
+                tag: "${CI_COMMIT_REF_SLUG}-${SHA8}",
+                name: utils.containerName(self.repo, self.tag),
+            },
         },
 
         ci: {
@@ -45,11 +50,6 @@ local utils = import "utils.libsonnet";
                 tag: "${CI_COMMIT_REF_SLUG}",
                 name: utils.containerName(self.repo, self.tag),
             },
-            catalog: {
-                repo: "quay.io/coreos/catalog-ci",
-                tag: "${CI_COMMIT_REF_SLUG}",
-                name: utils.containerName(self.repo, self.tag),
-            },
         },
 
         e2e: {
@@ -75,6 +75,11 @@ local utils = import "utils.libsonnet";
                 tag: "${CI_COMMIT_REF_SLUG}-pre",
                 name: utils.containerName(self.repo, self.tag),
             },
+            servicebroker: {
+                repo: "quay.io/coreos/alm-service-broker-ci",
+                tag: "${CI_COMMIT_REF_SLUG}-pre",
+                name: utils.containerName(self.repo, self.tag),
+            },
         },
     },
 }

Dockerfile

@@ -0,0 +1,38 @@
+FROM quay.io/coreos/alm-ci:base as builder
+LABEL builder=true
+WORKDIR /go/src/github.com/operator-framework/operator-lifecycle-manager
+RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 -o /bin/jq
+# Cache Dep first
+COPY Gopkg.toml Gopkg.lock Makefile ./
+RUN make vendor
+COPY . .
+RUN make build
+RUN go test -c -o /bin/e2e ./test/e2e/...
+
+FROM alpine:latest as olm
+LABEL olm=true
+WORKDIR /
+COPY --from=builder /go/src/github.com/operator-framework/operator-lifecycle-manager/bin/alm /bin/alm
+EXPOSE 8080
+CMD ["/bin/alm"]
+
+FROM alpine:latest as catalog
+LABEL catalog=true
+WORKDIR /
+COPY --from=builder /go/src/github.com/operator-framework/operator-lifecycle-manager/bin/catalog /bin/catalog
+EXPOSE 8080
+CMD ["/bin/catalog"]
+
+FROM alpine:latest as broker
+LABEL broker=true
+WORKDIR /
+COPY --from=builder /go/src/github.com/operator-framework/operator-lifecycle-manager/bin/servicebroker /bin/servicebroker
+EXPOSE 8080
+EXPOSE 8005
+CMD ["/bin/servicebroker"]
+
+FROM quay.io/coreos/alm-ci:base
+LABEL e2e=true
+COPY --from=builder /bin/e2e /bin/e2e
+COPY --from=builder /bin/jq /bin/jq
+CMD ["/bin/e2e"]

Documentation/install/install.md

@@ -72,6 +72,19 @@ catalog:
   service:
     # port for readiness/liveness probes
     internalPort: 8080
+
+# alm service broker run configuration
+servicebroker:
+  # OLM Service Broker doesn't need to be HA; scale with load
+  replicaCount: 1
+  image:
+    ref: quay.io/coreos/olm-service-broker:local
+    pullPolicy: IfNotPresent
+  service:
+    # port for readiness/liveness probes
+    internalPort: 8080
+    # port exposing OLM's service broker API
+    brokerPort: 8005
 ```
 
 To configure a release of OLM for installation in a cluster:
@@ -101,7 +114,7 @@ apiVersion: app.coreos.com/v1alpha1
 kind: Subscription-v1
 metadata:
   name: etcd
-  namespace: local 
+  namespace: local
 spec:
   channel: alpha
   name: etcd

Documentation/install/local-values-shift.yaml

@@ -17,3 +17,12 @@ catalog:
     pullPolicy: IfNotPresent
   service:
     internalPort: 8080
+
+servicebroker:
+  replicaCount: 1
+  image:
+    ref: quay.io/coreos/alm-service-broker:local
+    pullPolicy: IfNotPresent
+  service:
+    internalPort: 8080
+    brokerPort: 8005

Documentation/install/local-values.yaml

@@ -17,3 +17,12 @@ catalog:
     pullPolicy: IfNotPresent
   service:
     internalPort: 8080
+
+servicebroker:
+  replicaCount: 1
+  image:
+    ref: quay.io/coreos/alm-service-broker:local
+    pullPolicy: IfNotPresent
+  service:
+    internalPort: 8080
+    brokerPort: 8005