Convert kubeadm pull job to scenario.

This also changes the --kubeadm boolean flag in kubernetes_e2e.py to an
option to specify the type of job (ci, pull, periodic) so that it knows
where to find the .deb artifacts created by the previous bazel job,
since the output location can differ by type.

jenkins/bootstrap_test.py

@@ -1692,15 +1692,20 @@ def LoadProwYaml(self, path):
         if 'periodics' not in doc:
             self.fail('No periodics in prow config!')
 
+        if 'presubmits' not in doc:
+            self.fail('No presubmits in prow config!')
+
         for item in doc.get('periodics'):
             self.AddProwJob(item)
 
         if 'postsubmits' not in doc:
             self.fail('No postsubmits in prow config!')
 
+        presubmits = doc.get('presubmits')
         postsubmits = doc.get('postsubmits')
-        for repo in postsubmits:
-            for job in postsubmits.get(repo):
+
+        for repo, joblist in presubmits.items() + postsubmits.items():
+            for job in joblist:
                 self.AddProwJob(job)
 
     def LoadBootstrapYaml(self, path):
@@ -2002,6 +2007,10 @@ def testValidJobEnvs(self):
                         if 'gke' in job:
                             stage = 'gs://kubernetes-release-dev/ci'
                             suffix = True
+                        elif 'kubeadm' in job:
+                            # kubeadm-based jobs use out-of-band .deb artifacts,
+                            # not the --stage flag.
+                            continue
                         else:
                             stage = 'gs://kubernetes-release-pull/ci/%s' % job
                             suffix = False

jobs/config.json

@@ -2112,7 +2112,7 @@
       "--cluster=", 
       "--env-file=platforms/gce.env", 
       "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce.env", 
-      "--kubeadm", 
+      "--kubeadm=ci", 
       "--mode=local"
     ], 
     "scenario": "kubernetes_e2e"
@@ -2122,7 +2122,7 @@
       "--cluster=", 
       "--env-file=platforms/gce.env", 
       "--env-file=jobs/ci-kubernetes-e2e-kubeadm-gce-1-6.env", 
-      "--kubeadm", 
+      "--kubeadm=ci", 
       "--mode=local"
     ], 
     "scenario": "kubernetes_e2e"
@@ -2705,7 +2705,7 @@
       "--cluster=", 
       "--env-file=platforms/gce.env", 
       "--env-file=jobs/periodic-kubernetes-e2e-kubeadm-gce-1-6.env", 
-      "--kubeadm", 
+      "--kubeadm=periodic", 
       "--mode=local"
     ], 
     "scenario": "kubernetes_e2e"
@@ -2802,6 +2802,16 @@
     ], 
     "scenario": "kubernetes_e2e"
   }, 
+  "pull-kubernetes-e2e-kubeadm-gce": {
+    "args": [
+      "--cluster=", 
+      "--env-file=platforms/gce.env", 
+      "--env-file=jobs/pull-kubernetes-e2e-kubeadm-gce.env", 
+      "--kubeadm=pull", 
+      "--mode=local"
+    ], 
+    "scenario": "kubernetes_e2e"
+  }, 
   "pull-kubernetes-federation-e2e-gce": {
     "args": [
       "--env-file=platforms/gce.env", 

jobs/pull-kubernetes-e2e-kubeadm-gce.env

@@ -0,0 +1,15 @@
+### job-env
+
+PROJECT=k8s-jkns-pr-kubeadm
+KUBERNETES_PROVIDER=kubernetes-anywhere
+
+GINKGO_PARALLEL=y
+GINKGO_TEST_ARGS=--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]
+
+# Resource leak detection is disabled because prow runs multiple instances of
+# this job in the same project concurrently, and resource leak detection will
+# make the job flaky.
+FAIL_ON_GCP_RESOURCE_LEAK=false
+
+# After post-env
+KUBEKINS_TIMEOUT=55m

prow/config.yaml

@@ -91,6 +91,8 @@ presubmits:
         - "--pull=$(PULL_REFS)"
         - "--upload=gs://kubernetes-jenkins/pr-logs"
         - "--git-cache=/root/.cache/git"
+        - "--timeout=75"
+        - "--json"
         - "--clean"
         env:
         - name: GOOGLE_APPLICATION_CREDENTIALS
@@ -251,6 +253,8 @@ presubmits:
         - "--pull=$(PULL_REFS)"
         - "--upload=gs://kubernetes-jenkins/pr-logs"
         - "--git-cache=/root/.cache/git"
+        - "--timeout=75"
+        - "--json"
         - "--clean"
         env:
         - name: GOOGLE_APPLICATION_CREDENTIALS

scenarios/kubernetes_e2e.py

@@ -63,6 +63,32 @@ def parse_env(env):
     """Returns (FOO, BAR=MORE) for FOO=BAR=MORE."""
     return env.split('=', 1)
 
+def kubeadm_version(mode):
+    """Return string to use for kubeadm version, given the job's mode (ci/pull/periodic)."""
+    version = ''
+    if mode in ['ci', 'periodic']:
+        # This job only runs against the kubernetes repo, and bootstrap.py leaves the
+        # current working directory at the repository root. Grab the SCM_REVISION so we
+        # can use the .debs built during the bazel-build job that should have already
+        # succeeded.
+        status = re.search(
+            r'STABLE_BUILD_SCM_REVISION ([^\n]+)',
+            check_output('hack/print-workspace-status.sh')
+        )
+        if not status:
+            raise ValueError('STABLE_BUILD_SCM_REVISION not found')
+        version = status.group(1)
+
+    elif mode == 'pull':
+        version = '%s/%s' % (os.environ['PULL_NUMBER'], os.getenv('PULL_REFS'))
+
+    else:
+        raise ValueError("Unknown kubeadm mode given: %s" % mode)
+
+    # The path given here should match jobs/ci-kubernetes-bazel-build.sh
+    return 'gs://kubernetes-release-dev/bazel/%s/bin/linux/amd64/' % version
+
+
 class LocalMode(object):
     """Runs e2e tests by calling e2e-runner.sh."""
     def __init__(self, workspace):
@@ -335,25 +361,12 @@ def main(args):
     if args.kubeadm:
         # Not from Jenkins
         cluster = args.cluster or 'e2e-kubeadm-%s' % os.getenv('BUILD_NUMBER', 0)
-
-        # This job only runs against the kubernetes repo, and bootstrap.py leaves the
-        # current working directory at the repository root. Grab the SCM_REVISION so we
-        # can use the .debs built during the bazel-build job that should have already
-        # succeeded.
-        status = re.search(
-            r'STABLE_BUILD_SCM_REVISION ([^\n]+)',
-            check_output('hack/print-workspace-status.sh')
-        )
-        if not status:
-            raise ValueError('STABLE_BUILD_SCM_REVISION not found')
-
+        version = kubeadm_version(args.kubeadm)
         opt = '--deployment kubernetes-anywhere' \
             ' --kubernetes-anywhere-path /workspace/kubernetes-anywhere' \
             ' --kubernetes-anywhere-phase2-provider kubeadm' \
             ' --kubernetes-anywhere-cluster %s' \
-            ' --kubernetes-anywhere-kubeadm-version' \
-            ' gs://kubernetes-release-dev/bazel/%s/build/debs/' % (cluster, status.group(1))
-            # The gs:// path given here should match jobs/ci-kubernetes-bazel-build.sh
+            ' --kubernetes-anywhere-kubeadm-version %s' % (cluster, version)
         mode.add_environment('E2E_OPT=%s' % opt)
 
     # TODO(fejta): delete this?
@@ -441,7 +454,7 @@ def create_parser():
     parser.add_argument(
         '--down', default='true', help='If we need to set --down in e2e.go')
     parser.add_argument(
-        '--kubeadm', action='store_true', help='If the test is a kubeadm job')
+        '--kubeadm', choices=['ci', 'periodic', 'pull'])
     parser.add_argument(
         '--soak-test', action='store_true', help='If the test is a soak test job')
     parser.add_argument(

scenarios/kubernetes_e2e_test.py

@@ -20,6 +20,7 @@
 """Test for kubernetes_e2e.py"""
 
 import json
+import os
 import re
 import shutil
 import string
@@ -111,24 +112,26 @@ def test_local(self):
         for call in self.callstack:
             self.assertFalse(call.startswith('docker'))
 
-    def test_kubeadm(self):
-        """Make sure kubeadm mode is fine overall."""
-        args = self.parser.parse_args(['--mode=local', '--kubeadm'])
+    def test_kubeadm_ci(self):
+        """Make sure kubeadm ci mode is fine overall."""
+        args = self.parser.parse_args(['--mode=local', '--kubeadm=ci'])
         self.assertEqual(args.mode, 'local')
-        self.assertEqual(args.kubeadm, True)
+        self.assertEqual(args.kubeadm, 'ci')
         with Stub(kubernetes_e2e, 'check_env', self.fake_check_env):
             with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status):
                 kubernetes_e2e.main(args)
 
         self.assertIn('E2E_OPT', self.envs)
-        self.assertIn('v1.7.0-alpha.0.1320+599539dc0b9997', self.envs['E2E_OPT'])
+        self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/'
+                      'v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT'])
         called = False
         for call in self.callstack:
             self.assertFalse(call.startswith('docker'))
             if call == 'hack/print-workspace-status.sh':
                 called = True
         self.assertTrue(called)
 
+
     def test_include_host_env(self):
         """Ensure that host variables (such as GOPATH) are included."""
         mode = kubernetes_e2e.LocalMode('/orig-workspace')
@@ -138,6 +141,46 @@ def test_include_host_env(self):
         self.assertIn(['WORKSPACE', '/new/workspace'], mode.env)
         self.assertIn(['GOPATH', '/go/path'], mode.env)
 
+    def test_kubeadm_periodic(self):
+        """Make sure kubeadm periodic mode is fine overall."""
+        args = self.parser.parse_args(['--mode=local', '--kubeadm=periodic'])
+        self.assertEqual(args.mode, 'local')
+        self.assertEqual(args.kubeadm, 'periodic')
+        with Stub(kubernetes_e2e, 'check_env', self.fake_check_env):
+            with Stub(kubernetes_e2e, 'check_output', self.fake_output_work_status):
+                kubernetes_e2e.main(args)
+
+        self.assertIn('E2E_OPT', self.envs)
+        self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/'
+                      'v1.7.0-alpha.0.1320+599539dc0b9997/bin/linux/amd64/', self.envs['E2E_OPT'])
+        called = False
+        for call in self.callstack:
+            self.assertFalse(call.startswith('docker'))
+            if call == 'hack/print-workspace-status.sh':
+                called = True
+        self.assertTrue(called)
+
+    def test_kubeadm_pull(self):
+        """Make sure kubeadm pull mode is fine overall."""
+        args = self.parser.parse_args(['--mode=local', '--kubeadm=pull'])
+        self.assertEqual(args.mode, 'local')
+        self.assertEqual(args.kubeadm, 'pull')
+        fake_env = {'PULL_NUMBER': 1234, 'PULL_REFS': 'master:abcd'}
+        with Stub(kubernetes_e2e, 'check_env', self.fake_check_env):
+            with Stub(os, 'environ', fake_env):
+                kubernetes_e2e.main(args)
+
+        self.assertIn('E2E_OPT', self.envs)
+        self.assertIn('--kubernetes-anywhere-kubeadm-version gs://kubernetes-release-dev/bazel/'
+                      '1234/master:abcd/bin/linux/amd64/', self.envs['E2E_OPT'])
+
+    def test_kubeadm_invalid(self):
+        """Make sure kubeadm invalid mode exits unsuccessfully."""
+        with self.assertRaises(SystemExit) as sysexit:
+            self.parser.parse_args(['--mode=local', '--kubeadm=deploy'])
+
+        self.assertEqual(sysexit.exception.code, 2)
+
 class DockerTest(ScenarioTest):
     """Class for testing e2e scenario in docker mode."""
     def test_docker(self):