diff --git a/images/bootstrap/Dockerfile b/images/bootstrap/Dockerfile index e679e60cfdb9c..bef45ef6585d5 100644 --- a/images/bootstrap/Dockerfile +++ b/images/bootstrap/Dockerfile @@ -64,9 +64,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ENV PATH=/google-cloud-sdk/bin:/workspace:${PATH} \ CLOUDSDK_CORE_DISABLE_PROMPTS=1 -RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \ - tar xzf google-cloud-sdk.tar.gz -C / && \ - rm google-cloud-sdk.tar.gz && \ +ENV GCLOUD_VERSION=349.0.0 + +RUN set -eux; \ + \ + case $(uname -m) in \ + x86_64) export GCLOUD_TAR_FILE="google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz" ;; \ + aarch64) export GCLOUD_TAR_FILE="google-cloud-sdk-${GCLOUD_VERSION}-linux-arm.tar.gz" ;; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac; \ + \ + wget "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/${GCLOUD_TAR_FILE}"; && \ + tar xzf ${GCLOUD_TAR_FILE} -C / && \ + rm ${GCLOUD_TAR_FILE} && \ /google-cloud-sdk/install.sh \ --disable-installation-options \ --bash-completion=false \ @@ -93,10 +103,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ rm -rf /var/lib/apt/lists/* # Add the Docker apt-repository +ARG TARGETARCH RUN curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg \ | apt-key add - && \ add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \ + "deb [arch=${TARGETARCH:-amd64}] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \ $(lsb_release -cs) stable" # Install Docker diff --git a/images/bootstrap/scenarios/BUILD.bazel b/images/bootstrap/scenarios/BUILD.bazel new file mode 100644 index 0000000000000..82c35c0fb1ce9 --- /dev/null +++ b/images/bootstrap/scenarios/BUILD.bazel @@ -0,0 +1,37 @@ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "scenarios", + srcs = glob(["*.py"]), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) + +py_test( + name = "kubernetes_e2e_test", + srcs = [ + "kubernetes_e2e.py", + "kubernetes_e2e_test.py", + ], + python_version = "PY2", +) + +py_test( + name = "kubernetes_bazel_test", + srcs = [ + "kubernetes_bazel.py", + "kubernetes_bazel_test.py", + ], + python_version = "PY2", +) diff --git a/images/bootstrap/scenarios/OWNERS b/images/bootstrap/scenarios/OWNERS new file mode 100644 index 0000000000000..c854e7d6bf865 --- /dev/null +++ b/images/bootstrap/scenarios/OWNERS @@ -0,0 +1,18 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: +- amwat +- BenTheElder +- MushuEE +- spiffxp +approvers: +- amwat +- BenTheElder +- spiffxp +emeritus_approvers: +- cjwagner +- fejta +- krzyzacy + +labels: +- area/scenarios diff --git a/images/bootstrap/scenarios/README.md b/images/bootstrap/scenarios/README.md new file mode 100644 index 0000000000000..6b58067821118 --- /dev/null +++ b/images/bootstrap/scenarios/README.md @@ -0,0 +1,49 @@ +# DEPRECATION NOTICE + +*October 9, 2018* `scenarios/*.py` will be moved to become part of kubetest v2, so we are +not taking PRs except for urgent bug fixes. + +Also please bump [bootstrap image](/images/bootstrap) and +[kubekins image](/images/kubekins-e2e) to take in any future changes. + +# Test scenarios + +Place scripts to run test scenarios inside this location. + +Test jobs are composed of two things: +1) A scenario to test +2) Configuration options for the scenario. + +Three example scenarios are: + +* Unit tests +* Node e2e tests +* e2e tests + +Example configurations are: + +* Parallel tests on gce +* Build all platforms + +The assumption is that each scenario will be called a variety of times with +different configuration options. For example at the time of this writing there +are over 300 e2e jobs, each run with a slightly different set of options. + +## Contract + +The scenario assumes the calling process (bootstrap.py) has setup all +prerequisites, such as checking out the right repository, setting pwd, +activating service accounts, etc. + +The scenario also assumes that the calling process will handle all post-job +works, such as recording log output, copying logs to gcs, etc. + +The scenario should exit 0 if and only on success. + +The calling process can configure the scenario by calling the scenario +with different arguments. For example: `kubernetes\_build.py --fast` +configures the scenario to do a fast build (one platform) and/or +`kubernetes\_build.py --federation=random-project` configures the scenario +to do a federation build using the random-project. + +Call the scenario with `-h` to see configuration options. diff --git a/images/bootstrap/scenarios/canarypush.py b/images/bootstrap/scenarios/canarypush.py new file mode 100755 index 0000000000000..45b1cc1d481f3 --- /dev/null +++ b/images/bootstrap/scenarios/canarypush.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""Executes a command.""" + +import argparse +import os +import subprocess +import sys + +def check_with_log(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + print >>sys.stderr, subprocess.check_call(cmd) + +def check_no_log(*cmd): + """Run the command, raising on errors, no logs""" + try: + subprocess.check_call(cmd) + except: + raise subprocess.CalledProcessError(cmd='subprocess.check_call', returncode=1) + +def check_output(*cmd): + """Log and run the command, return output, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + return subprocess.check_output(cmd) + + +def main(target, buildfile): + """Build & push to canary.""" + check_with_log( + 'docker', 'build', '-t', target, '--no-cache=true', + '--pull=true', '--file=%s' % buildfile, '.' + ) + check_with_log('docker', 'inspect', target) + + user = None + if os.path.exists(os.environ.get('DOCKER_USER')): + with open(os.environ.get('DOCKER_USER'), 'r') as content_file: + user = content_file.read() + + pwd = None + if os.path.exists(os.environ.get('DOCKER_PASSWORD')): + with open(os.environ.get('DOCKER_PASSWORD'), 'r') as content_file: + pwd = content_file.read() + + if not user or not pwd: + print >>sys.stderr, 'Logging info not exist' + sys.exit(1) + print >>sys.stderr, 'Logging in as %r' % user + check_no_log('docker', 'login', '--username=%s' % user, '--password=%s' % pwd) + + os.environ.pop('DOCKER_USER', None) + os.environ.pop('DOCKER_PASSWORD', None) + + check_with_log('docker', 'push', target) + check_with_log('docker', 'logout') + + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument( + '--owner', help='Owner of the job') + PARSER.add_argument( + '--target', help='Build target') + PARSER.add_argument( + '--file', help='Build files') + ARGS = PARSER.parse_args() + if not ARGS.target or not ARGS.file: + raise ValueError('--target and --file must be set!') + if ARGS.owner: + os.environ['OWNER'] = ARGS.owner + main(ARGS.target, ARGS.file) diff --git a/images/bootstrap/scenarios/dindind_execute.py b/images/bootstrap/scenarios/dindind_execute.py new file mode 100755 index 0000000000000..61284d7446fe6 --- /dev/null +++ b/images/bootstrap/scenarios/dindind_execute.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""Prepares for nested docker, and executes a command.""" +# TODO(Q-Lee): check the necessity of this once MountPropagation is available in +# prow: https://github.com/kubernetes/kubernetes/pull/59252 + +import argparse +import os +import subprocess +import sys + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + + +def main(envs, cmd): + """Make important mounts r-shared, then run script and verify it exits 0.""" + check("mount", "--make-rshared", "/lib/modules") + check("mount", "--make-rshared", "/sys") + check("mount", "--make-rshared", "/") + + for env in envs: + key, val = env.split('=', 1) + print >>sys.stderr, '%s=%s' % (key, val) + os.environ[key] = val + if not cmd: + raise ValueError(cmd) + check(*cmd) + + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument('--env', default=[], action='append') + PARSER.add_argument('cmd', nargs=1) + PARSER.add_argument('args', nargs='*') + ARGS = PARSER.parse_args() + main(ARGS.env, ARGS.cmd + ARGS.args) diff --git a/images/bootstrap/scenarios/execute.py b/images/bootstrap/scenarios/execute.py new file mode 100755 index 0000000000000..4a3eb16e733c1 --- /dev/null +++ b/images/bootstrap/scenarios/execute.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""Executes a command.""" + +import argparse +import os +import subprocess +import sys + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + + +def main(envs, cmd): + """Run script and verify it exits 0.""" + for env in envs: + key, val = env.split('=', 1) + print >>sys.stderr, '%s=%s' % (key, val) + os.environ[key] = val + if not cmd: + raise ValueError(cmd) + check(*cmd) + + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument('--env', default=[], action='append') + PARSER.add_argument('cmd', nargs=1) + PARSER.add_argument('args', nargs='*') + ARGS = PARSER.parse_args() + main(ARGS.env, ARGS.cmd + ARGS.args) diff --git a/images/bootstrap/scenarios/kubernetes_bazel.py b/images/bootstrap/scenarios/kubernetes_bazel.py new file mode 100755 index 0000000000000..d81cfe2852c2d --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_bazel.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs bazel build/test for current repo.""" + +import argparse +import os +import subprocess +import sys + +ORIG_CWD = os.getcwd() + +def test_infra(*paths): + """Return path relative to root of test-infra repo.""" + return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths) + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + +def check_output(*cmd): + """Log and run the command, raising on errors, return output""" + print >>sys.stderr, 'Run:', cmd + return subprocess.check_output(cmd) + + +class Bazel(object): + def __init__(self, cfgs): + self.cfgs = cfgs or [] + + def _commands(self, cmd, *args, **kw): + commands = ['bazel', cmd] + if self.cfgs and kw.get('config', True): + commands.extend(['--config=%s' % c for c in self.cfgs]) + if args: + commands.extend(args) + return commands + + def check(self, cmd, *args, **kw): + """wrapper for check('bazel', *cmd).""" + check(*self._commands(cmd, *args, **kw)) + + def check_output(self, cmd, *args, **kw): + """wrapper for check_output('bazel', *cmd).""" + return check_output(*self._commands(cmd, *args, **kw)) + + def query(self, kind, selected_pkgs, changed_pkgs): + """ + Run a bazel query against target kind, include targets from args. + + Returns a list of kind objects from bazel query. + """ + + # Changes are calculated and no packages found, return empty list. + if changed_pkgs == []: + return [] + + selection = '//...' + if selected_pkgs: + # targets without a '-' operator prefix are implicitly additive + # when specifying build targets + selection = selected_pkgs[0] + for pkg in selected_pkgs[1:]: + if pkg.startswith('-'): + selection += ' '+pkg + else: + selection += ' +'+pkg + + + changes = '//...' + if changed_pkgs: + changes = 'set(%s)' % ' '.join(changed_pkgs) + + query_pat = 'kind(%s, rdeps(%s, %s)) except attr(\'tags\', \'manual\', //...)' + return [target for target in self.check_output( + 'query', + '--keep_going', + '--noshow_progress', + query_pat % (kind, selection, changes), + config=False, + ).split('\n') if target.startswith("//")] + + +def upload_string(gcs_path, text): + """Uploads text to gcs_path""" + cmd = ['gsutil', '-q', '-h', 'Content-Type:text/plain', 'cp', '-', gcs_path] + print >>sys.stderr, 'Run:', cmd, 'stdin=%s'%text + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) + proc.communicate(input=text) + +def echo_result(res): + """echo error message bazed on value of res""" + echo_map = { + 0:'Success', + 1:'Build failed', + 2:'Bad environment or flags', + 3:'Build passed, tests failed or timed out', + 4:'Build passed, no tests found', + 5:'Interrupted' + } + print echo_map.get(res, 'Unknown exit code : %s' % res) + +def get_version(): + """Return kubernetes version""" + # The check for version in bazel-genfiles can be removed once everyone is + # off of versions before 0.25.0. + # https://github.com/bazelbuild/bazel/issues/8651 + if os.path.isfile('bazel-genfiles/version'): + with open('bazel-genfiles/version') as fp: + return fp.read().strip() + with open('bazel-bin/version') as fp: + return fp.read().strip() + +def get_changed(base, pull): + """Get affected packages between base sha and pull sha.""" + diff = check_output( + 'git', 'diff', '--name-only', + '--diff-filter=d', '%s...%s' % (base, pull)) + return check_output( + 'bazel', 'query', + '--noshow_progress', + 'set(%s)' % diff).split('\n') + +def clean_file_in_dir(dirname, filename): + """Recursively remove all file with filename in dirname.""" + for parent, _, filenames in os.walk(dirname): + for name in filenames: + if name == filename: + os.remove(os.path.join(parent, name)) + +def main(args): + """Trigger a bazel build/test run, and upload results.""" + # pylint:disable=too-many-branches, too-many-statements, too-many-locals + if args.install: + for install in args.install: + if not os.path.isfile(install): + raise ValueError('Invalid install path: %s' % install) + check('pip', 'install', '-r', install) + + bazel = Bazel(args.config) + + bazel.check('version', config=False) + + res = 0 + try: + affected = None + if args.affected: + base = os.getenv('PULL_BASE_SHA', '') + pull = os.getenv('PULL_PULL_SHA', 'HEAD') + if not base: + raise ValueError('PULL_BASE_SHA must be set!') + affected = get_changed(base, pull) + + build_pkgs = [] + manual_build_targets = [] + test_pkgs = [] + manual_test_targets = [] + if args.build: + build_pkgs = args.build.split(' ') + if args.manual_build: + manual_build_targets = args.manual_build.split(' ') + if args.test: + test_pkgs = args.test.split(' ') + if args.manual_test: + manual_test_targets = args.manual_test.split(' ') + + buildables = [] + if build_pkgs or manual_build_targets or affected: + buildables = bazel.query('.*_binary', build_pkgs, affected) + manual_build_targets + + if args.release: + buildables.extend(args.release.split(' ')) + + if buildables: + bazel.check('build', *buildables) + else: + # Call bazel build regardless, to establish bazel symlinks + bazel.check('build') + + # clean up previous test.xml + clean_file_in_dir('./bazel-testlogs', 'test.xml') + + if test_pkgs or manual_test_targets or affected: + tests = bazel.query('test', test_pkgs, affected) + manual_test_targets + if tests: + if args.test_args: + tests = args.test_args + tests + bazel.check('test', *tests) + except subprocess.CalledProcessError as exp: + res = exp.returncode + + if args.push or args.release and res == 0: + version = get_version() + if not version: + print 'Kubernetes version missing; not uploading ci artifacts.' + res = 1 + else: + try: + if args.version_suffix: + version += args.version_suffix + gcs_build = '%s/%s' % (args.gcs, version) + bazel.check('run', '//:push-build', '--', gcs_build) + # log push-build location to path child jobs can find + # (gs:///$PULL_REFS/bazel-build-location.txt) + pull_refs = os.getenv('PULL_REFS', '') + gcs_shared = os.path.join(args.gcs_shared, pull_refs, 'bazel-build-location.txt') + if pull_refs: + upload_string(gcs_shared, gcs_build) + if args.publish_version: + upload_string(args.publish_version, version) + except subprocess.CalledProcessError as exp: + res = exp.returncode + + # Coalesce test results into one file for upload. + check(test_infra('hack/coalesce.py')) + + echo_result(res) + if res != 0: + sys.exit(res) + + +def create_parser(): + """Create argparser.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--affected', action='store_true', + help='If build/test affected targets. Filtered by --build and --test flags.') + parser.add_argument( + '--build', help='Bazel build target patterns, split by one space') + parser.add_argument( + '--manual-build', + help='Bazel build targets that should always be manually included, split by one space' + ) + parser.add_argument( + '--config', action='append', help='--config=foo rules to apply to bazel commands') + # TODO(krzyzacy): Convert to bazel build rules + parser.add_argument( + '--install', action="append", help='Python dependency(s) that need to be installed') + parser.add_argument( + '--push', action='store_true', help='Push release without building it') + parser.add_argument( + '--release', help='Run bazel build, and push release build to --gcs bucket') + parser.add_argument( + '--gcs-shared', + default="gs://kubernetes-jenkins/shared-results/", + help='If $PULL_REFS is set push build location to this bucket') + parser.add_argument( + '--publish-version', + help='publish GCS file here with the build version, like ci/latest.txt', + ) + parser.add_argument( + '--test', help='Bazel test target patterns, split by one space') + parser.add_argument( + '--manual-test', + help='Bazel test targets that should always be manually included, split by one space' + ) + parser.add_argument( + '--test-args', action="append", help='Bazel test args') + parser.add_argument( + '--gcs', + default='gs://k8s-release-dev/bazel', + help='GCS path for where to push build') + parser.add_argument( + '--version-suffix', + help='version suffix for build pushing') + return parser + +def parse_args(args=None): + """Return parsed args.""" + parser = create_parser() + return parser.parse_args(args) + +if __name__ == '__main__': + main(parse_args()) diff --git a/images/bootstrap/scenarios/kubernetes_bazel_test.py b/images/bootstrap/scenarios/kubernetes_bazel_test.py new file mode 100755 index 0000000000000..d42f3a65cd1d5 --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_bazel_test.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=too-few-public-methods + +"""Test for kubernetes_bazel.py""" + +import os +import string +import tempfile +import unittest + +import kubernetes_bazel + + +def fake_pass(*_unused, **_unused2): + """Do nothing.""" + pass + +def fake_bomb(*a, **kw): + """Always raise.""" + raise AssertionError('Should not happen', a, kw) + + +class Stub(object): + """Replace thing.param with replacement until exiting with.""" + def __init__(self, thing, param, replacement): + self.thing = thing + self.param = param + self.replacement = replacement + self.old = getattr(thing, param) + setattr(thing, param, self.replacement) + + def __enter__(self, *a, **kw): + return self.replacement + + def __exit__(self, *a, **kw): + setattr(self.thing, self.param, self.old) + + +class ScenarioTest(unittest.TestCase): # pylint: disable=too-many-public-methods + """Tests for bazel scenario.""" + callstack = [] + + def setUp(self): + self.boiler = { + 'check': Stub(kubernetes_bazel, 'check', self.fake_check), + 'check_output': Stub(kubernetes_bazel, 'check_output', self.fake_check), + 'query': Stub(kubernetes_bazel.Bazel, 'query', self.fake_query), + 'get_version': Stub(kubernetes_bazel, 'get_version', self.fake_version), + 'clean_file_in_dir': Stub(kubernetes_bazel, 'clean_file_in_dir', self.fake_clean), + } + + def tearDown(self): + for _, stub in self.boiler.items(): + with stub: # Leaving with restores things + pass + self.callstack[:] = [] + + def fake_check(self, *cmd): + """Log the command.""" + self.callstack.append(string.join(cmd)) + + @staticmethod + def fake_sha(key, default): + """fake base and pull sha.""" + if key == 'PULL_BASE_SHA': + return '12345' + if key == 'PULL_PULL_SHA': + return '67890' + return os.getenv(key, default) + + @staticmethod + def fake_version(): + """return a fake version""" + return 'v1.0+abcde' + + @staticmethod + def fake_query(_self, _kind, selected, changed): + """Simple filter selected by changed.""" + if changed == []: + return changed + if not changed: + return selected + if not selected: + return changed + + ret = [] + for pkg in selected: + if pkg in changed: + ret.append(pkg) + + return ret + + @staticmethod + def fake_changed_valid(_base, _pull): + """Return fake affected targets.""" + return ['//foo', '//bar'] + + @staticmethod + def fake_changed_empty(_base, _pull): + """Return fake affected targets.""" + return [] + + @staticmethod + def fake_clean(_dirname, _filename): + """Don't clean""" + pass + + def test_expand(self): + """Make sure flags are expanded properly.""" + args = kubernetes_bazel.parse_args([ + '--build=//b/... -//b/bb/... //c/...' + ]) + kubernetes_bazel.main(args) + + call = self.callstack[-2] + self.assertIn('//b/...', call) + self.assertIn('-//b/bb/...', call) + self.assertIn('//c/...', call) + + + def test_query(self): + """Make sure query is constructed properly.""" + args = kubernetes_bazel.parse_args([ + '--build=//b/... -//b/bb/... //c/...' + ]) + # temporarily un-stub query + with Stub(kubernetes_bazel.Bazel, 'query', self.boiler['query'].old): + def check_query(*cmd): + self.assertIn( + 'kind(.*_binary, rdeps(//b/... -//b/bb/... +//c/..., //...))' + ' except attr(\'tags\', \'manual\', //...)', + cmd + ) + return '//b/aa/...\n//c/...' + with Stub(kubernetes_bazel, 'check_output', check_query): + kubernetes_bazel.main(args) + + def test_expand_arg(self): + """Make sure flags are expanded properly.""" + args = kubernetes_bazel.parse_args([ + '--test-args=--foo', + '--test-args=--bar', + '--test=//b/... //c/...' + ]) + kubernetes_bazel.main(args) + + call = self.callstack[-2] + self.assertIn('--foo', call) + self.assertIn('--bar', call) + self.assertIn('//b/...', call) + self.assertIn('//c/...', call) + + def test_all_bazel(self): + """Make sure all commands starts with bazel except for coarse.""" + args = kubernetes_bazel.parse_args([ + '--build=//a', + '--test=//b', + '--release=//c' + ]) + kubernetes_bazel.main(args) + + for call in self.callstack[:-2]: + self.assertTrue(call.startswith('bazel'), call) + + def test_install(self): + """Make sure install is called as 1st scenario call.""" + with tempfile.NamedTemporaryFile(delete=False) as fp: + install = fp.name + args = kubernetes_bazel.parse_args([ + '--install=%s' % install, + ]) + kubernetes_bazel.main(args) + + self.assertIn(install, self.callstack[0]) + + def test_install_fail(self): + """Make sure install fails if path does not exist.""" + args = kubernetes_bazel.parse_args([ + '--install=foo', + ]) + with self.assertRaises(ValueError): + kubernetes_bazel.main(args) + + def test_affected(self): + """--test=affected will work.""" + args = kubernetes_bazel.parse_args([ + '--affected', + ]) + with self.assertRaises(ValueError): + kubernetes_bazel.main(args) + + with Stub(kubernetes_bazel, 'get_changed', self.fake_changed_valid): + with Stub(os, 'getenv', self.fake_sha): + kubernetes_bazel.main(args) + test = self.callstack[-2] + self.assertIn('//foo', test) + self.assertIn('//bar', test) + + build = self.callstack[-3] + self.assertIn('//foo', build) + self.assertIn('//bar', build) + + def test_affected_empty(self): + """if --affected returns nothing, then nothing should be triggered""" + args = kubernetes_bazel.parse_args([ + '--affected', + ]) + with Stub(kubernetes_bazel, 'get_changed', self.fake_changed_empty): + with Stub(os, 'getenv', self.fake_sha): + kubernetes_bazel.main(args) + # trigger empty build + self.assertIn('bazel build', self.callstack) + # nothing to test + for call in self.callstack: + self.assertNotIn('bazel test', call) + + def test_affected_filter(self): + """--test=affected will work.""" + args = kubernetes_bazel.parse_args([ + '--affected', + '--build=//foo', + '--test=//foo', + ]) + with Stub(kubernetes_bazel, 'get_changed', self.fake_changed_valid): + with Stub(os, 'getenv', self.fake_sha): + kubernetes_bazel.main(args) + test = self.callstack[-2] + self.assertIn('//foo', test) + self.assertNotIn('//bar', test) + + build = self.callstack[-3] + self.assertIn('//foo', build) + self.assertNotIn('//bar', build) + + +if __name__ == '__main__': + unittest.main() diff --git a/images/bootstrap/scenarios/kubernetes_build.py b/images/bootstrap/scenarios/kubernetes_build.py new file mode 100755 index 0000000000000..fdab957794db0 --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_build.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""Builds kubernetes with specified config""" + +import argparse +import os +import re +import subprocess +import sys + + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + +def check_no_stdout(*cmd): + """Log and run the command, suppress stdout & stderr, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + null = open(os.devnull, 'w') + subprocess.check_call(cmd, stdout=null, stderr=null) + +def check_output(*cmd): + """Log and run the command, raising on errors, return output""" + print >>sys.stderr, 'Run:', cmd + return subprocess.check_output(cmd, stderr=subprocess.STDOUT) + +def check_build_exists(gcs, suffix, fast): + """ check if a k8s build with same version + already exists in remote path + """ + if not os.path.exists('hack/print-workspace-status.sh'): + print >>sys.stderr, 'hack/print-workspace-status.sh not found, continue' + return False + + version = '' + try: + match = re.search( + r'gitVersion ([^\n]+)', + check_output('hack/print-workspace-status.sh') + ) + if match: + version = match.group(1) + except subprocess.CalledProcessError as exc: + # fallback with doing a real build + print >>sys.stderr, 'Failed to get k8s version, continue: %s' % exc + return False + + if version: + if not gcs: + gcs = 'k8s-release-dev' + gcs = 'gs://' + gcs + mode = 'ci' + if fast: + mode += '/fast' + if suffix: + mode += suffix + gcs = os.path.join(gcs, mode, version) + try: + check_no_stdout('gsutil', 'ls', gcs) + check_no_stdout('gsutil', 'ls', gcs + "/kubernetes.tar.gz") + check_no_stdout('gsutil', 'ls', gcs + "/bin") + return True + except subprocess.CalledProcessError as exc: + print >>sys.stderr, ( + 'gcs path %s (or some files under it) does not exist yet, continue' % gcs) + return False + + +def main(args): + # pylint: disable=too-many-branches + """Build and push kubernetes. + + This is a python port of the kubernetes/hack/jenkins/build.sh script. + """ + if os.path.split(os.getcwd())[-1] != 'kubernetes': + print >>sys.stderr, ( + 'Scenario should only run from either kubernetes directory!') + sys.exit(1) + + # pre-check if target build exists in gcs bucket or not + # if so, don't make duplicated builds + if check_build_exists(args.release, args.suffix, args.fast): + print >>sys.stderr, 'build already exists, exit' + sys.exit(0) + + env = { + # Skip gcloud update checking; do we still need this? + 'CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK': 'true', + # Don't run any unit/integration tests when building + 'KUBE_RELEASE_RUN_TESTS': 'n', + } + push_build_args = ['--nomock', '--verbose', '--ci'] + if args.suffix: + push_build_args.append('--gcs-suffix=%s' % args.suffix) + if args.release: + push_build_args.append('--bucket=%s' % args.release) + if args.registry: + push_build_args.append('--docker-registry=%s' % args.registry) + if args.extra_publish_file: + push_build_args.append('--extra-publish-file=%s' % args.extra_publish_file) + if args.extra_version_markers: + push_build_args.append('--extra-version-markers=%s' % args.extra_version_markers) + if args.fast: + push_build_args.append('--fast') + if args.allow_dup: + push_build_args.append('--allow-dup') + if args.skip_update_latest: + push_build_args.append('--noupdatelatest') + if args.register_gcloud_helper: + # Configure docker client for gcr.io authentication to allow communication + # with non-public registries. + check_no_stdout('gcloud', 'auth', 'configure-docker') + + for key, value in env.items(): + os.environ[key] = value + check('make', 'clean') + if args.fast: + check('make', 'quick-release') + else: + check('make', 'release') + output = check_output(args.push_build_script, *push_build_args) + print >>sys.stderr, 'Push build result: ', output + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser( + 'Build and push.') + PARSER.add_argument( + '--release', help='Upload binaries to the specified gs:// path') + PARSER.add_argument( + '--suffix', help='Append suffix to the upload path if set') + PARSER.add_argument( + '--registry', help='Push images to the specified docker registry') + PARSER.add_argument( + '--extra-publish-file', help='Additional version file uploads to') + PARSER.add_argument( + '--extra-version-markers', help='Additional version file uploads to') + PARSER.add_argument( + '--fast', action='store_true', help='Specifies a fast build') + PARSER.add_argument( + '--allow-dup', action='store_true', help='Allow overwriting if the build exists on gcs') + PARSER.add_argument( + '--skip-update-latest', action='store_true', help='Do not update the latest file') + PARSER.add_argument( + '--push-build-script', default='../release/push-build.sh', help='location of push-build.sh') + PARSER.add_argument( + '--register-gcloud-helper', action='store_true', + help='Register gcloud as docker credentials helper') + ARGS = PARSER.parse_args() + main(ARGS) diff --git a/images/bootstrap/scenarios/kubernetes_e2e.py b/images/bootstrap/scenarios/kubernetes_e2e.py new file mode 100755 index 0000000000000..440079bbffe16 --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_e2e.py @@ -0,0 +1,723 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""Runs kubernetes e2e test with specified config""" + +import argparse +import hashlib +import os +import random +import re +import shutil +import subprocess +import sys +import urllib2 +import time + +ORIG_CWD = os.getcwd() # Checkout changes cwd + +# The zones below are the zones available in the CNCF account (in theory, zones vary by account) +# We aim for 3 zones per region to try to maintain even spreading. +# We also remove a few zones where our preferred instance type is not available, +# though really this needs a better fix (likely in kops) +DEFAULT_AWS_ZONES = [ + 'ap-northeast-1a', + 'ap-northeast-1c', + 'ap-northeast-1d', + 'ap-northeast-2a', + #'ap-northeast-2b' - AZ does not exist, so we're breaking the 3 AZs per region target here + 'ap-northeast-2c', + 'ap-south-1a', + 'ap-south-1b', + 'ap-southeast-1a', + 'ap-southeast-1b', + 'ap-southeast-1c', + 'ap-southeast-2a', + 'ap-southeast-2b', + 'ap-southeast-2c', + 'ca-central-1a', + 'ca-central-1b', + 'eu-central-1a', + 'eu-central-1b', + 'eu-central-1c', + 'eu-west-1a', + 'eu-west-1b', + 'eu-west-1c', + 'eu-west-2a', + 'eu-west-2b', + 'eu-west-2c', + #'eu-west-3a', documented to not support c4 family + #'eu-west-3b', documented to not support c4 family + #'eu-west-3c', documented to not support c4 family + 'sa-east-1a', + #'sa-east-1b', AZ does not exist, so we're breaking the 3 AZs per region target here + 'sa-east-1c', + #'us-east-1a', # temporarily removing due to lack of quota #10043 + #'us-east-1b', # temporarily removing due to lack of quota #10043 + #'us-east-1c', # temporarily removing due to lack of quota #10043 + #'us-east-1d', # limiting to 3 zones to not overallocate + #'us-east-1e', # limiting to 3 zones to not overallocate + #'us-east-1f', # limiting to 3 zones to not overallocate + #'us-east-2a', InsufficientInstanceCapacity for c4.large 2018-05-30 + #'us-east-2b', InsufficientInstanceCapacity for c4.large 2018-05-30 + #'us-east-2c', InsufficientInstanceCapacity for c4.large 2018-05-30 + 'us-west-1a', + 'us-west-1b', + #'us-west-1c', AZ does not exist, so we're breaking the 3 AZs per region target here + #'us-west-2a', # temporarily removing due to lack of quota #10043 + #'us-west-2b', # temporarily removing due to lack of quota #10043 + #'us-west-2c', # temporarily removing due to lack of quota #10043 +] + +def test_infra(*paths): + """Return path relative to root of test-infra repo.""" + return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths) + + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + + +def check_output(*cmd): + """Log and run the command, raising on errors, return output""" + print >>sys.stderr, 'Run:', cmd + return subprocess.check_output(cmd) + + +def check_env(env, *cmd): + """Log and run the command with a specific env, raising on errors.""" + print >>sys.stderr, 'Environment:' + for key, value in sorted(env.items()): + print >>sys.stderr, '%s=%s' % (key, value) + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd, env=env) + + +def kubekins(tag): + """Return full path to kubekins-e2e:tag.""" + return 'gcr.io/k8s-testimages/kubekins-e2e:%s' % tag + +def parse_env(env): + """Returns (FOO, BAR=MORE) for FOO=BAR=MORE.""" + return env.split('=', 1) + +def aws_role_config(profile, arn): + return (('[profile jenkins-assumed-role]\n' + + 'role_arn = %s\n' + + 'source_profile = %s\n') % + (arn, profile)) + +class LocalMode(object): + """Runs e2e tests by calling kubetest.""" + def __init__(self, workspace, artifacts): + self.command = 'kubetest' + self.workspace = workspace + self.artifacts = artifacts + self.env = [] + self.os_env = [] + self.env_files = [] + self.add_environment( + 'HOME=%s' % workspace, + 'WORKSPACE=%s' % workspace, + 'PATH=%s' % os.getenv('PATH'), + ) + + def add_environment(self, *envs): + """Adds FOO=BAR to the list of environment overrides.""" + self.env.extend(parse_env(e) for e in envs) + + def add_os_environment(self, *envs): + """Adds FOO=BAR to the list of os environment overrides.""" + self.os_env.extend(parse_env(e) for e in envs) + + def add_file(self, env_file): + """Reads all FOO=BAR lines from env_file.""" + with open(env_file) as fp: + for line in fp: + line = line.rstrip() + if not line or line.startswith('#'): + continue + self.env_files.append(parse_env(line)) + + def add_env(self, env): + self.env_files.append(parse_env(env)) + + def add_aws_cred(self, priv, pub, cred): + """Sets aws keys and credentials.""" + ssh_dir = os.path.join(self.workspace, '.ssh') + if not os.path.isdir(ssh_dir): + os.makedirs(ssh_dir) + + cred_dir = os.path.join(self.workspace, '.aws') + if not os.path.isdir(cred_dir): + os.makedirs(cred_dir) + + aws_ssh = os.path.join(ssh_dir, 'kube_aws_rsa') + aws_pub = os.path.join(ssh_dir, 'kube_aws_rsa.pub') + aws_cred = os.path.join(cred_dir, 'credentials') + shutil.copy(priv, aws_ssh) + shutil.copy(pub, aws_pub) + shutil.copy(cred, aws_cred) + + self.add_environment( + 'AWS_SSH_PRIVATE_KEY_FILE=%s' % priv, + 'AWS_SSH_PUBLIC_KEY_FILE=%s' % pub, + 'AWS_SHARED_CREDENTIALS_FILE=%s' % cred, + ) + + def add_aws_role(self, profile, arn): + with open(os.path.join(self.workspace, '.aws', 'config'), 'w') as cfg: + cfg.write(aws_role_config(profile, arn)) + self.add_environment('AWS_SDK_LOAD_CONFIG=true') + return 'jenkins-assumed-role' + + def add_gce_ssh(self, priv, pub): + """Copies priv, pub keys to $WORKSPACE/.ssh.""" + ssh_dir = os.path.join(self.workspace, '.ssh') + if not os.path.isdir(ssh_dir): + os.makedirs(ssh_dir) + + gce_ssh = os.path.join(ssh_dir, 'google_compute_engine') + gce_pub = os.path.join(ssh_dir, 'google_compute_engine.pub') + shutil.copy(priv, gce_ssh) + shutil.copy(pub, gce_pub) + self.add_environment( + 'JENKINS_GCE_SSH_PRIVATE_KEY_FILE=%s' % gce_ssh, + 'JENKINS_GCE_SSH_PUBLIC_KEY_FILE=%s' % gce_pub, + ) + + @staticmethod + def add_service_account(path): + """Returns path.""" + return path + + def add_k8s(self, *a, **kw): + """Add specified k8s.io repos (noop).""" + pass + + def add_aws_runner(self): + """Start with kops-e2e-runner.sh""" + # TODO(Krzyzacy):retire kops-e2e-runner.sh + self.command = os.path.join(self.workspace, 'kops-e2e-runner.sh') + + def start(self, args): + """Starts kubetest.""" + print >>sys.stderr, 'starts with local mode' + env = {} + env.update(self.os_env) + env.update(self.env_files) + env.update(self.env) + check_env(env, self.command, *args) + + +def cluster_name(cluster, tear_down_previous=False): + """Return or select a cluster name.""" + if cluster: + return cluster + # Create a suffix based on the build number and job name. + # This ensures no conflict across runs of different jobs (see #7592). + # For PR jobs, we use PR number instead of build number to ensure the + # name is constant across different runs of the presubmit on the PR. + # This helps clean potentially leaked resources from earlier run that + # could've got evicted midway (see #7673). + job_type = os.getenv('JOB_TYPE') + if job_type == 'batch': + suffix = 'batch-%s' % os.getenv('BUILD_ID', 0) + elif job_type == 'presubmit' and tear_down_previous: + suffix = '%s' % os.getenv('PULL_NUMBER', 0) + else: + suffix = '%s' % os.getenv('BUILD_ID', 0) + if len(suffix) > 10: + suffix = hashlib.md5(suffix).hexdigest()[:10] + job_hash = hashlib.md5(os.getenv('JOB_NAME', '')).hexdigest()[:5] + return 'e2e-%s-%s' % (suffix, job_hash) + + +# TODO(krzyzacy): Move this into kubetest +def build_kops(kops, mode): + """Build kops, set kops related envs.""" + if not os.path.basename(kops) == 'kops': + raise ValueError(kops) + version = 'pull-' + check_output('git', 'describe', '--always').strip() + job = os.getenv('JOB_NAME', 'pull-kops-e2e-kubernetes-aws') + gcs = 'gs://kops-ci/pulls/%s' % job + gapi = 'https://storage.googleapis.com/kops-ci/pulls/%s' % job + mode.add_environment( + 'KOPS_BASE_URL=%s/%s' % (gapi, version), + 'GCS_LOCATION=%s' % gcs + ) + check('make', 'gcs-publish-ci', 'VERSION=%s' % version, 'GCS_LOCATION=%s' % gcs) + + +def set_up_kops_gce(workspace, args, mode, cluster, runner_args): + """Set up kops on GCE envs.""" + for path in [args.gce_ssh, args.gce_pub]: + if not os.path.isfile(os.path.expandvars(path)): + raise IOError(path, os.path.expandvars(path)) + mode.add_gce_ssh(args.gce_ssh, args.gce_pub) + + gce_ssh = os.path.join(workspace, '.ssh', 'google_compute_engine') + + zones = args.kops_zones or random.choice([ + 'us-central1-a', + 'us-central1-b', + 'us-central1-c', + 'us-central1-f', + ]) + + runner_args.extend([ + '--kops-cluster=%s' % cluster, + '--kops-zones=%s' % zones, + '--kops-state=%s' % args.kops_state_gce, + '--kops-nodes=%s' % args.kops_nodes, + '--kops-ssh-key=%s' % gce_ssh, + ]) + + +def set_up_kops_aws(workspace, args, mode, cluster, runner_args): + """Set up aws related envs for kops. Will replace set_up_aws.""" + for path in [args.aws_ssh, args.aws_pub, args.aws_cred]: + if not os.path.isfile(os.path.expandvars(path)): + raise IOError(path, os.path.expandvars(path)) + mode.add_aws_cred(args.aws_ssh, args.aws_pub, args.aws_cred) + + aws_ssh = os.path.join(workspace, '.ssh', 'kube_aws_rsa') + profile = args.aws_profile + if args.aws_role_arn: + profile = mode.add_aws_role(profile, args.aws_role_arn) + + # kubetest for kops now support select random regions and zones. + # For initial testing we are not sending in zones when the + # --kops-multiple-zones flag is set. If the flag is not set then + # we use the older functionality of passing in zones. + if args.kops_multiple_zones: + runner_args.extend(["--kops-multiple-zones"]) + else: + # TODO(@chrislovecnm): once we have tested we can remove the zones + # and region logic from this code and have kubetest handle that + # logic + zones = args.kops_zones or random.choice(DEFAULT_AWS_ZONES) + regions = ','.join([zone[:-1] for zone in zones.split(',')]) + runner_args.extend(['--kops-zones=%s' % zones]) + mode.add_environment( + 'KOPS_REGIONS=%s' % regions, + ) + + mode.add_environment( + 'AWS_PROFILE=%s' % profile, + 'AWS_DEFAULT_PROFILE=%s' % profile, + ) + + if args.aws_cluster_domain: + cluster = '%s.%s' % (cluster, args.aws_cluster_domain) + + # AWS requires a username (and it varies per-image) + ssh_user = args.kops_ssh_user or 'admin' + + runner_args.extend([ + '--kops-cluster=%s' % cluster, + '--kops-state=%s' % args.kops_state, + '--kops-nodes=%s' % args.kops_nodes, + '--kops-ssh-key=%s' % aws_ssh, + '--kops-ssh-user=%s' % ssh_user, + ]) + + +def set_up_aws(workspace, args, mode, cluster, runner_args): + """Set up aws related envs. Legacy; will be replaced by set_up_kops_aws.""" + for path in [args.aws_ssh, args.aws_pub, args.aws_cred]: + if not os.path.isfile(os.path.expandvars(path)): + raise IOError(path, os.path.expandvars(path)) + mode.add_aws_cred(args.aws_ssh, args.aws_pub, args.aws_cred) + + aws_ssh = os.path.join(workspace, '.ssh', 'kube_aws_rsa') + profile = args.aws_profile + if args.aws_role_arn: + profile = mode.add_aws_role(profile, args.aws_role_arn) + + zones = args.kops_zones or random.choice(DEFAULT_AWS_ZONES) + regions = ','.join([zone[:-1] for zone in zones.split(',')]) + + mode.add_environment( + 'AWS_PROFILE=%s' % profile, + 'AWS_DEFAULT_PROFILE=%s' % profile, + 'KOPS_REGIONS=%s' % regions, + ) + + if args.aws_cluster_domain: + cluster = '%s.%s' % (cluster, args.aws_cluster_domain) + + # AWS requires a username (and it varies per-image) + ssh_user = args.kops_ssh_user or 'admin' + + runner_args.extend([ + '--kops-cluster=%s' % cluster, + '--kops-zones=%s' % zones, + '--kops-state=%s' % args.kops_state, + '--kops-nodes=%s' % args.kops_nodes, + '--kops-ssh-key=%s' % aws_ssh, + '--kops-ssh-user=%s' % ssh_user, + ]) + # TODO(krzyzacy):Remove after retire kops-e2e-runner.sh + mode.add_aws_runner() + +def read_gcs_path(gcs_path): + """reads a gcs path (gs://...) by HTTP GET to storage.googleapis.com""" + link = gcs_path.replace('gs://', 'https://storage.googleapis.com/') + loc = urllib2.urlopen(link).read() + print >>sys.stderr, "Read GCS Path: %s" % loc + return loc + +def get_shared_gcs_path(gcs_shared, use_shared_build): + """return the shared path for this set of jobs using args and $PULL_REFS.""" + build_file = '' + if use_shared_build: + build_file += use_shared_build + '-' + build_file += 'build-location.txt' + return os.path.join(gcs_shared, os.getenv('PULL_REFS', ''), build_file) + +def inject_bazelrc(lines): + if not lines: + return + lines = [l + '\n' for l in lines] + with open('/etc/bazel.bazelrc', 'a') as fp: + fp.writelines(lines) + path = os.path.join(os.getenv('HOME'), '.bazelrc') + with open(path, 'a') as fp: + fp.writelines(lines) + +def main(args): + """Set up env, start kubekins-e2e, handle termination. """ + # pylint: disable=too-many-branches,too-many-statements,too-many-locals + + # Rules for env var priority here in docker: + # -e FOO=a -e FOO=b -> FOO=b + # --env-file FOO=a --env-file FOO=b -> FOO=b + # -e FOO=a --env-file FOO=b -> FOO=a(!!!!) + # --env-file FOO=a -e FOO=b -> FOO=b + # + # So if you overwrite FOO=c for a local run it will take precedence. + # + + # Set up workspace/artifacts dir + workspace = os.environ.get('WORKSPACE', os.getcwd()) + artifacts = os.environ.get('ARTIFACTS', os.path.join(workspace, '_artifacts')) + if not os.path.isdir(artifacts): + os.makedirs(artifacts) + + inject_bazelrc(args.inject_bazelrc) + + mode = LocalMode(workspace, artifacts) + + for env_file in args.env_file: + mode.add_file(test_infra(env_file)) + for env in args.env: + mode.add_env(env) + + # TODO(fejta): remove after next image push + mode.add_environment('KUBETEST_MANUAL_DUMP=y') + if args.dump_before_and_after: + before_dir = os.path.join(mode.artifacts, 'before') + if not os.path.exists(before_dir): + os.makedirs(before_dir) + after_dir = os.path.join(mode.artifacts, 'after') + if not os.path.exists(after_dir): + os.makedirs(after_dir) + + runner_args = [ + '--dump-pre-test-logs=%s' % before_dir, + '--dump=%s' % after_dir, + ] + else: + runner_args = [ + '--dump=%s' % mode.artifacts, + ] + + if args.service_account: + runner_args.append( + '--gcp-service-account=%s' % mode.add_service_account(args.service_account)) + + shared_build_gcs_path = "" + if args.use_shared_build is not None: + # find shared build location from GCS + gcs_path = get_shared_gcs_path(args.gcs_shared, args.use_shared_build) + print >>sys.stderr, 'Getting shared build location from: '+gcs_path + # retry loop for reading the location + attempts_remaining = 12 + while True: + attempts_remaining -= 1 + try: + # tell kubetest to extract from this location + shared_build_gcs_path = read_gcs_path(gcs_path) + args.kubetest_args.append('--extract=' + shared_build_gcs_path) + args.build = None + break + except urllib2.URLError as err: + print >>sys.stderr, 'Failed to get shared build location: %s' % err + if attempts_remaining > 0: + print >>sys.stderr, 'Waiting 5 seconds and retrying...' + time.sleep(5) + else: + raise RuntimeError('Failed to get shared build location too many times!') + + elif args.build is not None: + if args.build == '': + # Empty string means --build was passed without any arguments; + # if --build wasn't passed, args.build would be None + runner_args.append('--build') + else: + runner_args.append('--build=%s' % args.build) + k8s = os.getcwd() + if not os.path.basename(k8s) == 'kubernetes': + raise ValueError(k8s) + mode.add_k8s(os.path.dirname(k8s), 'kubernetes', 'release') + + if args.build_federation is not None: + if args.build_federation == '': + runner_args.append('--build-federation') + else: + runner_args.append('--build-federation=%s' % args.build_federation) + fed = os.getcwd() + if not os.path.basename(fed) == 'federation': + raise ValueError(fed) + mode.add_k8s(os.path.dirname(fed), 'federation', 'release') + + if args.kops_build: + build_kops(os.getcwd(), mode) + + if args.stage is not None: + runner_args.append('--stage=%s' % args.stage) + if args.aws: + for line in check_output('hack/print-workspace-status.sh').split('\n'): + if 'gitVersion' in line: + _, version = line.strip().split(' ') + break + else: + raise ValueError('kubernetes version not found in workspace status') + runner_args.append('--kops-kubernetes-version=%s/%s' % ( + args.stage.replace('gs://', 'https://storage.googleapis.com/'), + version)) + + # TODO(fejta): move these out of this file + if args.up == 'true': + runner_args.append('--up') + if args.down == 'true': + runner_args.append('--down') + if args.test == 'true': + runner_args.append('--test') + + # Passthrough some args to kubetest + if args.deployment: + runner_args.append('--deployment=%s' % args.deployment) + if args.provider: + runner_args.append('--provider=%s' % args.provider) + + cluster = cluster_name(args.cluster, args.tear_down_previous) + runner_args.append('--cluster=%s' % cluster) + runner_args.append('--gcp-network=%s' % cluster) + runner_args.extend(args.kubetest_args) + + if args.use_logexporter: + runner_args.append('--logexporter-gcs-path=%s' % args.logexporter_gcs_path) + + if args.aws: + # Legacy - prefer passing --deployment=kops, --provider=aws, + # which does not use kops-e2e-runner.sh + set_up_aws(mode.workspace, args, mode, cluster, runner_args) + elif args.deployment == 'kops' and args.provider == 'aws': + set_up_kops_aws(mode.workspace, args, mode, cluster, runner_args) + elif args.deployment == 'kops' and args.provider == 'gce': + set_up_kops_gce(mode.workspace, args, mode, cluster, runner_args) + elif args.deployment != 'kind' and args.gce_ssh: + mode.add_gce_ssh(args.gce_ssh, args.gce_pub) + + # TODO(fejta): delete this? + mode.add_os_environment(*( + '%s=%s' % (k, v) for (k, v) in os.environ.items())) + + mode.add_environment( + # Boilerplate envs + # Skip gcloud update checking + 'CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true', + # Use default component update behavior + 'CLOUDSDK_EXPERIMENTAL_FAST_COMPONENT_UPDATE=false', + # AWS + 'KUBE_AWS_INSTANCE_PREFIX=%s' % cluster, + # GCE + 'INSTANCE_PREFIX=%s' % cluster, + 'KUBE_GCE_INSTANCE_PREFIX=%s' % cluster, + ) + + mode.start(runner_args) + +def create_parser(): + """Create argparser.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--env-file', default=[], action="append", + help='Job specific environment file') + parser.add_argument( + '--env', default=[], action="append", + help='Job specific environment setting ' + + '(usage: "--env=VAR=SETTING" will set VAR to SETTING).') + parser.add_argument( + '--gce-ssh', + default=os.environ.get('JENKINS_GCE_SSH_PRIVATE_KEY_FILE'), + help='Path to .ssh/google_compute_engine keys') + parser.add_argument( + '--gce-pub', + default=os.environ.get('JENKINS_GCE_SSH_PUBLIC_KEY_FILE'), + help='Path to pub gce ssh key') + parser.add_argument( + '--service-account', + default=os.environ.get('GOOGLE_APPLICATION_CREDENTIALS'), + help='Path to service-account.json') + parser.add_argument( + '--build', nargs='?', default=None, const='', + help='Build kubernetes binaries if set, optionally specifying strategy') + parser.add_argument( + '--inject-bazelrc', default=[], action='append', + help='Inject /etc/bazel.bazelrc and ~/.bazelrc lines') + parser.add_argument( + '--build-federation', nargs='?', default=None, const='', + help='Build federation binaries if set, optionally specifying strategy') + parser.add_argument( + '--use-shared-build', nargs='?', default=None, const='', + help='Use prebuilt kubernetes binaries if set, optionally specifying strategy') + parser.add_argument( + '--gcs-shared', + default='gs://kubernetes-jenkins/shared-results/', + help='Get shared build from this bucket') + parser.add_argument( + '--cluster', default='bootstrap-e2e', help='Name of the cluster') + parser.add_argument( + '--stage', default=None, help='Stage release to GCS path provided') + parser.add_argument( + '--test', default='true', help='If we need to run any actual test within kubetest') + parser.add_argument( + '--down', default='true', help='If we need to tear down the e2e cluster') + parser.add_argument( + '--up', default='true', help='If we need to bring up a e2e cluster') + parser.add_argument( + '--tear-down-previous', action='store_true', + help='If we need to tear down previous e2e cluster') + parser.add_argument( + '--use-logexporter', + action='store_true', + help='If we need to use logexporter tool to upload logs from nodes to GCS directly') + parser.add_argument( + '--logexporter-gcs-path', + default=os.environ.get('GCS_ARTIFACTS_DIR',''), + help='GCS path where logexporter tool will upload logs if enabled') + parser.add_argument( + '--kubetest_args', + action='append', + default=[], + help='Send unrecognized args directly to kubetest') + parser.add_argument( + '--dump-before-and-after', action='store_true', + help='Dump artifacts from both before and after the test run') + + + # kops & aws + # TODO(justinsb): replace with --provider=aws --deployment=kops + parser.add_argument( + '--aws', action='store_true', help='E2E job runs in aws') + parser.add_argument( + '--aws-profile', + default=( + os.environ.get('AWS_PROFILE') or + os.environ.get('AWS_DEFAULT_PROFILE') or + 'default' + ), + help='Profile within --aws-cred to use') + parser.add_argument( + '--aws-role-arn', + default=os.environ.get('KOPS_E2E_ROLE_ARN'), + help='Use --aws-profile to run as --aws-role-arn if set') + parser.add_argument( + '--aws-ssh', + default=os.environ.get('AWS_SSH_PRIVATE_KEY_FILE'), + help='Path to private aws ssh keys') + parser.add_argument( + '--aws-pub', + default=os.environ.get('AWS_SSH_PUBLIC_KEY_FILE'), + help='Path to pub aws ssh key') + parser.add_argument( + '--aws-cred', + default=os.environ.get('AWS_SHARED_CREDENTIALS_FILE'), + help='Path to aws credential file') + parser.add_argument( + '--aws-cluster-domain', help='Domain of the aws cluster for aws-pr jobs') + parser.add_argument( + '--kops-nodes', default=4, type=int, help='Number of nodes to start') + parser.add_argument( + '--kops-ssh-user', default='', + help='Username for ssh connections to instances') + parser.add_argument( + '--kops-state', default='s3://k8s-kops-prow/', + help='Name of the aws state storage') + parser.add_argument( + '--kops-state-gce', default='gs://k8s-kops-gce/', + help='Name of the kops state storage for GCE') + parser.add_argument( + '--kops-zones', help='Comma-separated list of zones else random choice') + parser.add_argument( + '--kops-build', action='store_true', help='If we need to build kops locally') + parser.add_argument( + '--kops-multiple-zones', action='store_true', help='Use multiple zones') + + + # kubetest flags that also trigger behaviour here + parser.add_argument( + '--provider', help='provider flag as used by kubetest') + parser.add_argument( + '--deployment', help='deployment flag as used by kubetest') + + return parser + + +def parse_args(args=None): + """Return args, adding unrecognized args to kubetest_args.""" + parser = create_parser() + args, extra = parser.parse_known_args(args) + args.kubetest_args += extra + + if args.aws or args.provider == 'aws': + # If aws keys are missing, try to fetch from HOME dir + if not args.aws_ssh or not args.aws_pub or not args.aws_cred: + home = os.environ.get('HOME') + if not home: + raise ValueError('HOME dir not set!') + if not args.aws_ssh: + args.aws_ssh = '%s/.ssh/kube_aws_rsa' % home + print >>sys.stderr, '-aws-ssh key not set. Defaulting to %s' % args.aws_ssh + if not args.aws_pub: + args.aws_pub = '%s/.ssh/kube_aws_rsa.pub' % home + print >>sys.stderr, '--aws-pub key not set. Defaulting to %s' % args.aws_pub + if not args.aws_cred: + args.aws_cred = '%s/.aws/credentials' % home + print >>sys.stderr, '--aws-cred not set. Defaulting to %s' % args.aws_cred + return args + + +if __name__ == '__main__': + main(parse_args()) diff --git a/images/bootstrap/scenarios/kubernetes_e2e_test.py b/images/bootstrap/scenarios/kubernetes_e2e_test.py new file mode 100755 index 0000000000000..c6e48caa1a3e3 --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_e2e_test.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=too-few-public-methods + +"""Test for kubernetes_e2e.py""" + +import os +import shutil +import string +import tempfile +import urllib2 +import unittest +import time + +import kubernetes_e2e + +FAKE_WORKSPACE_STATUS = 'STABLE_BUILD_GIT_COMMIT 599539dc0b99976fda0f326f4ce47e93ec07217c\n' \ +'STABLE_BUILD_SCM_STATUS clean\n' \ +'STABLE_BUILD_SCM_REVISION v1.7.0-alpha.0.1320+599539dc0b9997\n' \ +'STABLE_BUILD_MAJOR_VERSION 1\n' \ +'STABLE_BUILD_MINOR_VERSION 7+\n' \ +'STABLE_gitCommit 599539dc0b99976fda0f326f4ce47e93ec07217c\n' \ +'STABLE_gitTreeState clean\n' \ +'STABLE_gitVersion v1.7.0-alpha.0.1320+599539dc0b9997\n' \ +'STABLE_gitMajor 1\n' \ +'STABLE_gitMinor 7+\n' + +FAKE_WORKSPACE_STATUS_V1_6 = 'STABLE_BUILD_GIT_COMMIT 84febd4537dd190518657405b7bdb921dfbe0387\n' \ +'STABLE_BUILD_SCM_STATUS clean\n' \ +'STABLE_BUILD_SCM_REVISION v1.6.4-beta.0.18+84febd4537dd19\n' \ +'STABLE_BUILD_MAJOR_VERSION 1\n' \ +'STABLE_BUILD_MINOR_VERSION 6+\n' \ +'STABLE_gitCommit 84febd4537dd190518657405b7bdb921dfbe0387\n' \ +'STABLE_gitTreeState clean\n' \ +'STABLE_gitVersion v1.6.4-beta.0.18+84febd4537dd19\n' \ +'STABLE_gitMajor 1\n' \ +'STABLE_gitMinor 6+\n' + +FAKE_DESCRIBE_FROM_FAMILY_RESPONSE = """ +archiveSizeBytes: '1581831882' +creationTimestamp: '2017-06-16T10:37:57.681-07:00' +description: 'Google, Container-Optimized OS, 59-9460.64.0 stable, Kernel: ChromiumOS-4.4.52 + Kubernetes: 1.6.4 Docker: 1.11.2' +diskSizeGb: '10' +family: cos-stable +id: '2388425242502080922' +kind: compute#image +labelFingerprint: 42WmSpB8rSM= +licenses: +- https://www.googleapis.com/compute/v1/projects/cos-cloud/global/licenses/cos +name: cos-stable-59-9460-64-0 +rawDisk: + containerType: TAR + source: '' +selfLink: https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-59-9460-64-0 +sourceType: RAW +status: READY +""" + +def fake_pass(*_unused, **_unused2): + """Do nothing.""" + pass + +def fake_bomb(*a, **kw): + """Always raise.""" + raise AssertionError('Should not happen', a, kw) + +def raise_urllib2_error(*_unused, **_unused2): + """Always raise a urllib2.URLError""" + raise urllib2.URLError("test failure") + +def always_kubernetes(*_unused, **_unused2): + """Always return 'kubernetes'""" + return 'kubernetes' + +class Stub(object): + """Replace thing.param with replacement until exiting with.""" + def __init__(self, thing, param, replacement): + self.thing = thing + self.param = param + self.replacement = replacement + self.old = getattr(thing, param) + setattr(thing, param, self.replacement) + + def __enter__(self, *a, **kw): + return self.replacement + + def __exit__(self, *a, **kw): + setattr(self.thing, self.param, self.old) + + +class ClusterNameTest(unittest.TestCase): + def test_name_filled(self): + """Return the cluster name if set.""" + name = 'foo' + build = '1984' + os.environ['BUILD_ID'] = build + actual = kubernetes_e2e.cluster_name(name) + self.assertTrue(actual) + self.assertIn(name, actual) + self.assertNotIn(build, actual) + + def test_name_empty_short_build(self): + """Return the build number if name is empty.""" + name = '' + build = '1984' + os.environ['BUILD_ID'] = build + actual = kubernetes_e2e.cluster_name(name) + self.assertTrue(actual) + self.assertIn(build, actual) + + def test_name_empty_long_build(self): + """Return a short hash of a long build number if name is empty.""" + name = '' + build = '0' * 63 + os.environ['BUILD_ID'] = build + actual = kubernetes_e2e.cluster_name(name) + self.assertTrue(actual) + self.assertNotIn(build, actual) + if len(actual) > 32: # Some firewall names consume half the quota + self.fail('Name should be short: %s' % actual) + + def test_name_presubmit(self): + """Return the build number if name is empty.""" + name = '' + build = '1984' + pr = '12345' + os.environ['BUILD_ID'] = build + os.environ['JOB_TYPE'] = 'presubmit' + os.environ['PULL_NUMBER'] = pr + actual = kubernetes_e2e.cluster_name(name, False) + self.assertTrue(actual) + self.assertIn(build, actual) + self.assertNotIn(pr, actual) + + actual = kubernetes_e2e.cluster_name(name, True) + self.assertTrue(actual) + self.assertIn(pr, actual) + self.assertNotIn(build, actual) + + +class ScenarioTest(unittest.TestCase): # pylint: disable=too-many-public-methods + """Test for e2e scenario.""" + callstack = [] + envs = {} + + def setUp(self): + self.boiler = [ + Stub(kubernetes_e2e, 'check', self.fake_check), + Stub(shutil, 'copy', fake_pass), + ] + + def tearDown(self): + for stub in self.boiler: + with stub: # Leaving with restores things + pass + self.callstack[:] = [] + self.envs.clear() + + def fake_check(self, *cmd): + """Log the command.""" + self.callstack.append(string.join(cmd)) + + def fake_check_env(self, env, *cmd): + """Log the command with a specific env.""" + self.envs.update(env) + self.callstack.append(string.join(cmd)) + + def fake_output_work_status(self, *cmd): + """fake a workstatus blob.""" + self.callstack.append(string.join(cmd)) + return FAKE_WORKSPACE_STATUS + + def fake_output_work_status_v1_6(self, *cmd): + """fake a workstatus blob for v1.6.""" + self.callstack.append(string.join(cmd)) + return FAKE_WORKSPACE_STATUS_V1_6 + + def fake_output_get_latest_image(self, *cmd): + """fake a `gcloud compute images describe-from-family` response.""" + self.callstack.append(string.join(cmd)) + return FAKE_DESCRIBE_FROM_FAMILY_RESPONSE + + def test_local(self): + """Make sure local mode is fine overall.""" + args = kubernetes_e2e.parse_args() + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + + self.assertNotEqual(self.envs, {}) + for call in self.callstack: + self.assertFalse(call.startswith('docker')) + + def test_check_leaks(self): + """Ensure --check-leaked-resources=true sends flag to kubetest.""" + args = kubernetes_e2e.parse_args(['--check-leaked-resources=true']) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + self.assertIn('--check-leaked-resources=true', self.callstack[-1]) + + def test_check_leaks_false(self): + """Ensure --check-leaked-resources=true sends flag to kubetest.""" + args = kubernetes_e2e.parse_args(['--check-leaked-resources=false']) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + self.assertIn('--check-leaked-resources=false', self.callstack[-1]) + + def test_check_leaks_default(self): + """Ensure --check-leaked-resources=true sends flag to kubetest.""" + args = kubernetes_e2e.parse_args(['--check-leaked-resources']) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + self.assertIn('--check-leaked-resources', self.callstack[-1]) + + def test_check_leaks_unset(self): + """Ensure --check-leaked-resources=true sends flag to kubetest.""" + args = kubernetes_e2e.parse_args(['--mode=local']) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + self.assertNotIn('--check-leaked-resources', self.callstack[-1]) + + def test_migrated_kubetest_args(self): + migrated = [ + '--stage-suffix=panda', + '--random-flag', 'random-value', + '--multiple-federations', + 'arg1', 'arg2', + '--federation', + '--kubemark', + '--extract=this', + '--extract=that', + '--save=somewhere', + '--skew', + '--publish=location', + '--timeout=42m', + '--upgrade_args=ginkgo', + '--check-leaked-resources=true', + '--charts', + ] + explicit_passthrough_args = [ + '--deployment=yay', + '--provider=gce', + ] + args = kubernetes_e2e.parse_args(migrated + + explicit_passthrough_args + + ['--test=false']) + self.assertEqual(migrated, args.kubetest_args) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + lastcall = self.callstack[-1] + for arg in migrated: + self.assertIn(arg, lastcall) + for arg in explicit_passthrough_args: + self.assertIn(arg, lastcall) + + def test_updown_default(self): + args = kubernetes_e2e.parse_args([]) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + lastcall = self.callstack[-1] + self.assertIn('--up', lastcall) + self.assertIn('--down', lastcall) + + def test_updown_set(self): + args = kubernetes_e2e.parse_args(['--up=false', '--down=true']) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + lastcall = self.callstack[-1] + self.assertNotIn('--up', lastcall) + self.assertIn('--down', lastcall) + + def test_local_env(self): + """ + Ensure that host variables (such as GOPATH) are included, + and added envs/env files overwrite os environment. + """ + mode = kubernetes_e2e.LocalMode('/orig-workspace', '/random-artifacts') + mode.add_environment(*( + 'FOO=BAR', 'GOPATH=/go/path', 'WORKSPACE=/new/workspace')) + mode.add_os_environment(*('USER=jenkins', 'FOO=BAZ', 'GOOS=linux')) + with tempfile.NamedTemporaryFile() as temp: + temp.write('USER=prow') + temp.flush() + mode.add_file(temp.name) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + mode.start([]) + self.assertIn(('FOO', 'BAR'), self.envs.viewitems()) + self.assertIn(('WORKSPACE', '/new/workspace'), self.envs.viewitems()) + self.assertIn(('GOPATH', '/go/path'), self.envs.viewitems()) + self.assertIn(('USER', 'prow'), self.envs.viewitems()) + self.assertIn(('GOOS', 'linux'), self.envs.viewitems()) + self.assertNotIn(('USER', 'jenkins'), self.envs.viewitems()) + self.assertNotIn(('FOO', 'BAZ'), self.envs.viewitems()) + + def test_parse_args_order_agnostic(self): + args = kubernetes_e2e.parse_args([ + '--some-kubetest-arg=foo', + '--cluster=test']) + self.assertEqual(args.kubetest_args, ['--some-kubetest-arg=foo']) + self.assertEqual(args.cluster, 'test') + + def test_gcp_network(self): + args = kubernetes_e2e.parse_args(['--mode=local', '--cluster=test']) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + lastcall = self.callstack[-1] + self.assertIn('--gcp-network=test', lastcall) + + def test_env_local(self): + env = 'FOO' + value = 'BLAT' + args = kubernetes_e2e.parse_args([ + '--mode=local', + '--env={env}={value}'.format(env=env, value=value), + ]) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + self.assertIn(env, self.envs) + self.assertEqual(self.envs[env], value) + + def test_aws(self): + temp = tempfile.NamedTemporaryFile() + args = kubernetes_e2e.parse_args([ + '--aws', + '--cluster=foo', + '--aws-cluster-domain=test-aws.k8s.io', + '--aws-ssh=%s' % temp.name, + '--aws-pub=%s' % temp.name, + '--aws-cred=%s' % temp.name, + ]) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + + lastcall = self.callstack[-1] + self.assertIn('kops-e2e-runner.sh', lastcall) + self.assertIn('--kops-cluster=foo.test-aws.k8s.io', lastcall) + self.assertIn('--kops-zones', lastcall) + self.assertIn('--kops-state=s3://k8s-kops-prow/', lastcall) + self.assertIn('--kops-nodes=4', lastcall) + self.assertIn('--kops-ssh-key', lastcall) + + self.assertNotIn('kubetest', lastcall) + self.assertIn('kops-e2e-runner.sh', lastcall) + + self.assertEqual( + self.envs['AWS_SSH_PRIVATE_KEY_FILE'], temp.name) + self.assertEqual( + self.envs['AWS_SSH_PUBLIC_KEY_FILE'], temp.name) + self.assertEqual( + self.envs['AWS_SHARED_CREDENTIALS_FILE'], temp.name) + + def test_kops_aws(self): + temp = tempfile.NamedTemporaryFile() + args = kubernetes_e2e.parse_args([ + '--provider=aws', + '--deployment=kops', + '--cluster=foo.example.com', + '--aws-ssh=%s' % temp.name, + '--aws-pub=%s' % temp.name, + '--aws-cred=%s' % temp.name, + ]) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + + lastcall = self.callstack[-1] + self.assertIn('kubetest', lastcall) + self.assertIn('--provider=aws', lastcall) + self.assertIn('--deployment=kops', lastcall) + self.assertIn('--kops-cluster=foo.example.com', lastcall) + self.assertIn('--kops-zones', lastcall) + self.assertIn('--kops-state=s3://k8s-kops-prow/', lastcall) + self.assertIn('--kops-nodes=4', lastcall) + self.assertIn('--kops-ssh-key', lastcall) + self.assertIn('kubetest', lastcall) + self.assertNotIn('kops-e2e-runner.sh', lastcall) + + def test_kops_gce(self): + temp = tempfile.NamedTemporaryFile() + args = kubernetes_e2e.parse_args([ + '--provider=gce', + '--deployment=kops', + '--cluster=foo.example.com', + '--gce-ssh=%s' % temp.name, + '--gce-pub=%s' % temp.name, + ]) + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + kubernetes_e2e.main(args) + + lastcall = self.callstack[-1] + self.assertIn('kubetest', lastcall) + self.assertIn('--provider=gce', lastcall) + self.assertIn('--deployment=kops', lastcall) + self.assertIn('--kops-cluster=foo.example.com', lastcall) + self.assertIn('--kops-zones', lastcall) + self.assertIn('--kops-state=gs://k8s-kops-gce/', lastcall) + self.assertIn('--kops-nodes=4', lastcall) + self.assertIn('--kops-ssh-key', lastcall) + + def test_use_shared_build(self): + # normal path + args = kubernetes_e2e.parse_args([ + '--use-shared-build=bazel' + ]) + def expect_bazel_gcs(path): + bazel_default = os.path.join( + 'gs://kubernetes-jenkins/shared-results', 'bazel-build-location.txt') + self.assertEqual(path, bazel_default) + return always_kubernetes() + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(kubernetes_e2e, 'read_gcs_path', expect_bazel_gcs): + with Stub(time, 'sleep', fake_pass): + kubernetes_e2e.main(args) + lastcall = self.callstack[-1] + self.assertIn('--extract=kubernetes', lastcall) + # normal path, not bazel + args = kubernetes_e2e.parse_args([ + '--use-shared-build' + ]) + def expect_normal_gcs(path): + bazel_default = os.path.join( + 'gs://kubernetes-jenkins/shared-results', 'build-location.txt') + self.assertEqual(path, bazel_default) + return always_kubernetes() + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(kubernetes_e2e, 'read_gcs_path', expect_normal_gcs): + kubernetes_e2e.main(args) + lastcall = self.callstack[-1] + self.assertIn('--extract=kubernetes', lastcall) + # test failure to read shared path from GCS + with Stub(kubernetes_e2e, 'check_env', self.fake_check_env): + with Stub(kubernetes_e2e, 'read_gcs_path', raise_urllib2_error): + with Stub(os, 'getcwd', always_kubernetes): + with Stub(time, 'sleep', fake_pass): + try: + kubernetes_e2e.main(args) + except RuntimeError as err: + if not err.message.startswith('Failed to get shared build location'): + raise err + +if __name__ == '__main__': + unittest.main() diff --git a/images/bootstrap/scenarios/kubernetes_execute_bazel.py b/images/bootstrap/scenarios/kubernetes_execute_bazel.py new file mode 100755 index 0000000000000..557a45916622c --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_execute_bazel.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Executes a command, afterwards executes coalesce.py, preserving the return code. + +Also supports configuring bazel remote caching.""" + +import argparse +import os +import subprocess +import sys + +ORIG_CWD = os.getcwd() + +def test_infra(*paths): + """Return path relative to root of test-infra repo.""" + return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths) + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + +def call(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + return subprocess.call(cmd) + + +def main(cmd): + """Run script and preserve return code after running coalesce.py.""" + if not cmd: + raise ValueError(cmd) + # update bazel caching configuration if enabled + # TODO(fejta): migrate all jobs to use RBE instead of this + if os.environ.get('BAZEL_REMOTE_CACHE_ENABLED', 'false') == 'true': + print 'Bazel remote cache is enabled, generating .bazelrcs ...' + # TODO: consider moving this once we've migrated all users + # of the remote cache to this script + check(test_infra('images/bootstrap/create_bazel_cache_rcs.sh')) + # call the user supplied command + return_code = call(*cmd) + # Coalesce test results into one file for upload. + check(test_infra('hack/coalesce.py')) + # preserve the exit code + sys.exit(return_code) + + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument('cmd', nargs=1) + PARSER.add_argument('args', nargs='*') + ARGS = PARSER.parse_args() + main(ARGS.cmd + ARGS.args) diff --git a/images/bootstrap/scenarios/kubernetes_janitor.py b/images/bootstrap/scenarios/kubernetes_janitor.py new file mode 100755 index 0000000000000..3d3f9f3653319 --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_janitor.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""Dig through jobs/FOO.env, and execute a janitor pass for each of the project""" + +import argparse +import json +import os +import re +import subprocess +import sys + +try: + from junit_xml import TestSuite, TestCase + HAS_JUNIT = True +except ImportError: + HAS_JUNIT = False + +ORIG_CWD = os.getcwd() # Checkout changes cwd + +def test_infra(*paths): + """Return path relative to root of test-infra repo.""" + return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths) + + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + + +def parse_project(path): + """Parse target env file and return GCP project name.""" + with open(path, 'r') as fp: + env = fp.read() + match = re.search(r'PROJECT=([^\n"]+)', env) + if match: + project = match.group(1) + return project + return None + + +def clean_project(project, hours=24, dryrun=False, ratelimit=None, filt=None): + """Execute janitor for target GCP project """ + # Multiple jobs can share the same project, woooo + if project in CHECKED: + return + CHECKED.add(project) + + cmd = ['python', test_infra('boskos/cmd/janitor/gcp_janitor.py'), '--project=%s' % project] + cmd.append('--hour=%d' % hours) + if dryrun: + cmd.append('--dryrun') + if ratelimit: + cmd.append('--ratelimit=%d' % ratelimit) + if VERBOSE: + cmd.append('--verbose') + if filt: + cmd.append('--filter=%s' % filt) + + try: + check(*cmd) + except subprocess.CalledProcessError: + FAILED.append(project) + + +EXEMPT_PROJECTS = [ + 'kubernetes-scale', # Let it's up/down job handle the resources + 'k8s-scale-testing', # As it can be running some manual experiments + 'k8s-jkns-e2e-gce-f8n-1-7', # federation projects should use fedtidy to clean up + 'k8s-jkns-e2e-gce-f8n-1-8', # federation projects should use fedtidy to clean up +] + +PR_PROJECTS = { + # k8s-jkns-pr-bldr-e2e-gce-fdrtn + # k8s-jkns-pr-cnry-e2e-gce-fdrtn + # cleans up resources older than 3h + # which is more than enough for presubmit jobs to finish. + 'k8s-jkns-pr-gce': 3, + 'k8s-jkns-pr-gce-bazel': 3, + 'k8s-jkns-pr-gce-etcd3': 3, + 'k8s-jkns-pr-gci-gce': 3, + 'k8s-jkns-pr-gci-gke': 3, + 'k8s-jkns-pr-gci-kubemark': 3, + 'k8s-jkns-pr-gke': 3, + 'k8s-jkns-pr-kubeadm': 3, + 'k8s-jkns-pr-kubemark': 3, + 'k8s-jkns-pr-node-e2e': 3, + 'k8s-jkns-pr-gce-gpus': 3, +} + +SCALE_PROJECT = { + 'k8s-presubmit-scale': 3, +} + +def check_predefine_jobs(jobs, ratelimit): + """Handle predefined jobs""" + for project, expire in jobs.iteritems(): + clean_project(project, hours=expire, ratelimit=ratelimit) + +def check_ci_jobs(): + """Handle CI jobs""" + with open(test_infra('jobs/config.json')) as fp: + config = json.load(fp) + + match_re = re.compile(r'--gcp-project=(.+)') + for value in config.values(): + clean_hours = 24 + found = None + for arg in value.get('args', []): + # lifetime for soak cluster should be 7 days + # clean up everything older than 10 days to prevent leak + if '--soak' in arg: + clean_hours = 24 * 10 + mat = match_re.match(arg) + if not mat: + continue + project = mat.group(1) + if any(b in project for b in EXEMPT_PROJECTS): + print >>sys.stderr, 'Project %r is exempted in ci-janitor' % project + continue + if project in PR_PROJECTS or project in SCALE_PROJECT: + continue # CI janitor skips all PR jobs + found = project + if found: + clean_project(found, clean_hours) + + +def main(mode, ratelimit, projects, age, artifacts, filt): + """Run janitor for each project.""" + if mode == 'pr': + check_predefine_jobs(PR_PROJECTS, ratelimit) + elif mode == 'scale': + check_predefine_jobs(SCALE_PROJECT, ratelimit) + elif mode == 'custom': + projs = str.split(projects, ',') + for proj in projs: + clean_project(proj.strip(), hours=age, ratelimit=ratelimit, filt=filt) + else: + check_ci_jobs() + + # Summary + print 'Janitor checked %d project, %d failed to clean up.' % (len(CHECKED), len(FAILED)) + print HAS_JUNIT + if artifacts: + output = os.path.join(artifacts, 'junit_janitor.xml') + if not HAS_JUNIT: + print 'Please install junit-xml (https://pypi.org/project/junit-xml/)' + else: + print 'Generating junit output:' + tcs = [] + for project in CHECKED: + tc = TestCase(project, 'kubernetes_janitor') + if project in FAILED: + # TODO(krzyzacy): pipe down stdout here as well + tc.add_failure_info('failed to clean up gcp project') + tcs.append(tc) + + ts = TestSuite('janitor', tcs) + with open(output, 'w') as f: + TestSuite.to_file(f, [ts]) + if FAILED: + print >>sys.stderr, 'Failed projects: %r' % FAILED + exit(1) + + +if __name__ == '__main__': + # keep some metric + CHECKED = set() + FAILED = [] + VERBOSE = False + PARSER = argparse.ArgumentParser() + PARSER.add_argument( + '--mode', default='ci', choices=['ci', 'pr', 'scale', 'custom'], + help='Which type of projects to clear') + PARSER.add_argument( + '--ratelimit', type=int, + help='Max number of resources to clear in one gcloud delete call ' + '(passed into gcp_janitor.py)') + PARSER.add_argument( + '--projects', type=str, + help='Comma separated list of projects to clean up. Only applicable in custom mode.') + PARSER.add_argument( + '--age', type=int, + help='Expiry age for projects, in hours. Only applicable in custom mode.') + PARSER.add_argument( + '--verbose', action='store_true', + help='If want more detailed logs from the janitor script.') + PARSER.add_argument( + '--artifacts', + help='generate junit style xml to target path', + default=os.environ.get('ARTIFACTS', None)) + PARSER.add_argument( + '--filter', + default=None, + help='Filter down to these instances(passed into gcp_janitor.py)') + ARGS = PARSER.parse_args() + VERBOSE = ARGS.verbose + main(ARGS.mode, ARGS.ratelimit, ARGS.projects, ARGS.age, ARGS.artifacts, ARGS.filter) diff --git a/images/bootstrap/scenarios/kubernetes_verify.py b/images/bootstrap/scenarios/kubernetes_verify.py new file mode 100755 index 0000000000000..2f25819df7314 --- /dev/null +++ b/images/bootstrap/scenarios/kubernetes_verify.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""Runs verify/test-go checks for kubernetes/kubernetes.""" + +import argparse +import os +import re +import subprocess +import sys + +# This is deprecated from 1.14 onwards. +VERSION_TAG = { + '1.11': '1.11-v20190318-2ac98e338', + '1.12': '1.12-v20190318-2ac98e338', + '1.13': '1.13-v20190817-cc05229', + '1.14': '1.14-v20190817-cc05229', + # this is master, feature branches... + 'default': '1.14-v20190817-cc05229', +} + + +def check_output(*cmd): + """Log and run the command, return output, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + return subprocess.check_output(cmd) + + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + + +def retry(func, times=5): + """call func until it returns true at most times times""" + success = False + for _ in range(0, times): + success = func() + if success: + return success + return success + + +def try_call(cmds): + """returns true if check(cmd) does not throw an exception + over all cmds where cmds = [[cmd, arg, arg2], [cmd2, arg]]""" + try: + for cmd in cmds: + check(*cmd) + return True + # pylint: disable=bare-except + except: + return False + + +def get_git_cache(k8s): + git = os.path.join(k8s, ".git") + if not os.path.isfile(git): + return None + with open(git) as git_file: + return git_file.read().replace("gitdir: ", "").rstrip("\n") + + +def branch_to_tag(branch): + verify_branch = re.match(r'release-(\d+\.\d+)', branch) + key = 'default' + if verify_branch and verify_branch.group(1) in VERSION_TAG: + key = verify_branch.group(1) + return VERSION_TAG[key] + + +def main(branch, script, force, on_prow, exclude_typecheck, exclude_godep, exclude_files_remake): + """Test branch using script, optionally forcing verify checks.""" + tag = branch_to_tag(branch) + + force = 'y' if force else 'n' + exclude_typecheck = 'y' if exclude_typecheck else 'n' + exclude_godep = 'y' if exclude_godep else 'n' + exclude_files_remake = 'y' if exclude_files_remake else 'n' + artifacts = '%s/_artifacts' % os.environ['WORKSPACE'] + k8s = os.getcwd() + if not os.path.basename(k8s) == 'kubernetes': + raise ValueError(k8s) + + check('rm', '-rf', '.gsutil') + remote = 'bootstrap-upstream' + uri = 'https://github.com/kubernetes/kubernetes.git' + + current_remotes = check_output('git', 'remote') + if re.search('^%s$' % remote, current_remotes, flags=re.MULTILINE): + check('git', 'remote', 'remove', remote) + check('git', 'remote', 'add', remote, uri) + check('git', 'remote', 'set-url', '--push', remote, 'no_push') + # If .git is cached between runs this data may be stale + check('git', 'fetch', remote) + + if not os.path.isdir(artifacts): + os.makedirs(artifacts) + + if on_prow: + # TODO: on prow REPO_DIR should be /go/src/k8s.io/kubernetes + # however these paths are brittle enough as is... + git_cache = get_git_cache(k8s) + cmd = [ + 'docker', 'run', '--rm=true', '--privileged=true', + '-v', '/var/run/docker.sock:/var/run/docker.sock', + '-v', '/etc/localtime:/etc/localtime:ro', + '-v', '%s:/go/src/k8s.io/kubernetes' % k8s, + ] + if git_cache is not None: + cmd.extend(['-v', '%s:%s' % (git_cache, git_cache)]) + cmd.extend([ + '-v', '/workspace/k8s.io/:/workspace/k8s.io/', + '-v', '%s:/workspace/artifacts' % artifacts, + '-e', 'KUBE_FORCE_VERIFY_CHECKS=%s' % force, + '-e', 'KUBE_VERIFY_GIT_BRANCH=%s' % branch, + '-e', 'EXCLUDE_TYPECHECK=%s' % exclude_typecheck, + '-e', 'EXCLUDE_FILES_REMAKE=%s' % exclude_files_remake, + '-e', 'EXCLUDE_GODEP=%s' % exclude_godep, + '-e', 'REPO_DIR=%s' % k8s, # hack/lib/swagger.sh depends on this + '--tmpfs', '/tmp:exec,mode=1777', + 'gcr.io/k8s-testimages/kubekins-test:%s' % tag, + 'bash', '-c', 'cd kubernetes && %s' % script, + ]) + check(*cmd) + else: + check( + 'docker', 'run', '--rm=true', '--privileged=true', + '-v', '/var/run/docker.sock:/var/run/docker.sock', + '-v', '/etc/localtime:/etc/localtime:ro', + '-v', '%s:/go/src/k8s.io/kubernetes' % k8s, + '-v', '%s:/workspace/artifacts' % artifacts, + '-e', 'KUBE_FORCE_VERIFY_CHECKS=%s' % force, + '-e', 'KUBE_VERIFY_GIT_BRANCH=%s' % branch, + '-e', 'EXCLUDE_TYPECHECK=%s' % exclude_typecheck, + '-e', 'EXCLUDE_FILES_REMAKE=%s' % exclude_files_remake, + '-e', 'EXCLUDE_GODEP=%s' % exclude_godep, + '-e', 'REPO_DIR=%s' % k8s, # hack/lib/swagger.sh depends on this + 'gcr.io/k8s-testimages/kubekins-test:%s' % tag, + 'bash', '-c', 'cd kubernetes && %s' % script, + ) + + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser( + 'Runs verification checks on the kubernetes repo') + PARSER.add_argument( + '--branch', default='master', help='Upstream target repo') + PARSER.add_argument( + '--force', action='store_true', help='Force all verify checks') + PARSER.add_argument( + '--exclude-typecheck', action='store_true', help='Exclude typecheck from verify') + PARSER.add_argument( + '--exclude-godep', action='store_true', help='Exclude godep checks from verify') + PARSER.add_argument( + '--exclude-files-remake', action='store_true', help='Exclude files remake from verify') + PARSER.add_argument( + '--script', + default='./hack/jenkins/test-dockerized.sh', + help='Script in kubernetes/kubernetes that runs checks') + PARSER.add_argument( + '--prow', action='store_true', help='Force Prow mode' + ) + ARGS = PARSER.parse_args() + main(ARGS.branch, ARGS.script, ARGS.force, ARGS.prow, + ARGS.exclude_typecheck, ARGS.exclude_godep, ARGS.exclude_files_remake) diff --git a/images/bootstrap/scenarios/maintenance.py b/images/bootstrap/scenarios/maintenance.py new file mode 100755 index 0000000000000..12051b207b081 --- /dev/null +++ b/images/bootstrap/scenarios/maintenance.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Need to figure out why this only fails on travis +# pylint: disable=bad-continuation + +"""update gcloud on Jenkins vms.""" + +import os +import subprocess +import sys + +def check(*cmd): + """Log and run the command, raising on errors.""" + print >>sys.stderr, 'Run:', cmd + subprocess.check_call(cmd) + + +def main(): + """update gcloud for Jenkins vms""" + host = os.environ.get('HOSTNAME') + if host == 'jenkins-master' or host == 'pull-jenkins-master': + check('sudo', 'gcloud', 'components', 'update') + check('sudo', 'gcloud', 'components', 'update', 'beta') + check('sudo', 'gcloud', 'components', 'update', 'alpha') + else: + try: + check('sudo', 'apt-get', 'update') + except subprocess.CalledProcessError: + check('sudo', 'rm', '/var/lib/apt/lists/partial/*') + check('sudo', 'apt-get', 'update') + check('sudo', 'apt-get', 'install', '-y', 'google-cloud-sdk') + + +if __name__ == '__main__': + main()