diff --git a/backend/src/apiserver/client_manager.go b/backend/src/apiserver/client_manager.go index 7004857380c..bd7525b6d32 100644 --- a/backend/src/apiserver/client_manager.go +++ b/backend/src/apiserver/client_manager.go @@ -41,15 +41,15 @@ import ( ) const ( - minioServiceHost = "MINIO_SERVICE_SERVICE_HOST" - minioServicePort = "MINIO_SERVICE_SERVICE_PORT" - mysqlServiceHost = "MYSQL_SERVICE_HOST" - mysqlUser = "DBConfig.User" - mysqlPassword = "DBConfig.Password" - mysqlServicePort = "MYSQL_SERVICE_PORT" + minioServiceHost = "MINIO_SERVICE_SERVICE_HOST" + minioServicePort = "MINIO_SERVICE_SERVICE_PORT" + mysqlServiceHost = "MYSQL_SERVICE_HOST" + mysqlServicePort = "MYSQL_SERVICE_PORT" + mysqlUser = "DBConfig.User" + mysqlPassword = "DBConfig.Password" + mysqlDBName = "DBConfig.DBName" podNamespace = "POD_NAMESPACE" - dbName = "mlpipeline" initConnectionTimeout = "InitConnectionTimeout" ) @@ -255,6 +255,7 @@ func initMysql(driverName string, initConnectionTimeout time.Duration) string { util.TerminateIfError(err) // Create database if not exist + dbName := getStringConfig(mysqlDBName) operation = func() error { _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", dbName)) if err != nil { @@ -280,7 +281,7 @@ func initMinioClient(initConnectionTimeout time.Duration) storage.ObjectStoreInt accessKey := getStringConfig("ObjectStoreConfig.AccessKey") secretKey := getStringConfig("ObjectStoreConfig.SecretAccessKey") bucketName := getStringConfig("ObjectStoreConfig.BucketName") - disableMultipart := getBoolConfigWithDefault("ObjectStoreConfig.Multipart.Disable", false) + disableMultipart := getBoolConfigWithDefault("ObjectStoreConfig.Multipart.Disable", true) minioClient := client.CreateMinioClientOrFatal(minioServiceHost, minioServicePort, accessKey, secretKey, initConnectionTimeout) diff --git a/backend/src/apiserver/config/config.json b/backend/src/apiserver/config/config.json index c8f9c530506..00ee8eb121c 100644 --- a/backend/src/apiserver/config/config.json +++ b/backend/src/apiserver/config/config.json @@ -1,7 +1,8 @@ { "DBConfig": { "DriverName": "mysql", - "DataSourceName": "" + "DataSourceName": "", + "DBName": "mlpipeline" }, "ObjectStoreConfig":{ "AccessKey": "minio", diff --git a/contrib/components/openvino/ovms-deployer/containers/evaluate.py b/contrib/components/openvino/ovms-deployer/containers/evaluate.py index 1f49a0acffd..ad934474439 100755 --- a/contrib/components/openvino/ovms-deployer/containers/evaluate.py +++ b/contrib/components/openvino/ovms-deployer/containers/evaluate.py @@ -59,7 +59,7 @@ def getJpeg(path, size, path_prefix): img = img.astype('float32') img = img.transpose(2,0,1).reshape(1,3,size,size) print(path, img.shape, "; data range:",np.amin(img),":",np.amax(img)) - except e: + except Exception as e: print("Can not read the image file", e) img = None else: @@ -138,4 +138,4 @@ def getJpeg(path, size, path_prefix): json.dump(metrics, f) f.close print("\nOverall accuracy=",matched/i*100,"%") -print("Average latency=",latency,"ms") \ No newline at end of file +print("Average latency=",latency,"ms") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1bac6456b88..ae282897763 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -34,6 +34,27 @@ } } }, + "@babel/polyfill": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.4.4.tgz", + "integrity": "sha512-WlthFLfhQQhh+A2Gn5NSFl0Huxz36x86Jn+E9OW7ibK8edKPq+KLy4apM1yDpQ8kJOVi1OVjpP4vSDLdrI04dg==", + "requires": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.2" + }, + "dependencies": { + "core-js": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz", + "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==" + }, + "regenerator-runtime": { + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz", + "integrity": "sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==" + } + } + }, "@babel/runtime": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.2.0.tgz", @@ -2255,6 +2276,11 @@ } } }, + "brace": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/brace/-/brace-0.11.1.tgz", + "integrity": "sha1-SJb8ydVE7vRfS7dmDbMg07N5/lg=" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4105,6 +4131,11 @@ "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==" }, + "diff-match-patch": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.4.tgz", + "integrity": "sha512-Uv3SW8bmH9nAtHKaKSanOQmj2DnlH65fUpcrMdfdaOxUG02QQ4YGZ8AE7kKOMisF7UqvOlGKVYWRvezdncW9lg==" + }, "diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -5319,7 +5350,8 @@ }, "ansi-regex": { "version": "2.1.1", - "bundled": true + "bundled": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -5684,7 +5716,8 @@ }, "safe-buffer": { "version": "5.1.1", - "bundled": true + "bundled": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -5732,6 +5765,7 @@ "strip-ansi": { "version": "3.0.1", "bundled": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5770,11 +5804,13 @@ }, "wrappy": { "version": "1.0.2", - "bundled": true + "bundled": true, + "optional": true }, "yallist": { "version": "3.0.2", - "bundled": true + "bundled": true, + "optional": true } } }, @@ -8418,6 +8454,11 @@ "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" + }, "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -8433,8 +8474,7 @@ "lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", - "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=", - "dev": true + "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" }, "lodash.isfunction": { "version": "3.0.9", @@ -11194,6 +11234,36 @@ "scheduler": "^0.12.0" } }, + "react-ace": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/react-ace/-/react-ace-7.0.2.tgz", + "integrity": "sha512-+TFuO1nO6dme/q+qEHjb7iOuWI8jRDzeALs9JyH8HoyHb9+A2bC8WHuJyNU3pmPo8623bytgAgzEJAzDMkzjlw==", + "requires": { + "@babel/polyfill": "^7.4.4", + "brace": "^0.11.1", + "diff-match-patch": "^1.0.4", + "lodash.get": "^4.4.2", + "lodash.isequal": "^4.5.0", + "prop-types": "^15.7.2" + }, + "dependencies": { + "prop-types": { + "version": "15.7.2", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", + "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + } + }, + "react-is": { + "version": "16.9.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.9.0.tgz", + "integrity": "sha512-tJBzzzIgnnRfEm046qRcURvwQnZVXmuCbscxUO5RWrGTXpon2d4c8mI0D8WE6ydVIm29JiLB6+RslkIvym9Rjw==" + } + } + }, "react-codemirror2": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/react-codemirror2/-/react-codemirror2-5.1.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index ee01ec5ea47..168405881c7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,6 +6,7 @@ "@material-ui/core": "3.7.1", "@material-ui/icons": "^3.0.1", "@types/js-yaml": "^3.11.2", + "brace": "^0.11.1", "codemirror": "^5.40.2", "d3": "^5.7.0", "d3-dsv": "^1.0.10", @@ -18,6 +19,7 @@ "portable-fetch": "^3.0.0", "re-resizable": "^4.9.0", "react": "^16.7.0", + "react-ace": "^7.0.2", "react-codemirror2": "^5.1.0", "react-dom": "^16.5.2", "react-dropzone": "^5.1.0", diff --git a/frontend/src/components/Editor.test.tsx b/frontend/src/components/Editor.test.tsx new file mode 100644 index 00000000000..2245fb6ba66 --- /dev/null +++ b/frontend/src/components/Editor.test.tsx @@ -0,0 +1,52 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as React from 'react'; +import { mount } from 'enzyme'; +import Editor from './Editor'; + +/* + These tests mimic https://github.com/securingsincity/react-ace/blob/master/tests/src/ace.spec.js + to ensure that editor properties (placeholder and value) can be properly + tested. +*/ + +describe('Editor', () => { + it('renders without a placeholder and value', () => { + const tree = mount(); + expect(tree.html()).toMatchSnapshot(); + }); + + it('renders with a placeholder', () => { + const placeholder = 'I am a placeholder.'; + const tree = mount(); + expect(tree.html()).toMatchSnapshot(); + }); + + it ('renders a placeholder that contains HTML', () => { + const placeholder = 'I am a placeholder with HTML.'; + const tree = mount(); + expect(tree.html()).toMatchSnapshot(); + }); + + it('has its value set to the provided value', () => { + const value = 'I am a value.'; + const tree = mount(); + expect(tree).not.toBeNull(); + const editor = (tree.instance() as any).editor; + expect(editor.getValue()).toBe(value); + }); +}); \ No newline at end of file diff --git a/frontend/src/components/Editor.tsx b/frontend/src/components/Editor.tsx new file mode 100644 index 00000000000..11dec1c990d --- /dev/null +++ b/frontend/src/components/Editor.tsx @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AceEditor from 'react-ace'; + +// Modified AceEditor that supports HTML within provided placeholder. This is +// important because it allows for the usage of multi-line placeholders. +class Editor extends AceEditor { + public updatePlaceholder(): void { + const editor = this.editor; + const { placeholder } = this.props; + + const showPlaceholder = !editor.session.getValue().length; + let node = editor.renderer.placeholderNode; + if (!showPlaceholder && node) { + editor.renderer.scroller.removeChild(editor.renderer.placeholderNode); + editor.renderer.placeholderNode = null; + } else if (showPlaceholder && !node) { + node = editor.renderer.placeholderNode = document.createElement('div'); + node.innerHTML = placeholder || ''; + node.className = 'ace_comment ace_placeholder'; + node.style.padding = '0 9px'; + node.style.position = 'absolute'; + node.style.zIndex = '3'; + editor.renderer.scroller.appendChild(node); + } else if (showPlaceholder && node) { + node.innerHTML = placeholder; + } + } +} + +export default Editor; \ No newline at end of file diff --git a/frontend/src/components/__snapshots__/Editor.test.tsx.snap b/frontend/src/components/__snapshots__/Editor.test.tsx.snap new file mode 100644 index 00000000000..10762a86460 --- /dev/null +++ b/frontend/src/components/__snapshots__/Editor.test.tsx.snap @@ -0,0 +1,7 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Editor renders a placeholder that contains HTML 1`] = `"
I am a placeholder with HTML.
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
"`; + +exports[`Editor renders with a placeholder 1`] = `"
I am a placeholder.
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
"`; + +exports[`Editor renders without a placeholder and value 1`] = `"
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
"`; diff --git a/go.mod b/go.mod index 8e7440e44eb..5cd5d23a99a 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/kubeflow/pipelines require ( - github.com/BurntSushi/toml v0.3.1 // indirect + cloud.google.com/go v0.44.3 // indirect github.com/Masterminds/squirrel v0.0.0-20190107164353-fa735ea14f09 github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f github.com/argoproj/argo v2.3.0+incompatible @@ -28,18 +28,19 @@ require ( github.com/gogo/protobuf v1.1.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 // indirect - github.com/golang/protobuf v1.2.0 - github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect - github.com/google/go-cmp v0.2.0 + github.com/golang/lint v0.0.0-20180702182130-06c8688daad7 // indirect + github.com/golang/protobuf v1.3.2 + github.com/google/go-cmp v0.3.1 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect github.com/google/ml-metadata v0.0.0-20190214221617-0fb82dc56ff7 - github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57 // indirect + github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect github.com/google/uuid v1.0.0 github.com/googleapis/gnostic v0.2.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect + github.com/gorilla/websocket v1.4.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/grpc-gateway v1.6.3 - github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 // indirect + github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/imdario/mergo v0.3.5 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/iris-contrib/go.uuid v2.0.0+incompatible // indirect @@ -49,6 +50,7 @@ require ( github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be // indirect github.com/jtolds/gls v4.2.1+incompatible // indirect github.com/kataras/iris v10.6.7+incompatible + github.com/kr/pty v1.1.8 // indirect github.com/lib/pq v1.0.0 // indirect github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a // indirect github.com/mattn/go-sqlite3 v1.9.0 @@ -75,14 +77,20 @@ require ( go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 // indirect golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045 // indirect - golang.org/x/net v0.0.0-20181201002055-351d144fa1fc - golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect - google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 - google.golang.org/grpc v1.16.0 + golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect + golang.org/x/mobile v0.0.0-20190814143026-e8b3e6111d02 // indirect + golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 + golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect + golang.org/x/tools v0.0.0-20190815212832-922a4ee32d1a // indirect + google.golang.org/api v0.8.0 + google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 + google.golang.org/grpc v1.23.0 gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.39.3 // indirect + gopkg.in/yaml.v2 v2.2.2 + honnef.co/go/tools v0.0.1-2019.2.2 // indirect k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 k8s.io/apiextensions-apiserver v0.0.0-20190103235604-e7617803aceb // indirect k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d diff --git a/go.sum b/go.sum index 79bf1c8959b..0b7b2a92e3e 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,13 @@ cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.3 h1:0sMegbmn/8uTwpNkB0q9cLEpZ2W5a6kl+wtBQgPWBJQ= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/squirrel v0.0.0-20170825200431-a6b93000bd21 h1:7+Xpo0TxPIBqSHVktrR1VYU2LSNTrJ3B79nVzE74nPA= github.com/Masterminds/squirrel v0.0.0-20170825200431-a6b93000bd21/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= github.com/Masterminds/squirrel v0.0.0-20190107164353-fa735ea14f09 h1:enWVS77aJkLWVIUExiqF6A8eWTVzCXUKUvkST3/wyKI= @@ -31,8 +38,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f h1:WH0w/R4Yoey+04HhFxqZ6VX6I0d7RMyw5aXQ9UTvQPs= github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -42,6 +51,7 @@ github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29/go.mod h1:Qh8CwZ github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v2.8.0+incompatible h1:wN8GCRDPGHguIynsnBartv5GUgGUg1LAU7+xnSn1j7Q= github.com/emicklei/go-restful v2.8.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -90,28 +100,51 @@ github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+A github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu4i+jD32SE9jQXyfnOvwhHqlT0= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/ml-metadata v0.0.0-20190214221617-0fb82dc56ff7 h1:Db+CbWW+XCYzfL662n+i2/xQGmLy4nRFFX3fEscNstk= github.com/google/ml-metadata v0.0.0-20190214221617-0fb82dc56ff7/go.mod h1:yO0xrdRxF2VbZGmBEPUsnKnANnPE/3kULpqDAFRPCmg= +github.com/google/ml-metadata v0.13.2/go.mod h1:yO0xrdRxF2VbZGmBEPUsnKnANnPE/3kULpqDAFRPCmg= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.6.3 h1:oQ+8y59SMDn8Ita1Sh4f94XCUVp8AB84sppXP8Qgiow= github.com/grpc-ecosystem/grpc-gateway v1.6.3/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -125,17 +158,24 @@ github.com/jinzhu/gorm v1.9.1 h1:lDSDtsCt5AGGSKTs8AHlSDbbgif4G4+CKJ8ETBDVHTA= github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a h1:eeaG9XMUvRBYXJi4pg1ZKM7nxc5AfXfojeLLW7O5J3k= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v0.0.0-20181116074157-8ec929ed50c3 h1:xvj06l8iSwiWpYgm8MbPp+naBg+pwfqmdXabzqPCn/8= github.com/jinzhu/now v0.0.0-20181116074157-8ec929ed50c3/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be h1:AHimNtVIpiBjPUhEF5KNCkrUyqTSA5zWUl8sQ2bfGBE= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kataras/iris v10.6.7+incompatible h1:K0K2vh7wJTkMCjCbWmuPDP7YpPAOj9L2AtU7AN79pdM= github.com/kataras/iris v10.6.7+incompatible/go.mod h1:ki9XPua5SyAJbIxDdsssxevgGrbpBmmvoQmo/A0IodY= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -182,6 +222,7 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nL github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 h1:x7xEyJDP7Hv3LVgvWhzioQqbC/KtuUhTigKlH/8ehhE= github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.0.6 h1:hcP1GmhGigz/O7h1WVUM5KklBp1JoNS9FggWKdj/j3s= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -206,6 +247,9 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4 h1:gKMu1Bf6QINDnvyZuTaACm9ofY+PRh+5vFz4oxBZeF8= github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4/go.mod h1:50wTf68f99/Zt14pr046Tgt3Lp2vLyFZKzbFXTOabXw= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= @@ -215,37 +259,122 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190814143026-e8b3e6111d02/go.mod h1:z5wpDCy2wbnXyFdvEuY3LhY9gBUL86/IOILm+Hsjx+E= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190815212832-922a4ee32d1a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0 h1:VGGbLNyPF7dvYHhcUGYBBGCRDDK0RRJAI6KCvo0CL+E= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 h1:yvw+zsSmSM02Z5H3ZdEV7B7Ql7eFrjQTnmByJvK+3J8= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 h1:9oFlwfEGIvmxXTcY53ygNyxIQtWciRHjrnUvZJCYXYU= google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 h1:iKtrH9Y8mcbADOP0YFaEMth7OfuHY9xHOwNj4znpM1A= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.16.0 h1:dz5IJGuC2BB7qXR5AyHNwAUBhZscK2xVez7mznh72sY= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -256,6 +385,11 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 h1:hROmpFC7JMobXFXMmD7ZKZLhDKvr1IKfFJoYS/45G/8= k8s.io/api v0.0.0-20180712090710-2d6f90ab1293/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apiextensions-apiserver v0.0.0-20190103235604-e7617803aceb h1:3yElwSbnV34qIVTlGgkbWwWM+wq7fw6i7EKRtLV6z58= @@ -272,5 +406,6 @@ k8s.io/kube-openapi v0.0.0-20180719232738-d8ea2fe547a4 h1:C8xi0mJeE8wOFsLofmG7JV k8s.io/kube-openapi v0.0.0-20180719232738-d8ea2fe547a4/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kubernetes v1.11.1 h1:wHOPX+teuYaSlUWfL/b24jMH0n7HECbj4Xt8i7kSZIw= k8s.io/kubernetes v1.11.1/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/controller-runtime v0.0.0-20181121180216-5558165425ef h1:3cR5xS/Oii/CbVbZ9qeModvTG83520jdkSKUdTJkwGo= sigs.k8s.io/controller-runtime v0.0.0-20181121180216-5558165425ef/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= diff --git a/samples/contrib/ibm-samples/watson/source/model-source-code/tf-model/input_data.py b/samples/contrib/ibm-samples/watson/source/model-source-code/tf-model/input_data.py index fdf46df45d9..9c0427473e0 100644 --- a/samples/contrib/ibm-samples/watson/source/model-source-code/tf-model/input_data.py +++ b/samples/contrib/ibm-samples/watson/source/model-source-code/tf-model/input_data.py @@ -4,6 +4,7 @@ """Functions for downloading and reading MNIST data.""" import gzip import os +from six.moves import xrange from six.moves.urllib.request import urlretrieve import numpy @@ -144,4 +145,4 @@ class DataSets(object): data_sets.train = DataSet(train_images, train_labels) data_sets.validation = DataSet(validation_images, validation_labels) data_sets.test = DataSet(test_images, test_labels) - return data_sets \ No newline at end of file + return data_sets diff --git a/samples/core/tfx_cab_classification/README.md b/samples/core/tfx_cab_classification/README.md index 6c27b45f0a9..bdfc1c6236c 100644 --- a/samples/core/tfx_cab_classification/README.md +++ b/samples/core/tfx_cab_classification/README.md @@ -56,20 +56,14 @@ Open the Kubeflow pipelines UI. Create a new pipeline, and then upload the compi 1. The name of a GCP project. 2. An output directory in a Google Cloud Storage bucket, of the form `gs:///`. - On-Premise - For On-Premise cluster, the pipeline will create a Persistent Volume Claim (PVC), and download - automatically the - [source data](https://github.com/kubeflow/pipelines/tree/master/samples/core/tfx_cab_classification/taxi-cab-classification) - to the PVC. + For On-Premise cluster, the pipeline will create a Persistent Volume Claim (PVC), and download automatically the [source date](https://github.com/kubeflow/pipelines/tree/master/samples/core/tfx_cab_classification/taxi-cab-classification) to the PVC. 1. The `output` is PVC mount point for the containers, can be set to `/mnt`. 2. The `project` can be set to `taxi-cab-classification-pipeline-onprem`. 3. If the PVC mounted to `/mnt`, the value of below parameters need to be set as following: - - `column-names`: ` -/mnt/pipelines/samples/tfx/taxi-cab-classification/column-names.json` - - `train`: `/mnt/pipelines/samples/tfx/taxi-cab-classification/train.csv` - - `evaluation`: `/mnt/pipelines/samples/tfx/taxi-cab-classification/eval.csv` - - `preprocess-module`: `/mnt/pipelines/samples/tfx/taxi-cab-classification/preprocessing.py` - - + - `column-names`: `/mnt/pipelines/samples/core/tfx_cab_classification/taxi-cab-classification/column-names.json` + - `train`: `/mnt/pipelines/samples/core/tfx_cab_classification/taxi-cab-classification/train.csv` + - `evaluation`: `/mnt/pipelines/samples/core/tfx_cab_classification/taxi-cab-classification/eval.csv` + - `preprocess-module`: `/mnt/pipelines/samples/core/tfx_cab_classification/taxi-cab-classification/preprocessing.py` ## Components source diff --git a/sdk/python/kfp/compiler/_container_builder.py b/sdk/python/kfp/compiler/_container_builder.py index b4d16318828..d8a9f785d12 100644 --- a/sdk/python/kfp/compiler/_container_builder.py +++ b/sdk/python/kfp/compiler/_container_builder.py @@ -41,7 +41,7 @@ def __init__(self, gcs_staging=None, gcr_image_tag=None, namespace=None): try: gcs_bucket = self._get_project_id() except: - raise ValueError('Please provide the gcr_staging.') + raise ValueError('Cannot get the Google Cloud project ID, please specify the gcs_staging argument.') self._gcs_staging = 'gs://' + gcs_bucket + '/' + GCS_STAGING_BLOB_DEFAULT_PREFIX else: from pathlib import PurePath @@ -160,7 +160,7 @@ def build(self, local_dir, docker_filename, target_image=None, timeout=1000): target_image (str): the target image tag to push the final image. timeout (int): time out in seconds. Default: 1000 """ - target_image = self._gcr_image_tag if target_image is None else target_image + target_image = target_image or self._gcr_image_tag # Prepare build context with tempfile.TemporaryDirectory() as local_build_dir: from ._gcs_helper import GCSHelper diff --git a/sdk/python/kfp/gcp.py b/sdk/python/kfp/gcp.py index 6e1cf30273d..c8eab7529d5 100644 --- a/sdk/python/kfp/gcp.py +++ b/sdk/python/kfp/gcp.py @@ -14,8 +14,7 @@ from kubernetes.client import V1Toleration - -def use_gcp_secret(secret_name='user-gcp-sa', secret_file_path_in_volume='/user-gcp-sa.json', volume_name=None, secret_volume_mount_path='/secret/gcp-credentials'): +def use_gcp_secret(secret_name='user-gcp-sa', secret_file_path_in_volume=None, volume_name=None, secret_volume_mount_path='/secret/gcp-credentials'): """An operator that configures the container to use GCP service account. The user-gcp-sa secret is created as part of the kubeflow deployment that @@ -32,8 +31,13 @@ def use_gcp_secret(secret_name='user-gcp-sa', secret_file_path_in_volume='/user- service account access permission. """ + # permitted values for secret_name = ['admin-gcp-sa', 'user-gcp-sa'] + if secret_file_path_in_volume is None: + secret_file_path_in_volume = '/' + secret_name + '.json' + if volume_name is None: volume_name = 'gcp-credentials-' + secret_name + else: import warnings warnings.warn('The volume_name parameter is deprecated and will be removed in next release. The volume names are now generated automatically.', DeprecationWarning) @@ -107,4 +111,4 @@ def _set_preemptible(task): task.add_node_selector_constraint("cloud.google.com/gke-preemptible", "true") return task - return _set_preemptible \ No newline at end of file + return _set_preemptible diff --git a/sdk/python/tests/compiler/container_builder_test.py b/sdk/python/tests/compiler/container_builder_test.py index d72bb16575c..3bf7509bb49 100644 --- a/sdk/python/tests/compiler/container_builder_test.py +++ b/sdk/python/tests/compiler/container_builder_test.py @@ -21,6 +21,7 @@ from kfp.compiler._component_builder import ContainerBuilder GCS_BASE = 'gs://kfp-testing/' +GCR_IMAGE_TAG = 'gcr.io/kfp-testing/image' @mock.patch('kfp.compiler._gcs_helper.GCSHelper') class TestContainerBuild(unittest.TestCase): @@ -39,7 +40,7 @@ def test_wrap_dir_in_tarball(self, mock_gcshelper): f.write('temporary file two content') # check - builder = ContainerBuilder(gcs_staging=GCS_BASE, namespace='') + builder = ContainerBuilder(gcs_staging=GCS_BASE, gcr_image_tag=GCR_IMAGE_TAG, namespace='') builder._wrap_dir_in_tarball(temp_tarball, test_data_dir) self.assertTrue(os.path.exists(temp_tarball)) with tarfile.open(temp_tarball) as temp_tarball_handle: @@ -57,7 +58,7 @@ def test_generate_kaniko_yaml(self, mock_gcshelper): test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') # check - builder = ContainerBuilder(gcs_staging=GCS_BASE, namespace='default') + builder = ContainerBuilder(gcs_staging=GCS_BASE, gcr_image_tag=GCR_IMAGE_TAG, namespace='default') generated_yaml = builder._generate_kaniko_spec(docker_filename='dockerfile', context='gs://mlpipeline/kaniko_build.tar.gz', target_image='gcr.io/mlpipeline/kaniko_image:latest') with open(os.path.join(test_data_dir, 'kaniko.basic.yaml'), 'r') as f: diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000000..fb86d0d665c --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +# temporary folder used in tests +bin diff --git a/test/README.md b/test/README.md index 8add7f2eb35..8b605edee66 100644 --- a/test/README.md +++ b/test/README.md @@ -72,6 +72,11 @@ Run the following commands from root of kubeflow/pipelines repo. #$PULL_PULL_SHA and $WORKSPACE are env variables set by Prow export PULL_PULL_SHA=pull-sha-placeholder export WORKSPACE=$(pwd) # root of kubeflow/pipelines git repo +export SA_KEY_FILE=PATH/TO/YOUR/GCP/PROJECT/SERVICE/ACCOUNT/KEY +# (optional) uncomment the following to keep reusing the same cluster +# export TEST_CLUSTER=YOUR_PRECONFIGURED_CLUSTER_NAME +# (optional) uncomment the following to disable built image caching +# export DISABLE_IMAGE_CACHING=true ./test/presubmit-tests-with-pipeline-deployment.sh \ --workflow_file e2e_test_gke_v2.yaml \ # You can specify other workflows you want to test too. diff --git a/test/build-images.sh b/test/build-images.sh new file mode 100755 index 00000000000..472c219948e --- /dev/null +++ b/test/build-images.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +IMAGE_BUILDER_ARG="" +if [ "$PROJECT" != "ml-pipeline-test" ]; then + COPIED_IMAGE_BUILDER_IMAGE=${GCR_IMAGE_BASE_DIR}/image-builder + echo "Copy image builder image to ${COPIED_IMAGE_BUILDER_IMAGE}" + yes | gcloud container images add-tag \ + gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4 \ + ${COPIED_IMAGE_BUILDER_IMAGE}:latest + IMAGE_BUILDER_ARG="-p image-builder-image=${COPIED_IMAGE_BUILDER_IMAGE}" +fi + +# Image caching can be turned off by setting $DISABLE_IMAGE_CACHING env flag. +# Note that GCR_IMAGE_BASE_DIR contains commit hash, so whenever there's a code +# change, we won't use caches for sure. +BUILT_IMAGES=$(gcloud container images list --repository=${GCR_IMAGE_BASE_DIR}) +if + test -z "$DISABLE_IMAGE_CACHING" && \ + echo "$BUILT_IMAGES" | grep api-server && \ + echo "$BUILT_IMAGES" | grep frontend && \ + echo "$BUILT_IMAGES" | grep scheduledworkflow && \ + echo "$BUILT_IMAGES" | grep persistenceagent; +then + echo "docker images for api-server, frontend, scheduledworkflow and \ + persistenceagent are already built in ${GCR_IMAGE_BASE_DIR}." +else + echo "submitting argo workflow to build docker images for commit ${PULL_PULL_SHA}..." + # Build Images + ARGO_WORKFLOW=`argo submit ${DIR}/build_image.yaml \ + -p image-build-context-gcs-uri="$remote_code_archive_uri" \ + ${IMAGE_BUILDER_ARG} \ + -p api-image="${GCR_IMAGE_BASE_DIR}/api-server" \ + -p frontend-image="${GCR_IMAGE_BASE_DIR}/frontend" \ + -p scheduledworkflow-image="${GCR_IMAGE_BASE_DIR}/scheduledworkflow" \ + -p persistenceagent-image="${GCR_IMAGE_BASE_DIR}/persistenceagent" \ + -n ${NAMESPACE} \ + --serviceaccount test-runner \ + -o name + ` + echo "build docker images workflow submitted successfully" + source "${DIR}/check-argo-status.sh" + echo "build docker images workflow completed" +fi diff --git a/test/check-argo-status.sh b/test/check-argo-status.sh index 64f99841825..80c4d4803cb 100755 --- a/test/check-argo-status.sh +++ b/test/check-argo-status.sh @@ -25,7 +25,8 @@ echo "check status of argo workflow $ARGO_WORKFLOW...." # probing the argo workflow status until it completed. Timeout after 30 minutes for i in $(seq 1 ${PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT}) do - WORKFLOW_STATUS=`kubectl get workflow $ARGO_WORKFLOW -n ${NAMESPACE} --show-labels` + WORKFLOW_STATUS=`kubectl get workflow $ARGO_WORKFLOW -n ${NAMESPACE} --show-labels 2>&1` \ + || echo kubectl get workflow failed with "$WORKFLOW_STATUS" # Tolerate temporary network failure during kubectl get workflow echo $WORKFLOW_STATUS | grep ${WORKFLOW_COMPLETE_KEYWORD} && s=0 && break || s=$? && printf "Workflow ${ARGO_WORKFLOW} is not finished.\n${WORKFLOW_STATUS}\nSleep for 20 seconds...\n" && sleep 20 done @@ -54,4 +55,4 @@ if [[ $WORKFLOW_STATUS = *"${WORKFLOW_FAILED_KEYWORD}"* ]]; then exit 1 else argo get ${ARGO_WORKFLOW} -n ${NAMESPACE} -fi \ No newline at end of file +fi diff --git a/test/deploy-cluster.sh b/test/deploy-cluster.sh new file mode 100755 index 00000000000..10f6e0e5d2a --- /dev/null +++ b/test/deploy-cluster.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Specify TEST_CLUSTER env variable to use an existing cluster. +TEST_CLUSTER_PREFIX=${WORKFLOW_FILE%.*} +TEST_CLUSTER_DEFAULT=$(echo $TEST_CLUSTER_PREFIX | cut -d _ -f 1)-${PULL_PULL_SHA:0:7}-${RANDOM} +TEST_CLUSTER=${TEST_CLUSTER:-${TEST_CLUSTER_DEFAULT}} +SHOULD_CLEANUP_CLUSTER=false + +function clean_up { + set +e # the following clean up commands shouldn't exit on error + + echo "Status of pods before clean up:" + kubectl get pods --all-namespaces + + echo "Clean up..." + if [ $SHOULD_CLEANUP_CLUSTER == true ]; then + # --async doesn't wait for this operation to complete, so we can get test + # results faster + yes | gcloud container clusters delete ${TEST_CLUSTER} --async + fi +} +trap clean_up EXIT SIGINT SIGTERM + +cd ${DIR} +# test if ${TEST_CLUSTER} exists or not +if gcloud container clusters describe ${TEST_CLUSTER} &>/dev/null; then + echo "Use existing test cluster: ${TEST_CLUSTER}" +else + echo "Creating a new test cluster: ${TEST_CLUSTER}" + SHOULD_CLEANUP_CLUSTER=true + # "storage-rw" is needed to allow VMs to push to gcr.io + # reference: https://cloud.google.com/compute/docs/access/service-accounts#accesscopesiam + SCOPE_ARG="--scopes=storage-rw" + # Machine type and cluster size is the same as kubeflow deployment to + # easily compare performance. We can reduce usage later. + NODE_POOL_CONFIG_ARG="--num-nodes=2 --machine-type=n1-standard-8 \ + --enable-autoscaling --max-nodes=8 --min-nodes=2" + gcloud container clusters create ${TEST_CLUSTER} ${SCOPE_ARG} ${NODE_POOL_CONFIG_ARG} +fi + +gcloud container clusters get-credentials ${TEST_CLUSTER} + +# when we reuse a cluster when debugging, clean up its kfp installation first +# this does nothing with a new cluster +kubectl delete namespace ${NAMESPACE} --wait || echo "No need to delete ${NAMESPACE} namespace. It doesn't exist." +kubectl create namespace ${NAMESPACE} --dry-run -o yaml | kubectl apply -f - + +if [ -z $SA_KEY_FILE ]; then + SA_KEY_FILE=${DIR}/key.json + # The service account key is for default VM service account. + # ref: https://cloud.google.com/compute/docs/access/service-accounts#compute_engine_default_service_account + # It was generated by the following command + # `gcloud iam service-accounts keys create $SA_KEY_FILE --iam-account ${VM_SERVICE_ACCOUNT}` + # Because there's a limit of 10 keys per service account, we are reusing the same key stored in the following bucket. + gsutil cp "gs://ml-pipeline-test-keys/ml-pipeline-test-sa-key.json" $SA_KEY_FILE +fi +kubectl create secret -n ${NAMESPACE} generic user-gcp-sa --from-file=user-gcp-sa.json=$SA_KEY_FILE --dry-run -o yaml | kubectl apply -f - diff --git a/test/deploy-pipeline-lite.sh b/test/deploy-pipeline-lite.sh new file mode 100755 index 00000000000..ae141d680be --- /dev/null +++ b/test/deploy-pipeline-lite.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +if ! which kustomize; then + # Download kustomize cli tool + TOOL_DIR=${DIR}/bin + mkdir -p ${TOOL_DIR} + wget https://github.com/kubernetes-sigs/kustomize/releases/download/v3.1.0/kustomize_3.1.0_linux_amd64 -O ${TOOL_DIR}/kustomize + chmod +x ${TOOL_DIR}/kustomize + PATH=${PATH}:${TOOL_DIR} +fi + +# delete argo first because KFP comes with argo too +kubectl delete namespace argo --wait || echo "No argo installed" + +KFP_MANIFEST_DIR=${DIR}/manifests +pushd ${KFP_MANIFEST_DIR} + +# This is the recommended approach to do this. +# reference: https://github.com/kubernetes-sigs/kustomize/blob/master/docs/eschewedFeatures.md#build-time-side-effects-from-cli-args-or-env-variables +kustomize edit set image gcr.io/ml-pipeline/api-server=${GCR_IMAGE_BASE_DIR}/api-server:latest +kustomize edit set image gcr.io/ml-pipeline/persistenceagent=${GCR_IMAGE_BASE_DIR}/persistenceagent:latest +kustomize edit set image gcr.io/ml-pipeline/scheduledworkflow=${GCR_IMAGE_BASE_DIR}/scheduledworkflow:latest +kustomize edit set image gcr.io/ml-pipeline/frontend=${GCR_IMAGE_BASE_DIR}/frontend:latest +cat kustomization.yaml + +kustomize build . | kubectl apply -f - +# show current info +echo "Status of pods after kubectl apply" +kubectl get pods -n ${NAMESPACE} + +popd diff --git a/test/install-argo.sh b/test/install-argo.sh index 6e2ef523d19..fad8af483b0 100755 --- a/test/install-argo.sh +++ b/test/install-argo.sh @@ -20,20 +20,25 @@ set -ex kubectl config set-context $(kubectl config current-context) --namespace=default echo "Add necessary cluster role bindings" ACCOUNT=$(gcloud info --format='value(config.account)') -kubectl create clusterrolebinding PROW_BINDING --clusterrole=cluster-admin --user=$ACCOUNT -kubectl create clusterrolebinding DEFAULT_BINDING --clusterrole=cluster-admin --serviceaccount=default:default +kubectl create clusterrolebinding PROW_BINDING --clusterrole=cluster-admin --user=$ACCOUNT --dry-run -o yaml | kubectl apply -f - +kubectl create clusterrolebinding DEFAULT_BINDING --clusterrole=cluster-admin --serviceaccount=default:default --dry-run -o yaml | kubectl apply -f - -echo "install argo" ARGO_VERSION=v2.3.0 -mkdir -p ~/bin/ -export PATH=~/bin/:$PATH -curl -sSL -o ~/bin/argo https://github.com/argoproj/argo/releases/download/$ARGO_VERSION/argo-linux-amd64 -chmod +x ~/bin/argo -#kubectl create ns argo -#kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/$ARGO_VERSION/manifests/install.yaml + +# if argo is not installed +if ! which argo; then + echo "install argo" + mkdir -p ~/bin/ + export PATH=~/bin/:$PATH + curl -sSL -o ~/bin/argo https://github.com/argoproj/argo/releases/download/$ARGO_VERSION/argo-linux-amd64 + chmod +x ~/bin/argo +fi + +kubectl create ns argo --dry-run -o yaml | kubectl apply -f - +kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/$ARGO_VERSION/manifests/install.yaml # Some workflows are deployed to the non-default namespace where the GCP credential secret is stored # In this case, the default service account in that namespace doesn't have enough permission echo "add service account for running the test workflow" -kubectl create serviceaccount test-runner -n ${NAMESPACE} -kubectl create clusterrolebinding test-admin-binding --clusterrole=cluster-admin --serviceaccount=${NAMESPACE}:test-runner +kubectl create serviceaccount test-runner -n ${NAMESPACE} --dry-run -o yaml | kubectl apply -f - +kubectl create clusterrolebinding test-admin-binding --clusterrole=cluster-admin --serviceaccount=${NAMESPACE}:test-runner --dry-run -o yaml | kubectl apply -f - diff --git a/test/manifests/kustomization.yaml b/test/manifests/kustomization.yaml new file mode 100644 index 00000000000..d785bf95edf --- /dev/null +++ b/test/manifests/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Actual image overrides will be added in test scripts. +images: [] +resources: +- ../../manifests/kustomize/namespaced-install diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh index 7e75642af51..d1a506e06e6 100755 --- a/test/presubmit-tests-with-pipeline-deployment.sh +++ b/test/presubmit-tests-with-pipeline-deployment.sh @@ -68,45 +68,25 @@ GCR_IMAGE_BASE_DIR=gcr.io/${PROJECT}/${PULL_PULL_SHA} TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${PULL_PULL_SHA}/${TEST_RESULT_FOLDER} DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" +# Configure `time` command output format. +TIMEFORMAT="[test-timing] It took %lR." + echo "presubmit test starts" -source "${DIR}/test-prep.sh" +time source "${DIR}/test-prep.sh" +echo "test env prepared" -# Deploy Kubeflow -source "${DIR}/deploy-kubeflow.sh" +time source "${DIR}/deploy-cluster.sh" +echo "cluster deployed" # Install Argo CLI and test-runner service account -source "${DIR}/install-argo.sh" +time source "${DIR}/install-argo.sh" +echo "argo installed" -IMAGE_BUILDER_ARG="" -# When project is not ml-pipeline-test, VMs need permission to fetch some images in gcr.io/ml-pipeline-test. -if [ "$PROJECT" != "ml-pipeline-test" ]; then - COPIED_IMAGE_BUILDER_IMAGE=${GCR_IMAGE_BASE_DIR}/image-builder - echo "Copy image builder image to ${COPIED_IMAGE_BUILDER_IMAGE}" - yes | gcloud container images add-tag \ - gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4 \ - ${COPIED_IMAGE_BUILDER_IMAGE}:latest - IMAGE_BUILDER_ARG="-p image-builder-image=${COPIED_IMAGE_BUILDER_IMAGE}" -fi +time source "${DIR}/build-images.sh" +echo "KFP images built" -# Build Images -echo "submitting argo workflow to build docker images for commit ${PULL_PULL_SHA}..." -ARGO_WORKFLOW=`argo submit ${DIR}/build_image.yaml \ --p image-build-context-gcs-uri="$remote_code_archive_uri" \ -${IMAGE_BUILDER_ARG} \ --p api-image="${GCR_IMAGE_BASE_DIR}/api-server" \ --p frontend-image="${GCR_IMAGE_BASE_DIR}/frontend" \ --p scheduledworkflow-image="${GCR_IMAGE_BASE_DIR}/scheduledworkflow" \ --p persistenceagent-image="${GCR_IMAGE_BASE_DIR}/persistenceagent" \ --n ${NAMESPACE} \ ---serviceaccount test-runner \ --o name -` -echo "build docker images workflow submitted successfully" -source "${DIR}/check-argo-status.sh" -echo "build docker images workflow completed" - -# Deploy the pipeline -source ${DIR}/deploy-pipeline.sh --gcr_image_base_dir ${GCR_IMAGE_BASE_DIR} +time source "${DIR}/deploy-pipeline-lite.sh" +echo "KFP lite deployed" echo "submitting argo workflow to run tests for commit ${PULL_PULL_SHA}..." ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ @@ -119,7 +99,6 @@ ${IMAGE_BUILDER_ARG} \ --serviceaccount test-runner \ -o name ` - echo "test workflow submitted successfully" -source "${DIR}/check-argo-status.sh" +time source "${DIR}/check-argo-status.sh" echo "test workflow completed" diff --git a/test/tools/project-cleaner/main.go b/test/tools/project-cleaner/main.go new file mode 100644 index 00000000000..d59d44e4720 --- /dev/null +++ b/test/tools/project-cleaner/main.go @@ -0,0 +1,164 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "time" + + "google.golang.org/api/container/v1" + "gopkg.in/yaml.v2" +) + +// project_cleanup tool provides support to cleanup different resources insides a GCP project. +// Types of resources to be cleaned up in a single execution of the tool is specified as YAML file +// (run-time parameter flag) to the tool. Each GCP resource is associated with a handler function +// that performs cleanup. Adding a new resource for cleanup involves - adding a helper function to +// ProjectCleaner type and updating the switch condition in the CleanupProject method. + +var ( + resourceSpec = flag.String("resource_spec", "", "resource spec yaml file") +) + +// ResourceSpec is the top-level spec specifying GCP resources to be cleaned up in a project. +type ResourceSpec struct { + // ProjectId - GCP Project Id + ProjectId string `yaml:"project-id"` + // Resources - Spec for each resource to be cleaned up by the tool. + Resources []GCPResource `yaml:"resources"` +} + +// GCPResource specifies an individual GCP resource to be cleaned up. +type GCPResource struct { + // Resource - name of the resource is used as key to the handler function. + Resource string `yaml:"resource"` + // Zones - Multiple zones support. + Zones []string `yaml:"zones"` + // NamePrefixes - Only support for pre-fixed based filtering as go-client doesn't support regex + // based filtering. + NamePrefixes []string `yaml:"name-prefixes"` + // TimeLapseInHours - Number of hours before the resource needs to be cleaned up. + TimeLapseInHours int `yaml:"time-lapse-hours"` +} + +// ProjectCleaner - top level type providing handlers for different GCP resources. +type ProjectCleaner struct { + ProjectId string + Resources []GCPResource +} + +// GKEClusterHandler provides support to cleanup GCP resources within the project. +func (p *ProjectCleaner) GKEClusterHandler(resource GCPResource) { + // See https://cloud.google.com/docs/authentication/. + // Use GOOGLE_APPLICATION_CREDENTIALS environment variable to specify a service account key file + // to authenticate to the API. + ctx := context.Background() + svc, err := container.NewService(ctx) + if err != nil { + log.Fatalf("Could not initialize gke client: %v", err) + } + + elapsedTime := time.Now().Add(time.Hour * (-1) * time.Duration(resource.TimeLapseInHours)) + for _, zone := range resource.Zones { + log.Printf("Listing gke clusters in zone: %v", zone) + listResponse, err := svc.Projects.Zones.Clusters.List(p.ProjectId, zone).Do() + if err != nil { + log.Printf("Failed listing gke clusters in zone: %v", zone) + } + + for _, cluster := range listResponse.Clusters { + createdTime, err := time.Parse(time.RFC3339, cluster.CreateTime) + if err != nil { + log.Printf("Unable to parse created time for cluster: %s. Ignoring cluster", + cluster.Name) + continue + } + + if p.checkForPrefix(cluster.Name, resource.NamePrefixes) && createdTime.Before(elapsedTime) { + log.Printf("Found cluster: %s for deletion", cluster.Name) + if op, err := svc.Projects.Zones.Clusters.Delete(p.ProjectId, zone, cluster.Name).Do(); + err != nil { + log.Printf("Encountered error calling delete on cluster: %s Error: %v", + cluster.Name, err) + } else { + log.Printf("Kicked off cluster delete : %v", op.TargetLink) + } + } + } + } +} + +// checkForPrefix - helper function to check if testStr string has any of the prefix specified in +// prefixes +func (p *ProjectCleaner) checkForPrefix(testStr string, prefixes []string) bool { + log.Printf("Performing prefix check for String: %s for Prefixes: %v", testStr, prefixes) + for _, prefix := range prefixes { + if strings.HasPrefix(testStr, prefix) { + return true + } + } + return false +} + +// CleanupProject iterates over each resource specified in the spec and calls the corresponding +// cleanup handler method. +func (p *ProjectCleaner) CleanupProject() { + for _, resource := range p.Resources { + switch resource.Resource { + case "gke-cluster": + p.GKEClusterHandler(resource) + default: + log.Printf("Un-identified resource: %v found in spec. Ignoring", resource.Resource) + } + } +} + +func main() { + flag.Parse() + log.SetOutput(os.Stdout) + + if *resourceSpec == "" { + fmt.Fprintln(os.Stderr, "missing resource_spec parameter") + flag.Usage() + os.Exit(2) + } + + if _, err := os.Stat(*resourceSpec); err != nil { + log.Fatalf("missing resource spec file") + } + + var r ResourceSpec + yamlFile, err := ioutil.ReadFile(*resourceSpec) + if err != nil { + log.Fatalf("Error reading yaml file") + } + + err = yaml.Unmarshal(yamlFile, &r) + if err != nil { + log.Fatalf("Unmarshal: %v", err) + } + + p := ProjectCleaner{ + ProjectId: r.ProjectId, + Resources: r.Resources, + } + p.CleanupProject() +} diff --git a/test/tools/project-cleaner/project_cleaner.sh b/test/tools/project-cleaner/project_cleaner.sh new file mode 100755 index 00000000000..c6cee72ebb5 --- /dev/null +++ b/test/tools/project-cleaner/project_cleaner.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +SCRIPT_DIR=$(dirname "${BASH_SOURCE}") +pushd "${SCRIPT_DIR}" + +echo "Building project cleaner tool" +go build . + +echo "Executing project cleaner" +./project-cleaner --resource_spec resource_spec.yaml +popd diff --git a/test/tools/project-cleaner/resource_spec.yaml b/test/tools/project-cleaner/resource_spec.yaml new file mode 100644 index 00000000000..726d7c5abbf --- /dev/null +++ b/test/tools/project-cleaner/resource_spec.yaml @@ -0,0 +1,23 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project-id: ml-pipeline-test +resources: + - resource: gke-cluster + zones: + - us-east1-b + name-prefixes: + - e2e- + - sample- + time-lapse-hours: 24