From 32e6018f10f9d6de58fd6c3d6844a68a5be7723b Mon Sep 17 00:00:00 2001 From: Kuat Date: Wed, 18 Sep 2019 20:51:02 -0700 Subject: [PATCH] api: add go proto generation script (#8155) Adds a script to create a go module from the generated protobufs as part of #8151. The module appears to build with the following module declaration: module github.com/envoyproxy/data-plane-api/api go 1.12 require ( github.com/census-instrumentation/opencensus-proto v0.2.1 github.com/envoyproxy/protoc-gen-validate v0.1.0 github.com/gogo/protobuf v1.3.0 github.com/golang/protobuf v1.3.2 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 google.golang.org/grpc v1.23.0 ) Add CI automation to trigger the script after the merge to master in envoyproxy. Risk Level: low Testing: local build Docs Changes: none Release Notes: none Fixes #8151 Signed-off-by: Kuat Yessenov --- .circleci/config.yml | 1 + api/bazel/api_build_system.bzl | 2 +- api/bazel/repositories.bzl | 2 +- api/test/build/go_build_test.go | 16 ++--- ci/go_mirror.sh | 8 +++ tools/api/generate_go_protobuf.py | 115 ++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 10 deletions(-) create mode 100755 ci/go_mirror.sh create mode 100755 tools/api/generate_go_protobuf.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 14cf64272abf..e0c239560e31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,6 +64,7 @@ jobs: fingerprints: - "fb:f3:fe:be:1c:b2:ec:b6:25:f9:7b:a6:87:54:02:8c" - run: ci/api_mirror.sh + - run: ci/go_mirror.sh - store_artifacts: path: /build/envoy/generated destination: / diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index a37f1d9d14c4..25bca4673541 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -9,7 +9,7 @@ _CC_SUFFIX = "_cc" _CC_GRPC_SUFFIX = "_cc_grpc" _CC_EXPORT_SUFFIX = "_export_cc" _GO_PROTO_SUFFIX = "_go_proto" -_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/" +_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/go-control-plane/" _COMMON_PROTO_DEPS = [ "@com_google_protobuf//:any_proto", diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index 1362c5671acb..27d0219ae8af 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -54,7 +54,7 @@ api_proto_library( go_proto_library( name = "client_model_go_proto", - importpath = "client_model", + importpath = "github.com/prometheus/client_model/go", proto = ":client_model", visibility = ["//visibility:public"], ) diff --git a/api/test/build/go_build_test.go b/api/test/build/go_build_test.go index c5c15becff35..638ef478b8c7 100644 --- a/api/test/build/go_build_test.go +++ b/api/test/build/go_build_test.go @@ -3,14 +3,14 @@ package go_build_test import ( "testing" - _ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2" - _ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2/auth" - _ "github.com/envoyproxy/data-plane-api/api/envoy/config/bootstrap/v2" - _ "github.com/envoyproxy/data-plane-api/api/envoy/service/accesslog/v2" - _ "github.com/envoyproxy/data-plane-api/api/envoy/service/discovery/v2" - _ "github.com/envoyproxy/data-plane-api/api/envoy/service/metrics/v2" - _ "github.com/envoyproxy/data-plane-api/api/envoy/service/ratelimit/v2" - _ "github.com/envoyproxy/data-plane-api/api/envoy/service/trace/v2" + _ "github.com/envoyproxy/go-control-plane/envoy/api/v2" + _ "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth" + _ "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2" + _ "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v2" + _ "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" + _ "github.com/envoyproxy/go-control-plane/envoy/service/metrics/v2" + _ "github.com/envoyproxy/go-control-plane/envoy/service/ratelimit/v2" + _ "github.com/envoyproxy/go-control-plane/envoy/service/trace/v2" ) func TestNoop(t *testing.T) { diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh new file mode 100755 index 000000000000..80be4cc0b532 --- /dev/null +++ b/ci/go_mirror.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] +then + tools/api/generate_go_protobuf.py +fi diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py new file mode 100755 index 000000000000..3c49dd19d895 --- /dev/null +++ b/tools/api/generate_go_protobuf.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +from subprocess import check_output +from subprocess import check_call +import glob +import os +import shutil +import sys +import re + +TARGETS = '@envoy_api//...' +IMPORT_BASE = 'github.com/envoyproxy/go-control-plane' +OUTPUT_BASE = 'build_go' +REPO_BASE = 'go-control-plane' +BRANCH = 'master' +MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ ' +USER_NAME = 'go-control-plane(CircleCI)' +USER_EMAIL = 'go-control-plane@users.noreply.github.com' + + +def generateProtobufs(output): + bazel_bin = check_output(['bazel', 'info', 'bazel-bin']).decode().strip() + go_protos = check_output([ + 'bazel', + 'query', + 'kind("go_proto_library", %s)' % TARGETS, + ]).split() + + # Each rule has the form @envoy_api//foo/bar:baz_go_proto. + # First build all the rules to ensure we have the output files. + check_call(['bazel', 'build', '-c', 'fastbuild'] + go_protos) + + for rule in go_protos: + # Example rule: + # @envoy_api//envoy/config/bootstrap/v2:pkg_go_proto + # + # Example generated directory: + # bazel-bin/external/envoy_api/envoy/config/bootstrap/v2/linux_amd64_stripped/pkg_go_proto%/github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2/ + # + # Example output directory: + # go_out/envoy/config/bootstrap/v2 + rule_dir, proto = rule.decode()[len('@envoy_api//'):].rsplit(':', 1) + input_dir = os.path.join(bazel_bin, 'external', 'envoy_api', rule_dir, 'linux_amd64_stripped', + proto + '%', IMPORT_BASE, rule_dir) + input_files = glob.glob(os.path.join(input_dir, '*.go')) + output_dir = os.path.join(output, rule_dir) + + # Ensure the output directory exists + os.makedirs(output_dir, 0o755, exist_ok=True) + for generated_file in input_files: + shutil.copy(generated_file, output_dir) + print('Go artifacts placed into: ' + output) + + +def git(repo, *args): + cmd = ['git'] + if repo: + cmd = cmd + ['-C', repo] + for arg in args: + cmd = cmd + [arg] + return check_output(cmd).decode() + + +def cloneGoProtobufs(repo): + # Create a local clone of go-control-plane + git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo) + git(repo, 'fetch') + git(repo, 'checkout', '-B', BRANCH, 'origin/master') + + +def findLastSyncSHA(repo): + # Determine last envoyproxy/envoy SHA in envoyproxy/go-control-plane + last_commit = git(repo, 'log', '--grep=' + MIRROR_MSG, '-n', '1', '--format=%B').strip() + # Initial SHA from which the APIs start syncing. Prior to that it was done manually. + if last_commit == "": + return 'e7f0b7176efdc65f96eb1697b829d1e6187f4502' + m = re.search(MIRROR_MSG + '(\w+)', last_commit) + return m.group(1) + + +def updatedSinceSHA(repo, last_sha): + # Determine if there are changes to API since last SHA + return git(None, 'rev-list', '%s..HEAD' % last_sha, 'api/envoy').split() + + +def syncGoProtobufs(output, repo): + # Sync generated content against repo and return true if there is a commit necessary + dst = os.path.join(repo, 'envoy') + # Remove subtree at envoy in repo + git(repo, 'rm', '-r', 'envoy') + # Copy subtree at envoy from output to repo + shutil.copytree(os.path.join(output, 'envoy'), dst) + + +def publishGoProtobufs(repo, sha): + # Publish generated files with the last SHA changes to API + git(repo, 'config', 'user.name', USER_NAME) + git(repo, 'config', 'user.email', USER_EMAIL) + git(repo, 'add', 'envoy') + git(repo, 'commit', '-s', '-m', MIRROR_MSG + sha) + git(repo, 'push', 'origin', BRANCH) + + +if __name__ == "__main__": + workspace = check_output(['bazel', 'info', 'workspace']).decode().strip() + output = os.path.join(workspace, OUTPUT_BASE) + generateProtobufs(output) + repo = os.path.join(workspace, REPO_BASE) + cloneGoProtobufs(repo) + last_sha = findLastSyncSHA(repo) + changes = updatedSinceSHA(repo, last_sha) + if changes: + print('Changes detected: %s' % changes) + syncGoProtobufs(output, repo) + publishGoProtobufs(repo, changes[0])