api: add go proto generation script (#8155)

Adds a script to create a go module from the generated protobufs as part of #8151.
The module appears to build with the following module declaration:

module github.com/envoyproxy/data-plane-api/api

go 1.12

require (
        github.com/census-instrumentation/opencensus-proto v0.2.1
        github.com/envoyproxy/protoc-gen-validate v0.1.0
        github.com/gogo/protobuf v1.3.0
        github.com/golang/protobuf v1.3.2
        github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
        google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55
        google.golang.org/grpc v1.23.0
)
Add CI automation to trigger the script after the merge to master in envoyproxy.

Risk Level: low
Testing: local build
Docs Changes: none
Release Notes: none

Fixes #8151 

Signed-off-by: Kuat Yessenov <[email protected]>

.circleci/config.yml

@@ -64,6 +64,7 @@ jobs:
            fingerprints:
              - "fb:f3:fe:be:1c:b2:ec:b6:25:f9:7b:a6:87:54:02:8c"
        - run: ci/api_mirror.sh
+       - run: ci/go_mirror.sh
        - store_artifacts:
            path: /build/envoy/generated
            destination: /

api/bazel/api_build_system.bzl

@@ -9,7 +9,7 @@ _CC_SUFFIX = "_cc"
 _CC_GRPC_SUFFIX = "_cc_grpc"
 _CC_EXPORT_SUFFIX = "_export_cc"
 _GO_PROTO_SUFFIX = "_go_proto"
-_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/"
+_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/go-control-plane/"
 
 _COMMON_PROTO_DEPS = [
     "@com_google_protobuf//:any_proto",

api/bazel/repositories.bzl

@@ -54,7 +54,7 @@ api_proto_library(
 
 go_proto_library(
     name = "client_model_go_proto",
-    importpath = "client_model",
+    importpath = "github.com/prometheus/client_model/go",
     proto = ":client_model",
     visibility = ["//visibility:public"],
 )

api/test/build/go_build_test.go

@@ -3,14 +3,14 @@ package go_build_test
 import (
 	"testing"
 
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2/auth"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/config/bootstrap/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/accesslog/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/discovery/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/metrics/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/ratelimit/v2"
-	_ "github.com/envoyproxy/data-plane-api/api/envoy/service/trace/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
+	_ "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/metrics/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/ratelimit/v2"
+	_ "github.com/envoyproxy/go-control-plane/envoy/service/trace/v2"
 )
 
 func TestNoop(t *testing.T) {

ci/go_mirror.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
+then
+  tools/api/generate_go_protobuf.py
+fi

tools/api/generate_go_protobuf.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+from subprocess import check_output
+from subprocess import check_call
+import glob
+import os
+import shutil
+import sys
+import re
+
+TARGETS = '@envoy_api//...'
+IMPORT_BASE = 'github.com/envoyproxy/go-control-plane'
+OUTPUT_BASE = 'build_go'
+REPO_BASE = 'go-control-plane'
+BRANCH = 'master'
+MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ '
+USER_NAME = 'go-control-plane(CircleCI)'
+USER_EMAIL = '[email protected]'
+
+
+def generateProtobufs(output):
+  bazel_bin = check_output(['bazel', 'info', 'bazel-bin']).decode().strip()
+  go_protos = check_output([
+      'bazel',
+      'query',
+      'kind("go_proto_library", %s)' % TARGETS,
+  ]).split()
+
+  # Each rule has the form @envoy_api//foo/bar:baz_go_proto.
+  # First build all the rules to ensure we have the output files.
+  check_call(['bazel', 'build', '-c', 'fastbuild'] + go_protos)
+
+  for rule in go_protos:
+    # Example rule:
+    # @envoy_api//envoy/config/bootstrap/v2:pkg_go_proto
+    #
+    # Example generated directory:
+    # bazel-bin/external/envoy_api/envoy/config/bootstrap/v2/linux_amd64_stripped/pkg_go_proto%/github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2/
+    #
+    # Example output directory:
+    # go_out/envoy/config/bootstrap/v2
+    rule_dir, proto = rule.decode()[len('@envoy_api//'):].rsplit(':', 1)
+    input_dir = os.path.join(bazel_bin, 'external', 'envoy_api', rule_dir, 'linux_amd64_stripped',
+                             proto + '%', IMPORT_BASE, rule_dir)
+    input_files = glob.glob(os.path.join(input_dir, '*.go'))
+    output_dir = os.path.join(output, rule_dir)
+
+    # Ensure the output directory exists
+    os.makedirs(output_dir, 0o755, exist_ok=True)
+    for generated_file in input_files:
+      shutil.copy(generated_file, output_dir)
+  print('Go artifacts placed into: ' + output)
+
+
+def git(repo, *args):
+  cmd = ['git']
+  if repo:
+    cmd = cmd + ['-C', repo]
+  for arg in args:
+    cmd = cmd + [arg]
+  return check_output(cmd).decode()
+
+
+def cloneGoProtobufs(repo):
+  # Create a local clone of go-control-plane
+  git(None, 'clone', '[email protected]:envoyproxy/go-control-plane', repo)
+  git(repo, 'fetch')
+  git(repo, 'checkout', '-B', BRANCH, 'origin/master')
+
+
+def findLastSyncSHA(repo):
+  # Determine last envoyproxy/envoy SHA in envoyproxy/go-control-plane
+  last_commit = git(repo, 'log', '--grep=' + MIRROR_MSG, '-n', '1', '--format=%B').strip()
+  # Initial SHA from which the APIs start syncing. Prior to that it was done manually.
+  if last_commit == "":
+    return 'e7f0b7176efdc65f96eb1697b829d1e6187f4502'
+  m = re.search(MIRROR_MSG + '(\w+)', last_commit)
+  return m.group(1)
+
+
+def updatedSinceSHA(repo, last_sha):
+  # Determine if there are changes to API since last SHA
+  return git(None, 'rev-list', '%s..HEAD' % last_sha, 'api/envoy').split()
+
+
+def syncGoProtobufs(output, repo):
+  # Sync generated content against repo and return true if there is a commit necessary
+  dst = os.path.join(repo, 'envoy')
+  # Remove subtree at envoy in repo
+  git(repo, 'rm', '-r', 'envoy')
+  # Copy subtree at envoy from output to repo
+  shutil.copytree(os.path.join(output, 'envoy'), dst)
+
+
+def publishGoProtobufs(repo, sha):
+  # Publish generated files with the last SHA changes to API
+  git(repo, 'config', 'user.name', USER_NAME)
+  git(repo, 'config', 'user.email', USER_EMAIL)
+  git(repo, 'add', 'envoy')
+  git(repo, 'commit', '-s', '-m', MIRROR_MSG + sha)
+  git(repo, 'push', 'origin', BRANCH)
+
+
+if __name__ == "__main__":
+  workspace = check_output(['bazel', 'info', 'workspace']).decode().strip()
+  output = os.path.join(workspace, OUTPUT_BASE)
+  generateProtobufs(output)
+  repo = os.path.join(workspace, REPO_BASE)
+  cloneGoProtobufs(repo)
+  last_sha = findLastSyncSHA(repo)
+  changes = updatedSinceSHA(repo, last_sha)
+  if changes:
+    print('Changes detected: %s' % changes)
+    syncGoProtobufs(output, repo)
+    publishGoProtobufs(repo, changes[0])