diff --git a/ci-operator/populate-owners.sh b/ci-operator/populate-owners.sh index fe33b9af6ae8..ec23ccaff05a 100755 --- a/ci-operator/populate-owners.sh +++ b/ci-operator/populate-owners.sh @@ -1,42 +1,6 @@ -#!/bin/bash +#!/bin/sh -# This script populates ConfigMaps using the configuration -# files in these directories. To be used to bootstrap the -# build cluster after a redeploy. +# This script runs /tools/populate-owners -set -o errexit -set -o nounset -set -o pipefail - - -temp_workdir=$( mktemp -d ) -trap "rm -rf ${temp_workdir}" EXIT - -function populate_owners() { - local org="$1" - local repo="$2" - local target_dir="${temp_workdir}/${org}/${repo}" - mkdir -p "${target_dir}" - git clone --depth 1 --single-branch "git@github.com:${org}/${repo}.git" "${target_dir}" - if [[ -f "${target_dir}/OWNERS" ]]; then - cp "${target_dir}/OWNERS" "${jobs}/${org}/${repo}" - if [[ -d "${config}/${org}/${repo}" ]]; then - cp "${target_dir}/OWNERS" "${config}/${org}/${repo}" - fi - fi -} - -jobs="$( dirname "${BASH_SOURCE[0]}" )/jobs" -config="$( dirname "${BASH_SOURCE[0]}" )/config" - -for org_dir in $( find "${jobs}" -mindepth 1 -maxdepth 1 -type d ); do - org="$( basename "${org_dir}" )" - for repo_dir in $( find "${jobs}/${org}" -mindepth 1 -maxdepth 1 -type d ); do - repo="$( basename "${repo_dir}" )" - populate_owners "${org}" "${repo}" & - done -done - -for job in $( jobs -p ); do - wait "${job}" -done \ No newline at end of file +REPO_ROOT="$(git rev-parse --show-toplevel)" && +exec go run "${REPO_ROOT}/tools/populate-owners/main.go" diff --git a/tools/populate-owners/OWNERS b/tools/populate-owners/OWNERS new file mode 100644 index 000000000000..fbd52a8f5b04 --- /dev/null +++ b/tools/populate-owners/OWNERS @@ -0,0 +1,3 @@ +approvers: +- stevekuznetsov +- wking diff --git a/tools/populate-owners/README.md b/tools/populate-owners/README.md new file mode 100644 index 000000000000..d3128c2ab161 --- /dev/null +++ b/tools/populate-owners/README.md @@ -0,0 +1,30 @@ +# Populating `OWNERS` and `OWNERS_ALIASES` + +This utility pulls `OWNERS` and `OWNERS_ALIASES` from upstream OpenShift repositories. +Usage: + +```console +$ go run main.go +``` + +Or, equivalently, execute [`populate-owners.sh`](../../ci-operator/populate-owners.sh) from anywhere in this repository. + +Upstream repositories are calculated from `ci-operator/jobs/{organization}/{repository}`. +For example, the presence of [`ci-operator/jobs/openshift/origin`](../../ci-operator/jobs/openshift/origin) inserts [openshift/origin][] as an upstream repository. + +The `HEAD` branch for each upstream repository is pulled to extract its `OWNERS` and `OWNERS_ALIASES`. +If `OWNERS` is missing, the utility will ignore `OWNERS_ALIASES`, even if it is present upstream. + +Once all the upstream content has been fetched, the utility namespaces any colliding upstream aliases. +Collisions only occur if multiple upstreams define the same alias with different member sets. +When that happens, the utility replaces the upstream alias with a `{organization}-{repository}-{upstream-alias}`. +For example, if [openshift/origin][] and [openshift/installer][] both defined an alias for `security` with different member sets, the utility would rename them to `openshift-origin-security` and `openshift-installer-security` respectively. + +After namespacing aliases, the utility writes `OWNERS_ALIASES` to the root of this repository. +If no upstreams define aliases, then the utility removes `OWNER_ALIASES` from the root of this repository. + +The utility also iterates through the `ci-operator/jobs/{organization}/{repository}` directories, writing `OWNERS` to reflect the upstream configuration. +If the upstream did not have an `OWNERS` file, the utility removes the associated `ci-operator/jobs/{organization}/{repository}/OWNERS`. + +[openshift/origin]: https://github.com/openshift/origin +[openshift/installer]: https://github.com/openshift/installer diff --git a/tools/populate-owners/main.go b/tools/populate-owners/main.go new file mode 100644 index 000000000000..b3adc39f4921 --- /dev/null +++ b/tools/populate-owners/main.go @@ -0,0 +1,332 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + + "gopkg.in/yaml.v2" +) + +const ( + doNotEdit = "# DO NOT EDIT; this file is auto-generated using tools/populate-owners.\n" + ownersComment = "# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md\n" + ownersAliasesComment = "# See the OWNERS_ALIASES docs: https://git.k8s.io/community/contributors/guide/owners.md#owners_aliases\n" +) + +// owners is copied from k8s.io/test-infra/prow/repoowners's Config +type owners struct { + Approvers []string `json:"approvers,omitempty" yaml:"approvers,omitempty"` + Reviewers []string `json:"reviewers,omitempty" yaml:"reviewers,omitempty"` + RequiredReviewers []string `json:"required_reviewers,omitempty" yaml:"required_reviewers,omitempty"` + Labels []string `json:"labels,omitempty" yaml:"labels,omitempty"` +} + +type aliases struct { + Aliases map[string][]string `json:"aliases,omitempty" yaml:"aliases,omitempty"` +} + +type orgRepo struct { + Directories []string `json:"directories,omitempty" yaml:"directories,omitempty"` + Organization string `json:"organization,omitempty" yaml:"organization,omitempty"` + Repository string `json:"repository,omitempty" yaml:"repository,omitempty"` + Owners *owners `json:"owners,omitempty" yaml:"owners,omitempty"` + Aliases *aliases `json:"aliases,omitempty" yaml:"aliases,omitempty"` + Commit string `json:"commit,omitempty" yaml:"commit,omitempty"` +} + +func getRepoRoot(directory string) (root string, err error) { + initialDir, err := filepath.Abs(directory) + if err != nil { + return "", err + } + + path := initialDir + for { + info, err := os.Stat(filepath.Join(path, ".git")) + if err == nil { + if info.IsDir() { + break + } + } else if !os.IsNotExist(err) { + return "", err + } + + parent := filepath.Dir(path) + if parent == path { + return "", fmt.Errorf("no .git found under %q", initialDir) + } + + path = parent + } + + return path, nil +} + +func orgRepos(dir string) (orgRepos []*orgRepo, err error) { + matches, err := filepath.Glob(filepath.Join(dir, "*", "*")) + if err != nil { + return nil, err + } + sort.Strings(matches) + + orgRepos = make([]*orgRepo, len(matches)) + for i, path := range matches { + relpath, err := filepath.Rel(dir, path) + if err != nil { + return nil, err + } + org, repo := filepath.Split(relpath) + org = strings.TrimSuffix(org, string(filepath.Separator)) + orgRepos[i] = &orgRepo{ + Directories: []string{path}, + Organization: org, + Repository: repo, + } + } + + return orgRepos, err +} + +func (orgRepo *orgRepo) getOwners() (err error) { + dir, err := ioutil.TempDir("", "populate-owners-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + gitURL := fmt.Sprintf("ssh://git@github.com/%s/%s.git", orgRepo.Organization, orgRepo.Repository) + cmd := exec.Command("git", "clone", "--depth=1", "--single-branch", gitURL, dir) + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + return err + } + + return orgRepo.extractOwners(dir) +} + +func (orgRepo *orgRepo) extractOwners(repoRoot string) (err error) { + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Stderr = os.Stderr + cmd.Dir = repoRoot + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + err = cmd.Start() + if err != nil { + return err + } + hash, err := ioutil.ReadAll(stdout) + if err != nil { + return err + } + err = cmd.Wait() + if err != nil { + return err + } + orgRepo.Commit = strings.TrimSuffix(string(hash), "\n") + + data, err := ioutil.ReadFile(filepath.Join(repoRoot, "OWNERS")) + if err != nil { + return err + } + + err = yaml.Unmarshal(data, &orgRepo.Owners) + if err != nil { + return err + } + + data, err = ioutil.ReadFile(filepath.Join(repoRoot, "OWNERS_ALIASES")) + if err != nil { + return err + } + + err = yaml.Unmarshal(data, &orgRepo.Aliases) + if err != nil { + return err + } + + return nil +} + +// namespaceAliases collects a set of aliases including all upstream +// aliases. If multiple upstreams define the same alias with different +// member sets, namespaceAliases renames the colliding aliases in both +// the input 'orgRepos' and the output 'collected' to use +// unique-to-each-upstream alias names. +func namespaceAliases(orgRepos []*orgRepo) (collected *aliases, err error) { + consumerMap := map[string][]*orgRepo{} + for _, orgRepo := range orgRepos { + if orgRepo.Aliases == nil { + continue + } + + for alias := range orgRepo.Aliases.Aliases { + consumerMap[alias] = append(consumerMap[alias], orgRepo) + } + } + + if len(consumerMap) == 0 { + return nil, nil + } + + collected = &aliases{ + Aliases: map[string][]string{}, + } + + for alias, consumers := range consumerMap { + namespace := false + members := consumers[0].Aliases.Aliases[alias] + for _, consumer := range consumers[1:] { + otherMembers := consumer.Aliases.Aliases[alias] + if !reflect.DeepEqual(members, otherMembers) { + namespace = true + break + } + } + + for i, consumer := range consumers { + newAlias := alias + if namespace { + newAlias = fmt.Sprintf("%s-%s-%s", consumer.Organization, consumer.Repository, alias) + consumer.Aliases.Aliases[newAlias] = consumer.Aliases.Aliases[alias] + delete(consumer.Aliases.Aliases, alias) + } + fmt.Fprintf( + os.Stderr, + "injecting alias %q from https://github.com/%s/%s/blob/%s/OWNERS_ALIASES\n", + alias, + consumer.Organization, + consumer.Repository, + consumer.Commit, + ) + if i == 0 || namespace { + _, ok := collected.Aliases[newAlias] + if ok { + return nil, fmt.Errorf("namespaced alias collision: %q", newAlias) + } + collected.Aliases[newAlias] = consumer.Aliases.Aliases[newAlias] + } + } + } + + return collected, nil +} + +func writeYAML(path string, data interface{}, prefix []string) (err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return err + } + defer file.Close() + + for _, line := range prefix { + _, err := file.Write([]byte(line)) + if err != nil { + return err + } + } + + encoder := yaml.NewEncoder(file) + return encoder.Encode(data) +} + +func (orgRepo *orgRepo) writeOwners() (err error) { + for _, directory := range orgRepo.Directories { + path := filepath.Join(directory, "OWNERS") + if orgRepo.Owners == nil { + err := os.Remove(path) + if err != nil && !os.IsNotExist(err) { + return err + } + continue + } + + err = writeYAML(path, orgRepo.Owners, []string{ + doNotEdit, + fmt.Sprintf( + "# from https://github.com/%s/%s/blob/%s/OWNERS\n", + orgRepo.Organization, + orgRepo.Repository, + orgRepo.Commit, + ), + ownersComment, + "\n", + }) + if err != nil { + return err + } + } + + return nil +} + +func writeOwnerAliases(repoRoot string, aliases *aliases) (err error) { + path := filepath.Join(repoRoot, "OWNERS_ALIASES") + if aliases == nil || len(aliases.Aliases) == 0 { + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) { + return err + } + return nil + } + + return writeYAML(path, aliases, []string{ + doNotEdit, + ownersAliasesComment, + "\n", + }) +} + +func pullOwners(directory string) (err error) { + repoRoot, err := getRepoRoot(directory) + if err != nil { + return err + } + + orgRepos, err := orgRepos(filepath.Join(repoRoot, "ci-operator", "jobs")) + if err != nil { + return err + } + + for _, orgRepo := range orgRepos { + err := orgRepo.getOwners() + if err != nil && !os.IsNotExist(err) { + return err + } + } + + aliases, err := namespaceAliases(orgRepos) + if err != nil { + return err + } + + err = writeOwnerAliases(repoRoot, aliases) + if err != nil { + return err + } + + for _, orgRepo := range orgRepos { + err = orgRepo.writeOwners() + if err != nil { + return err + } + } + + return nil +} + +func main() { + err := pullOwners(".") + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } +} diff --git a/tools/populate-owners/main_test.go b/tools/populate-owners/main_test.go new file mode 100644 index 000000000000..a084aef8e0e5 --- /dev/null +++ b/tools/populate-owners/main_test.go @@ -0,0 +1,454 @@ +package main + +import ( + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "testing" +) + +func assertEqual(t *testing.T, actual, expected interface{}) { + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("unexpected result: %+v != %+v", actual, expected) + } +} + +func TestGetRepoRoot(t *testing.T) { + dir, err := ioutil.TempDir("", "populate-owners-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + root := filepath.Join(dir, "root") + deep := filepath.Join(root, "a", "b", "c") + git := filepath.Join(root, ".git") + err = os.MkdirAll(deep, 0777) + if err != nil { + t.Fatal(err) + } + err = os.Mkdir(git, 0777) + if err != nil { + t.Fatal(err) + } + + t.Run("from inside the repository", func(t *testing.T) { + found, err := getRepoRoot(deep) + if err != nil { + t.Fatal(err) + } + if found != root { + t.Fatalf("unexpected root: %q != %q", found, root) + } + }) + + t.Run("from outside the repository", func(t *testing.T) { + _, err := getRepoRoot(dir) + if err == nil { + t.Fatal(err) + } + }) +} + +func TestOrgRepos(t *testing.T) { + dir, err := ioutil.TempDir("", "populate-owners-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + repoAB := filepath.Join(dir, "a", "b") + repoCD := filepath.Join(dir, "c", "d") + err = os.MkdirAll(repoAB, 0777) + if err != nil { + t.Fatal(err) + } + err = os.MkdirAll(repoCD, 0777) + if err != nil { + t.Fatal(err) + } + + orgRepos, err := orgRepos(dir) + if err != nil { + t.Fatal(err) + } + + expected := []*orgRepo{ + { + Directories: []string{repoAB}, + Organization: "a", + Repository: "b", + }, + { + Directories: []string{repoCD}, + Organization: "c", + Repository: "d", + }, + } + + assertEqual(t, orgRepos, expected) +} + +func TestExtractOwners(t *testing.T) { + dir, err := ioutil.TempDir("", "populate-owners-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + err = ioutil.WriteFile(filepath.Join(dir, "README"), []byte("Hello, World!\n"), 0666) + if err != nil { + t.Fatal(err) + } + + for _, args := range [][]string{ + {"git", "init"}, + {"git", "add", "README"}, + {"git", "commit", "-m", "Begin versioning"}, + } { + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = dir + cmd.Env = []string{ // for stable commit hashes + "GIT_COMMITTER_DATE=1112911993 -0700", + "GIT_AUTHOR_DATE=1112911993 -0700", + } + err = cmd.Run() + if err != nil { + t.Fatal(err) + } + } + + for _, test := range []struct { + name string + setup string + expected *orgRepo + error *regexp.Regexp + }{ + { + name: "no OWNERS", + expected: &orgRepo{ + Commit: "3e7341c55330a127038bfc8d7a396d4951049b85", + }, + error: regexp.MustCompile("^open .*/populate-owners-[0-9]*/OWNERS: no such file or directory"), + }, + { + name: "only OWNERS", + setup: "OWNERS", + expected: &orgRepo{ + Owners: &owners{Approvers: []string{"alice", "bob"}}, + Commit: "3e7341c55330a127038bfc8d7a396d4951049b85", + }, + error: regexp.MustCompile("^open .*/populate-owners-[0-9]*/OWNERS_ALIASES: no such file or directory"), + }, + { + name: "OWNERS and OWNERS_ALIASES", + setup: "OWNERS_ALIASES", + expected: &orgRepo{ + Owners: &owners{Approvers: []string{"sig-alias"}}, + Aliases: &aliases{Aliases: map[string][]string{"sig-alias": {"alice", "bob"}}}, + Commit: "3e7341c55330a127038bfc8d7a396d4951049b85", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + switch test.setup { + case "": // nothing to do + case "OWNERS": + err = ioutil.WriteFile( + filepath.Join(dir, "OWNERS"), + []byte("approvers:\n- alice\n- bob\n"), + 0666, + ) + if err != nil { + t.Fatal(err) + } + case "OWNERS_ALIASES": + err = ioutil.WriteFile( + filepath.Join(dir, "OWNERS"), + []byte("approvers:\n- sig-alias\n"), + 0666, + ) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile( + filepath.Join(dir, "OWNERS_ALIASES"), + []byte("aliases:\n sig-alias:\n - alice\n - bob\n"), + 0666, + ) + if err != nil { + t.Fatal(err) + } + default: + t.Fatalf("unrecognized setup: %q", test.setup) + } + + orgrepo := &orgRepo{} + err := orgrepo.extractOwners(dir) + if test.error == nil { + if err != nil { + t.Fatal(err) + } + } else if !test.error.MatchString(err.Error()) { + t.Fatalf("unexpected error: %v does not match %v", err, test.error) + } + + assertEqual(t, orgrepo, test.expected) + }) + } +} + +func TestNamespaceAliases(t *testing.T) { + for _, test := range []struct { + name string + input []*orgRepo + namespaced []*orgRepo + collected *aliases + error *regexp.Regexp + }{ + { + name: "no alias name collisions, so no namespaced aliases created", + input: []*orgRepo{ + { + Organization: "a", + Repository: "b", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + { + Organization: "c", + Repository: "d", + Aliases: &aliases{Aliases: map[string][]string{ + "cd": {"bob", "charlie"}, + }}, + }, + }, + namespaced: []*orgRepo{ + { + Organization: "a", + Repository: "b", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + { + Organization: "c", + Repository: "d", + Aliases: &aliases{Aliases: map[string][]string{ + "cd": {"bob", "charlie"}, + }}, + }, + }, + collected: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + "cd": {"bob", "charlie"}, + }}, + }, + { + name: "matching members, so no namespaced aliases created", + input: []*orgRepo{ + { + Organization: "a", + Repository: "b", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + { + Organization: "c", + Repository: "d", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + }, + namespaced: []*orgRepo{ + { + Organization: "a", + Repository: "b", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + { + Organization: "c", + Repository: "d", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + }, + collected: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + { + name: "different members, so namespaced aliases created", + input: []*orgRepo{ + { + Organization: "a", + Repository: "b", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + { + Organization: "c", + Repository: "d", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"bob", "charlie"}, + }}, + }, + }, + namespaced: []*orgRepo{ + { + Organization: "a", + Repository: "b", + Aliases: &aliases{Aliases: map[string][]string{ + "a-b-ab": {"alice", "bob"}, + }}, + }, + { + Organization: "c", + Repository: "d", + Aliases: &aliases{Aliases: map[string][]string{ + "c-d-ab": {"bob", "charlie"}, + }}, + }, + }, + collected: &aliases{Aliases: map[string][]string{ + "a-b-ab": {"alice", "bob"}, + "c-d-ab": {"bob", "charlie"}, + }}, + }, + { + name: "collisions after namespacing", + input: []*orgRepo{ + { + Organization: "a", + Repository: "b", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"alice", "bob"}, + }}, + }, + { + Organization: "c", + Repository: "d", + Aliases: &aliases{Aliases: map[string][]string{ + "ab": {"bob", "charlie"}, + }}, + }, + { + Organization: "e", + Repository: "f", + Aliases: &aliases{Aliases: map[string][]string{ + "a-b-ab": {"bob", "charlie"}, + }}, + }, + }, + error: regexp.MustCompile("^namespaced alias collision: \"a-b-ab\"$"), + }, + } { + t.Run(test.name, func(t *testing.T) { + collected, err := namespaceAliases(test.input) + if test.error == nil { + if err != nil { + t.Fatal(err) + } + } else if !test.error.MatchString(err.Error()) { + t.Fatalf("unexpected error: %v does not match %v", err, test.error) + } + + assertEqual(t, collected, test.collected) + if test.namespaced != nil { + assertEqual(t, test.input, test.namespaced) + } + }) + } +} + +func TestWriteYAML(t *testing.T) { + dir, err := ioutil.TempDir("", "populate-owners-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + for _, test := range []struct { + name string + filename string + data interface{} + expected string + }{ + { + name: "OWNERS", + filename: "OWNERS", + data: &owners{ + Approvers: []string{"alice", "bob"}, + }, + expected: `# prefix 1 +# prefix 2 + +approvers: +- alice +- bob +`, + }, + { + name: "OWNERS overwrite", + filename: "OWNERS", + data: &owners{ + Approvers: []string{"bob", "charlie"}, + }, + expected: `# prefix 1 +# prefix 2 + +approvers: +- bob +- charlie +`, + }, + { + name: "OWNERS_ALIASES", + filename: "OWNERS_ALIASES", + data: &aliases{ + Aliases: map[string][]string{ + "group-1": {"alice", "bob"}, + }, + }, + expected: `# prefix 1 +# prefix 2 + +aliases: + group-1: + - alice + - bob +`, + }, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(dir, test.filename) + err = writeYAML( + path, + test.data, + []string{"# prefix 1\n", "# prefix 2\n", "\n"}, + ) + if err != nil { + t.Fatal(err) + } + + data, err := ioutil.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + if string(data) != test.expected { + t.Fatalf("unexpected result:\n---\n%s\n--- != ---\n%s\n---\n", string(data), test.expected) + } + }) + } +}