From 8170caaba9ca0678daa7eec503d842671e1a8eae Mon Sep 17 00:00:00 2001 From: Kevin Broch Date: Tue, 5 Nov 2024 17:00:55 -0800 Subject: [PATCH] initial git repo tool POC --- projects/rvi-repo-manifest/Makefile | 43 ++++++ projects/rvi-repo-manifest/README.adoc | 52 +++++++ projects/rvi-repo-manifest/_common.xml | 5 + .../rvi-repo-manifest/generate-manifests.sh | 74 ++++++++++ .../rvi-repo-manifest/generated/_common.xml | 5 + .../generated/all-rvi-orgs.xml | 11 ++ .../generated/riscv-admin.xml | 104 ++++++++++++++ .../generated/riscv-android-src.xml | 136 ++++++++++++++++++ .../generated/riscv-collab.xml | 11 ++ .../generated/riscv-non-isa.xml | 41 ++++++ .../generated/riscv-software-src.xml | 23 +++ .../rvi-repo-manifest/generated/riscv.xml | 59 ++++++++ 12 files changed, 564 insertions(+) create mode 100644 projects/rvi-repo-manifest/Makefile create mode 100644 projects/rvi-repo-manifest/README.adoc create mode 100644 projects/rvi-repo-manifest/_common.xml create mode 100755 projects/rvi-repo-manifest/generate-manifests.sh create mode 100644 projects/rvi-repo-manifest/generated/_common.xml create mode 100644 projects/rvi-repo-manifest/generated/all-rvi-orgs.xml create mode 100644 projects/rvi-repo-manifest/generated/riscv-admin.xml create mode 100644 projects/rvi-repo-manifest/generated/riscv-android-src.xml create mode 100644 projects/rvi-repo-manifest/generated/riscv-collab.xml create mode 100644 projects/rvi-repo-manifest/generated/riscv-non-isa.xml create mode 100644 projects/rvi-repo-manifest/generated/riscv-software-src.xml create mode 100644 projects/rvi-repo-manifest/generated/riscv.xml diff --git a/projects/rvi-repo-manifest/Makefile b/projects/rvi-repo-manifest/Makefile new file mode 100644 index 0000000..986a11d --- /dev/null +++ b/projects/rvi-repo-manifest/Makefile @@ -0,0 +1,43 @@ + +ORGS := \ + riscv-collab \ + riscv-software-src \ + riscv \ + riscv-non-isa \ + riscv-admin \ + riscv-software-src \ + riscv-collab \ + riscv-android-src + +TOOL := ./generate-manifests.sh +BUILD_DIR := generated +ALL_ORGS_XML := all-rvi-orgs.xml + +ORGS_XML := $(addsuffix .xml, $(ORGS)) + +.PHONY: build-xml build-all-rvi-orgs-xml + +default: build-all-rvi-orgs-xml + +build-xml: + $(MAKE) $(ORGS_XML) + +build-all-rvi-orgs-xml: build-xml + @echo '' > $(BUILD_DIR)/$(ALL_ORGS_XML) + @echo '' >> $(BUILD_DIR)/$(ALL_ORGS_XML) + @for item in $(ORGS_XML); do echo " " >> $(BUILD_DIR)/$(ALL_ORGS_XML); done + @echo '' >> $(BUILD_DIR)/$(ALL_ORGS_XML) + +%.xml: + $(TOOL) $(BUILD_DIR)/$@ $* + cp _common.xml $(BUILD_DIR)/. + +clean: + rm -rf $(BUILD_DIR/*.xml) + +clean-all: + rm -rf $(BUILD_DIR) + +# Debug variables +print-%: + @echo $* = $($*) diff --git a/projects/rvi-repo-manifest/README.adoc b/projects/rvi-repo-manifest/README.adoc new file mode 100644 index 0000000..00ce7a5 --- /dev/null +++ b/projects/rvi-repo-manifest/README.adoc @@ -0,0 +1,52 @@ += RVI repos manifest + +RVI has many link:https://lf-riscv.atlassian.net/wiki/spaces/HOME/pages/16154706/GitHub+Repo+Map[github org's] with many repositories that represent the work being done. There are tools that make it easier to work with multiple repos. This POC provides one example. + +== Details + +This POC show how to do the following: + +* use github CLI tool `gh` and `jq` to generate a manifest (XML listing of multiple repos) +* use link:https://gerrit.googlesource.com/git-repo/[git `repo` tool] to work with the repos in that manifest + +NOTE: `repo` has many features that won't be covered in here or even needed (like `upload`ing changeset to gerrit server) + +=== Prerequisites + +* install and configure (authorize) link:https://cli.github.com/[`gh`] +* install link:https://jqlang.github.io/jq/[`jq`] +* Install link:https://gerrit.googlesource.com/git-repo/#install[git `repo` tool] + +=== Usage + +=== Generating a manifest file + +Although manifest files can be created manually, a bash script is used here to do it automatically. + +==== Using git `repo` tool + +Similarly to cloning a single repo, the first step in using `repo` is to link:https://source.android.com/docs/setup/reference/repo#init[initialize] a directory with the configuration (manifest file). This should be done in an empty dir. + +[source, bash] +---- +mkdir rvi-repos && cd rvi-repos +repo init -u https://github.com/riscv-admin/docs-sig -m projects/rvi-repo-manifest/generated/all-rvi-orgs.xml -g all,-riscv-android-src +---- + +=== Pros and cons + +* Pros +** perform git commands on multiple repos at the same time +* Cons +** manifests manually generated so additions or deletions of repos aren't automatic +*** deletions especially will generate a `repo rsync` failure for that given repo + +== References + +List of multi-repo tools: + +* https://gerrit.googlesource.com/git-repo/ +* https://manicli.com/ +* https://github.com/nosarthur/gita +* https://github.com/fabioz/mu-repo +* https://github.com/asottile/all-repos diff --git a/projects/rvi-repo-manifest/_common.xml b/projects/rvi-repo-manifest/_common.xml new file mode 100644 index 0000000..8ad60d9 --- /dev/null +++ b/projects/rvi-repo-manifest/_common.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/projects/rvi-repo-manifest/generate-manifests.sh b/projects/rvi-repo-manifest/generate-manifests.sh new file mode 100755 index 0000000..0a59931 --- /dev/null +++ b/projects/rvi-repo-manifest/generate-manifests.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +MAX_REPOS=1000 + +################################################################# +# SW PREREQ's +# Check if gh is installed +if ! command -v gh &> /dev/null +then + echo "ERROR: gh CLI could not be found. Please install it first." + exit 1 +fi + +# Check if jq is installed +if ! command -v jq &> /dev/null +then + echo "ERROR: jq could not be found. Please install it first." + exit 1 +fi + +################################################################# +# Check if correct number of arguments are provided +if [ $# -lt 2 ]; then + echo "Usage: $0 [org2] [org3] ..." + exit 1 +fi + +# Get the output file name from the first argument +output_file="$1" +shift # Remove the first argument (output file) from the list +mkdir -p "$(dirname "$output_file")" + +# Create or clear the output file +> "$output_file" + +# List of organizations from command line arguments +org_list="$@" + +# XML header +xml_header=' + + + + ' + +echo "$xml_header" > "$output_file" + +# Loop through each organization +for org in "${org_list[@]}"; do + echo "Fetching repositories for organization: $org" + + # Get list of repositories for the organization + repos=$(gh repo list -L $MAX_REPOS --no-archived --source $org --json name --jq '.[].name') + + # Loop through each repository + for repo in $repos; do + # Get default branch and topics + repo_info=$(gh repo view "$org/$repo" --json defaultBranchRef --jq '{db: .defaultBranchRef.name}') + + default_branch=$(echo $repo_info | jq -r '.db') + + # Add repository information to the XML file + repo_entry=" " + echo "$repo_entry" >> $output_file + done +done + +xml_footer=" +" + +# Close the root element +echo $xml_footer >> $output_file + +echo "XML manifest generated: $output_file" diff --git a/projects/rvi-repo-manifest/generated/_common.xml b/projects/rvi-repo-manifest/generated/_common.xml new file mode 100644 index 0000000..8ad60d9 --- /dev/null +++ b/projects/rvi-repo-manifest/generated/_common.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/projects/rvi-repo-manifest/generated/all-rvi-orgs.xml b/projects/rvi-repo-manifest/generated/all-rvi-orgs.xml new file mode 100644 index 0000000..18b94ed --- /dev/null +++ b/projects/rvi-repo-manifest/generated/all-rvi-orgs.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/projects/rvi-repo-manifest/generated/riscv-admin.xml b/projects/rvi-repo-manifest/generated/riscv-admin.xml new file mode 100644 index 0000000..15bea21 --- /dev/null +++ b/projects/rvi-repo-manifest/generated/riscv-admin.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rvi-repo-manifest/generated/riscv-android-src.xml b/projects/rvi-repo-manifest/generated/riscv-android-src.xml new file mode 100644 index 0000000..08be4f3 --- /dev/null +++ b/projects/rvi-repo-manifest/generated/riscv-android-src.xml @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rvi-repo-manifest/generated/riscv-collab.xml b/projects/rvi-repo-manifest/generated/riscv-collab.xml new file mode 100644 index 0000000..9941b46 --- /dev/null +++ b/projects/rvi-repo-manifest/generated/riscv-collab.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/projects/rvi-repo-manifest/generated/riscv-non-isa.xml b/projects/rvi-repo-manifest/generated/riscv-non-isa.xml new file mode 100644 index 0000000..d49464a --- /dev/null +++ b/projects/rvi-repo-manifest/generated/riscv-non-isa.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rvi-repo-manifest/generated/riscv-software-src.xml b/projects/rvi-repo-manifest/generated/riscv-software-src.xml new file mode 100644 index 0000000..00fc51b --- /dev/null +++ b/projects/rvi-repo-manifest/generated/riscv-software-src.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rvi-repo-manifest/generated/riscv.xml b/projects/rvi-repo-manifest/generated/riscv.xml new file mode 100644 index 0000000..89161e9 --- /dev/null +++ b/projects/rvi-repo-manifest/generated/riscv.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +