From 80b96b4625cfea9875f6d0f68414774fd99f5c72 Mon Sep 17 00:00:00 2001 From: bosong Date: Thu, 31 Mar 2022 18:15:58 +0800 Subject: [PATCH 1/4] feat(repo): add private for repo create --- README.md | 2 +- README_en.md | 1 + action.yml | 4 ++++ entrypoint.sh | 3 ++- hub-mirror/hub.py | 7 ++++--- hub-mirror/hubmirror.py | 1 + 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index be4390bf..62786aef 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ steps: - `debug` 默认为false, 配置后，启用debug开关，会显示所有执行命令。 - `timeout` 默认为'30m', 用于设置每个git命令的超时时间，'600'=>600s, '30m'=>30 mins, '1h'=>1 hours - `mappings` 源仓库映射规则，比如'A=>B, C=>CC', A会被映射为B，C会映射为CC，映射不具有传递性。主要用于源和目的仓库名不同的镜像。 - +- `dst_private` 默认为false，配置后，如果gitee仓库不存在，则创建公开仓库，否则创建私有仓库 ## 举些例子 #### 组织同步 diff --git a/README_en.md b/README_en.md index 1b39367c..14222f29 100644 --- a/README_en.md +++ b/README_en.md @@ -47,6 +47,7 @@ More than [100+](https://github.com/search?p=2&q=hub-mirror-action+%22account_ty - `force_update` (optional) Force to update the destination repo, use '-f' flag do 'git push' - `timeout` (optional) Default is '30m', set the timeout for every git command, like '600'=>600s, '30m'=>30 mins, '1h'=>1 hours - `mappings` (optional) Default is empty, the source repos mappings, such as 'A=>B, C=>CC', source repo name would be mapped follow the rule: A to B, C to CC. Mapping is not transitive. +- `dst_private` (optional) Default is false. After configuration, if the gitee repository does not exist, create a public repository, otherwise create a private repository ## Scenarios diff --git a/action.yml b/action.yml index a7d49ea9..85c23bf7 100644 --- a/action.yml +++ b/action.yml @@ -53,6 +53,9 @@ inputs: mappings: description: "The source repos mappings, such as 'A=>B, C=>CC', source repo name would be mapped follow the rule: A to B, C to CC. Mapping is not transitive." default: '' + dst_private: + description: "set create repo to public or private" + default: false runs: using: "docker" image: "Dockerfile" @@ -73,3 +76,4 @@ runs: - ${{ inputs.debug}} - ${{ inputs.timeout}} - ${{ inputs.mappings}} + - ${{ inputs.dst_private }} diff --git a/entrypoint.sh b/entrypoint.sh index b6566fde..0bc09868 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -25,7 +25,8 @@ python3 /hub-mirror/hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \ --force-update "${INPUT_FORCE_UPDATE}" \ --debug "${INPUT_DEBUG}" \ --timeout "${INPUT_TIMEOUT}" \ ---mappings "${INPUT_MAPPINGS}" +--mappings "${INPUT_MAPPINGS}" \ +--dst_private "${INPUT_DST_PRIVATE}" # Skip original code exit $? diff --git a/hub-mirror/hub.py b/hub-mirror/hub.py index b88bad3c..63767af2 100644 --- a/hub-mirror/hub.py +++ b/hub-mirror/hub.py @@ -7,7 +7,7 @@ class Hub(object): def __init__( - self, src, dst, dst_token, account_type="user", + self, src, dst, dst_token,dst_private, account_type="user", clone_style="https", src_account_type=None, dst_account_type=None, @@ -19,6 +19,7 @@ def __init__( self.src_type, self.src_account = src.split('/') self.dst_type, self.dst_account = dst.split('/') self.dst_token = dst_token + self.dst_private = dst_private self.session = requests.Session() if self.dst_type == "gitee": self.dst_base = 'https://gitee.com/api/v5' @@ -62,7 +63,7 @@ def create_dst_repo(self, repo_name): if self.dst_type == 'gitee': data = {'name': repo_name} elif self.dst_type == 'github': - data = json.dumps({'name': repo_name}) + data = json.dumps({'name': repo_name,'private': self.dst_private}) if not self.has_dst_repo(repo_name): print(repo_name + " doesn't exist, create it...") if self.dst_type == "github": @@ -80,7 +81,7 @@ def create_dst_repo(self, repo_name): response = requests.post( url, headers={'Content-Type': 'application/json;charset=UTF-8'}, - params={"name": repo_name, "access_token": self.dst_token} + params={"name": repo_name, "access_token": self.dst_token ,"private": self.dst_private} ) result = response.status_code == 201 if result: diff --git a/hub-mirror/hubmirror.py b/hub-mirror/hubmirror.py index e87eab4c..907ef423 100644 --- a/hub-mirror/hubmirror.py +++ b/hub-mirror/hubmirror.py @@ -54,6 +54,7 @@ def run(self): self.args.src, self.args.dst, self.args.dst_token, + self.args.dst_private, account_type=self.args.account_type, clone_style=self.args.clone_style, src_account_type=self.args.src_account_type, From 78bebeeb5abf19b1068fb0a78c66e2690799070f Mon Sep 17 00:00:00 2001 From: bosong Date: Thu, 31 Mar 2022 18:44:44 +0800 Subject: [PATCH 2/4] feat(repo): fix _ to - --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 0bc09868..7eaf1004 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -26,7 +26,7 @@ python3 /hub-mirror/hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \ --debug "${INPUT_DEBUG}" \ --timeout "${INPUT_TIMEOUT}" \ --mappings "${INPUT_MAPPINGS}" \ ---dst_private "${INPUT_DST_PRIVATE}" +--dst-private "${INPUT_DST_PRIVATE}" # Skip original code exit $? From 673d3e7e407a681dfcd9da4e22d45023333a5df9 Mon Sep 17 00:00:00 2001 From: bosong Date: Wed, 18 May 2022 19:00:36 +0800 Subject: [PATCH 3/4] fix(remove code): remove more code for public --- .github/workflows/verify-on-ubuntu-org.yml | 25 --- .../workflows/verify-on-ubuntu-user-cache.yml | 3 +- .github/workflows/verify-on-ubuntu-user.yml | 23 --- LICENSE | 2 +- README.md | 188 +++--------------- README_en.md | 153 ++------------ action.yml | 44 +--- entrypoint.sh | 10 +- hub-mirror/hub.py | 120 +---------- hub-mirror/hubmirror.py | 56 ++---- hub-mirror/mirror.py | 3 +- 11 files changed, 77 insertions(+), 550 deletions(-) delete mode 100644 .github/workflows/verify-on-ubuntu-org.yml delete mode 100644 .github/workflows/verify-on-ubuntu-user.yml diff --git a/.github/workflows/verify-on-ubuntu-org.yml b/.github/workflows/verify-on-ubuntu-org.yml deleted file mode 100644 index 88bed90c..00000000 --- a/.github/workflows/verify-on-ubuntu-org.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: - push: - pull_request_target: - branches: [master] -name: Tests / test-org-mirror -jobs: - run: - name: Run - runs-on: ubuntu-latest - steps: - - name: Checkout source code - uses: actions/checkout@v1 - - name: Mirror Github to Gitee - uses: ./. - with: - src: github/hub-mirror-action - dst: gitee/hub-mirror-action - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - account_type: org - # Only sync normal repo - black_list: 'test' - white_list: 'normal,test,empty' - force_update: true - debug: true diff --git a/.github/workflows/verify-on-ubuntu-user-cache.yml b/.github/workflows/verify-on-ubuntu-user-cache.yml index 46c2bfa1..f90f47c3 100644 --- a/.github/workflows/verify-on-ubuntu-user-cache.yml +++ b/.github/workflows/verify-on-ubuntu-user-cache.yml @@ -35,7 +35,8 @@ jobs: cache_path: /github/workspace/hub-mirror-cache white_list: 'hub-mirror-action, hexo-migrator-github-issue' force_update: true - + mappings: 'hashes=>hashes_gitee' + debug: false - name: Print cache path run: | ls -la ${{ github.workspace }}/hub-mirror-cache diff --git a/.github/workflows/verify-on-ubuntu-user.yml b/.github/workflows/verify-on-ubuntu-user.yml deleted file mode 100644 index 2032cd40..00000000 --- a/.github/workflows/verify-on-ubuntu-user.yml +++ /dev/null @@ -1,23 +0,0 @@ -on: - push: - pull_request_target: - branches: [master] -name: Tests / test-user-mirror -jobs: - run: - name: Run - runs-on: ubuntu-latest - steps: - - name: Checkout source codes - uses: actions/checkout@v1 - - name: Mirror Github to Gitee with white list - uses: ./. - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - static_list: 'hub-mirror-action,hashes, simple-spring' - force_update: true - debug: true - mappings: 'hashes=>hashes_gitee' diff --git a/LICENSE b/LICENSE index 26222c9a..ce3014aa 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2020 +Copyright (c) 2022 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 62786aef..c84eb7e7 100644 --- a/README.md +++ b/README.md @@ -1,192 +1,56 @@ -# Hub Mirror Action +# Pub Hub Mirror Action 简体中文 | [English](./README_en.md) -一个用于在hub间（例如Github，Gitee）账户代码仓库同步的action +一个用于在hub间（例如Github，Gitee、Coding，不局限，可以是所有）账户代码仓库同步的action，这个项目脱胎于Yikun/hub-mirror-action@master。 + +简单做几点说明： +1、由于我是想要一个纯粹的不同的hub之间 同步的脚本，所以将该脚本进行了删减，不是作者做的不好，只是我仅仅需要简单的功能罢了 +2、目前只支持，也只会支持两个仓库必须在两个hub之间存在的情况，不再创建新的仓库（由于创建仓库需要api支持，但是为了更通用，所以决定不支持对应的功能） +3、根据能量守恒定律，失去些什么，必然能得到些什么，这样就可以在不同的hub之间同步数据，不管是 从 github->gitee 还是 gitee-github 都可以支持到 +4、src、dst 都需要写全路径了，例如：github.com/kunpengcompute +5、static_list 是必传参数，因为不会再动态获取对应的repos了 +6、dst_key 也是必传参数，因为为了安全考虑，我决定全部使用ssh的方式进行同步，如果后期有需要，可以兼容https ## 用法 ```yaml steps: - name: Mirror the Github organization repos to Gitee. - uses: Yikun/hub-mirror-action@master + uses: Dislazy/hub-mirror-action@master with: - src: github/kunpengcompute - dst: gitee/kunpengcompute + src: github.com/kunpengcompute + dst: gitee.com/kunpengcompute dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - account_type: org - # src_account_type: org - # dst_account_type: org + static_list: repo1,repo2,repo3 + cache_path: /github/workspace/hub-mirror-cache + force_update: true + debug: true + timeout: '1h' + mappings: "yikun.github.com=>blog" ``` -上面的配置完成了kunpencompute组织从github到gitee的同步，你可以在[测试和demo](https://github.com/Yikun/hub-mirror-action/tree/master/.github/workflows)找到完整用法。 - -有疑问、想法、问题、建议，可以通过[![Gitter](https://badges.gitter.im/hub-mirror-action/community.svg)](https://gitter.im/hub-mirror-action/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)找到我们。 - -## 谁在使用？ -超过[100+](https://github.com/search?p=2&q=hub-mirror-action+%22account_type%22+%22org%22&type=Code)组织，[4000+](https://github.com/search?l=YAML&q=%22hub-mirror-action%22&type=Code)使用者正在使用，[50+](https://github.com/search?l=Markdown&q=%22hub-mirror-action%22&type=code)来自使用者的使用教程： - - +上面的配置完成了kunpencompute组织从github到gitee的同步，你可以在[测试和demo](https://github.com/dislazy/hub-mirror-action/tree/master/.github/workflows)找到完整用法。 ## 参数详解 #### 必选参数 -- `src` 需要被同步的源端账户名，如github/kunpengcompute，表示Github的kunpengcompute账户。 -- `dst` 需要同步到的目的端账户名，如gitee/kunpengcompute，表示Gitee的kunpengcompute账户。 +- `src` 需要被同步的源端账户名，如github.com/kunpengcompute，表示Github的kunpengcompute账户。 +- `dst` 需要同步到的目的端账户名，如gitee.com/kunpengcompute，表示Gitee的kunpengcompute账户。 - `dst_key` 用于在目的端上传代码的私钥(默认可以从~/.ssh/id_rsa获取），可参考[生成/添加SSH公钥](https://gitee.com/help/articles/4181)或[generating SSH keys](https://docs.github.com/articles/generating-an-ssh-key/)生成，并确认对应公钥已经被正确配置在目的端。对应公钥，Github可以在[这里](https://github.com/settings/keys)配置，Gitee可以[这里](https://gitee.com/profile/sshkeys)配置。 -- `dst_token` 创建仓库的API tokens， 用于自动创建不存在的仓库，Github可以在[这里](https://github.com/settings/tokens)找到，Gitee可以在[这里](https://gitee.com/profile/personal_access_tokens)找到。 +- `static_list` 默认为'', 配置后，仅同步静态列表，不会再动态获取需同步列表，如“repo1,repo2,repo3”。 #### 可选参数 -- `account_type` 默认为user，源和目的的账户类型，可以设置为org（组织）或者user（用户），该参数支持**同类型账户**（即组织到组织，或用户到用户）的同步。如果源目的仓库是不同类型，请单独使用`src_account_type`和`dst_account_type`配置。 -- `src_account_type` 默认为`account_type`，源账户类型，可以设置为org（组织）或者user（用户）。 -- `dst_account_type` 默认为`account_type`，目的账户类型，可以设置为org（组织）或者user（用户）。 -- `clone_style` 默认为https，可以设置为ssh或者https。当设置为ssh时，你需要将`dst_key`所对应的公钥同时配置到源端和目的端。 - `cache_path` 默认为'', 将代码缓存在指定目录，用于与actions/cache配合以加速镜像过程。 -- `black_list` 默认为'', 配置后，黑名单中的repos将不会被同步，如“repo1,repo2,repo3”。 -- `white_list` 默认为'', 配置后，仅同步白名单中的repos，如“repo1,repo2,repo3”。 -- `static_list` 默认为'', 配置后，仅同步静态列表，不会再动态获取需同步列表（黑白名单机制依旧生效），如“repo1,repo2,repo3”。 - `force_update` 默认为false, 配置后，启用git push -f强制同步，**注意：开启后，会强制覆盖目的端仓库**。 - `debug` 默认为false, 配置后，启用debug开关，会显示所有执行命令。 - `timeout` 默认为'30m', 用于设置每个git命令的超时时间，'600'=>600s, '30m'=>30 mins, '1h'=>1 hours - `mappings` 源仓库映射规则，比如'A=>B, C=>CC', A会被映射为B，C会映射为CC，映射不具有传递性。主要用于源和目的仓库名不同的镜像。 -- `dst_private` 默认为false，配置后，如果gitee仓库不存在，则创建公开仓库，否则创建私有仓库 -## 举些例子 - -#### 组织同步 - -同步Github的kunpengcompute组织到Gitee - -```yaml -- name: Organization mirror - uses: Yikun/hub-mirror-action@master - with: - src: github/kunpengcompute - dst: gitee/kunpengcompute - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - account_type: org -``` - -#### 黑/白名单 - -动态获取源端github/Yikun的repos，但仅同步名为hub-mirror-action，不同步hashes这个repo到Gitee - -```yaml -- name: Single repo mirror - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - white_list: "hub-mirror-action" - black_list: "hashes" -``` - -#### 静态名单（可用于单一仓库同步） - -不会动态获取源端github/Yikun的repos，仅同步hub-mirror-action这个repo - -```yaml -- name: Black list - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - static_list: "hub-mirror-action" -``` - -#### clone方式 - -使用ssh方式进行clone - -说明：请把`dst_key`所的公钥配置到源端（在这里为github）及目的端（在这里为gitee） - -```yaml -- name: ssh clone style - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - clone_style: "ssh" -``` - -#### 指定目录cache -```yaml -- name: Mirror with specific cache - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - cache_path: /github/workspace/hub-mirror-cache -``` - -#### 强制更新，并打开debug日志开关 -```yaml -- name: Mirror with force push (git push -f) - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - force_update: true - debug: true -``` - -#### 设置命令行超时时间为1小时 -```yaml -- name: Mirror with force push (git push -f) - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - force_update: true - timeout: '1h' -``` - -#### 仓库名不同时同步（github/yikun/yikun.github.com to gitee/yikunkero/blog） -```yaml -- name: mirror with mappings - uses: Yikun/hub-mirror-action@mappings - with: - src: github/yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - mappings: "yikun.github.com=>blog" - static_list: "yikun.github.com" -``` - -#### 仅同步非fork且非private的仓库 -```yaml -- name: Get repository list exclude private and fork - id: repo - uses: yi-Xu-0100/repo-list-generator@v1.0.1 -- name: Mirror repository list exclude private and fork - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - static_list: ${{ steps.repo.outputs.repoList }} -``` ## FAQ -- 如何在secrets添加dst_token和dst_key？ +- 如何在secrets添加dst_key？ 下面是添加secrets的方法，也可以参考[secrets官方文档](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets)了解更多： - 1. **获取Token和Key**，分别获取[ssh key](https://gitee.com/profile/sshkeys)和[token](https://gitee.com/profile/personal_access_tokens)。 2. **增加Secrets配置**，在配置仓库的Setting-Secrets中新增Secrets，例如GITEE_PRIVATE_KEY、GITEE_TOKEN 3. **在Workflow中引用**， 可以用过类似`${{ secrets.GITEE_PRIVATE_KEY }}`来访问 @@ -194,10 +58,10 @@ steps: - [Hub mirror template](https://github.com/yi-Xu-0100/hub-mirror): 一个用于展示如何使用这个action的模板仓库. from @yi-Xu-0100 - [自动镜像 GitHub 仓库到 Gitee](https://github.com/ShixiangWang/sync2gitee): 一个关于如何使用这个action的介绍. from @ShixiangWang - [巧用Github Action同步代码到Gitee](http://yikun.github.io/2020/01/17/%E5%B7%A7%E7%94%A8Github-Action%E5%90%8C%E6%AD%A5%E4%BB%A3%E7%A0%81%E5%88%B0Gitee/): Github Action第一篇软文 - +- [Hub Mirror Action](https://github.com/Yikun/hub-mirror-action): 该仓库的前身仓库，棒极了 ## 最后 如果觉得不错，**来个star**支持下作者吧！你的Star是我更新代码的动力！：） -想任何想吐槽或者建议的都可以直接飞个[issue](https://github.com/Yikun/hub-mirror-action/issues). +想任何想吐槽或者建议的都可以直接飞个[issue](https://github.com/dislazy/hub-mirror-action/issues). diff --git a/README_en.md b/README_en.md index 14222f29..41d63f4b 100644 --- a/README_en.md +++ b/README_en.md @@ -9,15 +9,17 @@ Action for mirroring repos between Hubs (like Github and Gitee) ```yaml steps: - name: Mirror the Github organization repos to Gitee. - uses: Yikun/hub-mirror-action@master + uses: Dislazy/hub-mirror-action@master with: - src: github/kunpengcompute - dst: gitee/kunpengcompute + src: github.com/kunpengcompute + dst: gitee.com/kunpengcompute dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - account_type: org - # src_account_type: org - # dst_account_type: org + static_list: repo1,repo2,repo3 + cache_path: /github/workspace/hub-mirror-cache + force_update: true + debug: true + timeout: '1h' + mappings: "yikun.github.com=>blog" ``` Here is a workflow to mirror the kunpengcompute org repos from Github to Gitee, see more complete workflows in [here](https://github.com/Yikun/hub-mirror-action/tree/master/.github/workflows). @@ -33,151 +35,24 @@ More than [100+](https://github.com/search?p=2&q=hub-mirror-action+%22account_ty - `src` source account, such as `github/kunpengcompute`, is the Github kunpengcompute account. - `dst` Destination account, such as `/kunpengcompute`, is the Gitee kunpengcompute account. - `dst_key` the private key to push code in destination account (default in ~/.ssh/id_rsa), you can see [generating SSH keys](https://docs.github.com/articles/generating-an-ssh-key/) to generate the pri/pub key, and make sure the pub key has been added in destination. You can set Github ssh key in [here](https://github.com/settings/keys)，set the Gitee ssh key in [here](https://gitee.com/profile/sshkeys). -- `dst_token` the API token to create non-existent repo, You can get Github token in [here](https://github.com/settings/tokens), and the Gitee in [here](https://gitee.com/profile/personal_access_tokens). +- `static_list` Only mirror repos in the static list, like 'repo1,repo2,repo3' #### Optional -- `account_type` (optional) default is `user`, the account type of src and dst account, can be set to `org` or `user`，only support mirror between same account type (that is "org to org" or "user to user"). if u wanna mirror difference account type, use the `src_account_type` and `dst_account_type` please. -- `src_account_type` (optional) default is `account_type`, the account type of src account, can be set to `org` or `user`. -- `dst_account_type` (optional) default is `account_type`, the account type of dst account, can be set to `org` or `user`. -- `clone_style` (optional) default is `https`, can be set to `ssh` or `https`.When you are using ssh clone style, you need to configure the public key of `dst_key` to both source end and destination end. - `cache_path` (optional) let code clone in specific path, can be used with actions/cache to speed up mirror. -- `black_list` (optional) the black list, such as “repo1,repo2,repo3”. -- `white_list` (optional) the white list, such as “repo1,repo2,repo3”. -- `static_list` (optional) Only mirror repos in the static list, but don't get list from repo api dynamically (the white/black list is still available). like 'repo1,repo2,repo3' - `force_update` (optional) Force to update the destination repo, use '-f' flag do 'git push' - `timeout` (optional) Default is '30m', set the timeout for every git command, like '600'=>600s, '30m'=>30 mins, '1h'=>1 hours - `mappings` (optional) Default is empty, the source repos mappings, such as 'A=>B, C=>CC', source repo name would be mapped follow the rule: A to B, C to CC. Mapping is not transitive. -- `dst_private` (optional) Default is false. After configuration, if the gitee repository does not exist, create a public repository, otherwise create a private repository - -## Scenarios - -#### Organization mirror, mirror the Github/kunpengcompute to Gitee/kunpengcompute -```yaml -- name: Organization mirror - uses: Yikun/hub-mirror-action@master - with: - src: github/kunpengcompute - dst: gitee/kunpengcompute - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - account_type: org -``` - -#### White/Black list, only mirror the Yikun/hub-mirror-action but not Yikun/hashes -```yaml -- name: Single repo mirror - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - white_list: "hub-mirror-action" - white_list: "hashes" -``` - -#### Static list, only mirror the repos `hub-mirror-action` and `hashes` -```yaml -- name: Black list - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - static_list: "hub-mirror-action,hashes" -``` - -#### clone style, use `ssh` clone style -Note: please configure the public key of `dst_key` to the source (github in here) and destination(gitee in here) - -```yaml -- name: ssh clone style - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - clone_style: "ssh" -``` - -#### set sepecific cache -```yaml -- name: Mirror with specific cache - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - cache_path: /github/workspace/hub-mirror-cache -``` - -#### Force udpate and enable the debug flag -```yaml -- name: Mirror with force push (git push -f) - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - force_update: true - debug: true -``` - -#### Set command timeout to an hour -```yaml -- name: Mirror with force push (git push -f) - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - force_update: true - timeout: '1h' -``` - -#### Sync between different repo name（github/yikun/yikun.github.com to gitee/yikunkero/blog） -``` -- name: mirror with mappings - uses: Yikun/hub-mirror-action@mappings - with: - src: github/yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - mappings: "yikun.github.com=>blog" - static_list: "yikun.github.com" -``` - -### Only sync repository list exclude private and fork -```yaml -- name: Get repository list exclude private and fork - id: repo - uses: yi-Xu-0100/repo-list-generator@v1.0.1 -- name: Mirror repository list exclude private and fork - uses: Yikun/hub-mirror-action@master - with: - src: github/Yikun - dst: gitee/yikunkero - dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} - dst_token: ${{ secrets.GITEE_TOKEN }} - static_list: ${{ steps.repo.outputs.repoList }} -``` ## FAQ -- How to use `secrets` to add token and key? +- How to use `secrets` to add key? You can use below steps to add secrets, you can also see more in [Secrets](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets). - 1. **Get Token and Key**，you can get them in [ssh key](https://gitee.com/profile/sshkeys) and [token](https://gitee.com/profile/personal_access_tokens). - 2. **Add Secrets**，add settings-secrets in repo，like `GITEE_PRIVATE_KEY`、`GITEE_TOKEN` - 3. **Add workflow**，add the workflow file into .github/workflows. + 1**Add Secrets**，add settings-secrets in repo，like `GITEE_PRIVATE_KEY`、`GITEE_TOKEN` + 2**Add workflow**，add the workflow file into .github/workflows. ## Reference - [Hub mirror template](https://github.com/yi-Xu-0100/hub-mirror): A template repo to show how to use this action. from @yi-Xu-0100 - [Auto-Sync GitHub Repositories to Gitee](https://github.com/ShixiangWang/sync2gitee): An introduction about how to use this action. from @ShixiangWang - [Use Github Action to sync reois to Gitee](http://yikun.github.io/2020/01/17/%E5%B7%A7%E7%94%A8Github-Action%E5%90%8C%E6%AD%A5%E4%BB%A3%E7%A0%81%E5%88%B0Gitee/): The blog for this action. +- [Hub Mirror Action](https://github.com/Yikun/hub-mirror-action): This before repo,greate diff --git a/action.yml b/action.yml index 85c23bf7..ede41268 100644 --- a/action.yml +++ b/action.yml @@ -1,6 +1,6 @@ -name: "Hub Mirror Action." -description: "Mirror the organization repos between hub (github/gitee)." -author: "yikun" +name: "More Hub Mirror Action." +description: "Mirror the list repos between hub (github/gitee/other)." +author: "Jack" branding: icon: "upload-cloud" color: "gray-dark" @@ -8,72 +8,46 @@ inputs: dst_key: description: "The private SSH key which is used to to push code in destination hub." required: true - dst_token: - description: "The app token which is used to create repo in destination hub." - required: true dst: description: "Destination name. Such as `gitee/kunpengcompute`." required: true src: description: "Source name. Such as `github/kunpengcompute`." required: true - account_type: - description: "The account type. Such as org, user." - default: 'user' - src_account_type: - description: "The src account type. Such as org, user." - default: '' - dst_account_type: - description: "The dst account type. Such as org, user." - default: '' - clone_style: - description: "The git clone style, https or ssh." - default: 'https' cache_path: description: "The path to cache the source repos code." default: '/github/workspace/hub-mirror-cache' - black_list: - description: "Hight priority, the back list of mirror repo. like 'repo1,repo2,repo3'" - default: '' - white_list: - description: "Low priority, the white list of mirror repo. like 'repo1,repo2,repo3'" - default: '' + required: false static_list: description: "Only mirror repo in the static list, but don't get from repo api (the white/black list is still available). like 'repo1,repo2,repo3'" default: '' + required: true force_update: description: "Force to update the destination repo, use '-f' flag do 'git push'" default: false + required: false debug: description: "Enable the debug flag to show detail log" default: false + required: false timeout: description: "Set the timeout for every git command, like '600'=>600s, '30m'=>30 mins, '1h'=>1 hours" default: '30m' + required: false mappings: description: "The source repos mappings, such as 'A=>B, C=>CC', source repo name would be mapped follow the rule: A to B, C to CC. Mapping is not transitive." default: '' - dst_private: - description: "set create repo to public or private" - default: false + required: false runs: using: "docker" image: "Dockerfile" args: - ${{ inputs.dst_key }} - - ${{ inputs.dst_token }} - ${{ inputs.src }} - ${{ inputs.dst }} - - ${{ inputs.account_type }} - - ${{ inputs.src_account_type }} - - ${{ inputs.dst_account_type }} - - ${{ inputs.clone_style }} - ${{ inputs.cache_path }} - - ${{ inputs.black_list }} - - ${{ inputs.white_list }} - ${{ inputs.static_list }} - ${{ inputs.force_update}} - ${{ inputs.debug}} - ${{ inputs.timeout}} - ${{ inputs.mappings}} - - ${{ inputs.dst_private }} diff --git a/entrypoint.sh b/entrypoint.sh index 7eaf1004..248ce6be 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,20 +13,12 @@ chmod 600 /root/.ssh/id_rsa pip3 install -r /hub-mirror/requirements.txt python3 /hub-mirror/hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \ ---dst-token "${INPUT_DST_TOKEN}" \ ---account-type "${INPUT_ACCOUNT_TYPE}" \ ---src-account-type "${INPUT_SRC_ACCOUNT_TYPE}" \ ---dst-account-type "${INPUT_DST_ACCOUNT_TYPE}" \ ---clone-style "${INPUT_CLONE_STYLE}" \ --cache-path "${INPUT_CACHE_PATH}" \ ---black-list "${INPUT_BLACK_LIST}" \ ---white-list "${INPUT_WHITE_LIST}" \ --static-list "${INPUT_STATIC_LIST}" \ --force-update "${INPUT_FORCE_UPDATE}" \ --debug "${INPUT_DEBUG}" \ --timeout "${INPUT_TIMEOUT}" \ ---mappings "${INPUT_MAPPINGS}" \ ---dst-private "${INPUT_DST_PRIVATE}" +--mappings "${INPUT_MAPPINGS}" # Skip original code exit $? diff --git a/hub-mirror/hub.py b/hub-mirror/hub.py index 63767af2..f4ade6f5 100644 --- a/hub-mirror/hub.py +++ b/hub-mirror/hub.py @@ -1,121 +1,13 @@ -import time -import functools -import json - -import requests class Hub(object): def __init__( - self, src, dst, dst_token,dst_private, account_type="user", - clone_style="https", - src_account_type=None, - dst_account_type=None, + self, src, dst ): - # TODO: check invalid type - self.account_type = account_type - self.src_account_type = src_account_type or account_type - self.dst_account_type = dst_account_type or account_type + # 切分/前后的 域名 和 用户 self.src_type, self.src_account = src.split('/') self.dst_type, self.dst_account = dst.split('/') - self.dst_token = dst_token - self.dst_private = dst_private - self.session = requests.Session() - if self.dst_type == "gitee": - self.dst_base = 'https://gitee.com/api/v5' - elif self.dst_type == "github": - self.dst_base = 'https://api.github.com' - - prefix = "https://" if clone_style == 'https' else 'git@' - suffix = "/" if clone_style == 'https' else ':' - if self.src_type == "gitee": - self.src_base = 'https://gitee.com/api/v5' - self.src_repo_base = prefix + 'gitee.com' + suffix - elif self.src_type == "github": - self.src_base = 'https://api.github.com' - self.src_repo_base = prefix + 'github.com' + suffix - self.src_repo_base = self.src_repo_base + self.src_account - # TODO: toekn push support - prefix = "git@" + self.dst_type + ".com:" - self.dst_repo_base = prefix + self.dst_account - - def has_dst_repo(self, repo_name): - url = '/'.join( - [ - self.dst_base, self.dst_account_type+'s', self.dst_account, - 'repos' - ] - ) - repo_names = self._get_all_repo_names(url) - if not repo_names: - print("Warning: destination repos is []") - return False - return repo_name in repo_names - - def create_dst_repo(self, repo_name): - suffix = 'user/repos' - if self.dst_account_type == "org": - suffix = 'orgs/%s/repos' % self.dst_account - url = '/'.join( - [self.dst_base, suffix] - ) - result = None - if self.dst_type == 'gitee': - data = {'name': repo_name} - elif self.dst_type == 'github': - data = json.dumps({'name': repo_name,'private': self.dst_private}) - if not self.has_dst_repo(repo_name): - print(repo_name + " doesn't exist, create it...") - if self.dst_type == "github": - response = self.session.post( - url, - data=data, - headers={'Authorization': 'token ' + self.dst_token} - ) - result = response.status_code == 201 - if result: - print("Destination repo creating accepted.") - else: - print("Destination repo creating failed: " + response.text) - elif self.dst_type == "gitee": - response = requests.post( - url, - headers={'Content-Type': 'application/json;charset=UTF-8'}, - params={"name": repo_name, "access_token": self.dst_token ,"private": self.dst_private} - ) - result = response.status_code == 201 - if result: - print("Destination repo creating accepted.") - else: - print("Destination repo creating failed: " + response.text) - else: - print(repo_name + " repo exist, skip creating...") - # TODO(snowyu): Cleanup 2s sleep - if result: - time.sleep(2) - return result - - def dynamic_list(self): - url = '/'.join( - [ - self.src_base, self.src_account_type+'s', self.src_account, - 'repos', - ] - ) - return self._get_all_repo_names(url) - - @functools.lru_cache - def _get_all_repo_names(self, url, page=1): - per_page = 60 - api = url + f"?page={page}&per_page=" + str(per_page) - # TODO: src_token support - response = self.session.get(api) - all_items = [] - if response.status_code != 200: - print("Repo getting failed: " + response.text) - return all_items - items = response.json() - if items: - names = [i['name'] for i in items] - return names + self._get_all_repo_names(url, page=page+1) - return all_items + prefix = 'git@' + suffix = ':' + self.src_repo_base = prefix + self.src_type + suffix + self.src_account + self.dst_repo_base = prefix + self.dst_type + suffix + self.dst_account diff --git a/hub-mirror/hubmirror.py b/hub-mirror/hubmirror.py index 907ef423..03411442 100644 --- a/hub-mirror/hubmirror.py +++ b/hub-mirror/hubmirror.py @@ -11,8 +11,6 @@ class HubMirror(object): def __init__(self): self.parser = self._create_parser() self.args = self.parser.parse_args() - self.white_list = str2list(self.args.white_list) - self.black_list = str2list(self.args.black_list) self.static_list = str2list(self.args.static_list) self.mappings = str2map(self.args.mappings) @@ -38,33 +36,15 @@ def _create_parser(self): ) return parser - def test_black_white_list(self, repo): - if repo in self.black_list: - print("Skip, %s in black list: %s" % (repo, self.black_list)) - return False - - if self.white_list and repo not in self.white_list: - print("Skip, %s not in white list: %s" % (repo, self.white_list)) - return False - - return True - def run(self): hub = Hub( self.args.src, - self.args.dst, - self.args.dst_token, - self.args.dst_private, - account_type=self.args.account_type, - clone_style=self.args.clone_style, - src_account_type=self.args.src_account_type, - dst_account_type=self.args.dst_account_type, + self.args.dst ) - src_type, src_account = self.args.src.split('/') # Using static list when static_list is set repos = self.static_list - src_repos = repos if repos else hub.dynamic_list() + src_repos = repos if repos else [] total, success, skip = len(src_repos), 0, 0 failed_list = [] @@ -72,24 +52,22 @@ def run(self): # Set dst_repo to src_repo mapping or src_repo directly dst_repo = self.mappings.get(src_repo, src_repo) print("Map %s to %s" % (src_repo, dst_repo)) - if self.test_black_white_list(src_repo): - print("Backup %s" % src_repo) - try: - mirror = Mirror( - hub, src_repo, dst_repo, - cache=self.args.cache_path, - timeout=self.args.timeout, - force_update=self.args.force_update, + print("Backup %s" % src_repo) + try: + mirror = Mirror( + hub, src_repo, dst_repo, + cache=self.args.cache_path, + timeout=self.args.timeout, + force_update=self.args.force_update, ) - mirror.download() - mirror.create() - mirror.push() - success += 1 - except Exception as e: - print(e) - failed_list.append(src_repo) - else: - skip += 1 + mirror.download() + mirror.create() + mirror.push() + success += 1 + except Exception as e: + print(e) + failed_list.append(src_repo) + failed = total - success - skip res = (total, skip, success, failed) print("Total: %s, skip: %s, successed: %s, failed: %s." % res) diff --git a/hub-mirror/mirror.py b/hub-mirror/mirror.py index a9477800..23f737cb 100644 --- a/hub-mirror/mirror.py +++ b/hub-mirror/mirror.py @@ -27,7 +27,6 @@ def __init__( @retry(wait=wait_exponential(), reraise=True, stop=stop_after_attempt(3)) def _clone(self): - # TODO: process empty repo print("Starting git clone " + self.src_url) mygit = git.cmd.Git(os.getcwd()) mygit.clone( @@ -59,7 +58,7 @@ def download(self): def create(self): print("(2/3) Creating...") - self.hub.create_dst_repo(self.dst_name) + return False def _check_empty(self, repo): cmd = ["-n", "1", "--all"] From b440d2d6c947f4e0550e5a99b8062ee47650f799 Mon Sep 17 00:00:00 2001 From: bosong Date: Wed, 18 May 2022 19:28:07 +0800 Subject: [PATCH 4/4] fix(readme): fix README.md --- README.md | 22 ++++++++++++++-------- README_en.md | 11 +++-------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index c84eb7e7..1a3b9de5 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,18 @@ 一个用于在hub间（例如Github，Gitee、Coding，不局限，可以是所有）账户代码仓库同步的action，这个项目脱胎于Yikun/hub-mirror-action@master。 简单做几点说明： -1、由于我是想要一个纯粹的不同的hub之间 同步的脚本，所以将该脚本进行了删减，不是作者做的不好，只是我仅仅需要简单的功能罢了 -2、目前只支持，也只会支持两个仓库必须在两个hub之间存在的情况，不再创建新的仓库（由于创建仓库需要api支持，但是为了更通用，所以决定不支持对应的功能） -3、根据能量守恒定律，失去些什么，必然能得到些什么，这样就可以在不同的hub之间同步数据，不管是 从 github->gitee 还是 gitee-github 都可以支持到 -4、src、dst 都需要写全路径了，例如：github.com/kunpengcompute -5、static_list 是必传参数，因为不会再动态获取对应的repos了 -6、dst_key 也是必传参数，因为为了安全考虑，我决定全部使用ssh的方式进行同步，如果后期有需要，可以兼容https + +1. 由于我是想要一个纯粹的不同的hub之间 同步的脚本，所以将该脚本进行了删减，不是作者做的不好，只是我仅仅需要简单的功能罢了 + +2. 目前只支持，也只会支持两个仓库必须在两个hub之间存在的情况，不再创建新的仓库（由于创建仓库需要api支持，但是为了更通用，所以决定不支持对应的功能） + +3. 根据能量守恒定律，失去些什么，必然能得到些什么，这样就可以在不同的hub之间同步数据，不管是 从 github->gitee 还是 gitee-github 都可以支持到 + +4. src、dst 都需要写全路径了，例如：github.com/kunpengcompute + +5. static_list 是必传参数，因为不会再动态获取对应的repos了 + +6. dst_key 也是必传参数，因为为了安全考虑，我决定全部使用ssh的方式进行同步，如果后期有需要，可以兼容https ## 用法 @@ -51,8 +57,8 @@ steps: - 如何在secrets添加dst_key？ 下面是添加secrets的方法，也可以参考[secrets官方文档](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets)了解更多： - 2. **增加Secrets配置**，在配置仓库的Setting-Secrets中新增Secrets，例如GITEE_PRIVATE_KEY、GITEE_TOKEN - 3. **在Workflow中引用**， 可以用过类似`${{ secrets.GITEE_PRIVATE_KEY }}`来访问 + 1. **增加Secrets配置**，在配置仓库的Setting-Secrets中新增Secrets，例如GITEE_PRIVATE_KEY、GITEE_TOKEN + 2. **在Workflow中引用**， 可以用过类似`${{ secrets.GITEE_PRIVATE_KEY }}`来访问 ## 参考 - [Hub mirror template](https://github.com/yi-Xu-0100/hub-mirror): 一个用于展示如何使用这个action的模板仓库. from @yi-Xu-0100 diff --git a/README_en.md b/README_en.md index 41d63f4b..158ebf7d 100644 --- a/README_en.md +++ b/README_en.md @@ -2,7 +2,7 @@ English | [简体中文](./README.md) -Action for mirroring repos between Hubs (like Github and Gitee) +Action for mirroring repos between Hubs (like Github and Gitee and more) ## Tutorial @@ -24,11 +24,6 @@ steps: Here is a workflow to mirror the kunpengcompute org repos from Github to Gitee, see more complete workflows in [here](https://github.com/Yikun/hub-mirror-action/tree/master/.github/workflows). -## Who are using? -More than [100+](https://github.com/search?p=2&q=hub-mirror-action+%22account_type%22+%22org%22&type=Code) organizations，[4000+](https://github.com/search?l=YAML&q=%22hub-mirror-action%22&type=Code) users are using, [50+](https://github.com/search?l=Markdown&q=%22hub-mirror-action%22&type=code) related blogs from users： - - - ## Usage #### Required @@ -48,8 +43,8 @@ More than [100+](https://github.com/search?p=2&q=hub-mirror-action+%22account_ty You can use below steps to add secrets, you can also see more in [Secrets](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets). - 1**Add Secrets**，add settings-secrets in repo，like `GITEE_PRIVATE_KEY`、`GITEE_TOKEN` - 2**Add workflow**，add the workflow file into .github/workflows. + 1. **Add Secrets**，add settings-secrets in repo，like `GITEE_PRIVATE_KEY`、`GITEE_TOKEN` + 2. **Add workflow**，add the workflow file into .github/workflows. ## Reference - [Hub mirror template](https://github.com/yi-Xu-0100/hub-mirror): A template repo to show how to use this action. from @yi-Xu-0100