Rewrite hub-mirror-action using Python. (#90)

In this patch, we rewrite the hub-mirror-action using Python.

There are 3 core classes:
1. HubMirror, the main class to interate Mirror and Hub.
2. Mirror, the class for git operations.
3. Hub, the class for repo related remote api operations

Close: #88 This is the feature PR.
Close: #78 This is a bug fix.

.github/workflows/verify-on-ubuntu-org.yml

@@ -5,18 +5,18 @@ jobs:
     name: Run
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout source codes
+    - name: Checkout source code
       uses: actions/checkout@v1
     - name: Mirror Github to Gitee
       uses: ./.
       with:
-        src: github/kunpengcompute
-        dst: gitee/kunpengcompute
+        src: github/hub-mirror-action
+        dst: gitee/hub-mirror-action
         dst_key: ${{ secrets.GITEE_PRIVATE_KEY }}
         dst_token: ${{ secrets.GITEE_TOKEN }}
         account_type: org
-        # Only sync Kunpeng
-        black_list: 'KAE'
-        white_list: 'KAE,Kunpeng,kunpengcompute.github.io'
+        # Only sync normal repo
+        black_list: 'test'
+        white_list: 'normal,test'
         force_update: true
-        debug: true
+        debug: true

Dockerfile

@@ -1,8 +1,11 @@
-FROM alpine
+FROM ubuntu
 
-RUN apk add --no-cache git openssh-client bash jq curl&& \
+RUN apt update && apt install git python3 python3-pip -y && \
   echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
 
 ADD *.sh /
+ADD hubmirror.py /
+ADD requirements.txt /
+ADD action.yml /
 
 ENTRYPOINT ["/entrypoint.sh"]

entrypoint.sh

@@ -10,239 +10,19 @@ mkdir -p /root/.ssh
 echo "${INPUT_DST_KEY}" > /root/.ssh/id_rsa
 chmod 600 /root/.ssh/id_rsa
 
-DST_TOKEN="${INPUT_DST_TOKEN}"
-
-SRC_HUB="${INPUT_SRC}"
-DST_HUB="${INPUT_DST}"
-
-ACCOUNT_TYPE="${INPUT_ACCOUNT_TYPE}"
-
-SRC_TYPE=`dirname $SRC_HUB`
-DST_TYPE=`dirname $DST_HUB`
-
-SRC_ACCOUNT=`basename $SRC_HUB`
-DST_ACCOUNT=`basename $DST_HUB`
-
-CLONE_STYLE="${INPUT_CLONE_STYLE}"
-
-CACHE_PATH="${INPUT_CACHE_PATH}"
-
-WHITE_LIST="${INPUT_WHITE_LIST}"
-BLACK_LIST="${INPUT_BLACK_LIST}"
-STATIC_LIST="${INPUT_STATIC_LIST}"
-
-FORCE_UPDATE="${INPUT_FORCE_UPDATE}"
-
-DELAY_EXIT=false
-
-TIME_OUT="${INPUT_TIMEOUT}"
-RETRY_TIMES=3
-
-function err_exit {
-  echo -e "\033[31m $1 \033[0m"
-  exit 1
-}
-
-FAILED_LIST=()
-
-function delay_exit {
-  echo -e "\033[31m $1 \033[0m"
-  FAILED_LIST+=($2)
-  DELAY_EXIT=true
-  return 1
-}
-
-if [[ "$ACCOUNT_TYPE" == "org" ]]; then
-  SRC_LIST_URL_SUFFIX=orgs/$SRC_ACCOUNT/repos
-  DST_LIST_URL_SUFFIX=orgs/$DST_ACCOUNT/repos
-  DST_CREATE_URL_SUFFIX=orgs/$DST_ACCOUNT/repos
-elif [[ "$ACCOUNT_TYPE" == "user" ]]; then
-  SRC_LIST_URL_SUFFIX=users/$SRC_ACCOUNT/repos
-  DST_LIST_URL_SUFFIX=users/$DST_ACCOUNT/repos
-  DST_CREATE_URL_SUFFIX=user/repos
-else
-  err_exit "Unknown account type, the `account_type` should be `user` or `org`"
-fi
-
-if [[ "$SRC_TYPE" == "github" ]]; then
-  SRC_REPO_LIST_API=https://api.github.com/$SRC_LIST_URL_SUFFIX
-  if [[ "$CLONE_STYLE" == "ssh" ]]; then
-    [email protected]:
-  elif [[ "$CLONE_STYLE" == "https" ]]; then
-    SRC_REPO_BASE_URL=https://github.com/
-  fi
-elif [[ "$SRC_TYPE" == "gitee" ]]; then
-  SRC_REPO_LIST_API=https://gitee.com/api/v5/$SRC_LIST_URL_SUFFIX
-  if [[ "$CLONE_STYLE" == "ssh" ]]; then
-    [email protected]:
-  elif [[ "$CLONE_STYLE" == "https" ]]; then
-    SRC_REPO_BASE_URL=https://gitee.com/
-  fi
-else
-  err_exit "Unknown src args, the `src` should be `[github|gittee]/account`"
-fi
-
-function retry {
-  local retries=$RETRY_TIMES
-  local count=0
-  until timeout $TIME_OUT "$@"; do
-    exit=$?
-    wait=$((2 ** $count))
-    count=$(($count + 1))
-    if [ $count -lt $retries ]; then
-      echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-      sleep $wait
-    else
-      echo "Retry $count/$retries exited $exit, no more retries left."
-      return $exit
-    fi
-  done
-  return 0
-}
-
-function get_all_repo_names
-{
-  PAGE_NUM=100
-  URL=$1
-  HUB_TYPE=$2
-  if [[ "$HUB_TYPE" == "github" ]]; then
-    total=`curl -sI "$URL?page=1&per_page=$PAGE_NUM" | sed -nr "s/^[lL]ink:.*page=([0-9]+)&per_page=$PAGE_NUM.*/\1/p"`
-  elif [[ "$HUB_TYPE" == "gitee" ]]; then
-    total=`curl -sI "$URL?page=1&per_page=$PAGE_NUM" | grep total_page: |cut -d ' ' -f2 |tr -d '\r'`
-  fi
-
-  # use pagination?
-  if [ -z "$total" ]; then
-      # no - this result has only one page
-      total=1
-  fi
-
-  p=1
-  while [ "$p" -le "$total" ]; do
-    x=`curl -s "$URL?page=$p&per_page=$PAGE_NUM" | jq -r '.[] | .name'`
-    echo $x
-    p=$(($p + 1))
-  done
-}
-
-if [[ -z $STATIC_LIST ]]; then
-  SRC_REPOS=`get_all_repo_names $SRC_REPO_LIST_API $SRC_TYPE`
-else
-  SRC_REPOS=`echo $STATIC_LIST | tr ',' ' '`
-fi
-
-if [[ "$DST_TYPE" == "github" ]]; then
-  DST_REPO_CREATE_API=https://api.github.com/$DST_CREATE_URL_SUFFIX
-  DST_REPO_LIST_API=https://api.github.com/$DST_LIST_URL_SUFFIX
-elif [[ "$DST_TYPE" == "gitee" ]]; then
-  DST_REPO_CREATE_API=https://gitee.com/api/v5/$DST_CREATE_URL_SUFFIX
-  DST_REPO_LIST_API=https://gitee.com/api/v5/$DST_LIST_URL_SUFFIX
-else
-  err_exit "Unknown dst args, the `dst` should be `[github|gittee]/account`"
-fi
-
-DST_REPOS=`get_all_repo_names $DST_REPO_LIST_API $DST_TYPE`
-
-function clone_repo
-{
-  echo -e "\033[31m(0/3)\033[0m" "Downloading..."
-  if [ ! -d "$1" ]; then
-    retry git clone $SRC_REPO_BASE_URL$SRC_ACCOUNT/$1.git
-  fi
-  cd $1
-}
-
-function create_repo
-{
-  # Auto create non-existing repo
-  has_repo=`echo $DST_REPOS | sed 's/ /\n/g' | grep -Fx $1 | wc -l`
-  if [ $has_repo == 0 ]; then
-    echo "Create non-exist repo..."
-    if [[ "$DST_TYPE" == "github" ]]; then
-      curl -s -H "Authorization: token $2" --data '{"name":"'$1'"}' $DST_REPO_CREATE_API > /dev/null
-    elif [[ "$DST_TYPE" == "gitee" ]]; then
-      curl -s -X POST --header 'Content-Type: application/json;charset=UTF-8' $DST_REPO_CREATE_API -d '{"name": "'$1'","access_token": "'$2'"}' > /dev/null
-    fi
-  fi
-  git remote add $DST_TYPE git@$DST_TYPE.com:$DST_ACCOUNT/$1.git || echo "Remote already exists."
-}
-
-function update_repo
-{
-  echo -e "\033[31m(1/3)\033[0m" "Updating..."
-  retry git pull -p
-}
-
-function import_repo
-{
-  echo -e "\033[31m(2/3)\033[0m" "Importing..."
-  git remote set-head origin -d
-  if [[ "$FORCE_UPDATE" == "true" ]]; then
-    retry git push -f $DST_TYPE refs/remotes/origin/*:refs/heads/* --tags --prune
-  else
-    retry git push $DST_TYPE refs/remotes/origin/*:refs/heads/* --tags --prune
-  fi
-}
-
-function _check_in_list () {
-  local e match="$1"
-  shift
-  for e; do [[ "$e" == "$match" ]] && return 0; done
-  return 1
-}
-
-function test_black_white_list
-{
-  WHITE_ARR=(`echo $WHITE_LIST | tr ',' ' '`)
-  BLACK_ARR=(`echo $BLACK_LIST | tr ',' ' '`)
-  _check_in_list $1 "${WHITE_ARR[@]}";in_white_list=$?
-  _check_in_list $1 "${BLACK_ARR[@]}";in_back_list=$?
-
-  if [[ $in_back_list -ne 0 ]] ; then
-    if [[ -z $WHITE_LIST ]] || [[ $in_white_list -eq 0 ]] ; then
-      return 0
-    else
-      echo "Skip, "$1" not in non-empty white list"$WHITE_LIST
-      return 1
-    fi
-  else
-    echo "Skip, "$1 "in black list: "$BLACK_LIST
-    return 1
-  fi
-}
-
-if [ ! -d "$CACHE_PATH" ]; then
-  mkdir -p $CACHE_PATH
-fi
-cd $CACHE_PATH
-
-all=0
-success=0
-skip=0
-for repo in $SRC_REPOS
-{
-  all=$(($all + 1))
-  if test_black_white_list $repo ; then
-    echo -e "\n\033[31mBackup $repo ...\033[0m"
-
-    cd $CACHE_PATH
-
-    clone_repo $repo || delay_exit "clone and cd failed"  $repo || continue
-
-    create_repo $repo $DST_TOKEN || delay_exit "create failed" $repo || continue
-
-    update_repo || delay_exit "Update failed" $repo || continue
-
-    import_repo && success=$(($success + 1)) || delay_exit "Push failed" $repo || continue
-  else
-    skip=$(($skip + 1))
-  fi
-}
-
-failed=$(($all - $skip - $success))
-echo "Total: $all, skip: $skip, successed: $success, failed: $failed."
-echo "Failed: "$FAILED_LIST
-
-if [[ "$DELAY_EXIT" == "true" ]]; then
-  exit 1
-fi
+pip3 install -r /requirements.txt
+
+python3 /hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \
+--dst-token "${INPUT_DST_TOKEN}" \
+--account-type "${INPUT_ACCOUNT_TYPE}" \
+--clone-style "${INPUT_CLONE_STYLE}" \
+--cache-path "${INPUT_CACHE_PATH}" \
+--black-list "${INPUT_BLACK_LIST}" \
+--white-list "${INPUT_WHITE_LIST}" \
+--static-list "${INPUT_STATIC_LIST}" \
+--force-update "${INPUT_FORCE_UPDATE}" \
+--debug "${INPUT_DEBUG}" \
+--timeout  "${INPUT_TIMEOUT}"
+
+# Skip original code
+exit $?