From 8596213ba3fcabb9d9e59e0e69f65afda085d2bc Mon Sep 17 00:00:00 2001 From: kareem zarka Date: Fri, 14 Jul 2023 17:30:37 +0200 Subject: [PATCH 1/5] DockerFile: Add Git LFS to DockerFile Signed-off-by: kareem zarka --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d77a67d0..5f9bed3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu -RUN apt update && apt install git python3 python3-pip -y && \ +RUN apt update && apt install git git-lfs python3 python3-pip -y && \ echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config ADD *.sh / From a7cb1dda0a33df036dd9cb32cfe4fb9aade7675f Mon Sep 17 00:00:00 2001 From: kareem zarka Date: Fri, 14 Jul 2023 17:33:16 +0200 Subject: [PATCH 2/5] action.yml: add 'lfs' input parameter In order to use it to enable LFS during mirror action. Signed-off-by: kareem zarka --- action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/action.yml b/action.yml index a7d49ea9..50a52a78 100644 --- a/action.yml +++ b/action.yml @@ -53,6 +53,9 @@ inputs: mappings: description: "The source repos mappings, such as 'A=>B, C=>CC', source repo name would be mapped follow the rule: A to B, C to CC. Mapping is not transitive." default: '' + lfs: + description: "Enable Git LFS support." + default: false runs: using: "docker" image: "Dockerfile" @@ -73,3 +76,4 @@ runs: - ${{ inputs.debug}} - ${{ inputs.timeout}} - ${{ inputs.mappings}} + - ${{ inputs.lfs }} From 9befc4f4f986f0429c65b7183682b16980c49de2 Mon Sep 17 00:00:00 2001 From: kareem zarka Date: Fri, 14 Jul 2023 17:47:31 +0200 Subject: [PATCH 3/5] entry.sh: Add Git LFS installation and add lfs argument Signed-off-by: kareem zarka --- entrypoint.sh | 5 ++++- hub-mirror/mirror.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index b6566fde..ecca8795 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -12,6 +12,8 @@ chmod 600 /root/.ssh/id_rsa pip3 install -r /hub-mirror/requirements.txt +git lfs install + python3 /hub-mirror/hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \ --dst-token "${INPUT_DST_TOKEN}" \ --account-type "${INPUT_ACCOUNT_TYPE}" \ @@ -25,7 +27,8 @@ python3 /hub-mirror/hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \ --force-update "${INPUT_FORCE_UPDATE}" \ --debug "${INPUT_DEBUG}" \ --timeout "${INPUT_TIMEOUT}" \ ---mappings "${INPUT_MAPPINGS}" +--mappings "${INPUT_MAPPINGS}"\ +--lfs "${INPUT_LFS}" # Skip original code exit $? diff --git a/hub-mirror/mirror.py b/hub-mirror/mirror.py index a9477800..aec70184 100644 --- a/hub-mirror/mirror.py +++ b/hub-mirror/mirror.py @@ -93,4 +93,4 @@ def push(self, force=False): else: print("(3/3) Force pushing...") cmd = ['-f'] + cmd - local_repo.git.push(*cmd, kill_after_timeout=self.timeout) + local_repo.git.push(*cmd, kill_after_timeout=self.timeout) \ No newline at end of file From f050892eefd4efb97d2a11b943aa48a82540caa9 Mon Sep 17 00:00:00 2001 From: kareem zarka Date: Mon, 17 Jul 2023 23:49:09 +0200 Subject: [PATCH 4/5] mirror/hubmirror: add Git LFS support Implemented Git LFS handling, this includes: Installing LFS during clone/update operations, Pushing LFS objects during push operations. Signed-off-by: kareem zarka --- hub-mirror/hubmirror.py | 4 ++++ hub-mirror/mirror.py | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/hub-mirror/hubmirror.py b/hub-mirror/hubmirror.py index e87eab4c..3db43efe 100644 --- a/hub-mirror/hubmirror.py +++ b/hub-mirror/hubmirror.py @@ -79,6 +79,10 @@ def run(self): cache=self.args.cache_path, timeout=self.args.timeout, force_update=self.args.force_update, + lfs=( + self.args.lfs if hasattr(self.args, "lfs") + else False + ) ) mirror.download() mirror.create() diff --git a/hub-mirror/mirror.py b/hub-mirror/mirror.py index aec70184..9c28eb6b 100644 --- a/hub-mirror/mirror.py +++ b/hub-mirror/mirror.py @@ -11,7 +11,7 @@ class Mirror(object): def __init__( self, hub, src_name, dst_name, - cache='.', timeout='0', force_update=False + cache='.', timeout='0', force_update=False, lfs=False ): self.hub = hub self.src_name = src_name @@ -24,6 +24,7 @@ def __init__( else: self.timeout = 0 self.force_update = force_update + self.lfs = lfs @retry(wait=wait_exponential(), reraise=True, stop=stop_after_attempt(3)) def _clone(self): @@ -34,12 +35,17 @@ def _clone(self): git.cmd.Git.polish_url(self.src_url), self.repo_path, kill_after_timeout=self.timeout ) - print("Clone completed: %s" % os.getcwd() + self.repo_path) + local_repo = git.Repo(self.repo_path) + if self.lfs: + local_repo.git.lfs("fetch", "--all", "origin") + print("Clone completed: %s" % (os.getcwd() + self.repo_path)) @retry(wait=wait_exponential(), reraise=True, stop=stop_after_attempt(3)) def _update(self, local_repo): try: local_repo.git.pull(kill_after_timeout=self.timeout) + if self.lfs: + local_repo.git.lfs("fetch", "--all", "origin") except git.exc.GitCommandError: # Cleanup local repo and re-clone print('Updating failed, re-clone %s' % self.src_name) @@ -71,6 +77,7 @@ def _check_empty(self, repo): @retry(wait=wait_exponential(), reraise=True, stop=stop_after_attempt(3)) def push(self, force=False): local_repo = git.Repo(self.repo_path) + git_cmd = local_repo.git if self._check_empty(local_repo): print("Empty repo %s, skip pushing." % self.src_url) return @@ -79,7 +86,7 @@ def push(self, force=False): try: local_repo.create_remote(self.hub.dst_type, self.dst_url) except git.exc.GitCommandError: - print("Remote exsits, re-create: set %s to %s" % ( + print("Remote exists, re-create: set %s to %s" % ( self.hub.dst_type, self.dst_url)) local_repo.delete_remote(self.hub.dst_type) local_repo.create_remote(self.hub.dst_type, self.dst_url) @@ -90,7 +97,11 @@ def push(self, force=False): if not self.force_update: print("(3/3) Pushing...") local_repo.git.push(*cmd, kill_after_timeout=self.timeout) + if self.lfs: + git_cmd.lfs("push", self.hub.dst_type, "--all") else: print("(3/3) Force pushing...") + if self.lfs: + git_cmd.lfs("push", self.hub.dst_type, "--all") cmd = ['-f'] + cmd - local_repo.git.push(*cmd, kill_after_timeout=self.timeout) \ No newline at end of file + local_repo.git.push(*cmd, kill_after_timeout=self.timeout) From 2b75641475826b3263711c4af0786584a203d606 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Tue, 18 Jul 2023 09:34:13 +0800 Subject: [PATCH 5/5] Fix --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index ecca8795..fe6b3543 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -27,7 +27,7 @@ python3 /hub-mirror/hubmirror.py --src "${INPUT_SRC}" --dst "${INPUT_DST}" \ --force-update "${INPUT_FORCE_UPDATE}" \ --debug "${INPUT_DEBUG}" \ --timeout "${INPUT_TIMEOUT}" \ ---mappings "${INPUT_MAPPINGS}"\ +--mappings "${INPUT_MAPPINGS}" \ --lfs "${INPUT_LFS}" # Skip original code