From e2654f23ecd52d214ff549b77831ca31aaeda5c5 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 12:29:29 -0800 Subject: [PATCH 1/8] Don't try to encode diff using utf-8 as diff could be from binary file --- colcon_cache/task/lock/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colcon_cache/task/lock/git.py b/colcon_cache/task/lock/git.py index 67cf98e..0375a1c 100644 --- a/colcon_cache/task/lock/git.py +++ b/colcon_cache/task/lock/git.py @@ -124,5 +124,5 @@ def compute_current_checksums(self, args, lockfile): # noqa: D102 lockfile.checksums.reference = h.hexdigest() if diff: - h.update(diff.encode('utf-8')) + h.update(diff) lockfile.checksums.current = h.hexdigest() From 14f8feda70df3a021b905770677556d9175479b0 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 14:21:08 -0800 Subject: [PATCH 2/8] Add Dockerfile --- Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5ab1f71 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.8 + +WORKDIR /usr/src/colcon-cache + +COPY requirements.txt . +COPY test/requirements.txt ./test/ +RUN pip install -r test/requirements.txt + +COPY . . +RUN colcon build \ + --symlink-install + +RUN colcon test && \ + colcon test-result --verbose From a8b1c09a857b372ee7cf4cc706d18209e10ab66e Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 14:59:15 -0800 Subject: [PATCH 3/8] Use git hash object instead of hashing diff output to avoid issues with diffing binary files https://git-scm.com/docs/git-hash-object --- colcon_cache/task/lock/git.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/colcon_cache/task/lock/git.py b/colcon_cache/task/lock/git.py index 0375a1c..f1e0f06 100644 --- a/colcon_cache/task/lock/git.py +++ b/colcon_cache/task/lock/git.py @@ -110,12 +110,6 @@ def compute_current_checksums(self, args, lockfile): # noqa: D102 lockfile.metadata['reference_revision'] = reference_commit.hexsha - diff_args = [] - diff_args.append('--diff-filter={}'.format(args.git_diff_filter)) - diff_args.append(reference_commit.hexsha) - diff_args.append(args.path) - diff = repo.git.diff(diff_args) - h = hashlib.sha1() for _, checksum in lockfile.dependencies.items(): h.update(bytes.fromhex(checksum)) @@ -123,6 +117,12 @@ def compute_current_checksums(self, args, lockfile): # noqa: D102 h.update(bytes.fromhex(reference_commit.hexsha)) lockfile.checksums.reference = h.hexdigest() + diff = repo.index.diff(reference_commit, paths=[args.path]) if diff: - h.update(diff) + for change in diff.iter_change_type(args.git_diff_filter): + if not change.deleted_file: + hash_object = repo.git.hash_object(change.b_path) + h.update(bytes.fromhex(hash_object)) + h.update(change.a_path.encode('utf-8')) + h.update(change.b_path.encode('utf-8')) lockfile.checksums.current = h.hexdigest() From 045de2d6604b1f4404d54ef4877cb8cae881e7a0 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 15:36:05 -0800 Subject: [PATCH 4/8] Diff with respect to reference_commit --- colcon_cache/task/lock/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colcon_cache/task/lock/git.py b/colcon_cache/task/lock/git.py index f1e0f06..2f4953d 100644 --- a/colcon_cache/task/lock/git.py +++ b/colcon_cache/task/lock/git.py @@ -117,7 +117,7 @@ def compute_current_checksums(self, args, lockfile): # noqa: D102 h.update(bytes.fromhex(reference_commit.hexsha)) lockfile.checksums.reference = h.hexdigest() - diff = repo.index.diff(reference_commit, paths=[args.path]) + diff = reference_commit.diff(None, paths=[args.path]) if diff: for change in diff.iter_change_type(args.git_diff_filter): if not change.deleted_file: From 823c60dbe9527092e0edeb79c728adc1b8a0fd64 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 15:36:40 -0800 Subject: [PATCH 5/8] Iterate over change type filters --- colcon_cache/task/lock/git.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/colcon_cache/task/lock/git.py b/colcon_cache/task/lock/git.py index 2f4953d..9570f84 100644 --- a/colcon_cache/task/lock/git.py +++ b/colcon_cache/task/lock/git.py @@ -119,10 +119,11 @@ def compute_current_checksums(self, args, lockfile): # noqa: D102 diff = reference_commit.diff(None, paths=[args.path]) if diff: - for change in diff.iter_change_type(args.git_diff_filter): - if not change.deleted_file: - hash_object = repo.git.hash_object(change.b_path) - h.update(bytes.fromhex(hash_object)) - h.update(change.a_path.encode('utf-8')) - h.update(change.b_path.encode('utf-8')) + for change_type in sorted(list(args.git_diff_filter)): + for change in diff.iter_change_type(change_type): + if not change.deleted_file: + hash_object = repo.git.hash_object(change.b_path) + h.update(bytes.fromhex(hash_object)) + h.update(change.a_path.encode('utf-8')) + h.update(change.b_path.encode('utf-8')) lockfile.checksums.current = h.hexdigest() From b6b48e3fb95fb59f50eba0af7010bd6315e69b80 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 15:49:39 -0800 Subject: [PATCH 6/8] Update hash with change_type --- colcon_cache/task/lock/git.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/colcon_cache/task/lock/git.py b/colcon_cache/task/lock/git.py index 9570f84..f796d7d 100644 --- a/colcon_cache/task/lock/git.py +++ b/colcon_cache/task/lock/git.py @@ -121,9 +121,10 @@ def compute_current_checksums(self, args, lockfile): # noqa: D102 if diff: for change_type in sorted(list(args.git_diff_filter)): for change in diff.iter_change_type(change_type): + h.update(change.change_type.encode('utf-8')) + h.update(change.a_path.encode('utf-8')) + h.update(change.b_path.encode('utf-8')) if not change.deleted_file: hash_object = repo.git.hash_object(change.b_path) h.update(bytes.fromhex(hash_object)) - h.update(change.a_path.encode('utf-8')) - h.update(change.b_path.encode('utf-8')) lockfile.checksums.current = h.hexdigest() From 7c499aba5df15ac07e215278b990ad656c0ce709 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 16:17:33 -0800 Subject: [PATCH 7/8] linting --- colcon_cache/task/lock/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colcon_cache/task/lock/git.py b/colcon_cache/task/lock/git.py index f796d7d..dc52252 100644 --- a/colcon_cache/task/lock/git.py +++ b/colcon_cache/task/lock/git.py @@ -119,7 +119,7 @@ def compute_current_checksums(self, args, lockfile): # noqa: D102 diff = reference_commit.diff(None, paths=[args.path]) if diff: - for change_type in sorted(list(args.git_diff_filter)): + for change_type in sorted(args.git_diff_filter): for change in diff.iter_change_type(change_type): h.update(change.change_type.encode('utf-8')) h.update(change.a_path.encode('utf-8')) From 1f7cca248ed08583679c479af3e175f3aa1737e4 Mon Sep 17 00:00:00 2001 From: ruffsl Date: Wed, 24 Nov 2021 16:26:50 -0800 Subject: [PATCH 8/8] Test deleted file change type --- test/resources/test_src/test-repo/test-package-c/trash.txt | 0 test/test_main.py | 4 ++++ 2 files changed, 4 insertions(+) create mode 100644 test/resources/test_src/test-repo/test-package-c/trash.txt diff --git a/test/resources/test_src/test-repo/test-package-c/trash.txt b/test/resources/test_src/test-repo/test-package-c/trash.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/test_main.py b/test/test_main.py index 55b2234..dc8d900 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -100,6 +100,10 @@ def test_main(): with open(test_file, 'w') as f: f.write('def test_empty():\n assert False\n') + test_file = \ + ws_base / 'src' / 'test-repo' / 'test-package-c' / 'trash.txt' + os.remove(test_file) + main(argv=argv + ['cache', 'lock']) main(argv=argv + ['list', '--packages-select-cache-modified'])