diff --git a/src/deadline/client/cli/_groups/manifest_group.py b/src/deadline/client/cli/_groups/manifest_group.py index 867b5ed4..7c78e998 100644 --- a/src/deadline/client/cli/_groups/manifest_group.py +++ b/src/deadline/client/cli/_groups/manifest_group.py @@ -14,9 +14,13 @@ from typing import List import click +from deadline.job_attachments._diff import pretty_print_cli from deadline.job_attachments.api.manifest import ( + _glob_files, + _manifest_diff, _manifest_snapshot, ) +from deadline.job_attachments.models import ManifestDiff from ...exceptions import NonValidInputError from .._common import _handle_error @@ -139,6 +143,12 @@ def manifest_snapshot( default=None, help="Include and exclude config of files and directories to include and exclude. Can be a json file or json string.", ) +@click.option( + "--force-rehash", + default=False, + is_flag=True, + help="Rehash all files to compare using file hashes.", +) @click.option("--json", default=None, is_flag=True, help="Output is printed as JSON for scripting.") @_handle_error def manifest_diff( @@ -147,13 +157,43 @@ def manifest_diff( include: List[str], exclude: List[str], include_exclude_config: str, + force_rehash: bool, json: bool, **args, ): """ Check file differences between a directory and specified manifest. """ - raise NotImplementedError("This CLI is being implemented.") + logger: ClickLogger = ClickLogger(is_json=json) + if not os.path.isfile(manifest): + raise NonValidInputError(f"Specified manifest file {manifest} does not exist. ") + + if not os.path.isdir(root): + raise NonValidInputError(f"Specified root directory {root} does not exist. ") + + # Perform the diff. + differences: ManifestDiff = _manifest_diff( + manifest=manifest, + root=root, + include=include, + exclude=exclude, + include_exclude_config=include_exclude_config, + force_rehash=force_rehash, + logger=logger, + ) + + # Print results to console. + if json: + logger.json(dataclasses.asdict(differences), indent=4) + else: + logger.echo(f"Manifest Diff of root directory: {root}") + all_files = _glob_files( + root=root, + include=include, + exclude=exclude, + include_exclude_config=include_exclude_config, + ) + pretty_print_cli(root=root, all_files=all_files, manifest_diff=differences) @cli_manifest.command( diff --git a/src/deadline/job_attachments/_diff.py b/src/deadline/job_attachments/_diff.py index 0c3e3141..618512d5 100644 --- a/src/deadline/job_attachments/_diff.py +++ b/src/deadline/job_attachments/_diff.py @@ -1,6 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. import concurrent.futures +import logging import os from pathlib import Path, PurePosixPath from typing import Dict, List, Tuple @@ -12,7 +13,7 @@ BaseManifestPath, ) from deadline.job_attachments.caches.hash_cache import HashCache -from deadline.job_attachments.models import AssetRootManifest, FileStatus +from deadline.job_attachments.models import AssetRootManifest, FileStatus, ManifestDiff from deadline.job_attachments.upload import S3AssetManager @@ -117,16 +118,30 @@ def compare_manifest( def _fast_file_list_to_manifest_diff( - root: str, current_files: List[str], diff_manifest: BaseAssetManifest, logger: ClickLogger -) -> List[str]: + root: str, + current_files: List[str], + diff_manifest: BaseAssetManifest, + logger: ClickLogger, + return_root_relative_path: bool = True, +) -> List[Tuple[str, FileStatus]]: """ Perform a fast difference of the current list of files to a previous manifest to diff against using time stamps and file sizes. :param root: Root folder of files to diff against. :param current_files: List of files to compare with. :param diff_manifest: Manifest containing files to diff against. - :return List[str]: List of files that are new, or modified. + :param return_root_relative_path: File Path to return, either relative to root or full. + :param logger: logger. + :return List[Tuple[str, FileStatus]]: List of Tuple containing the file path and FileStatus pair. """ - changed_paths: List[str] = [] + + # Select either relative or absolut path for results. + def select_path(full_path: str, relative_path: str, return_root_relative_path: bool): + if return_root_relative_path: + return relative_path + else: + return full_path + + changed_paths: List[Tuple[str, FileStatus]] = [] input_files_map: Dict[str, BaseManifestPath] = {} for input_file in diff_manifest.paths: # Normalize paths so we can compare different OSes @@ -134,6 +149,7 @@ def _fast_file_list_to_manifest_diff( input_files_map[normalized_path] = input_file # Iterate for each file that we found in glob. + root_relative_paths: List[str] = [] for local_file in current_files: # Get the file's time stamp and size. We want to compare both. # From enabling CRT, sometimes timestamp update can fail. @@ -141,23 +157,146 @@ def _fast_file_list_to_manifest_diff( file_stat = local_file_path.stat() # Compare the glob against the relative path we store in the manifest. + # Save it to a list so we can look for deleted files. root_relative_path = str(PurePosixPath(*local_file_path.relative_to(root).parts)) + root_relative_paths.append(root_relative_path) + + return_path = select_path( + full_path=local_file, + relative_path=root_relative_path, + return_root_relative_path=return_root_relative_path, + ) if root_relative_path not in input_files_map: # This is a new file logger.echo(f"Found difference at: {root_relative_path}, Status: FileStatus.NEW") - changed_paths.append(local_file) + changed_paths.append((return_path, FileStatus.NEW)) else: # This is a modified file, compare with manifest relative timestamp. input_file = input_files_map[root_relative_path] # Check file size first as it is easier to test. Usually modified files will also have size diff. if file_stat.st_size != input_file.size: - changed_paths.append(local_file) + changed_paths.append((return_path, FileStatus.MODIFIED)) logger.echo( f"Found size difference at: {root_relative_path}, Status: FileStatus.MODIFIED" ) elif int(file_stat.st_mtime_ns // 1000) != input_file.mtime: - changed_paths.append(local_file) + changed_paths.append((return_path, FileStatus.MODIFIED)) logger.echo( f"Found time difference at: {root_relative_path}, Status: FileStatus.MODIFIED" ) + + # Find deleted files. Manifest store files in relative form. + for manifest_file_path in diff_manifest.paths: + if manifest_file_path.path not in root_relative_paths: + full_path = os.path.join(root, manifest_file_path.path) + return_path = select_path( + full_path=full_path, + relative_path=manifest_file_path.path, + return_root_relative_path=return_root_relative_path, + ) + changed_paths.append((return_path, FileStatus.DELETED)) return changed_paths + + +def pretty_print_cli(root: str, all_files: List[str], manifest_diff: ManifestDiff): + """ + Prints to command line a formatted file tree structure with corresponding file statuses + """ + + # ASCII characters for the tree structure + PIPE = "│" + HORIZONTAL = "──" + ELBOW = "└" + TEE = "├" + SPACE = " " + + # ANSI escape sequences for colors + COLORS = { + "MODIFIED": "\033[93m", # yellow + "NEW": "\033[92m", # green + "DELETED": "\033[91m", # red + "UNCHANGED": "\033[90m", # grey + "RESET": "\033[0m", # base color + "DIRECTORY": "\033[80m", # grey + } + + # Tooltips: + TOOLTIPS = { + FileStatus.NEW: " +", # added files + FileStatus.DELETED: " -", # deleted files + FileStatus.MODIFIED: " M", # modified files + FileStatus.UNCHANGED: "", # unchanged files + } + + class ColorFormatter(logging.Formatter): + def format(self, record): + message = super().format(record) + return f"{message}" + + # Configure logger + formatter = ColorFormatter("") + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger = logging.getLogger(__name__) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.propagate = False + + def print_tree(directory_tree, prefix=""): + sorted_entries = sorted(directory_tree.items()) + + for i, (entry, subtree) in enumerate(sorted_entries, start=1): + is_last_entry = i == len(sorted_entries) + symbol = ELBOW + HORIZONTAL if is_last_entry else TEE + HORIZONTAL + is_dir = isinstance(subtree, dict) + color = COLORS["DIRECTORY"] if is_dir else COLORS[subtree.name] + tooltip = TOOLTIPS[FileStatus.UNCHANGED] if is_dir else TOOLTIPS[subtree] + + message = f"{prefix}{symbol}{color}{entry}{tooltip}{COLORS['RESET']}{os.path.sep if is_dir else ''}" + logger.info(message) + + if is_dir: + new_prefix = prefix + (SPACE if is_last_entry else PIPE + SPACE) + print_tree(subtree, new_prefix) + + if not directory_tree: + symbol = ELBOW + HORIZONTAL + message = f"{prefix}{symbol}{COLORS['UNCHANGED']}. {COLORS['RESET']}" + logger.info(message) + + def get_file_status(file: str, manifest_diff: ManifestDiff): + print(file) + if file in manifest_diff.new: + return FileStatus.NEW + elif file in manifest_diff.modified: + return FileStatus.MODIFIED + elif file in manifest_diff.deleted: + return FileStatus.DELETED + else: + # Default, not in any diff list. + return FileStatus.UNCHANGED + + def build_directory_tree(all_files: List[str]) -> Dict[str, dict]: + directory_tree: dict = {} + + def add_to_tree(path, status): + parts = str(path).split(os.path.sep) + current_level = directory_tree + for i, part in enumerate(parts): + if i == len(parts) - 1: + current_level[part] = status + else: + current_level = current_level.setdefault(part, {}) + + for file in all_files: + print(f"{file} {root}") + relative_path = str(Path(file).relative_to(root)) + add_to_tree( + relative_path, + get_file_status(relative_path, manifest_diff), + ) + return directory_tree + + directory_tree = build_directory_tree(all_files) + print_tree(directory_tree) + logger.info("") diff --git a/src/deadline/job_attachments/api/manifest.py b/src/deadline/job_attachments/api/manifest.py index c5f149b7..1511616f 100644 --- a/src/deadline/job_attachments/api/manifest.py +++ b/src/deadline/job_attachments/api/manifest.py @@ -2,6 +2,7 @@ import datetime import os +from pathlib import Path from typing import List, Optional, Tuple from deadline.client.cli._groups.click_logger import ClickLogger @@ -11,16 +12,21 @@ _create_manifest_for_single_root, ) from deadline.job_attachments.asset_manifests.base_manifest import ( + BaseAssetManifest, BaseManifestPath, ) +from deadline.client.config import config_file from deadline.job_attachments.asset_manifests.decode import decode_manifest from deadline.job_attachments.asset_manifests.hash_algorithms import hash_data +from deadline.job_attachments.caches.hash_cache import HashCache from deadline.job_attachments.models import ( FileStatus, GlobConfig, + ManifestDiff, ManifestSnapshot, default_glob_all, ) +from deadline.job_attachments.upload import S3AssetManager """ APIs here should be business logic only. It should perform one thing, and one thing well. @@ -29,6 +35,37 @@ """ +def _glob_files( + root: str, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + include_exclude_config: Optional[str] = None, +) -> List[str]: + """ + :param include: Include glob to look for files to add to the manifest. + :param exclude: Exclude glob to exclude files from the manifest. + :param include_exclude_config: Config JSON or file containeing input and exclude config. + :returns: All files matching the include and exclude expressions. + """ + + # Get all files in the root. + glob_config: GlobConfig + if include or exclude: + include = include if include is not None else default_glob_all() + exclude = exclude if exclude is not None else [] + glob_config = GlobConfig(include_glob=include, exclude_glob=exclude) + elif include_exclude_config: + glob_config = _process_glob_inputs(include_exclude_config) + else: + # Default, include all. + glob_config = GlobConfig() + + input_files = _glob_paths( + root, include=glob_config.include_glob, exclude=glob_config.exclude_glob + ) + return input_files + + def _manifest_snapshot( root: str, destination: str, @@ -77,9 +114,17 @@ def _manifest_snapshot( # Fast comparison using time stamps and sizes. if not force_rehash: - changed_paths = _fast_file_list_to_manifest_diff( - root, current_files, source_manifest, logger + diff_list: List[Tuple[str, FileStatus]] = _fast_file_list_to_manifest_diff( + root=root, + current_files=current_files, + diff_manifest=source_manifest, + logger=logger, + return_root_relative_path=False, ) + for diff_file in diff_list: + # Add all new and modified + if diff_file[1] != FileStatus.DELETED: + changed_paths.append(diff_file[0]) else: # In "slow / thorough" mode, we check by hash, which is definitive. output_manifest = _create_manifest_for_single_root( @@ -128,3 +173,78 @@ def _manifest_snapshot( # No manifest generated. logger.echo("No manifest generated") return None + + +def _manifest_diff( + manifest: str, + root: str, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + include_exclude_config: Optional[str] = None, + force_rehash=False, + logger: ClickLogger = ClickLogger(False), +) -> ManifestDiff: + """ + BETA API - This API is still evolving but will be made public in the near future. + API to diff a manifest root with a previously snapshotted manifest. + :param manifest: Manifest file path to compare against. + :param root: Root directory to generate the manifest fileset. + :param include: Include glob to look for files to add to the manifest. + :param exclude: Exclude glob to exclude files from the manifest. + :param include_exclude_config: Config JSON or file containeing input and exclude config. + :param logger: Click Logger instance to print to CLI as test or JSON. + :returns: ManifestDiff object containing all new changed, deleted files. + """ + + # Find all files matching our regex + input_files = _glob_files( + root=root, include=include, exclude=exclude, include_exclude_config=include_exclude_config + ) + input_paths = [Path(p) for p in input_files] + + # Placeholder Asset Manager + asset_manager = S3AssetManager() + + # parse the given manifest to compare against. + local_manifest_object: BaseAssetManifest + with open(manifest) as input_file: + manifest_data_str = input_file.read() + local_manifest_object = decode_manifest(manifest_data_str) + + output: ManifestDiff = ManifestDiff() + + if force_rehash: + # hash and create manifest of local directory + cache_config = config_file.get_cache_directory() + with HashCache(cache_config) as hash_cache: + directory_manifest_object = asset_manager._create_manifest_file( + input_paths=input_paths, root_path=root, hash_cache=hash_cache + ) + + # Hash based compare manifests. + differences: List[Tuple[FileStatus, BaseManifestPath]] = compare_manifest( + reference_manifest=local_manifest_object, compare_manifest=directory_manifest_object + ) + # Map to output datastructure. + for item in differences: + if item[0] == FileStatus.MODIFIED: + output.modified.append(item[1].path) + elif item[0] == FileStatus.NEW: + output.new.append(item[1].path) + elif item[0] == FileStatus.DELETED: + output.deleted.append(item[1].path) + + else: + # File based comparisons. + fast_diff: List[Tuple[str, FileStatus]] = _fast_file_list_to_manifest_diff( + root=root, current_files=input_files, diff_manifest=local_manifest_object, logger=logger + ) + for fast_diff_item in fast_diff: + if fast_diff_item[1] == FileStatus.MODIFIED: + output.modified.append(fast_diff_item[0]) + elif fast_diff_item[1] == FileStatus.NEW: + output.new.append(fast_diff_item[0]) + elif fast_diff_item[1] == FileStatus.DELETED: + output.deleted.append(fast_diff_item[0]) + + return output diff --git a/src/deadline/job_attachments/asset_manifests/_create_manifest.py b/src/deadline/job_attachments/asset_manifests/_create_manifest.py index c6ed5a18..f2cb7e4f 100644 --- a/src/deadline/job_attachments/asset_manifests/_create_manifest.py +++ b/src/deadline/job_attachments/asset_manifests/_create_manifest.py @@ -7,7 +7,6 @@ from deadline.client.cli._groups.click_logger import ClickLogger from deadline.job_attachments.asset_manifests.base_manifest import BaseAssetManifest from deadline.job_attachments.exceptions import ManifestCreationException -from deadline.job_attachments.models import JobAttachmentS3Settings from deadline.job_attachments.upload import S3AssetManager @@ -22,9 +21,7 @@ def _create_manifest_for_single_root( :return """ # Placeholder Asset Manager - asset_manager = S3AssetManager( - farm_id=" ", queue_id=" ", job_attachment_settings=JobAttachmentS3Settings(" ", " ") - ) + asset_manager = S3AssetManager() hash_callback_manager = _ProgressBarCallbackManager(length=100, label="Hashing Attachments") diff --git a/src/deadline/job_attachments/upload.py b/src/deadline/job_attachments/upload.py index 2f45f27c..6d157e50 100644 --- a/src/deadline/job_attachments/upload.py +++ b/src/deadline/job_attachments/upload.py @@ -45,6 +45,7 @@ AssetSyncCancelledError, AssetSyncError, JobAttachmentS3BotoCoreError, + JobAttachmentsError, JobAttachmentsS3ClientError, MisconfiguredInputsError, MissingS3BucketError, @@ -744,25 +745,26 @@ class S3AssetManager: def __init__( self, - farm_id: str, - queue_id: str, - job_attachment_settings: JobAttachmentS3Settings, + farm_id: Optional[str] = None, + queue_id: Optional[str] = None, + job_attachment_settings: Optional[JobAttachmentS3Settings] = None, asset_uploader: Optional[S3AssetUploader] = None, session: Optional[boto3.Session] = None, asset_manifest_version: ManifestVersion = ManifestVersion.v2023_03_03, ) -> None: self.farm_id = farm_id self.queue_id = queue_id - self.job_attachment_settings: JobAttachmentS3Settings = job_attachment_settings + self.job_attachment_settings: Optional[JobAttachmentS3Settings] = job_attachment_settings - if not self.job_attachment_settings.s3BucketName: - raise MissingS3BucketError( - "To use Job Attachments, the 's3BucketName' must be set in your queue's JobAttachmentSettings" - ) - if not self.job_attachment_settings.rootPrefix: - raise MissingS3RootPrefixError( - "To use Job Attachments, the 'rootPrefix' must be set in your queue's JobAttachmentSettings" - ) + if self.job_attachment_settings: + if not self.job_attachment_settings.s3BucketName: + raise MissingS3BucketError( + "To use Job Attachments, the 's3BucketName' must be set in your queue's JobAttachmentSettings" + ) + if not self.job_attachment_settings.rootPrefix: + raise MissingS3RootPrefixError( + "To use Job Attachments, the 'rootPrefix' must be set in your queue's JobAttachmentSettings" + ) if asset_uploader is None: asset_uploader = S3AssetUploader(session=session) @@ -1241,6 +1243,11 @@ def upload_assets( a tuple with (1) the summary statistics of the upload operation, and (2) the S3 path to the asset manifest file. """ + # This is a programming error if the user did not construct the object with Farm and Queue IDs. + if not self.farm_id or not self.queue_id: + logger.error("upload_assets: Farm or Fleet ID is missing.") + raise JobAttachmentsError("upload_assets: Farm or Fleet ID is missing.") + # Sets up progress tracker to report upload progress back to the caller. (input_files, input_bytes) = self._get_total_input_size_from_manifests(manifests) progress_tracker = ProgressTracker( @@ -1269,9 +1276,9 @@ def upload_assets( if asset_root_manifest.asset_manifest: (partial_manifest_key, asset_manifest_hash) = self.asset_uploader.upload_assets( - job_attachment_settings=self.job_attachment_settings, + job_attachment_settings=self.job_attachment_settings, # type: ignore[arg-type] manifest=asset_root_manifest.asset_manifest, - partial_manifest_prefix=self.job_attachment_settings.partial_manifest_prefix( + partial_manifest_prefix=self.job_attachment_settings.partial_manifest_prefix( # type: ignore[union-attr] self.farm_id, self.queue_id ), source_root=Path(asset_root_manifest.root_path), diff --git a/src/hello b/src/hello new file mode 100644 index 00000000..e69de29b diff --git a/test/unit/deadline_client/cli/test_cli_manifest_diff.py b/test/unit/deadline_client/cli/test_cli_manifest_diff.py new file mode 100644 index 00000000..be17fd2c --- /dev/null +++ b/test/unit/deadline_client/cli/test_cli_manifest_diff.py @@ -0,0 +1,106 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +""" +Integ tests for the CLI asset commands. +""" +import json +import os +from pathlib import Path +from click.testing import CliRunner +from deadline.client.cli._groups.manifest_group import cli_manifest +import pytest +import tempfile + +from deadline.client.cli import main + + +@pytest.mark.skip("Random Failure with no credentials on Github") +class TestSnapshot: + + @pytest.fixture + def temp_dir(self): + with tempfile.TemporaryDirectory() as tmpdir_path: + yield tmpdir_path + + def _create_test_manifest(self, tmp_path: str, root_dir: str) -> str: + """ + Create some test files in the temp dir, snapshot it and return the manifest file. + """ + TEST_MANIFEST_DIR = "manifest_dir" + + # Given + manifest_dir = os.path.join(tmp_path, TEST_MANIFEST_DIR) + os.makedirs(manifest_dir) + + subdir1 = os.path.join(root_dir, "subdir1") + subdir2 = os.path.join(root_dir, "subdir2") + os.makedirs(subdir1) + os.makedirs(subdir2) + Path(os.path.join(subdir1, "file1.txt")).touch() + Path(os.path.join(subdir2, "file2.txt")).touch() + + # When snapshot is called. + runner = CliRunner() + main.add_command(cli_manifest) + result = runner.invoke( + main, + [ + "manifest", + "snapshot", + "--root", + root_dir, + "--destination", + manifest_dir, + "--name", + "test", + ], + ) + assert result.exit_code == 0, result.output + + manifest_files = os.listdir(manifest_dir) + assert ( + len(manifest_files) == 1 + ), f"Expected exactly one manifest file, but got {len(manifest_files)}" + manifest = manifest_files[0] + assert "test" in manifest, f"Expected test in manifest file name, got {manifest}" + + # Return the manifest that we found. + return os.path.join(manifest_dir, manifest) + + @pytest.mark.parametrize( + "json_output", + [ + pytest.param(True), + pytest.param(False), + ], + ) + def test_manifest_diff(self, tmp_path: str, json_output: bool): + """ + Tests if manifest diff CLI works, basic case. Variation on JSON as printout. + Business logic testing will be done at the API level where we can check the outputs. + """ + TEST_ROOT_DIR = "root_dir" + + # Given a created manifest file... + root_dir = os.path.join(tmp_path, TEST_ROOT_DIR) + manifest = self._create_test_manifest(tmp_path, root_dir) + + # Lets add another file. + new_file = "file3.txt" + Path(os.path.join(root_dir, new_file)).touch() + + # When + runner = CliRunner() + main.add_command(cli_manifest) + args = ["manifest", "diff", "--root", root_dir, "--manifest", manifest] + if json_output: + args.append("--json") + result = runner.invoke(main, args) + + # Then + assert result.exit_code == 0, result.output + if json_output: + # If JSON mode was specified, make sure the output is JSON and contains the new file. + diff = json.loads(result.output) + assert len(diff["new"]) == 1 + assert new_file in diff["new"] diff --git a/test/unit/deadline_job_attachments/api/test_manifest_diff.py b/test/unit/deadline_job_attachments/api/test_manifest_diff.py new file mode 100644 index 00000000..ca0e6257 --- /dev/null +++ b/test/unit/deadline_job_attachments/api/test_manifest_diff.py @@ -0,0 +1,130 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import json +import os +from pathlib import Path +import tempfile +from typing import Optional +from deadline.job_attachments.api.manifest import _manifest_diff, _manifest_snapshot +from deadline.job_attachments.models import ManifestDiff, ManifestSnapshot +import pytest + + +TEST_FILE = "test_file" + + +class TestDiffAPI: + + @pytest.fixture + def temp_dir(self): + with tempfile.TemporaryDirectory() as tmpdir_path: + yield tmpdir_path + + def _snapshot_folder_helper(self, temp_dir, root_dir) -> str: + """ + Snapshot with a folder and a single file in it. Should generate a manifest containing 1 file. + """ + + # Given snapshot folder and 1 test file + test_file_name = TEST_FILE + test_file = os.path.join(root_dir, test_file_name) + os.makedirs(os.path.dirname(test_file), exist_ok=True) + with open(test_file, "w") as f: + f.write("testing123") + + # When + manifest: Optional[ManifestSnapshot] = _manifest_snapshot( + root=root_dir, destination=temp_dir, name="test" + ) + + # Then + assert manifest is not None + assert manifest.manifest is not None + with open(manifest.manifest, "r") as manifest_file: + manifest_payload = json.load(manifest_file) + assert len(manifest_payload["paths"]) == 1 + assert manifest_payload["paths"][0]["path"] == test_file_name + + # Return the tested manifest. + return manifest.manifest + + def test_diff_no_change(self, temp_dir): + """ + Diff with the same folder, no new files. Should return with all empty no diff result. + """ + # Given + root_dir = os.path.join(temp_dir, "snapshot") + manifest_file = self._snapshot_folder_helper(temp_dir=temp_dir, root_dir=root_dir) + + # When + manifest_diff: ManifestDiff = _manifest_diff(root=root_dir, manifest=manifest_file) + assert len(manifest_diff.deleted) == 0 + assert len(manifest_diff.modified) == 0 + assert len(manifest_diff.new) == 0 + + def test_diff_new_files(self, temp_dir): + """ + Diff with the same folder, new files. Should return with all empty no diff result. + """ + # Given + root_dir = os.path.join(temp_dir, "snapshot") + manifest_file = self._snapshot_folder_helper(temp_dir=temp_dir, root_dir=root_dir) + + # When + # Make 2 new files, one in the snapshot dir, another in a nested dir. + new_file_name = "new_file" + new_file = os.path.join(root_dir, new_file_name) + Path(new_file).touch() + + new_dir = "new_dir" + new_file2_name = "new_file2" + new_file2 = os.path.join(root_dir, new_dir, new_file2_name) + os.makedirs(os.path.dirname(new_file2), exist_ok=True) + Path(new_file2).touch() + + # Then + manifest_diff: ManifestDiff = _manifest_diff(root=root_dir, manifest=manifest_file) + assert len(manifest_diff.deleted) == 0 + assert len(manifest_diff.modified) == 0 + assert len(manifest_diff.new) == 2 + assert new_file_name in manifest_diff.new + assert f"{new_dir}/{new_file2_name}" in manifest_diff.new + + def test_diff_deleted_file(self, temp_dir): + """ + Diff with the same folder, delete the test file. It should be found by delete. + """ + # Given + root_dir = os.path.join(temp_dir, "snapshot") + manifest_file = self._snapshot_folder_helper(temp_dir=temp_dir, root_dir=root_dir) + + # When + os.remove(os.path.join(root_dir, TEST_FILE)) + manifest_diff: ManifestDiff = _manifest_diff(root=root_dir, manifest=manifest_file) + + # Then + assert len(manifest_diff.modified) == 0 + assert len(manifest_diff.new) == 0 + assert len(manifest_diff.deleted) == 1 + assert TEST_FILE in manifest_diff.deleted + + def test_diff_modified_file_size(self, temp_dir): + """ + Diff with the same folder, modified the test file. It should be found by modified. + """ + # Given + root_dir = os.path.join(temp_dir, "snapshot") + manifest_file = self._snapshot_folder_helper(temp_dir=temp_dir, root_dir=root_dir) + + # When + test_file = os.path.join(root_dir, TEST_FILE) + with open(test_file, "w") as f: + f.write("something_different") + + manifest_diff: ManifestDiff = _manifest_diff(root=root_dir, manifest=manifest_file) + + # Then + assert len(manifest_diff.new) == 0 + assert len(manifest_diff.deleted) == 0 + assert len(manifest_diff.modified) == 1 + assert TEST_FILE in manifest_diff.modified