From 9df20fe7ad6fafb3b292c1d290d5e4ef33438616 Mon Sep 17 00:00:00 2001 From: Jakub Tymejczyk Date: Sat, 7 Oct 2023 10:48:18 +0200 Subject: [PATCH] Add support for `folder_format` config option --- README.md | 1 + src/config_parser.py | 10 ++++++++++ src/sync_photos.py | 26 +++++++++++++++++++++----- tests/test_config_parser.py | 11 +++++++++++ tests/test_sync_photos.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3fdebfd61..038a3ae07 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ photos: remove_obsolete: false sync_interval: 500 all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem + folder_format: "%Y/%m" # optional, if set put photos in subfolders according to format filters: # if all_albums is false list of albums to download, if all_albums is true list of ignored albums # if empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders diff --git a/src/config_parser.py b/src/config_parser.py index 58a58f707..03a596396 100644 --- a/src/config_parser.py +++ b/src/config_parser.py @@ -355,3 +355,13 @@ def get_region(config): region = "global" return region + + +def get_folder_format(config): + """Return filename format or None.""" + fmt = None + config_path = ["photos", "folder_format"] + if traverse_config_path(config=config, config_path=config_path): + fmt = get_config_value(config=config, config_path=config_path) + LOGGER.info(f"Using format {fmt}.") + return fmt diff --git a/src/sync_photos.py b/src/sync_photos.py index fbb864e75..95b8466c6 100644 --- a/src/sync_photos.py +++ b/src/sync_photos.py @@ -22,7 +22,7 @@ def photo_wanted(photo, extensions): return False -def generate_file_name(photo, file_size, destination_path): +def generate_file_name(photo, file_size, destination_path, folder_fmt): """Generate full path to file.""" filename = photo.filename name, extension = filename.rsplit(".", 1) if "." in filename else [filename, ""] @@ -40,6 +40,17 @@ def generate_file_name(photo, file_size, destination_path): else f'{"__".join([name, file_size, base64.urlsafe_b64encode(photo.id.encode()).decode()])}.{extension}', ) + if folder_fmt is not None: + folder = photo.created.strftime(folder_fmt) + file_size_id_path = os.path.join( + destination_path, + folder, + f'{"__".join([name, file_size, base64.urlsafe_b64encode(photo.id.encode()).decode()])}' + if extension == "" + else f'{"__".join([name, file_size, base64.urlsafe_b64encode(photo.id.encode()).decode()])}.{extension}', + ) + os.makedirs(os.path.join(destination_path, folder), exist_ok=True) + file_size_id_path_norm = unicodedata.normalize("NFC", file_size_id_path) if os.path.isfile(file_path): @@ -83,10 +94,10 @@ def download_photo(photo, file_size, destination_path): return True -def process_photo(photo, file_size, destination_path, files): +def process_photo(photo, file_size, destination_path, files, folder_fmt): """Process photo details.""" photo_path = generate_file_name( - photo=photo, file_size=file_size, destination_path=destination_path + photo=photo, file_size=file_size, destination_path=destination_path, folder_fmt=folder_fmt ) if file_size not in photo.versions: LOGGER.warning( @@ -101,7 +112,7 @@ def process_photo(photo, file_size, destination_path, files): return True -def sync_album(album, destination_path, file_sizes, extensions=None, files=None): +def sync_album(album, destination_path, file_sizes, extensions=None, files=None, folder_fmt=None): """Sync given album.""" if album is None or destination_path is None or file_sizes is None: return None @@ -110,7 +121,7 @@ def sync_album(album, destination_path, file_sizes, extensions=None, files=None) for photo in album: if photo_wanted(photo, extensions): for file_size in file_sizes: - process_photo(photo, file_size, destination_path, files) + process_photo(photo, file_size, destination_path, files, folder_fmt) else: LOGGER.debug(f"Skipping the unwanted photo {photo.filename}.") for subalbum in album.subalbums: @@ -120,6 +131,7 @@ def sync_album(album, destination_path, file_sizes, extensions=None, files=None) file_sizes, extensions, files, + folder_fmt ) return True @@ -145,6 +157,7 @@ def sync_photos(config, photos): filters = config_parser.get_photos_filters(config=config) files = set() download_all = config_parser.get_photos_all_albums(config=config) + folder_fmt = config_parser.get_folder_format(config=config) if download_all: for album in photos.albums.keys(): sync_album( @@ -153,6 +166,7 @@ def sync_photos(config, photos): file_sizes=filters["file_sizes"], extensions=filters["extensions"], files=files, + folder_fmt=folder_fmt, ) elif filters["albums"]: for album in iter(filters["albums"]): @@ -162,6 +176,7 @@ def sync_photos(config, photos): file_sizes=filters["file_sizes"], extensions=filters["extensions"], files=files, + folder_fmt=folder_fmt, ) else: sync_album( @@ -170,6 +185,7 @@ def sync_photos(config, photos): file_sizes=filters["file_sizes"], extensions=filters["extensions"], files=files, + folder_fmt=folder_fmt, ) if config_parser.get_photos_remove_obsolete(config=config): diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index 402bcc79f..f4df78bae 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -417,3 +417,14 @@ def test_get_all_albums_false(self): config["photos"]["all_albums"] = False self.assertFalse(config_parser.get_photos_all_albums(config=config)) + def test_folder_fmt_empty(self): + """Empty folder_format.""" + config = read_config(config_path=tests.CONFIG_PATH) + self.assertIsNone(config_parser.get_folder_format(config=config)) + + def test_folder_fmt_set(self): + """folder_format is set.""" + config = read_config(config_path=tests.CONFIG_PATH) + config["photos"]["folder_format"] = "%Y/%m" + self.assertEqual(config_parser.get_folder_format(config=config), "%Y/%m") + diff --git a/tests/test_sync_photos.py b/tests/test_sync_photos.py index d1821d887..3bed5002a 100644 --- a/tests/test_sync_photos.py +++ b/tests/test_sync_photos.py @@ -89,6 +89,34 @@ def test_sync_photos_all_albums( self.assertTrue(len(os.listdir(album_0_path)) > 1) self.assertTrue(len(os.listdir(album_1_path)) > 0) + @patch(target="keyring.get_password", return_value=data.VALID_PASSWORD) + @patch( + target="src.config_parser.get_username", return_value=data.AUTHENTICATED_USER + ) + @patch("icloudpy.ICloudPyService") + @patch("src.read_config") + def test_sync_photos_folder_format( + self, mock_read_config, mock_service, mock_get_username, mock_get_password + ): + """Test for successful original photo size download with folder format.""" + mock_service = self.service + config = self.config.copy() + config["photos"]["destination"] = self.destination_path + config["photos"]["folder_format"] = "%Y/%m" + mock_read_config.return_value = config + # Sync original photos + self.assertIsNone( + sync_photos.sync_photos(config=config, photos=mock_service.photos) + ) + album_0_path = os.path.join( + self.destination_path, config["photos"]["filters"]["albums"][0] + ) + album_1_path = os.path.join( + self.destination_path, config["photos"]["filters"]["albums"][1] + ) + self.assertTrue(os.path.isdir(os.path.join(album_0_path, "2020", "08"))) + self.assertTrue(os.path.isdir(os.path.join(album_1_path, "2020", "07"))) + @patch(target="keyring.get_password", return_value=data.VALID_PASSWORD) @patch( target="src.config_parser.get_username", return_value=data.AUTHENTICATED_USER @@ -491,6 +519,7 @@ def versions(self): file_size="medium", destination_path=self.destination_path, files=None, + folder_fmt=None, ) ) @@ -516,6 +545,7 @@ def versions(self): file_size="thumb", destination_path=self.destination_path, files=None, + folder_fmt=None, ) )