clean-assets: delete release in case all assets need to be deleted

In case a release has hundreds of files that need to be deleted this
requires quite a bit of time and also works against the API rate limiting.

In case we want to delete all assets of an release just delete and
re-create the whole release instead.

Fixes msys2#77

msys2_autobuild/cmd_clean_assets.py

@@ -1,60 +1,83 @@
+import re
 import fnmatch
-from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Dict, List
+from typing import Any, List, Tuple
 
 from github.GitReleaseAsset import GitReleaseAsset
+from github.GitRelease import GitRelease
 
 from .config import get_all_build_types
 from .gh import (get_asset_filename, get_current_repo, get_release,
                  get_release_assets, make_writable)
 from .queue import get_buildqueue
 
 
-def get_assets_to_delete() -> List[GitReleaseAsset]:
-    repo = get_current_repo()
+def get_assets_to_delete() -> Tuple[List[GitRelease], List[GitReleaseAsset]]:
 
     print("Fetching packages to build...")
-    patterns = []
+    keep_patterns = []
     for pkg in get_buildqueue():
         for build_type in pkg.get_build_types():
-            patterns.append(pkg.get_failed_name(build_type))
-            patterns.extend(pkg.get_build_patterns(build_type))
+            keep_patterns.append(pkg.get_failed_name(build_type))
+            keep_patterns.extend(pkg.get_build_patterns(build_type))
+    keep_pattern_regex = re.compile('|'.join(fnmatch.translate(p) for p in keep_patterns))
 
-    print("Fetching assets...")
-    assets: Dict[str, List[GitReleaseAsset]] = {}
-    for build_type in get_all_build_types():
-        release = get_release(repo, "staging-" + build_type)
-        for asset in get_release_assets(release, include_incomplete=True):
-            assets.setdefault(get_asset_filename(asset), []).append(asset)
+    def should_be_deleted(asset: GitReleaseAsset) -> bool:
+        filename = get_asset_filename(asset)
+        return not keep_pattern_regex.match(filename)
+
+    def get_to_delete(release: GitRelease) -> Tuple[List[GitRelease], List[GitReleaseAsset]]:
+        assets = get_release_assets(release, include_incomplete=True)
+        to_delete = []
+        for asset in assets:
+            if should_be_deleted(asset):
+                to_delete.append(asset)
+
+        # Deleting and re-creating a release requires two write calls, so delete
+        # the release if all assets should be deleted and there are more than 2.
+        if len(to_delete) > 2 and len(assets) == len(to_delete):
+            return [release], []
+        else:
+            return [], to_delete
+
+    def get_all_releases() -> List[GitRelease]:
+        repo = get_current_repo()
 
-    release = get_release(repo, "staging-failed")
-    for asset in get_release_assets(release, include_incomplete=True):
-        assets.setdefault(get_asset_filename(asset), []).append(asset)
+        releases = []
+        for build_type in get_all_build_types():
+            releases.append(get_release(repo, "staging-" + build_type))
+        releases.append(get_release(repo, "staging-failed"))
+        return releases
 
-    for pattern in patterns:
-        for key in fnmatch.filter(assets.keys(), pattern):
-            del assets[key]
+    print("Fetching assets...")
+    releases = []
+    assets = []
+    for release in get_all_releases():
+        r, a = get_to_delete(release)
+        releases.extend(r)
+        assets.extend(a)
 
-    result = []
-    for items in assets.values():
-        for asset in items:
-            result.append(asset)
-    return result
+    return releases, assets
 
 
 def clean_gha_assets(args: Any) -> None:
-    assets = get_assets_to_delete()
+    repo = get_current_repo()
+    releases, assets = get_assets_to_delete()
 
-    def delete_asset(asset: GitReleaseAsset) -> None:
+    print("Resetting releases...")
+    for release in releases:
+        print(f"Resetting {release.tag_name}...")
+        if not args.dry_run:
+            with make_writable(release):
+                release.delete_release()
+            get_release(repo, release.tag_name)
+
+    print("Deleting assets...")
+    for asset in assets:
         print(f"Deleting {get_asset_filename(asset)}...")
         if not args.dry_run:
             with make_writable(asset):
                 asset.delete_asset()
 
-    with ThreadPoolExecutor(2) as executor:
-        for item in executor.map(delete_asset, assets):
-            pass
-
 
 def add_parser(subparsers: Any) -> None:
     sub = subparsers.add_parser("clean-assets", help="Clean up GHA assets", allow_abbrev=False)