fixup! [u] Fix: Partition sizing ignores supplementary bundles (#5207)

DataBiosphere · May 20, 2023 · 657bb30 · 657bb30
1 parent 1881c3d
commit 657bb30
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 44 deletions.
diff --git a/deployments/anvilbox/sources.json b/deployments/anvilbox/sources.json
diff --git a/deployments/anvildev/sources.json b/deployments/anvildev/sources.json
diff --git a/deployments/anvilprod/environment.py b/deployments/anvilprod/environment.py
@@ -56,9 +56,9 @@ def mkdict(previous_catalog: dict[str, str],
 
 
 anvil_sources = mkdict({}, 3, mkdelta([
-    mksrc('datarepo-dev-43738c90', 'ANVIL_1000G_2019_Dev_20230302_ANV5_202303032342', 6404),
-    mksrc('datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 25),
-    mksrc('datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25),
+    mksrc('datarepo-dev-43738c90', 'ANVIL_1000G_2019_Dev_20230302_ANV5_202303032342', 22814),
+    mksrc('datarepo-dev-42c70e6a', 'ANVIL_CCDG_Sample_1_20230228_ANV5_202302281520', 28),
+    mksrc('datarepo-dev-97ad270b', 'ANVIL_CMG_Sample_1_20230225_ANV5_202302281509', 25)
 ]))
 
 

diff --git a/scripts/generate_sources.py → scripts/update_subgraph_counts.py b/scripts/generate_sources.py → scripts/update_subgraph_counts.py
@@ -1,12 +1,9 @@
 """
-Determine sizes
+Count the number of subgraphs per configured source and produce output to
+expedite updated source configurations.
 """
 
 import argparse
-import json
-from pathlib import (
-    Path,
-)
 import sys
 
 import attr
@@ -23,6 +20,9 @@
 from azul.indexer import (
     Prefix,
 )
+from azul.openapi import (
+    format_description,
+)
 from azul.terra import (
     TDRSourceSpec,
 )
@@ -52,27 +52,17 @@ def __str__(self) -> str:
 
 azul = AzulClient()
 
-active_deployment_dir = Path(config.project_root) / 'deployments' / '.active'
-with open(active_deployment_dir / 'sources.json') as f:
-    raw_sources = json.load(f)
-
 
 def generate_sources(catalog: str) -> list[SourceSpecArgs]:
-    catalog_sources = raw_sources[catalog]
     plugin = azul.repository_plugin(catalog)
     sources = []
-    for source in catalog_sources:
-        project, snapshot = source.split('.')
-        spec = TDRSourceSpec(project=project,
-                             name=snapshot,
-                             is_snapshot=True,
-                             prefix=Prefix.of_everything)
+    for spec in plugin.sources:
+        spec: TDRSourceSpec = attr.evolve(spec, prefix=Prefix.of_everything)
         ref = plugin.resolve_source(str(spec))
         partitions = plugin.list_partitions(ref)
-        sources.append(SourceSpecArgs(project=project,
-                                      snapshot=snapshot,
+        sources.append(SourceSpecArgs(project=spec.project,
+                                      snapshot=spec.name,
                                       subgraph_count=sum(partitions.values())))
-
     return sources
 
 
@@ -84,11 +74,18 @@ def main(args: list[str]):
         print('-' * len(catalog))
         spec_args_list = generate_sources(catalog)
         spec_args_list.sort(key=lambda spec_args: spec_args.snapshot)
-        print(',\n'.join(map(str, spec_args_list)))
-
-    msg = "Don't forget to manually specify flags for `ma` and `pop`."
-    print('-' * len(msg))
-    print(msg)
+        print(',\n'.join(map(str, spec_args_list)), end='\n\n')
+
+    print(format_description('''
+        -----------------
+        !!! IMPORTANT !!!
+        -----------------
+
+        This script does *not* populate the `ma` or `pop` flags for the source
+        specs. Do not copy/paste the above output without checking whether these
+        flags should be applied. If `mksrc` generates a common prefix, manual
+        adjustment of the generated common prefix may be required.
+    '''))
 
 
 if __name__ == '__main__':