Refactoring for 0.2.0

cultcargo/builder/build_cargo.py

@@ -56,7 +56,7 @@ def run(command, cwd=None, input=None):
     args = command.split()
     result = subprocess.run(args, cwd=cwd, input=input, text=True)
     if result.returncode:
-        print(f"{command} failed with exit code {result.returncode}")
+        print(f"[bold red]{command} failed with exit code {result.returncode}[/bold red]")
         sys.exit(1)
     return 0
 
@@ -75,10 +75,12 @@ def run(command, cwd=None, input=None):
 @click.option('-a', '--all', is_flag=True, help='Build and/or push all images in manifest.')
 @click.option('-E', '--experimental', is_flag=True, help='Enable experimental versions.')
 @click.option('-v', '--verbose', is_flag=True, help='Be verbose.')
+@click.option('--no-tests', is_flag=True, help='Skip image tests during the list or build.')
 @click.option('--ignore-latest-tag', is_flag=True, help='Neither require nor apply latest tag.')
 @click.option('--boring', is_flag=True, help='Be boring -- no progress bar.')
 @click.argument('imagenames', type=str, nargs=-1)
 def build_cargo(manifest: str, do_list=False, build=False, push=False, all=False, rebuild=False, boring=False,
+                no_tests=False,
                 experimental=False, ignore_latest_tag=False, verbose=False, imagenames: List[str] = []):
     if not (build or push or do_list):
         build = push = True
@@ -145,7 +147,6 @@ def build_cargo(manifest: str, do_list=False, build=False, push=False, all=False
                 print(f"  [red]Failed to fetch release info: {response.status_code}[/red]")
                 sys.exit(1)
 
-
         # get registry
         def resolve_config_reference(value):
             comps =  value.split("::")
@@ -248,7 +249,7 @@ def resolve_config_reference(value):
             image_info = conf.images[image]
             image_vars = global_vars.copy()
             image_vars.update(IMAGE=image, **(image_info.assign or {}))
-            image_vars.setdefault("CMD", image)
+            image_vars.setdefault("CMD", f"{image} --help")
 
             path = os.path.join(global_vars.BASE_IMAGE_PATH, image).format(**image_vars)
 
@@ -329,6 +330,11 @@ def resolve_config_reference(value):
                     # is this the latest version that needs to be tagged
                     if image_version == tag_latest.get(image):
                         run(f"docker tag {registry}/{image}:{image_version} {registry}/{image}:{BUNDLE_VERSION}")
+
+                # run the image tests
+                if not no_tests:
+                    print(f"Running sanity check of {full_image}")
+                    run(f"docker run {full_image}", input="")
 
                 # go push
                 if push:

cultcargo/builder/cargo-manifest.yml

@@ -18,15 +18,46 @@ assign:
   post_install: ''
   extra_deps: ''
 
-  # base image for generic Python-based packages
-  base_python_image: python-astro:3.9
-  # corresponding python binary
-  python: python3.9
+  default_python_version: '3.10'
+  # slim_python_image: python:3.10-slim
+  # # base cult image for generic Python-based packages
+  # base_python_image: python-base:3.10
+  # # corresponding python binary
+  # python: python3.10
 
 images:
+  # Bare-bones base images based off kern, with updates applied
+  # These are suitable as base images for binary installs
   base-cult:
     versions:
-      base: {}
+      'kern7':
+        kern_version: 7
+      'kern8':
+        kern_version: 8
+      'kern9':
+        kern_version: 9
+
+  # This adds a specific version of python on top of a base image, and upgrades pip
+  # These are suitable as base images for pip installs
+  python-base:
+    versions:
+#      '3.12': {}   # having some trouble with 3.12 so ignoring it for now
+      '3.11': {}
+      '3.10': {}
+      '3.9': {}
+      '3.8': {}
+
+  # Python base will all the astro goodies (and stimela) thrown in
+  # This is the default image stimela uses for python cabs
+  python-astro:
+    versions:
+#      '3.12': {}
+      '3.11': {}
+      '3.10': {}
+      '3.9': {}
+      '3.8': {}
+
+  # Package-specific images follow
 
   aimfast:
     versions:
@@ -42,33 +73,35 @@ images:
 
   wsclean:
     versions:
-      kern7:
-        dockerfile: Dockerfile.kern7
-      ## 2.9 won't build with this recipe, figure it out later
+      # these versions shipped in KERN
+      '2.10.1-kern7':  # frozen version
+        kern_version: 7
+        dockerfile: Dockerfile.kern
+      '3.1-kern8':     # frozen version
+        kern_version: 8
+        dockerfile: Dockerfile.kern
+      'kern9':
+        kern_version: 9
+        dockerfile: Dockerfile.kern
+      ## 2.9/10 won't build with this recipe, figure it out later
       # '2.9':
       #   dockerfile: Dockerfile.build.2x
       #   tag: wsclean2.9
-      '2.10.1':
-        dockerfile: Dockerfile.build
-        tag: v2.10.1
+      # '2.10.1':
+      #   dockerfile: Dockerfile.build.2x
+      #   tag: v2.10.1
       '3.0.1':
         dockerfile: Dockerfile.build
-        tag: v3.0.1
       '3.1':
         dockerfile: Dockerfile.build
-        tag: v3.1
       '3.2':
         dockerfile: Dockerfile.build
-        tag: v3.2
       '3.3':
         dockerfile: Dockerfile.build
-        tag: v3.3
-
-  python-astro:
-    versions:
-      '3.10': {}
-      '3.9': {}
-      '3.8': {}
+      '3.4':
+        dockerfile: Dockerfile.build
+      '3.5':
+        dockerfile: Dockerfile.build
 
   casa:
     assign:
@@ -85,10 +118,10 @@ images:
 
   casa6:
     versions:
-      '6.5':
-          wheel_version: 6.5.6.22
+      # '6.5':  # deprecated
+      #     wheel_version: 6.5.6.22
       '6.6':
-          wheel_version: 6.6.3.22
+          wheel_version: 6.6.5.31
 
   breizorro:
     versions:
@@ -109,24 +142,30 @@ images:
         package: git+https://github.com/ratt-ru/shadeMS
 
   crystalball:
+    assign:
+      extra_deps: -U matplotlib
     versions:
-      # '0.3.0':
-      #   package: crystalball==0.3.0
-#      latest:
-#         package: crystalball  # doesn't work?
       master:
         package: git+https://github.com/caracal-pipeline/crystalball
+      '0.4.2': 
+        package: crystalball==0.4.2
+      latest:
+        package: crystalball  
 
   tricolour:
-    # assign:
+    assign:
+      default_python_version: '3.8'
+      # python: python3.8
+      # extra_deps: -U scipy 
     #  pre_install: RUN pip uninstall -y bokeh # bokeh causes deps conflict with numpy pin in tricolour
-    #   extra_deps: numpy\<1.20 numba==0.54.0 bokeh\<3 psutil
+    #  extra_deps: numpy\<1.20 # numba==0.54.0 bokeh\<3 psutil
     versions:
       '0.1.8.1':
-        package: tricolour==0.1.8
+        package: tricolour==0.1.8.1
       latest:
         package: tricolour
       master:
+        # pre_install: RUN pip install -U
         package: git+https://github.com/ratt-ru/tricolour
 
   smops:
@@ -149,12 +188,16 @@ images:
 
   quartical:
     versions:
+      master:
+        package: git+https://github.com/ratt-ru/QuartiCal
       '0.2.2':
         package: quartical==0.2.2
+      '0.2.3':
+        package: quartical==0.2.3
+      '0.2.4':
+        package: quartical==0.2.4
       latest:
         package: quartical
-      master:
-        package: git+https://github.com/ratt-ru/QuartiCal
 
   cubical:
     versions:
@@ -165,10 +208,12 @@ images:
       master:
         package: git+https://github.com/ratt-ru/CubiCal
 
-  pfb-clean:
+  pfb-imaging:
     versions:
-      latest:
-        package: git+https://github.com/ratt-ru/pfb-clean
+      '0.0.4':
+        package: pfb-imaging==0.0.4
+      # master:
+      #   package: git+https://github.com/ratt-ru/pfb-imaging
 
   ddfacet:
     assign: