Merge pull request #97 from mwestphall/feature/release-series

Feature: Symlink Latest Release Series

distrepos/__main__.py

@@ -37,14 +37,15 @@
     Options,
     Tag,
     ActionType,
+    ReleaseSeries,
     format_tag,
     format_mirror,
     get_args,
     parse_config,
 )
 from distrepos.tag_run import run_one_tag
 from distrepos.mirror_run import update_mirrors_for_tag
-from distrepos.link_static import link_static_data
+from distrepos.symlink_utils import link_static_data, link_latest_release
 from distrepos.util import lock_context, check_rsync, log_ml, run_with_log
 from distrepos.tarball_sync import update_tarball_dirs
 
@@ -226,6 +227,24 @@ def link_static(options: Options) -> int:
         return ERR_FAILURES
 
 
+def link_release(options: Options, release_series: t.List[ReleaseSeries]) -> int:
+    """
+    Create symlinks for the latest "osg-release" rpm in the "release" repo of a release series
+    """
+    _log.info("Updating symlinks to latest osg-release rpm")
+    try:
+        ok, err = link_latest_release(options, release_series)
+        if ok:
+            _log.info("latest-release symlinks updated successfully")
+            return 0
+        else:
+            _log.warning(f"Unable to update latest-release symlinks: {err}")
+            return ERR_FAILURES
+    except Exception as e:
+        _log.exception(f"Unexpected error updating latest-release symlinks: {e}")
+        return ERR_FAILURES
+
+
 def sync_tarballs(options: Options) -> int:
     """
     Sync client tarballs from an upstream rsync server to repo
@@ -261,7 +280,7 @@ def main(argv: t.Optional[t.List[str]] = None) -> int:
     config = ConfigParser(interpolation=ExtendedInterpolation())
     config.read(config_path)
 
-    options, taglist = parse_config(args, config)
+    options, series, taglist = parse_config(args, config)
 
     if args.print_tags:
         for tag in taglist:
@@ -303,6 +322,9 @@ def main(argv: t.Optional[t.List[str]] = None) -> int:
     if ActionType.TARBALL_SYNC in args.action and not result:
         result = sync_tarballs(options)
 
+    if ActionType.LINK_RELEASE in args.action and not result:
+        result = link_release(options, series)
+
     # If all actions were successful, update the repo timestamp
     if not result:
         update_repo_timestamp(options)

distrepos/params.py

@@ -50,6 +50,15 @@ class SrcDst(t.NamedTuple):
     def __str__(self):
         return f"{self.src} -> {self.dst}"
 
+class ReleaseSeries(t.NamedTuple):
+    """
+    Parameters for a release series based on the [series] sections of the
+    config file.
+    """
+    name: str
+    dest: str
+    arches: t.List[str]
+    dvers: t.List[str]
 
 class Tag(t.NamedTuple):
     """
@@ -93,6 +102,7 @@ class ActionType(str, Enum):
     CADIST = "cadist"
     MIRROR = "mirror"
     LINK_STATIC = "link_static"
+    LINK_RELEASE = "link_release"
     TARBALL_SYNC = "tarball_sync"
 
 
@@ -307,6 +317,7 @@ def get_taglist(args: Namespace, config: ConfigParser) -> t.List[Tag]:
             "debug_rpms_subdir", fallback=arch_rpms_subdir
         ).strip("/")
         source_rpms_subdir = section["source_rpms_subdir"].strip("/")
+
         taglist.append(
             Tag(
                 name=tag_name,
@@ -323,10 +334,33 @@ def get_taglist(args: Namespace, config: ConfigParser) -> t.List[Tag]:
 
     return taglist
 
+def get_release_series(args: Namespace, config: ConfigParser) -> t.List[ReleaseSeries]:
+    """ 
+    Parse the 'series' sections of the config to return a list of ReleaseSeries objects.
+    Args:
+        args: command-line arguments parsed by argparse
+        config: ConfigParser configuration
+
+    Returns:
+        a list of ReleaseSeries objects
+    """
+
+    # Get the list of release series to which each tag might apply
+    release_series : t.List[ReleaseSeries] = []
+    for section_name, section in config.items():
+        if not section_name.startswith("series "):
+            continue
+        release_series.append(ReleaseSeries(
+            name=section_name.removeprefix("series "),
+            dest=section['dest'],
+            arches=section['arches'].split(),
+            dvers=section['dvers'].split()
+        ))
+    return release_series
 
 def parse_config(
     args: Namespace, config: ConfigParser
-) -> t.Tuple[Options, t.List[Tag]]:
+) -> t.Tuple[Options, t.List[ReleaseSeries], t.List[Tag]]:
     """
     Parse the config file and return the tag list and Options object from the parameters.
     Apply any overrides from the command-line.
@@ -351,8 +385,11 @@ def parse_config(
         raise ConfigError("No (matching) [tag ...] or [tagset ...] sections found")
 
     options = get_options(args, config)
+
+    release_series = get_release_series(args, config)
     return (
         options,
+        release_series,
         taglist,
     )
 

distrepos/symlink_utils.py

@@ -0,0 +1,115 @@
+"""
+Set of utilities for setting up symlinks within the repo webserver.
+"""
+
+import typing as t
+from pathlib import Path
+from distrepos.params import Options, ReleaseSeries
+import re
+
+
+def link_static_data(options: Options, repo_name: str = "osg") -> t.Tuple[bool, str]:
+    """
+    Utility function to create a symlink to each top-level directory under options.static_root
+    from options.dest_root
+    
+    Args:
+        options: The global options for the run
+        repo_name: The repo to link between dest_root and static_root
+
+    Returns:
+        An (ok, error message) tuple.
+
+    TODO: "osg" repo is essentially hardcoded by the default arg here, might want to specify
+          somewhere in config instead
+    """
+    if not options.static_root:
+        # no static data specified, no op
+        return True, ""
+
+    # This code assumes options.static_root is an absolute path
+    if not Path('/') in options.static_root.parents:
+        return False, f"Static data path must be absolute, got {options.static_root}"
+
+    static_src = options.static_root / repo_name
+    data_dst = options.dest_root / repo_name
+
+    if not static_src.exists():
+        return False, f"Static data path {static_src} does not exist"
+
+    if not data_dst.exists():
+        data_dst.mkdir(parents=False)
+
+
+    # clear out decayed symlinks to static_src in data_dst
+    for pth in data_dst.iterdir():
+        if pth.is_symlink() and static_src in pth.readlink().parents and not pth.readlink().exists():
+            pth.unlink()
+
+    # create missing symlinks to static_src in data_dist
+    for pth in static_src.iterdir():
+        dest = data_dst / pth.name
+        # Continue if symlink is already correct
+        if dest.is_symlink() and dest.readlink() == pth:
+            continue
+
+        if dest.is_symlink() and dest.readlink() != pth:
+            # Reassign incorrect symlinks
+            dest.unlink()
+        elif dest.exists():
+            # Fail if dest is not a symlink
+            return False, f"Expected static data symlink {dest} is not a symlink"
+
+        # Create the symlink
+        dest.symlink_to(pth.relative_to(dest.parent))
+
+    return True, ""
+
+
+RELEASE_RPM='osg-release'
+RELEASE_PATTERN = re.compile(r"-([0-9]+)\.osg")
+
+def _get_release_number(release_rpm: Path) -> int:
+    """
+    Extract the integer release number from the release rpm name. Assumes all release RPMs 
+    for a given series have the same semantic version and are only differentiated by integer 
+    release number.
+    """
+    release_match = RELEASE_PATTERN.search(release_rpm.name)
+    if not release_match:
+        return 0
+    return int(release_match[1])
+
+def link_latest_release(options: Options, release_series: t.List[ReleaseSeries]) -> t.Tuple[bool, str]:
+    """
+    For the given release series, find the latest-versioned `osg-release`
+    rpm within that series, then symlink <series>/osg-<series>-<dver>-release-latest.rpm to it
+
+    Args:
+        options: The global options for the run
+        release_series: The list of release series (eg. [23-main, 24-main] to create release symlinks for)
+
+    Returns:
+        An (ok, error message) tuple.
+    """
+
+    for series in release_series:
+        series_root = Path(options.dest_root) / series.dest
+        base_arch = series.arches[0]
+
+        for dver in series.dvers:
+            # Filter release rpms in the repo down to ones in the "primary" arch 
+            # with parse-able release numbers
+            release_rpms = [
+                rpm for rpm in (series_root / dver).rglob(f"release/{base_arch}/**/{RELEASE_RPM}*")
+                if _get_release_number(rpm) > 0
+            ]
+
+            if not release_rpms:
+                return False, f"No valid release RPMs found for series {series.name}"
+
+            release_rpms.sort(key = _get_release_number, reverse=True)
+            latest_symlink = series_root / f"osg-{series.name}-{dver}-release-latest.rpm"
+            latest_symlink.symlink_to(release_rpms[0].relative_to(latest_symlink.parent))
+
+    return True, ""

distrepos/tag_run.py

@@ -12,9 +12,10 @@
 import tempfile
 import typing as t
 from pathlib import Path
+import re
 
 from distrepos.error import DiskFullError, TagFailure
-from distrepos.params import Options, Tag
+from distrepos.params import Options, Tag, ReleaseSeries
 from distrepos.util import (
     RSYNC_NOT_FOUND,
     RSYNC_OK,
@@ -327,7 +328,6 @@ def create_compat_symlink(working_path: Path):
         raise TagFailure(f"Error {description}") from err
     _log.info("%s ok", description)
 
-
 def update_release_repos(release_path: Path, working_path: Path, previous_path: Path):
     """
     Update the published repos by moving the published dir to the 'previous' dir

docker-compose.yml

@@ -7,9 +7,11 @@ services:
     build: .
     volumes:
       - repo-data:/data
+      - static-data:/data/repo/osg/archive
     ports:
       - "8080:80"
 
 volumes:
   repo-data:
+  static-data:
 

docker/supervisor-distrepos.conf

@@ -1,5 +1,5 @@
 [program:distrepos]
-command=/bin/bash -c 'for action in link_static rsync mirror cadist tarball_sync; do /bin/distrepos --action $action; done; sleep 360'
+command=/bin/bash -c 'for action in link_static rsync mirror cadist tarball_sync link_release; do /bin/distrepos --action $action; done; sleep 360'
 autorestart=true
 
 # Log the output of distrepos to supervisord's stdout/err so k8s logging picks it up

etc/distrepos.conf

@@ -88,7 +88,7 @@ previous_root = /data/repo.previous
 logfile = /var/log/distrepos.log
 
 # The location of static repo files not managed by the distrepo script.
-static_root = /static-data/repo
+static_root = /data/repo/osg/archive/repo
 
 # The base location of mirrorlist files.
 mirror_root = /data/mirror
@@ -104,6 +104,19 @@ mirror_hosts =
 # Parent directory within dest_root for tarball client
 tarball_install = tarball-install
 
+#
+# Release series
+#
+# Each release series comprises a series of tagsets, as well as an osg-release <noarch>
+# rpm containing yum repository definitions for the rpms contained in each tagset
+
+[series 24-main]
+dest = osg/24-main
+dvers = el8 el9
+
+[series 23-main]
+dest = osg/23-main
+dvers = el8 el9
 
 #
 #