feat: Add support for .conda format (#981)

fix: Ignore Zstandard-packed repodata artifact fix: Ignore src_cache for container-built packages Signed-off-by: Marcel Bargull <[email protected]>
bioconda · Apr 16, 2024 · 125c9ea · 125c9ea
1 parent 17cbc7f
commit 125c9ea
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 38 deletions.
diff --git a/bioconda_utils/artifacts.py b/bioconda_utils/artifacts.py
@@ -74,15 +74,16 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe
                     platform_patterns.append("noarch")
 
                 for platform_pattern in platform_patterns:
-                    pattern = f"{tmpdir}/*/packages/{platform_pattern}/*.tar.bz2"
-                    logger.info(f"Checking for packages at {pattern}.")
-                    for pkg in glob.glob(pattern):
-                        if dryrun:
-                            logger.info(f"Would upload {pkg} to anaconda.org.")
-                        else:
-                            logger.info(f"Uploading {pkg} to anaconda.org.")
-                            # upload the package
-                            success.append(anaconda_upload(pkg, label=label))
+                    for ext in (".tar.bz2", ".conda"):
+                        pattern = f"{tmpdir}/*/packages/{platform_pattern}/*{ext}"
+                        logger.info(f"Checking for packages at {pattern}.")
+                        for pkg in glob.glob(pattern):
+                            if dryrun:
+                                logger.info(f"Would upload {pkg} to anaconda.org.")
+                            else:
+                                logger.info(f"Uploading {pkg} to anaconda.org.")
+                                # upload the package
+                                success.append(anaconda_upload(pkg, label=label))
 
                 if mulled_upload_target:
                     quay_login = os.environ['QUAY_LOGIN']
@@ -213,7 +214,7 @@ def get_circleci_artifacts(check_run, platform):
                 else:
                     for artifact in json_job["items"]:
                         artifact_url = artifact["url"]
-                        if artifact_url.endswith(".html") or artifact_url.endswith(".json") or artifact_url.endswith(".json.bz2"):
+                        if artifact_url.endswith((".html", ".json", ".json.bz2", ".json.zst")):
                             continue
                         else:
                             yield artifact_url

diff --git a/bioconda_utils/bioconda_utils-requirements.txt b/bioconda_utils/bioconda_utils-requirements.txt
@@ -31,6 +31,7 @@ anaconda-client=1.12.*        # anaconda_upload
 galaxy-tool-util=24.*         # mulled test and container build
 involucro=1.1.*               # mulled test and container build
 skopeo=1.15.*                 # docker upload
+findutils                     # find/xargs copy built packages from container
 git=2.*                       # well - git
 
 # hosters - special regex not supported by RE

diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py
@@ -238,23 +238,25 @@ def duplicates(config,
         check_fields += ['build']
 
     def remove_package(spec):
-        fn = '{}-{}-{}.tar.bz2'.format(*spec)
-        name, version = spec[:2]
-        subcmd = [
-            'remove', '-f',
-            '{channel}/{name}/{version}/{fn}'.format(
-                name=name, version=version, fn=fn, channel=our_channel
-            )
-        ]
-        if dryrun:
-            logger.info(" ".join([utils.bin_for('anaconda')] + subcmd))
-        else:
-            token = os.environ.get('ANACONDA_TOKEN')
-            if token is None:
-                token = []
+        for ext in (".tar.bz2", ".conda"):
+            name, version = spec[:2]
+            dist = '{}-{}-{}'.format(*spec)
+            fn = f"{dist}{ext}"
+            subcmd = [
+                'remove', '-f',
+                '{channel}/{name}/{version}/{fn}'.format(
+                    name=name, version=version, fn=fn, channel=our_channel
+                )
+            ]
+            if dryrun:
+                logger.info(" ".join([utils.bin_for('anaconda')] + subcmd))
             else:
-                token = ['-t', token]
-            logger.info(utils.run([utils.bin_for('anaconda')] + token + subcmd, mask=[token]).stdout)
+                token = os.environ.get('ANACONDA_TOKEN')
+                if token is None:
+                    token = []
+                else:
+                    token = ['-t', token]
+                logger.info(utils.run([utils.bin_for('anaconda')] + token + subcmd, mask=[token]).stdout)
 
     # packages in our channel
     repodata = utils.RepoData()

diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py
@@ -83,7 +83,7 @@
 # filled in here.
 #
 BUILD_SCRIPT_TEMPLATE = \
-"""
+r"""
 #!/bin/bash
 set -eo pipefail
 
@@ -111,9 +111,12 @@
 conda-build -c file://{self.container_staging} {self.conda_build_args} {self.container_recipe}/meta.yaml 2>&1
 
 # copy all built packages to the staging area
-cp /opt/conda/conda-bld/*/*.tar.bz2 {self.container_staging}/{arch}
+find /opt/conda/conda-bld \
+  -name src_cache -prune -o \
+  -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print0 |
+  xargs -0 -- cp -t '{self.container_staging}/{arch}' --
 #While technically better, this is slower and more prone to breaking
-#cp `conda-build {self.conda_build_args} {self.container_recipe}/meta.yaml --output | grep tar.bz2` {self.container_staging}/{arch}
+#cp `conda-build {self.conda_build_args} {self.container_recipe}/meta.yaml --output | grep -e '\.tar\.bz2$' -e '\.conda$')` {self.container_staging}/{arch}
 conda index {self.container_staging}
 # Ensure permissions are correct on the host.
 HOST_USER={self.user_info[uid]}

diff --git a/bioconda_utils/pkg_test.py b/bioconda_utils/pkg_test.py
@@ -70,12 +70,17 @@ def get_image_name(path):
     ----------
 
     path : str
-        Path to .tar.by2 package build by conda-build
+        Path to .tar.bz2 or .conda package build by conda-build
 
     """
-    assert path.endswith('.tar.bz2')
-
-    pkg = os.path.basename(path).replace('.tar.bz2', '')
+    if path.endswith(".tar.bz2"):
+        ext = ".tar.bz2"
+    elif path.endswith(".conda"):
+        ext = ".conda"
+    else:
+        raise ValueError()
+
+    pkg = os.path.basename(path).removesuffix(ext)
     toks = pkg.split('-')
     build_string = toks[-1]
     version = toks[-2]
@@ -100,7 +105,7 @@ def test_package(
     Parameters
     ----------
     path : str
-        Path to a .tar.bz2 package built by conda-build
+        Path to a .tar.bz2 or .conda package built by conda-build
 
     name_override : str
         Passed as the --name-override argument to mulled-build
@@ -125,7 +130,7 @@ def test_package(
         If True, enable live logging during the build process
     """
 
-    assert path.endswith('.tar.bz2'), "Unrecognized path {0}".format(path)
+    assert path.endswith((".tar.bz2", ".conda")), "Unrecognized path {0}".format(path)
     # assert os.path.exists(path), '{0} does not exist'.format(path)
 
     conda_bld_dir = os.path.abspath(os.path.dirname(os.path.dirname(path)))

diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py
@@ -1492,13 +1492,16 @@ def _load_channel_dataframe(self):
         def to_dataframe(json_data, meta_data):
             channel, platform = meta_data
             repo = json.loads(json_data)
-            df = pd.DataFrame.from_dict(repo['packages'], 'index',
-                                        columns=self._load_columns)
+            subdir = repo["info"]["subdir"]
+            packages = repo["packages"]
+            packages.update(repo.get("packages.conda", {}))
+
+            df = pd.DataFrame.from_dict(packages, 'index', columns=self._load_columns)
             # Ensure that version is always a string.
             df['version'] = df['version'].astype(str)
             df['channel'] = channel
             df['platform'] = platform
-            df['subdir'] = repo['info']['subdir']
+            df['subdir'] = subdir
             return df
 
         if urls: