feat: Publish: add argument --skip-existing to avoid overwrite (#2383)

news/2362.feature.md

@@ -0,0 +1 @@
+Add `--skip-existing` to `pdm publish` to ignore the uploading error if the package already exists.

src/pdm/cli/commands/publish/__init__.py

@@ -4,13 +4,6 @@
 import os
 from typing import TYPE_CHECKING
 
-from rich.progress import (
-    BarColumn,
-    DownloadColumn,
-    TimeRemainingColumn,
-    TransferSpeedColumn,
-)
-
 from pdm.cli.commands import build
 from pdm.cli.commands.base import BaseCommand
 from pdm.cli.commands.publish.package import PackageFile
@@ -69,6 +62,11 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
             dest="build",
             help="Don't build the package before publishing",
         )
+        parser.add_argument(
+            "--skip-existing",
+            action="store_true",
+            help="Skip uploading files that already exist. This may not work with some repository implementations.",
+        )
         group = parser.add_mutually_exclusive_group()
         group.add_argument(
             "--no-very-ssl", action="store_false", dest="verify_ssl", help="Disable SSL verification", default=None
@@ -89,6 +87,26 @@ def _make_package(filename: str, signatures: dict[str, str], options: argparse.N
             p.sign(options.identity)
         return p
 
+    @staticmethod
+    def _skip_upload(response: Response) -> bool:
+        status = response.status_code
+        reason = response.reason.lower()
+        text = response.text.lower()
+
+        # Borrowed from https://github.com/pypa/twine/blob/main/twine/commands/upload.py#L149
+        return (
+            # pypiserver (https://pypi.org/project/pypiserver)
+            status == 409
+            # PyPI / TestPyPI / GCP Artifact Registry
+            or (status == 400 and any("already exist" in x for x in [reason, text]))
+            # Nexus Repository OSS (https://www.sonatype.com/nexus-repository-oss)
+            or (status == 400 and any("updating asset" in x for x in [reason, text]))
+            # Artifactory (https://jfrog.com/artifactory/)
+            or (status == 403 and "overwrite artifact" in text)
+            # Gitlab Enterprise Edition (https://about.gitlab.com)
+            or (status == 400 and "already been taken" in text)
+        )
+
     @staticmethod
     def _check_response(response: Response) -> None:
         import requests
@@ -146,26 +164,23 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
 
         repository = self.get_repository(project, options)
         uploaded: list[PackageFile] = []
-        with project.core.ui.make_progress(
-            " [progress.percentage]{task.percentage:>3.0f}%",
-            BarColumn(),
-            DownloadColumn(),
-            "•",
-            TimeRemainingColumn(
-                compact=True,
-                elapsed_when_finished=True,
-            ),
-            "•",
-            TransferSpeedColumn(),
-        ) as progress, project.core.ui.logging("publish"):
+        with project.core.ui.logging("publish"):
             packages = sorted(
                 (self._make_package(p, signatures, options) for p in package_files),
                 # Upload wheels first if they exist.
                 key=lambda p: not p.base_filename.endswith(".whl"),
             )
             for package in packages:
-                resp = repository.upload(package, progress)
+                resp = repository.upload(package)
                 logger.debug("Response from %s:\n%s %s", resp.url, resp.status_code, resp.reason)
+
+                if options.skip_existing and self._skip_upload(resp):
+                    project.core.ui.echo(
+                        f"Skipping {package.base_filename} because it appears to already exist",
+                        style="warning",
+                        err=True,
+                    )
+                    continue
                 self._check_response(resp)
                 uploaded.append(package)
 

src/pdm/cli/commands/publish/repository.py

@@ -6,7 +6,7 @@
 from typing import TYPE_CHECKING, Any, Iterable
 from urllib.parse import urlparse, urlunparse
 
-import rich.progress
+from rich.progress import BarColumn, DownloadColumn, TimeRemainingColumn, TransferSpeedColumn
 
 from pdm import termui
 from pdm.cli.commands.publish.package import PackageFile
@@ -138,7 +138,7 @@ def get_release_urls(self, packages: list[PackageFile]) -> Iterable[str]:
             return set()
         return {f"{base}project/{package.metadata['name']}/{package.metadata['version']}/" for package in packages}
 
-    def upload(self, package: PackageFile, progress: rich.progress.Progress) -> Response:
+    def upload(self, package: PackageFile) -> Response:
         import requests_toolbelt
 
         payload = package.metadata_dict
@@ -149,24 +149,35 @@ def upload(self, package: PackageFile, progress: rich.progress.Progress) -> Resp
             }
         )
         field_parts = self._convert_to_list_of_tuples(payload)
-
-        progress.live.console.print(f"Uploading [success]{package.base_filename}")
-
-        with open(package.filename, "rb") as fp:
-            field_parts.append(("content", (package.base_filename, fp, "application/octet-stream")))
-
-            def on_upload(monitor: requests_toolbelt.MultipartEncoderMonitor) -> None:
-                progress.update(job, completed=monitor.bytes_read)
-
-            monitor = requests_toolbelt.MultipartEncoderMonitor.from_fields(field_parts, callback=on_upload)
-            job = progress.add_task("", total=monitor.len)
-            resp = self.session.post(
-                self.url,
-                data=monitor,
-                headers={"Content-Type": monitor.content_type},
-                allow_redirects=False,
-            )
-            if resp.status_code < 400 and self._credentials_to_save is not None:
-                self._save_credentials(*self._credentials_to_save)
-                self._credentials_to_save = None
-            return resp
+        with self.ui.make_progress(
+            " [progress.percentage]{task.percentage:>3.0f}%",
+            BarColumn(),
+            DownloadColumn(),
+            "•",
+            TimeRemainingColumn(
+                compact=True,
+                elapsed_when_finished=True,
+            ),
+            "•",
+            TransferSpeedColumn(),
+        ) as progress:
+            progress.console.print(f"Uploading [success]{package.base_filename}")
+
+            with open(package.filename, "rb") as fp:
+                field_parts.append(("content", (package.base_filename, fp, "application/octet-stream")))
+
+                def on_upload(monitor: requests_toolbelt.MultipartEncoderMonitor) -> None:
+                    progress.update(job, completed=monitor.bytes_read)
+
+                monitor = requests_toolbelt.MultipartEncoderMonitor.from_fields(field_parts, callback=on_upload)
+                job = progress.add_task("", total=monitor.len)
+                resp = self.session.post(
+                    self.url,
+                    data=monitor,
+                    headers={"Content-Type": monitor.content_type},
+                    allow_redirects=False,
+                )
+                if resp.status_code < 400 and self._credentials_to_save is not None:
+                    self._save_credentials(*self._credentials_to_save)
+                    self._credentials_to_save = None
+                return resp

src/pdm/cli/completions/pdm.bash

@@ -85,7 +85,7 @@ _pdm_a919b69078acdf0a_complete()
             ;;
 
             (publish)
-            opts="--ca-certs --comment --help --identity --no-build --no-very-ssl --password --project --quiet --repository --sign --skip --username --verbose"
+            opts="--ca-certs --comment --help --identity --no-build --no-very-ssl --password --project --quiet --repository --sign --skip --skip-existing --username --verbose"
             ;;
 
             (remove)

src/pdm/cli/completions/pdm.fish

@@ -239,7 +239,7 @@ complete -c pdm -A -n '__fish_seen_subcommand_from lock' -l quiet -d 'Suppress o
 complete -c pdm -A -n '__fish_seen_subcommand_from lock' -l refresh -d 'Don\'t update pinned versions, only refresh the lock file'
 complete -c pdm -A -n '__fish_seen_subcommand_from lock' -l skip -d 'Skip some tasks and/or hooks by their comma-separated names. Can be supplied multiple times. Use ":all" to skip all hooks. Use ":pre" and ":post" to skip all pre or post hooks.'
 complete -c pdm -A -n '__fish_seen_subcommand_from lock' -l static-urls -d '[DEPRECATED] Store static file URLs in the lockfile'
-complete -c pdm -A -n '__fish_seen_subcommand_from lock' -l strategy -d 'Specify lock strategy(cross_platform,static_urls,direct_minimal_versions). Add \'no_\' prefix to disable. Support given multiple times or split by comma.'
+complete -c pdm -A -n '__fish_seen_subcommand_from lock' -l strategy -d 'Specify lock strategy (cross_platform, static_urls, direct_minimal_versions). Add \'no_\' prefix to disable. Can be supplied multiple times or split by comma.'
 complete -c pdm -A -n '__fish_seen_subcommand_from lock' -l verbose -d 'Use `-v` for detailed output and `-vv` for more detailed'
 
 # plugin
@@ -294,6 +294,7 @@ complete -c pdm -A -n '__fish_seen_subcommand_from publish' -l quiet -d 'Suppres
 complete -c pdm -A -n '__fish_seen_subcommand_from publish' -l repository -d 'The repository name or url to publish the package to [env var: PDM_PUBLISH_REPO]'
 complete -c pdm -A -n '__fish_seen_subcommand_from publish' -l sign -d 'Upload the package with PGP signature'
 complete -c pdm -A -n '__fish_seen_subcommand_from publish' -l skip -d 'Skip some tasks and/or hooks by their comma-separated names. Can be supplied multiple times. Use ":all" to skip all hooks. Use ":pre" and ":post" to skip all pre or post hooks.'
+complete -c pdm -A -n '__fish_seen_subcommand_from publish' -l skip-existing -d 'Skip uploading files that already exist. This may not work with some repository implementations.'
 complete -c pdm -A -n '__fish_seen_subcommand_from publish' -l username -d 'The username to access the repository [env var: PDM_PUBLISH_USERNAME]'
 complete -c pdm -A -n '__fish_seen_subcommand_from publish' -l verbose -d 'Use `-v` for detailed output and `-vv` for more detailed'
 
@@ -465,7 +466,7 @@ complete -c pdm -A -n '__fish_seen_subcommand_from venv' -l python -d 'Show the
 # venv subcommands
 set -l venv_subcommands activate create list purge remove
 # venv activate
-complete -c pdm -f -n '__fish_seen_subcommand_from venv; and not __fish_seen_subcommand_from $venv_subcommands' -a activate -d 'Activate the virtualenv with the given name'
+complete -c pdm -f -n '__fish_seen_subcommand_from venv; and not __fish_seen_subcommand_from $venv_subcommands' -a activate -d 'Print the command to activate the virtualenv with the given name'
 complete -c pdm -A -n '__fish_seen_subcommand_from venv; and __fish_seen_subcommand_from activate' -l help -d 'Show this help message and exit.'
 complete -c pdm -A -n '__fish_seen_subcommand_from venv; and __fish_seen_subcommand_from activate' -l quiet -d 'Suppress output'
 complete -c pdm -A -n '__fish_seen_subcommand_from venv; and __fish_seen_subcommand_from activate' -l verbose -d 'Use `-v` for detailed output and `-vv` for more detailed'

src/pdm/cli/completions/pdm.ps1

@@ -361,7 +361,10 @@ function TabExpansion($line, $lastWord) {
             "publish" {
                 $completer.AddOpts(
                     @(
-                        [Option]::new(@("-r", "--repository", "-u", "--username", "-P", "--password", "-S", "--sign", "-i", "--identity", "-c", "--comment", "--no-build", "--ca-certs", "--no-verify-ssl")),
+                        [Option]::new(@(
+                            "-r", "--repository", "-u", "--username", "-P", "--password", "-S", "--sign", "-i", "--identity", "-c", "--comment",
+                            "--no-build", "--ca-certs", "--no-verify-ssl", "--skip-existing"
+                        )),
                         $skipOption,
                         $projectOption
                     ))

src/pdm/cli/completions/pdm.zsh

@@ -313,6 +313,7 @@ _pdm() {
         "--no-verify-ssl[Disable SSL verification]"
         "--ca-certs[The path to a PEM-encoded Certificate Authority bundle to use for publish server validation]:cacerts:_files"
         "--no-build[Don't build the package before publishing]"
+        "--skip-existing[Skip uploading files that already exist. This may not work with some repository implementations.]"
       )
       ;;
     remove)

tests/cli/conftest.py

@@ -57,7 +57,7 @@ def post(url, *, data, **kwargs):
 def uploaded(mocker: MockerFixture):
     packages = []
 
-    def fake_upload(package, progress):
+    def fake_upload(package):
         packages.append(package)
         resp = requests.Response()
         resp.status_code = 200