Filter tarfile entry mode bits

src/venvstacks/pack_venv.py

@@ -267,7 +267,7 @@ def create_archive(
             def report_progress(_: Any) -> None:
                 pass
         else:
-            progress_bar = ProgressBar()
+            progress_bar = _ProgressBar()
             progress_bar.show(0.0)
             num_archive_entries = 0
             total_entries_to_archive = sum(1 for __ in env_path.rglob("*"))
@@ -379,6 +379,15 @@ def _process_archive_entry(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
         tarinfo.uname = tarinfo.gname = "root"
         if _clamp_mtime is not None:
             tarinfo.mtime = _clamp_mtime(tarinfo.mtime)
+        # Ensure permissions are compatible with `tar_filter` extraction
+        # Layered environments will still need to be fully trusted when
+        # unpacking them (due to the external symlinks to the base runtime)
+        mode = tarinfo.mode
+        if mode is not None:
+            # Apply the same mode filtering as tarfile.tar_filter in 3.12+
+            # https://docs.python.org/3.13/library/tarfile.html#tarfile.tar_filter
+            # Clears high bits (e.g. setuid/)setgid, and the group/other write bits
+            tarinfo.mode = mode & 0o755
         # Report progress if requested
         if progress_callback is not None:
             progress_callback(tarinfo.name)
@@ -405,12 +414,12 @@ def _process_archive_entry(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
 
 if _WINDOWS_BUILD:
 
-    def set_mtime(fspath: str, mtime: int | float) -> None:
+    def _set_mtime(fspath: str, mtime: int | float) -> None:
         # There's no `follow_symlinks` option available on Windows
         os.utime(fspath, (mtime, mtime))
 else:
 
-    def set_mtime(fspath: str, mtime: int | float) -> None:
+    def _set_mtime(fspath: str, mtime: int | float) -> None:
         os.utime(fspath, (mtime, mtime), follow_symlinks=False)
 
 
@@ -476,7 +485,7 @@ def _add_zip_entry(fspath: str, arcname: str) -> None:
             fs_mtime = os.lstat(fspath).st_mtime
             zip_entry_mtime = adjust_mtime(min(fs_mtime, max_mtime))
             if zip_entry_mtime != fs_mtime:
-                set_mtime(fspath, zip_entry_mtime)
+                _set_mtime(fspath, zip_entry_mtime)
             zf.write(fspath, arcname)
 
         arcname = os.path.normpath(base_dir)
@@ -516,22 +525,25 @@ def _add_zip_entry(fspath: str, arcname: str) -> None:
     # Everywhere else, create XZ compressed tar archives
     _make_archive = _make_tar_archive
 
-# Basic progress bar support, taken from my SO answer at
+# Basic progress bar support, taken from ncoghlan's SO answer at
 # https://stackoverflow.com/questions/3160699/python-progress-bar/78590319#78590319
-# (since the code originated with me, it isn't subject to Stack Overflow's CC-BY-SA terms)
+# (since the code originated with her, it isn't subject to Stack Overflow's CC-BY-SA terms)
 #
-# I originally skipped this, but archiving pytorch (and similarly large AI/ML libraries)
-# takes a long time, so you really need some assurance that progress is being made.
+# Archiving pytorch (and similarly large AI/ML libraries) takes a long time,
+# so you really need some assurance that progress is being made.
 #
 # If compression times are a significant problem, it would be worth moving in the same
 # direction as conda-pack did, and implementing support for parallel compression (the
 # compression libraries all drop the GIL when compressing data chunks, so this approach
 # scales effectively up to the number of available CPUs)
-ProgressSummary = tuple[int, str]
-ProgressReport = tuple[str, ProgressSummary]
+#
+# See https://github.com/lmstudio-ai/venvstacks/issues/4
+
+_ProgressSummary = tuple[int, str]
+_ProgressReport = tuple[str, _ProgressSummary]
 
 
-class ProgressBar:
+class _ProgressBar:
     """Display & update a progress bar"""
 
     TEXT_ABORTING = "Aborting..."
@@ -541,7 +553,7 @@ class ProgressBar:
     bar_length: int
     stream: TextIO
     _last_displayed_text: str | None
-    _last_displayed_summary: ProgressSummary | None
+    _last_displayed_summary: _ProgressSummary | None
 
     def __init__(self, bar_length: int = 25, stream: TextIO = sys.stdout) -> None:
         self.bar_length = bar_length
@@ -554,7 +566,7 @@ def reset(self) -> None:
         self._last_displayed_text = None
         self._last_displayed_summary = None
 
-    def _format_progress(self, progress: float, aborting: bool) -> ProgressReport:
+    def _format_progress(self, progress: float, aborting: bool) -> _ProgressReport:
         """Internal helper that also reports the number of completed increments"""
         bar_length = self.bar_length
         progress = float(progress)

src/venvstacks/stacks.py

@@ -882,11 +882,9 @@ def _pdm_python_install(target_path: Path, request: str) -> Path | None:
         with tempfile.NamedTemporaryFile() as tf:
             tf.close()
             original_filename = download(python_file, tf.name, env.session)
-            # TODO: use "tar_filter" here instead of "fully_trusted"
-            # Currently blocked on Python 3.11 producing different results
-            # if Python 3.12+ enables a filter that actually makes any changes
-            # https://github.com/lmstudio-ai/venvstacks/issues/23
-            with default_tarfile_filter("fully_trusted"):
+            # Use "tar_filter" if stdlib tar extraction filters are available
+            # (they were only added in Python 3.12, so no filtering on 3.11)
+            with default_tarfile_filter("tar_filter"):
                 install_file(tf.name, destination, original_filename)
     if interpreter.exists():
         # Installation successful, return the path to the installation folder

tests/expected-output-config.toml

@@ -33,5 +33,4 @@ UV_EXCLUDE_NEWER="2024-10-15 00:00:00+00:00"
 # Metadata updates can also be requested when the
 # launch module content in the sample project changes
 
-# Last requested update: launch module autoformatting
-
+# Last requested update: tar archive mode filtering

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json

@@ -3,7 +3,7 @@
   "app_launch_module_hash": "sha256/344b1be70920bd9635ce38fa14fca86b531ce2e334f54968321469c5fbb5b608",
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "dc50b3c60b5a3b0f8e9b4c8c746129f4f7b2bef94b65935026643c814e40fa74"
+    "sha256": "737a6b60e3be2fed00a8d1385973bf9526b25250a8fa2dd65d288925fcfa341b"
   },
   "archive_name": "app-scipy-client.tar.xz",
   "archive_size": 1424,

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json

@@ -3,7 +3,7 @@
   "app_launch_module_hash": "sha256:6278ff255372146d752518ffdf49f3432667d7c93997ed980b3676fdc75406ee",
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "bb52646bffa6cebadc37f8f0d7463b8b581c96116c89b4b77135c72b0b4099c4"
+    "sha256": "2d8d19d109288cdc91a643e7b3fd9bcbc9fa38d40bbafe631074878245cb038f"
   },
   "archive_name": "app-scipy-import.tar.xz",
   "archive_size": 1336,

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-sklearn-import.json

@@ -3,7 +3,7 @@
   "app_launch_module_hash": "sha256:b6de2b52093004bcc39df16d115929021937f77b5feda45d090b06116ea34f49",
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "20d8aa01d2a0c36b2cfc00c7cc1798025349add5eddb9f053b06427bc43ae928"
+    "sha256": "dd778c5f82c1c1ee62765d876a063dba4b374b6f4e06842104c8945e66663748"
   },
   "archive_name": "app-sklearn-import.tar.xz",
   "archive_size": 1332,

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.11.json

@@ -1,10 +1,10 @@
 {
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "ab57f3a6fd70e1855773382bda7ddb3e4c2a520cd17ed7549e1983ca3410f47a"
+    "sha256": "552dbdf8a29c65ae31bc4786a6e754b146ed8965e5cf4d360f29b41d2f880dbb"
   },
   "archive_name": "[email protected]",
-  "archive_size": 29727160,
+  "archive_size": 29721908,
   "install_target": "[email protected]",
   "layer_name": "[email protected]",
   "lock_version": 1,

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/cpython@3.12.json

@@ -1,10 +1,10 @@
 {
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "bf3ad4fd54b7e0d5ec6f1df88838f3a0d809d4b4ea07ee37b6e073a96c2ac995"
+    "sha256": "d41dc43405cd5c1e96ce81396075f25bd1aec731827de6c1d5ead264d45f02b0"
   },
   "archive_name": "[email protected]",
-  "archive_size": 42714592,
+  "archive_size": 42726560,
   "install_target": "[email protected]",
   "layer_name": "[email protected]",
   "lock_version": 1,

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-http-client.json

@@ -1,10 +1,10 @@
 {
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "66c677c7e2d823df72107f50ac0a3ce107d532947c9b24464561f9d74801851f"
+    "sha256": "a68b0063f9149b0475faacf7bc6987db30e6ad1171438d6eac6a8d1c22cd8db4"
   },
   "archive_name": "framework-http-client.tar.xz",
-  "archive_size": 362600,
+  "archive_size": 362568,
   "install_target": "framework-http-client",
   "layer_name": "framework-http-client",
   "lock_version": 1,

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json

@@ -1,10 +1,10 @@
 {
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "b9cca34dcca0233adc912ab1cd637dc65e9bb2969dd74225a7a1c1951b61fd4d"
+    "sha256": "bd8bf5b409d03d78e878f3a5d7e350daea0e94cc983a6146997a1bd0d1834bf5"
   },
   "archive_name": "framework-scipy.tar.xz",
-  "archive_size": 23961856,
+  "archive_size": 23956040,
   "install_target": "framework-scipy",
   "layer_name": "framework-scipy",
   "lock_version": 1,

tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-sklearn.json

@@ -1,10 +1,10 @@
 {
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "babdbccb6c127a897aadce388b6bd9afa7a23c391548b024cf7b0aa7a3a2bfb5"
+    "sha256": "0faa4c3e0709d4ebb19e8f9a81c36d0d422f6ef77d31af7496738f9bb937da4b"
   },
   "archive_name": "framework-sklearn.tar.xz",
-  "archive_size": 30377392,
+  "archive_size": 30370864,
   "install_target": "framework-sklearn",
   "layer_name": "framework-sklearn",
   "lock_version": 1,

tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json

@@ -6,7 +6,7 @@
         "app_launch_module_hash": "sha256:6278ff255372146d752518ffdf49f3432667d7c93997ed980b3676fdc75406ee",
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "bb52646bffa6cebadc37f8f0d7463b8b581c96116c89b4b77135c72b0b4099c4"
+          "sha256": "2d8d19d109288cdc91a643e7b3fd9bcbc9fa38d40bbafe631074878245cb038f"
         },
         "archive_name": "app-scipy-import.tar.xz",
         "archive_size": 1336,
@@ -26,7 +26,7 @@
         "app_launch_module_hash": "sha256/344b1be70920bd9635ce38fa14fca86b531ce2e334f54968321469c5fbb5b608",
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "dc50b3c60b5a3b0f8e9b4c8c746129f4f7b2bef94b65935026643c814e40fa74"
+          "sha256": "737a6b60e3be2fed00a8d1385973bf9526b25250a8fa2dd65d288925fcfa341b"
         },
         "archive_name": "app-scipy-client.tar.xz",
         "archive_size": 1424,
@@ -47,7 +47,7 @@
         "app_launch_module_hash": "sha256:b6de2b52093004bcc39df16d115929021937f77b5feda45d090b06116ea34f49",
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "20d8aa01d2a0c36b2cfc00c7cc1798025349add5eddb9f053b06427bc43ae928"
+          "sha256": "dd778c5f82c1c1ee62765d876a063dba4b374b6f4e06842104c8945e66663748"
         },
         "archive_name": "app-sklearn-import.tar.xz",
         "archive_size": 1332,
@@ -67,10 +67,10 @@
       {
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "b9cca34dcca0233adc912ab1cd637dc65e9bb2969dd74225a7a1c1951b61fd4d"
+          "sha256": "bd8bf5b409d03d78e878f3a5d7e350daea0e94cc983a6146997a1bd0d1834bf5"
         },
         "archive_name": "framework-scipy.tar.xz",
-        "archive_size": 23961856,
+        "archive_size": 23956040,
         "install_target": "framework-scipy",
         "layer_name": "framework-scipy",
         "lock_version": 1,
@@ -82,10 +82,10 @@
       {
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "babdbccb6c127a897aadce388b6bd9afa7a23c391548b024cf7b0aa7a3a2bfb5"
+          "sha256": "0faa4c3e0709d4ebb19e8f9a81c36d0d422f6ef77d31af7496738f9bb937da4b"
         },
         "archive_name": "framework-sklearn.tar.xz",
-        "archive_size": 30377392,
+        "archive_size": 30370864,
         "install_target": "framework-sklearn",
         "layer_name": "framework-sklearn",
         "lock_version": 1,
@@ -97,10 +97,10 @@
       {
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "66c677c7e2d823df72107f50ac0a3ce107d532947c9b24464561f9d74801851f"
+          "sha256": "a68b0063f9149b0475faacf7bc6987db30e6ad1171438d6eac6a8d1c22cd8db4"
         },
         "archive_name": "framework-http-client.tar.xz",
-        "archive_size": 362600,
+        "archive_size": 362568,
         "install_target": "framework-http-client",
         "layer_name": "framework-http-client",
         "lock_version": 1,
@@ -114,10 +114,10 @@
       {
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "ab57f3a6fd70e1855773382bda7ddb3e4c2a520cd17ed7549e1983ca3410f47a"
+          "sha256": "552dbdf8a29c65ae31bc4786a6e754b146ed8965e5cf4d360f29b41d2f880dbb"
         },
         "archive_name": "[email protected]",
-        "archive_size": 29727160,
+        "archive_size": 29721908,
         "install_target": "[email protected]",
         "layer_name": "[email protected]",
         "lock_version": 1,
@@ -129,10 +129,10 @@
       {
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "bf3ad4fd54b7e0d5ec6f1df88838f3a0d809d4b4ea07ee37b6e073a96c2ac995"
+          "sha256": "d41dc43405cd5c1e96ce81396075f25bd1aec731827de6c1d5ead264d45f02b0"
         },
         "archive_name": "[email protected]",
-        "archive_size": 42714592,
+        "archive_size": 42726560,
         "install_target": "[email protected]",
         "layer_name": "[email protected]",
         "lock_version": 1,

tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.11.json

@@ -1,10 +1,10 @@
 {
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "bca47c4596578940d12523c4cdea9c54b80a4f3ff1bc74171320616d898ea11f"
+    "sha256": "671fbe66582efd0a4cfc0c837b303ef1e267b8929ff0b87aec648471f2c6392c"
   },
   "archive_name": "[email protected]",
-  "archive_size": 14960344,
+  "archive_size": 14965684,
   "install_target": "[email protected]",
   "layer_name": "[email protected]",
   "lock_version": 1,

tests/sample_project/expected_manifests/macosx_arm64/env_metadata/cpython@3.12.json

@@ -1,10 +1,10 @@
 {
   "archive_build": 1,
   "archive_hashes": {
-    "sha256": "0a6c9c612d2efee08d7f0d9b51ff2403f55c521826e7b135b1ab022911dbdf08"
+    "sha256": "f3e6749b5c07afde8468ca03457e1ea4bdda2adab8e510f1d353f21ee0f71981"
   },
   "archive_name": "[email protected]",
-  "archive_size": 13600552,
+  "archive_size": 13599424,
   "install_target": "[email protected]",
   "layer_name": "[email protected]",
   "lock_version": 1,

tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json

@@ -94,10 +94,10 @@
       {
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "bca47c4596578940d12523c4cdea9c54b80a4f3ff1bc74171320616d898ea11f"
+          "sha256": "671fbe66582efd0a4cfc0c837b303ef1e267b8929ff0b87aec648471f2c6392c"
         },
         "archive_name": "[email protected]",
-        "archive_size": 14960344,
+        "archive_size": 14965684,
         "install_target": "[email protected]",
         "layer_name": "[email protected]",
         "lock_version": 1,
@@ -109,10 +109,10 @@
       {
         "archive_build": 1,
         "archive_hashes": {
-          "sha256": "0a6c9c612d2efee08d7f0d9b51ff2403f55c521826e7b135b1ab022911dbdf08"
+          "sha256": "f3e6749b5c07afde8468ca03457e1ea4bdda2adab8e510f1d353f21ee0f71981"
         },
         "archive_name": "[email protected]",
-        "archive_size": 13600552,
+        "archive_size": 13599424,
         "install_target": "[email protected]",
         "layer_name": "[email protected]",
         "lock_version": 1,